max / makenotwork
16 files changed,
+1091 insertions,
-95 deletions
| @@ -112,7 +112,7 @@ | |||
| 112 | 112 | | `clippy` | build | `cargo clippy --all-targets -- -D warnings` over every `[[test_target]]`. | | |
| 113 | 113 | | `cargo_audit` | build | `cargo audit` in each `[[test_target]]` carrying a `.cargo/audit.toml`. | | |
| 114 | 114 | | `cargo_deny` | build | `cargo deny check` in each `[[test_target]]` carrying a `deny.toml`. | | |
| 115 | - | | `migration_dry_run` | build | Migrations apply cleanly to a restored production dump. Blocks if the newest fetched dump is older than `backup_max_age_hours` (default 48) — a stale dump proves nothing about today's schema. | | |
| 115 | + | | `migration_dry_run` | build | Every configured `[[migration_check]]` database's migrations apply cleanly to a restored production dump of *that* database. Blocks if a check's newest fetched dump is older than `backup_max_age_hours` (default 48) — a stale dump proves nothing about today's schema. | | |
| 116 | 116 | | `boot_smoke` | build | The staged artifact boots in minimal no-DB mode on the build host. | | |
| 117 | 117 | | `node_health` | post-deploy | Each deployed node's unit is active (and serves 2xx if `health_url` is set). Recorded at the end of a promote as the evidence the next promote checks. | | |
| 118 | 118 | | `burn_in` | promote | The tier has held its current version for N hours. Evaluated live against the clock. | | |
| @@ -163,6 +163,71 @@ | |||
| 163 | 163 | tip. If *no* target exists, the gate fails rather than reporting a pass over | |
| 164 | 164 | zero suites. | |
| 165 | 165 | ||
| 166 | + | ### What `migration_dry_run` restores | |
| 167 | + | ||
| 168 | + | The gate used to be hardcoded to `worktree/server/migrations` against the one | |
| 169 | + | `scratch_db_url`, so it gated exactly one database. multithreaded ships its own | |
| 170 | + | migrations and applies them with `sqlx::migrate!()` at boot against its own | |
| 171 | + | database, which meant it carried the server's exposure with none of the server's | |
| 172 | + | gate: an edited already-applied migration would not fail a dry run, it would fail | |
| 173 | + | to boot in prod. (That is not hypothetical — an exorcise sweep rewrote comments | |
| 174 | + | in 29 applied server migrations in July 2026, and sqlx checksums whole files. The | |
| 175 | + | gate caught it. mt was spared by luck.) | |
| 176 | + | ||
| 177 | + | Each database now gets a check in the daemon config, paired with its own dump in | |
| 178 | + | the topology: | |
| 179 | + | ||
| 180 | + | ```toml | |
| 181 | + | # sando.toml (topology) | |
| 182 | + | [[backup]] | |
| 183 | + | name = "server" | |
| 184 | + | source = "ssh://backup-puller@alpha-west-1:2200/makenotwork/latest.sql.gz" | |
| 185 | + | local_path = "/srv/sando/backups/latest.sql.gz" | |
| 186 | + | ||
| 187 | + | [[backup]] | |
| 188 | + | name = "multithreaded" | |
| 189 | + | source = "ssh://backup-puller@alpha-west-1:2200/multithreaded/latest.sql.gz" | |
| 190 | + | local_path = "/srv/sando/backups/multithreaded-latest.sql.gz" | |
| 191 | + | ``` | |
| 192 | + | ||
| 193 | + | ```toml | |
| 194 | + | # sando-daemon.toml | |
| 195 | + | [[migration_check]] | |
| 196 | + | dir = "server/migrations" | |
| 197 | + | backup = "server" | |
| 198 | + | ||
| 199 | + | [[migration_check]] | |
| 200 | + | dir = "multithreaded/migrations" | |
| 201 | + | backup = "multithreaded" | |
| 202 | + | scratch_db = "sando_scratch_mt" | |
| 203 | + | owner_role = "multithreaded" | |
| 204 | + | ``` | |
| 205 | + | ||
| 206 | + | Omitting either key keeps the historical behavior: one `server/migrations` check | |
| 207 | + | against one `[backup]`, which still parses as a single-entry list. | |
| 208 | + | ||
| 209 | + | Notes on the semantics: | |
| 210 | + | ||
| 211 | + | - **A check restores its own database's dump.** Restoring the server's dump under | |
| 212 | + | another service's migrations would fail on the first migration for the least | |
| 213 | + | interesting reason — a `_sqlx_migrations` table full of someone else's rows. | |
| 214 | + | - `scratch_db` is what keeps checks from clobbering each other. The server check | |
| 215 | + | leaves it unset, so it runs against `scratch_db_url` itself and leaves it in | |
| 216 | + | migrated state for `cargo_test` to reuse; every other check names its own | |
| 217 | + | database, which the daemon drops and recreates at the start of the check. Two | |
| 218 | + | checks sharing one is refused at config load. | |
| 219 | + | - `owner_role` defaults to `scratch_owner_role`. A dump carries | |
| 220 | + | `ALTER ... OWNER TO <role>` for every object, and the role has to exist in the | |
| 221 | + | scratch cluster before the restore, so a dump owned by anyone else needs this. | |
| 222 | + | - Freshness and the fetch's plausibility floor are both per-dump. A fresh server | |
| 223 | + | dump does not make a 45-day-old mt dump look current, and the server's size | |
| 224 | + | does not set mt's floor (they differ by two orders of magnitude). | |
| 225 | + | - All checks share one `gate_runs` row and one log file, sectioned by | |
| 226 | + | `==== migration_check: <dir> ====` banners. The gate stops at the first red | |
| 227 | + | check. | |
| 228 | + | - A `[[migration_check]]` naming a `backup` the topology does not declare fails | |
| 229 | + | at startup, not at the first promote. | |
| 230 | + | ||
| 166 | 231 | ### What `code_smoke` builds first | |
| 167 | 232 | ||
| 168 | 233 | Before it creates a database or boots anything, `code_smoke` compiles every | |
| @@ -237,7 +302,7 @@ | |||
| 237 | 302 | | POST | `/promote/{tier}` | `{version?, hotfix?, reset_burn_in?}` | Verify predecessor gates, deploy to tier nodes, advance state. `version` defaults to the predecessor tier's `current_version`. Red post-deploy gates advance the tier (the nodes really are running it) but return 409 and flag the tier `partial` — the rollout landed, the tier cannot promote onward. | | |
| 238 | 303 | | POST | `/rollback/{tier}` | — | Swap `current` symlink to `previous_version` on every node in the tier. One step only: `previous_version` is cleared afterwards, so a second `/rollback` returns 409 rather than rolling forward onto the version you just escaped. | | |
| 239 | 304 | | POST | `/confirm/{tier}` | — | Insert a passing `manual_confirm` gate row for the tier's `current_version`. Replaces hand-SQL. | | |
| 240 | - | | POST | `/backup/fetch` | `{force?}` | Pull the prod backup. Supports `file://`, `rsync://`, `ssh://user@host[:port]/path`. A fetch is rejected if the dump is under half the last verified one, which wedges the fetch permanently when the source legitimately shrinks; `force` accepts one undersized dump and makes it the new reference. `force` never skips the gzip integrity check. | | |
| 305 | + | | POST | `/backup/fetch` | `{force?, name?}` | Pull every configured prod dump, or just `name`. Supports `file://`, `rsync://`, `ssh://user@host[:port]/path`. A fetch is rejected if the dump is under half the last verified one *of the same name*, which wedges the fetch permanently when the source legitimately shrinks; `force` accepts one undersized dump and makes it the new reference. `force` never skips the gzip integrity check. Each dump is attempted even if another fails. | | |
| 241 | 306 | | GET | `/events` | — | WebSocket stream of typed events (RebuildRequested, BuildStart/Ok/Failed, GateStart/Done, DeployStart/Ok/Failed, PromoteComplete, Rollback, BackupFetched, ManualConfirm, BuildAborted). | | |
| 242 | 307 | ||
| 243 | 308 | ## TUI | |
| @@ -272,7 +337,8 @@ | |||
| 272 | 337 | ||
| 273 | 338 | - `migration_dry_run` requires a scratch Postgres at `scratch_db_url`. The | |
| 274 | 339 | gate drops every non-system schema on every run; do not point this at | |
| 275 | - | anything that matters. | |
| 340 | + | anything that matters. A check with its own `scratch_db` gets that whole | |
| 341 | + | database dropped and recreated instead. | |
| 276 | 342 | ||
| 277 | 343 | ## License | |
| 278 | 344 |
| @@ -47,13 +47,25 @@ | |||
| 47 | 47 | branch = "main" | |
| 48 | 48 | checkout_dir = "Libraries/docengine" | |
| 49 | 49 | ||
| 50 | - | [backup] | |
| 51 | - | # Source of the prod-backup clone used by migration_dry_run on the Sando host. | |
| 52 | - | # For localhost dev this can be a file:// path to a fixture dump. In prod we | |
| 53 | - | # pull directly from alpha-west-1 via a scoped `backup-puller` rrsync user. | |
| 54 | - | source = "ssh://backup-puller@alpha-west-1:2200/latest.sql.gz" | |
| 50 | + | # Prod-backup clones used by migration_dry_run on the Sando host, one per | |
| 51 | + | # database that has a [[migration_check]] in the daemon config. For localhost dev | |
| 52 | + | # a source can be a file:// path to a fixture dump. In prod we pull from | |
| 53 | + | # alpha-west-1 via a scoped `backup-puller` rrsync user, whose forced command is | |
| 54 | + | # `rrsync -ro /var/lib/mnw/backups` — so every path here is relative to that | |
| 55 | + | # directory, and the per-DB subdirs are what `server/deploy/backup-db.sh` writes. | |
| 56 | + | [[backup]] | |
| 57 | + | name = "server" | |
| 58 | + | source = "ssh://backup-puller@alpha-west-1:2200/makenotwork/latest.sql.gz" | |
| 55 | 59 | local_path = "/srv/sando/backups/latest.sql.gz" | |
| 56 | 60 | ||
| 61 | + | # multithreaded has its own database and applies its own migrations at boot | |
| 62 | + | # (multithreaded/src/main.rs, `sqlx::migrate!()`), so it needs its own dump to | |
| 63 | + | # dry-run against. Restoring the server's would prove nothing about it. | |
| 64 | + | [[backup]] | |
| 65 | + | name = "multithreaded" | |
| 66 | + | source = "ssh://backup-puller@alpha-west-1:2200/multithreaded/latest.sql.gz" | |
| 67 | + | local_path = "/srv/sando/backups/multithreaded-latest.sql.gz" | |
| 68 | + | ||
| 57 | 69 | # ---- host: fw13 local pre-staging gate ---- | |
| 58 | 70 | [[tier]] | |
| 59 | 71 | name = "host" |
| @@ -25,6 +25,31 @@ | |||
| 25 | 25 | # manual SQL is needed on a fresh box. Must match the prod DB owner. | |
| 26 | 26 | scratch_owner_role = "makenotwork" | |
| 27 | 27 | ||
| 28 | + | # Databases the migration_dry_run gate dry-runs, in order: restore that | |
| 29 | + | # database's prod dump into a scratch DB, then run the worktree's migrations on | |
| 30 | + | # top. `backup` names a [[backup]] entry in the topology (sando.toml). Omit the | |
| 31 | + | # whole key to get just the first entry, which is the historical behavior. | |
| 32 | + | # | |
| 33 | + | # The server check leaves `scratch_db` unset, so it runs against scratch_db_url | |
| 34 | + | # itself and leaves it in migrated state for the cargo_test gate to reuse. Every | |
| 35 | + | # other check must name its own database; the daemon creates it (DROP + CREATE) | |
| 36 | + | # at the start of the check, so a new entry owes no host bootstrap step. | |
| 37 | + | [[migration_check]] | |
| 38 | + | dir = "server/migrations" | |
| 39 | + | backup = "server" | |
| 40 | + | ||
| 41 | + | # multithreaded applies its own 36+ migrations at boot (multithreaded/src/main.rs, | |
| 42 | + | # `sqlx::migrate!()`) against its own database, so it carried the server's | |
| 43 | + | # exposure with none of the server's gate: an edited already-applied migration | |
| 44 | + | # would not fail a dry run, it would fail to boot in prod. `owner_role` is that | |
| 45 | + | # dump's owner — pg_dump emits `ALTER ... OWNER TO multithreaded` for every | |
| 46 | + | # object, and the role must exist in the scratch cluster before the restore. | |
| 47 | + | [[migration_check]] | |
| 48 | + | dir = "multithreaded/migrations" | |
| 49 | + | backup = "multithreaded" | |
| 50 | + | scratch_db = "sando_scratch_mt" | |
| 51 | + | owner_role = "multithreaded" | |
| 52 | + | ||
| 28 | 53 | # TypeScript frontends the code_smoke gate compiles. Both build scripts downgrade | |
| 29 | 54 | # a tsc error to a cargo::warning so the Rust build still succeeds against a | |
| 30 | 55 | # stale static/dist/; this is where that failure is fatal instead. See the |
| @@ -13,7 +13,7 @@ | |||
| 13 | 13 | | `bootstrap-node.sh` | a deploy target | One-time node setup (release dirs, deploy user, service). | | |
| 14 | 14 | | `sando-daemon.toml.example` | Sando host | Template for the daemon config (`sando.toml`). | | |
| 15 | 15 | | `post-receive` | git remote | Push-to-deploy hook. | | |
| 16 | - | | `sandod-backup-fetch.{service,timer}` | Sando host | Daily pull of the prod backup to `/srv/sando/backups/latest.sql.gz` (04:00 UTC). | | |
| 16 | + | | `sandod-backup-fetch.{service,timer}` | Sando host | Daily pull of every configured prod dump into `/srv/sando/backups/` (04:00 UTC). | | |
| 17 | 17 | | `mnw-testnot-seed.{sh,service}` | Sando host | Reset testnot.work to the fabricated example catalog (`--seed-examples`). On-demand, not scheduled. | | |
| 18 | 18 | | `sando-update@.service` + `sando-self-update.sh` | Sando host | Self-update: rebuild + restart `sandod` to a target sha. | | |
| 19 | 19 | | `10-sando-update.rules` | Sando host | polkit grant letting the `sando` user start (only) `sando-update@*`. | | |
| @@ -104,10 +104,14 @@ | |||
| 104 | 104 | serve a mismatched schema. | |
| 105 | 105 | ||
| 106 | 106 | **Does the backup actually restore?** The `migration_dry_run` gate answers this | |
| 107 | - | on every build: it resets a scratch database, restores the latest | |
| 108 | - | `/srv/sando/backups/latest.sql.gz`, and runs the migrator against it. A failed | |
| 109 | - | restore fails the gate. What is *not* automated is a full restore-to-serving | |
| 110 | - | drill (restore into a throwaway target and confirm the app boots and serves | |
| 107 | + | on every build, once per `[[migration_check]]`: it resets that check's scratch | |
| 108 | + | database, restores the latest dump of the database the check is for, and runs | |
| 109 | + | the migrator against it. A failed restore fails the gate. Both prod databases | |
| 110 | + | are covered — `makenotwork` and `multithreaded`, each from its own dump under | |
| 111 | + | `/srv/sando/backups/`. A check that names a scratch database of its own gets it | |
| 112 | + | created by the daemon on first run, so adding one owes no step here. What is | |
| 113 | + | *not* automated is a full restore-to-serving drill (restore into a throwaway | |
| 114 | + | target and confirm the app boots and serves | |
| 111 | 115 | against it) — that is tracked as an infra task, not wired into the pipeline. | |
| 112 | 116 | ||
| 113 | 117 | ## Companion services (deploying mnw-cli in lockstep) |
| @@ -113,6 +113,11 @@ | |||
| 113 | 113 | SQL | |
| 114 | 114 | ||
| 115 | 115 | # CREATE DATABASE can't be inside a DO block, hence the separate guard. | |
| 116 | + | # Only the primary scratch DB is created here. A `[[migration_check]]` with its | |
| 117 | + | # own `scratch_db` (e.g. sando_scratch_mt) is created by the daemon at the start | |
| 118 | + | # of that check — DROP + CREATE, so it is sando-owned and the PG15+ public-schema | |
| 119 | + | # grants below are applied by reset_scratch rather than by hand. Adding a gated | |
| 120 | + | # database must not depend on someone having remembered a step in this script. | |
| 116 | 121 | if ! sudo -u postgres psql -tAc \ | |
| 117 | 122 | "SELECT 1 FROM pg_database WHERE datname = 'sando_scratch'" \ | |
| 118 | 123 | | grep -q '^1$'; then |
| @@ -33,6 +33,31 @@ | |||
| 33 | 33 | # sando user must be able to write it; cargo creates it if absent. | |
| 34 | 34 | cargo_target_dir = "/srv/sando/cargo-target" | |
| 35 | 35 | ||
| 36 | + | # Databases the migration_dry_run gate dry-runs, in order: restore that | |
| 37 | + | # database's prod dump into a scratch DB, then run the worktree's migrations on | |
| 38 | + | # top. `backup` names a [[backup]] entry in the topology (sando.toml). Omit the | |
| 39 | + | # whole key to get just the first entry, which is the historical behavior. | |
| 40 | + | # | |
| 41 | + | # The server check leaves `scratch_db` unset, so it runs against scratch_db_url | |
| 42 | + | # itself and leaves it in migrated state for the cargo_test gate to reuse. Every | |
| 43 | + | # other check must name its own database; the daemon creates it (DROP + CREATE) | |
| 44 | + | # at the start of the check, so a new entry owes no host bootstrap step. | |
| 45 | + | [[migration_check]] | |
| 46 | + | dir = "server/migrations" | |
| 47 | + | backup = "server" | |
| 48 | + | ||
| 49 | + | # multithreaded applies its own 36+ migrations at boot (multithreaded/src/main.rs, | |
| 50 | + | # `sqlx::migrate!()`) against its own database, so it carried the server's | |
| 51 | + | # exposure with none of the server's gate: an edited already-applied migration | |
| 52 | + | # would not fail a dry run, it would fail to boot in prod. `owner_role` is that | |
| 53 | + | # dump's owner — pg_dump emits `ALTER ... OWNER TO multithreaded` for every | |
| 54 | + | # object, and the role must exist in the scratch cluster before the restore. | |
| 55 | + | [[migration_check]] | |
| 56 | + | dir = "multithreaded/migrations" | |
| 57 | + | backup = "multithreaded" | |
| 58 | + | scratch_db = "sando_scratch_mt" | |
| 59 | + | owner_role = "multithreaded" | |
| 60 | + | ||
| 36 | 61 | # TypeScript frontends the code_smoke gate compiles (npm run build), before it | |
| 37 | 62 | # creates a database or boots anything. Both crates compile these from a build | |
| 38 | 63 | # script that reports a tsc error as a cargo::warning and lets the Rust build |
| @@ -22,6 +22,9 @@ | |||
| 22 | 22 | ||
| 23 | 23 | #[derive(Debug, Clone)] | |
| 24 | 24 | pub struct FetchedBackup { | |
| 25 | + | /// Which configured dump this is (`BackupConfig::name`), so the caller can | |
| 26 | + | /// tell the server's from multithreaded's in one response. | |
| 27 | + | pub name: String, | |
| 25 | 28 | pub source: String, | |
| 26 | 29 | pub local_path: String, | |
| 27 | 30 | pub byte_size: Option<i64>, | |
| @@ -139,19 +142,70 @@ | |||
| 139 | 142 | Ok(()) | |
| 140 | 143 | } | |
| 141 | 144 | ||
| 142 | - | /// Pull the configured prod dump into `topo.backup.local_path`. | |
| 145 | + | /// Pull every configured prod dump, or just `only` when named. | |
| 146 | + | /// | |
| 147 | + | /// Each dump is fetched independently: one source being down must not leave the | |
| 148 | + | /// others un-refreshed, because a stale dump is a *blocked* gate and the whole | |
| 149 | + | /// point of having more than one is that each database gets its own. So every | |
| 150 | + | /// entry is attempted, and the errors are aggregated at the end — a caller | |
| 151 | + | /// (`/backup/fetch`, and the daily timer through it) still sees a failure, it | |
| 152 | + | /// just sees it after the work that could succeed did. | |
| 143 | 153 | /// | |
| 144 | 154 | /// `force` re-baselines the plausibility floor: see `MIN_BACKUP_FRACTION_DENOM`. | |
| 145 | 155 | /// Pass `false` for anything automated — it is an operator escape hatch, not a | |
| 146 | 156 | /// retry strategy. | |
| 147 | 157 | pub async fn fetch( | |
| 148 | 158 | pool: &SqlitePool, | |
| 149 | - | _cfg: &Arc<Config>, | |
| 159 | + | cfg: &Arc<Config>, | |
| 150 | 160 | topo: &Arc<Topology>, | |
| 151 | 161 | force: bool, | |
| 162 | + | only: Option<&str>, | |
| 163 | + | ) -> Result<Vec<FetchedBackup>> { | |
| 164 | + | let selected: Vec<&crate::topology::BackupConfig> = match only { | |
| 165 | + | Some(name) => vec![topo.backup_named(name).with_context(|| { | |
| 166 | + | format!( | |
| 167 | + | "no backup named {name:?} in the topology (have: {})", | |
| 168 | + | topo.backup | |
| 169 | + | .iter() | |
| 170 | + | .map(|b| b.name.as_str()) | |
| 171 | + | .collect::<Vec<_>>() | |
| 172 | + | .join(", ") | |
| 173 | + | ) | |
| 174 | + | })?], | |
| 175 | + | None => topo.backup.iter().collect(), | |
| 176 | + | }; | |
| 177 | + | ||
| 178 | + | let mut fetched = Vec::new(); | |
| 179 | + | let mut failures = Vec::new(); | |
| 180 | + | for backup in selected { | |
| 181 | + | match fetch_one(pool, cfg, backup, force).await { | |
| 182 | + | Ok(fb) => fetched.push(fb), | |
| 183 | + | Err(e) => { | |
| 184 | + | tracing::error!(backup = %backup.name, error = %e, "backup fetch failed"); | |
| 185 | + | failures.push(format!("{}: {e:#}", backup.name)); | |
| 186 | + | } | |
| 187 | + | } | |
| 188 | + | } | |
| 189 | + | anyhow::ensure!( | |
| 190 | + | failures.is_empty(), | |
| 191 | + | "{} of {} backup fetch(es) failed: {}", | |
| 192 | + | failures.len(), | |
| 193 | + | failures.len() + fetched.len(), | |
| 194 | + | failures.join("; "), | |
| 195 | + | ); | |
| 196 | + | Ok(fetched) | |
| 197 | + | } | |
| 198 | + | ||
| 199 | + | /// Pull one configured dump into its `local_path`. | |
| 200 | + | async fn fetch_one( | |
| 201 | + | pool: &SqlitePool, | |
| 202 | + | _cfg: &Arc<Config>, | |
| 203 | + | backup: &crate::topology::BackupConfig, | |
| 204 | + | force: bool, | |
| 152 | 205 | ) -> Result<FetchedBackup> { | |
| 153 | - | let source = topo.backup.source.clone(); | |
| 154 | - | let local_path = topo.backup.local_path.clone(); | |
| 206 | + | let name = backup.name.clone(); | |
| 207 | + | let source = backup.source.clone(); | |
| 208 | + | let local_path = backup.local_path.clone(); | |
| 155 | 209 | ||
| 156 | 210 | if let Some(parent) = Path::new(&local_path).parent() { | |
| 157 | 211 | tokio::fs::create_dir_all(parent).await?; | |
| @@ -168,12 +222,17 @@ | |||
| 168 | 222 | .is_some_and(|ext| ext.eq_ignore_ascii_case("gz")); | |
| 169 | 223 | ||
| 170 | 224 | // Plausibility floor: half the last verified backup's size, never below the | |
| 171 | - | // absolute floor. The first-ever fetch (no prior row) falls back to the | |
| 172 | - | // absolute floor. | |
| 173 | - | let last_size: Option<i64> = | |
| 174 | - | sqlx::query_scalar("SELECT byte_size FROM backups ORDER BY fetched_at DESC LIMIT 1") | |
| 175 | - | .fetch_optional(pool) | |
| 176 | - | .await?; | |
| 225 | + | // absolute floor. Scoped to this dump's name — the server's dump is two | |
| 226 | + | // orders of magnitude larger than multithreaded's, so a shared floor would | |
| 227 | + | // reject every mt fetch as implausibly small and, on the other side, let a | |
| 228 | + | // truncated server dump through. The first-ever fetch of a name (no prior | |
| 229 | + | // row) falls back to the absolute floor. | |
| 230 | + | let last_size: Option<i64> = sqlx::query_scalar( | |
| 231 | + | "SELECT byte_size FROM backups WHERE name = ? ORDER BY fetched_at DESC LIMIT 1", | |
| 232 | + | ) | |
| 233 | + | .bind(&name) | |
| 234 | + | .fetch_optional(pool) | |
| 235 | + | .await?; | |
| 177 | 236 | let min_bytes = if force { | |
| 178 | 237 | tracing::warn!( | |
| 179 | 238 | last_verified_bytes = last_size, | |
| @@ -267,8 +326,10 @@ | |||
| 267 | 326 | // reference a path that no longer exists — keep the table from growing. | |
| 268 | 327 | let mut tx = pool.begin().await?; | |
| 269 | 328 | sqlx::query( | |
| 270 | - | "INSERT INTO backups (fetched_at, source, local_path, byte_size) VALUES (?, ?, ?, ?)", | |
| 329 | + | "INSERT INTO backups (name, fetched_at, source, local_path, byte_size) \ | |
| 330 | + | VALUES (?, ?, ?, ?, ?)", | |
| 271 | 331 | ) | |
| 332 | + | .bind(&name) | |
| 272 | 333 | .bind(Utc::now().to_rfc3339()) | |
| 273 | 334 | .bind(&source) | |
| 274 | 335 | .bind(&local_path) | |
| @@ -281,6 +342,7 @@ | |||
| 281 | 342 | tx.commit().await?; | |
| 282 | 343 | ||
| 283 | 344 | Ok(FetchedBackup { | |
| 345 | + | name, | |
| 284 | 346 | source, | |
| 285 | 347 | local_path, | |
| 286 | 348 | byte_size: Some(size), | |
| @@ -311,7 +373,11 @@ | |||
| 311 | 373 | branch: "main".into(), | |
| 312 | 374 | upstream: None, | |
| 313 | 375 | }, | |
| 314 | - | backup: BackupConfig { source, local_path }, | |
| 376 | + | backup: vec![BackupConfig { | |
| 377 | + | name: "server".into(), | |
| 378 | + | source, | |
| 379 | + | local_path, | |
| 380 | + | }], | |
| 315 | 381 | tiers: vec![], | |
| 316 | 382 | aux_repos: Vec::new(), | |
| 317 | 383 | } | |
| @@ -352,7 +418,10 @@ | |||
| 352 | 418 | let pool = mem_pool().await; | |
| 353 | 419 | let cfg = Arc::new(Config::for_tests()); | |
| 354 | 420 | ||
| 355 | - | let fb = fetch(&pool, &cfg, &topo, false).await.unwrap(); | |
| 421 | + | let fb = fetch(&pool, &cfg, &topo, false, None) | |
| 422 | + | .await | |
| 423 | + | .unwrap() | |
| 424 | + | .remove(0); | |
| 356 | 425 | assert!(dest.exists(), "live backup written"); | |
| 357 | 426 | assert!( | |
| 358 | 427 | !dest.with_file_name("latest.sql.gz.partial").exists(), | |
| @@ -389,7 +458,7 @@ | |||
| 389 | 458 | .bind(dest.to_string_lossy().into_owned()) | |
| 390 | 459 | .execute(&pool).await.unwrap(); | |
| 391 | 460 | ||
| 392 | - | let err = fetch(&pool, &cfg, &topo, false) | |
| 461 | + | let err = fetch(&pool, &cfg, &topo, false, None) | |
| 393 | 462 | .await | |
| 394 | 463 | .unwrap_err() | |
| 395 | 464 | .to_string(); | |
| @@ -405,6 +474,102 @@ | |||
| 405 | 474 | assert_eq!(count.0, 1, "the rejected fetch records no new row"); | |
| 406 | 475 | } | |
| 407 | 476 | ||
| 477 | + | #[tokio::test] | |
| 478 | + | async fn the_plausibility_floor_is_scoped_to_one_dump() { | |
| 479 | + | // The server's dump is two orders of magnitude larger than | |
| 480 | + | // multithreaded's. A shared floor would reject every mt fetch as | |
| 481 | + | // implausibly small (and, the other way round, let a badly truncated | |
| 482 | + | // server dump through on mt's reference). Seed a large `server` row and | |
| 483 | + | // fetch a small `multithreaded` one: it must be accepted. | |
| 484 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 485 | + | let src = tmp.path().join("mt.sql.gz"); | |
| 486 | + | write_valid_gz(&src).await; // ~4 KB valid gzip | |
| 487 | + | let dest = tmp.path().join("backups/mt-latest.sql.gz"); | |
| 488 | + | let mut topo = topo_with_backup( | |
| 489 | + | format!("file://{}", src.display()), | |
| 490 | + | dest.to_string_lossy().into_owned(), | |
| 491 | + | ); | |
| 492 | + | topo.backup[0].name = "multithreaded".into(); | |
| 493 | + | let topo = Arc::new(topo); | |
| 494 | + | let pool = mem_pool().await; | |
| 495 | + | let cfg = Arc::new(Config::for_tests()); | |
| 496 | + | ||
| 497 | + | sqlx::query( | |
| 498 | + | "INSERT INTO backups (name, fetched_at, source, local_path, byte_size) \ | |
| 499 | + | VALUES ('server', ?, 'x', '/tmp/server.sql.gz', 1000000)", | |
| 500 | + | ) | |
| 501 | + | .bind(Utc::now().to_rfc3339()) | |
| 502 | + | .execute(&pool) | |
| 503 | + | .await | |
| 504 | + | .unwrap(); | |
| 505 | + | ||
| 506 | + | let fetched = fetch(&pool, &cfg, &topo, false, None) | |
| 507 | + | .await | |
| 508 | + | .expect("the server's size must not set multithreaded's floor"); | |
| 509 | + | assert_eq!(fetched.len(), 1); | |
| 510 | + | assert_eq!(fetched[0].name, "multithreaded"); | |
| 511 | + | let recorded: (String,) = | |
| 512 | + | sqlx::query_as("SELECT name FROM backups ORDER BY id DESC LIMIT 1") | |
| 513 | + | .fetch_one(&pool) | |
| 514 | + | .await | |
| 515 | + | .unwrap(); | |
| 516 | + | assert_eq!(recorded.0, "multithreaded", "the row is recorded by name"); | |
| 517 | + | } | |
| 518 | + | ||
| 519 | + | #[tokio::test] | |
| 520 | + | async fn fetching_an_unknown_name_is_an_error_not_a_silent_no_op() { | |
| 521 | + | // A typo'd `{"name":"mt"}` must not report success having fetched | |
| 522 | + | // nothing — the operator would read that as a refreshed dump. | |
| 523 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 524 | + | let topo = Arc::new(topo_with_backup( | |
| 525 | + | "file:///nope".into(), | |
| 526 | + | tmp.path().join("x.sql.gz").to_string_lossy().into_owned(), | |
| 527 | + | )); | |
| 528 | + | let pool = mem_pool().await; | |
| 529 | + | let cfg = Arc::new(Config::for_tests()); | |
| 530 | + | ||
| 531 | + | let err = fetch(&pool, &cfg, &topo, false, Some("mt")) | |
| 532 | + | .await | |
| 533 | + | .unwrap_err() | |
| 534 | + | .to_string(); | |
| 535 | + | assert!(err.contains("no backup named"), "{err}"); | |
| 536 | + | } | |
| 537 | + | ||
| 538 | + | #[tokio::test] | |
| 539 | + | async fn one_failing_source_does_not_skip_the_others() { | |
| 540 | + | // Each dump gates a different database, and a stale dump is a blocked | |
| 541 | + | // gate — so a broken source must not cost the working one its refresh. | |
| 542 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 543 | + | let good_src = tmp.path().join("good.sql.gz"); | |
| 544 | + | write_valid_gz(&good_src).await; | |
| 545 | + | let good_dest = tmp.path().join("backups/good.sql.gz"); | |
| 546 | + | let mut topo = topo_with_backup( | |
| 547 | + | "file:///nonexistent/sando-test-missing.sql.gz".into(), | |
| 548 | + | tmp.path() | |
| 549 | + | .join("backups/bad.sql.gz") | |
| 550 | + | .to_string_lossy() | |
| 551 | + | .into_owned(), | |
| 552 | + | ); | |
| 553 | + | topo.backup.push(crate::topology::BackupConfig { | |
| 554 | + | name: "multithreaded".into(), | |
| 555 | + | source: format!("file://{}", good_src.display()), | |
| 556 | + | local_path: good_dest.to_string_lossy().into_owned(), | |
| 557 | + | }); | |
| 558 | + | let topo = Arc::new(topo); | |
| 559 | + | let pool = mem_pool().await; | |
| 560 | + | let cfg = Arc::new(Config::for_tests()); | |
| 561 | + | ||
| 562 | + | let err = fetch(&pool, &cfg, &topo, false, None) | |
| 563 | + | .await | |
| 564 | + | .unwrap_err() | |
| 565 | + | .to_string(); | |
| 566 | + | assert!(err.contains("server:"), "the failure names its dump: {err}"); | |
| 567 | + | assert!( | |
| 568 | + | good_dest.exists(), | |
| 569 | + | "the reachable dump is still fetched after the unreachable one fails" | |
| 570 | + | ); | |
| 571 | + | } | |
| 572 | + | ||
| 408 | 573 | #[tokio::test] | |
| 409 | 574 | async fn force_rebaselines_the_floor_after_a_legitimate_shrink() { | |
| 410 | 575 | // The wedge this exists for: the floor is derived from a row that only a | |
| @@ -427,7 +592,10 @@ | |||
| 427 | 592 | .execute(&pool).await.unwrap(); | |
| 428 | 593 | ||
| 429 | 594 | // Same dump the un-forced fetch rejects above. | |
| 430 | - | let fb = fetch(&pool, &cfg, &topo, true).await.unwrap(); | |
| 595 | + | let fb = fetch(&pool, &cfg, &topo, true, None) | |
| 596 | + | .await | |
| 597 | + | .unwrap() | |
| 598 | + | .remove(0); | |
| 431 | 599 | assert!(dest.exists(), "the forced dump becomes the live backup"); | |
| 432 | 600 | let recorded = fb.byte_size.unwrap(); | |
| 433 | 601 | assert!( | |
| @@ -436,7 +604,7 @@ | |||
| 436 | 604 | ); | |
| 437 | 605 | ||
| 438 | 606 | // The new row is now the reference, so the next fetch passes unforced. | |
| 439 | - | fetch(&pool, &cfg, &topo, false) | |
| 607 | + | fetch(&pool, &cfg, &topo, false, None) | |
| 440 | 608 | .await | |
| 441 | 609 | .expect("floor re-baselined to the forced fetch's size"); | |
| 442 | 610 | } | |
| @@ -461,7 +629,7 @@ | |||
| 461 | 629 | let pool = mem_pool().await; | |
| 462 | 630 | let cfg = Arc::new(Config::for_tests()); | |
| 463 | 631 | ||
| 464 | - | let err = fetch(&pool, &cfg, &topo, true) | |
| 632 | + | let err = fetch(&pool, &cfg, &topo, true, None) | |
| 465 | 633 | .await | |
| 466 | 634 | .unwrap_err() | |
| 467 | 635 | .to_string(); | |
| @@ -495,7 +663,7 @@ | |||
| 495 | 663 | let pool = mem_pool().await; | |
| 496 | 664 | let cfg = Arc::new(Config::for_tests()); | |
| 497 | 665 | ||
| 498 | - | let res = fetch(&pool, &cfg, &topo, false).await; | |
| 666 | + | let res = fetch(&pool, &cfg, &topo, false, None).await; | |
| 499 | 667 | assert!(res.is_err(), "a truncated gzip must fail the fetch"); | |
| 500 | 668 | assert!( | |
| 501 | 669 | !dest.exists(), |
| @@ -695,6 +695,7 @@ | |||
| 695 | 695 | all_features: false, | |
| 696 | 696 | scratch_db: true, | |
| 697 | 697 | }], | |
| 698 | + | migration_checks: vec![], | |
| 698 | 699 | frontend_builds: vec![], | |
| 699 | 700 | backup_max_age_hours: 48, | |
| 700 | 701 | }; | |
| @@ -705,10 +706,11 @@ | |||
| 705 | 706 | branch: "main".into(), | |
| 706 | 707 | upstream: None, | |
| 707 | 708 | }, | |
| 708 | - | backup: BackupConfig { | |
| 709 | + | backup: vec![BackupConfig { | |
| 710 | + | name: "server".into(), | |
| 709 | 711 | source: "file:///tmp/test-backup.sql".into(), | |
| 710 | 712 | local_path: "/tmp/local-backup.sql".into(), | |
| 711 | - | }, | |
| 713 | + | }], | |
| 712 | 714 | tiers: vec![Tier { | |
| 713 | 715 | name: "host".into(), | |
| 714 | 716 | provisioned: true, | |
| @@ -775,6 +777,7 @@ | |||
| 775 | 777 | gate_timeout_secs: 2400, | |
| 776 | 778 | companions: Vec::new(), | |
| 777 | 779 | test_targets: vec![], | |
| 780 | + | migration_checks: vec![], | |
| 778 | 781 | frontend_builds: vec![], | |
| 779 | 782 | backup_max_age_hours: 48, | |
| 780 | 783 | } | |
| @@ -787,10 +790,11 @@ | |||
| 787 | 790 | branch: "main".into(), | |
| 788 | 791 | upstream: None, | |
| 789 | 792 | }, | |
| 790 | - | backup: BackupConfig { | |
| 793 | + | backup: vec![BackupConfig { | |
| 794 | + | name: "server".into(), | |
| 791 | 795 | source: "s".into(), | |
| 792 | 796 | local_path: "/tmp/d".into(), | |
| 793 | - | }, | |
| 797 | + | }], | |
| 794 | 798 | tiers: vec![], | |
| 795 | 799 | aux_repos, | |
| 796 | 800 | } |
| @@ -113,6 +113,18 @@ | |||
| 113 | 113 | /// project that configures nothing keeps today's behavior. | |
| 114 | 114 | #[serde(default, rename = "frontend_build")] | |
| 115 | 115 | pub frontend_builds: Vec<FrontendBuild>, | |
| 116 | + | /// Databases the `migration_dry_run` gate dry-runs, in order. The gate used | |
| 117 | + | /// to hardcode `worktree/server/migrations` against the one `scratch_db_url`, | |
| 118 | + | /// so every other database in the repo shipped with no gate at all — | |
| 119 | + | /// including multithreaded's, which `multithreaded/src/main.rs` migrates | |
| 120 | + | /// with `sqlx::migrate!()` at boot. That is the same exposure that blocked | |
| 121 | + | /// every server deploy 2026-07-27..07-30 (an exorcise sweep rewrote comments | |
| 122 | + | /// in 29 already-applied migrations; sqlx checksums whole files), except | |
| 123 | + | /// that with no gate it would not have failed a dry run — it would have | |
| 124 | + | /// failed to boot in prod. Default is the historical single `server` entry, | |
| 125 | + | /// so a project that configures nothing keeps today's behavior. | |
| 126 | + | #[serde(default = "default_migration_checks", rename = "migration_check")] | |
| 127 | + | pub migration_checks: Vec<MigrationCheck>, | |
| 116 | 128 | /// How old (hours) the fetched prod dump may be before `migration_dry_run` | |
| 117 | 129 | /// refuses to run against it. The gate restores whatever `backups` row is | |
| 118 | 130 | /// newest, and presence alone used to be the only check — so a fetch that | |
| @@ -163,6 +175,63 @@ | |||
| 163 | 175 | pub scratch_db: bool, | |
| 164 | 176 | } | |
| 165 | 177 | ||
| 178 | + | /// One database's migrations, as dry-run by the `migration_dry_run` gate: | |
| 179 | + | /// restore that database's prod dump into a scratch DB, then run the worktree's | |
| 180 | + | /// migrations on top. | |
| 181 | + | #[derive(Debug, Clone, Deserialize)] | |
| 182 | + | pub struct MigrationCheck { | |
| 183 | + | /// Migrations directory under the worktree (e.g. `server/migrations`). | |
| 184 | + | /// Passed to `sqlx::migrate::Migrator::new`. | |
| 185 | + | pub dir: PathBuf, | |
| 186 | + | /// Which configured dump to restore, by `[[backup]]` name in the topology. | |
| 187 | + | /// Defaults to `server`, matching the historical single `[backup]` table. | |
| 188 | + | /// | |
| 189 | + | /// It must be that database's *own* dump. Restoring the server's dump under | |
| 190 | + | /// another service's migrations would fail on the first migration for the | |
| 191 | + | /// least interesting reason (a `_sqlx_migrations` table full of someone | |
| 192 | + | /// else's rows), and if it somehow passed it would prove nothing. | |
| 193 | + | #[serde(default = "default_backup_name")] | |
| 194 | + | pub backup: String, | |
| 195 | + | /// Database name on the scratch cluster to restore into. `None` uses | |
| 196 | + | /// `scratch_db_url` as configured, which is what the server check does and | |
| 197 | + | /// what the `cargo_test` gate then reuses in migrated state. Any other check | |
| 198 | + | /// must name its own: two checks sharing a database would each drop the | |
| 199 | + | /// other's restore, and the last one to run would decide what `cargo_test` | |
| 200 | + | /// sees. The daemon creates it (DROP + CREATE) at the start of the check, so | |
| 201 | + | /// no host bootstrap step is owed for a new entry. | |
| 202 | + | #[serde(default)] | |
| 203 | + | pub scratch_db: Option<String>, | |
| 204 | + | /// Role that owns the objects in *this* dump — `pg_dump` emits | |
| 205 | + | /// `ALTER ... OWNER TO <role>` for every one, and the role must exist in the | |
| 206 | + | /// scratch cluster before the restore. Defaults to `scratch_owner_role` | |
| 207 | + | /// (the server's owner). multithreaded's dump is owned by `multithreaded`, | |
| 208 | + | /// so its check must say so or the restore fails on the first ALTER. | |
| 209 | + | /// | |
| 210 | + | /// Interpolated into DDL as an identifier, so it is restricted to | |
| 211 | + | /// `[A-Za-z0-9_]+` at load. | |
| 212 | + | #[serde(default)] | |
| 213 | + | pub owner_role: Option<String>, | |
| 214 | + | } | |
| 215 | + | ||
| 216 | + | fn default_backup_name() -> String { | |
| 217 | + | "server".into() | |
| 218 | + | } | |
| 219 | + | ||
| 220 | + | fn default_migration_checks() -> Vec<MigrationCheck> { | |
| 221 | + | vec![MigrationCheck { | |
| 222 | + | dir: PathBuf::from("server").join("migrations"), | |
| 223 | + | backup: default_backup_name(), | |
| 224 | + | scratch_db: None, | |
| 225 | + | owner_role: None, | |
| 226 | + | }] | |
| 227 | + | } | |
| 228 | + | ||
| 229 | + | /// The default check list, for the topology cross-check test. | |
| 230 | + | #[cfg(test)] | |
| 231 | + | pub(crate) fn default_migration_checks_for_test() -> Vec<MigrationCheck> { | |
| 232 | + | default_migration_checks() | |
| 233 | + | } | |
| 234 | + | ||
| 166 | 235 | fn default_test_targets() -> Vec<TestTarget> { | |
| 167 | 236 | vec![TestTarget { | |
| 168 | 237 | dir: PathBuf::from("server"), | |
| @@ -334,6 +403,51 @@ | |||
| 334 | 403 | t.dir.display(), | |
| 335 | 404 | ); | |
| 336 | 405 | } | |
| 406 | + | anyhow::ensure!( | |
| 407 | + | !self.migration_checks.is_empty(), | |
| 408 | + | "migration_check list is empty; migration_dry_run would restore nothing and pass. \ | |
| 409 | + | Omit the key entirely to get the default `server/migrations` check.", | |
| 410 | + | ); | |
| 411 | + | for (i, m) in self.migration_checks.iter().enumerate() { | |
| 412 | + | anyhow::ensure!( | |
| 413 | + | !m.backup.is_empty(), | |
| 414 | + | "migration_check {} has an empty backup name; omit the key for the default \ | |
| 415 | + | `server`", | |
| 416 | + | m.dir.display(), | |
| 417 | + | ); | |
| 418 | + | for role in [m.owner_role.as_deref(), m.scratch_db.as_deref()] | |
| 419 | + | .into_iter() | |
| 420 | + | .flatten() | |
| 421 | + | { | |
| 422 | + | anyhow::ensure!( | |
| 423 | + | !role.is_empty() | |
| 424 | + | && role.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'), | |
| 425 | + | "migration_check {} has {role:?} as an owner_role/scratch_db; both are \ | |
| 426 | + | interpolated into DDL as bare identifiers and must match [A-Za-z0-9_]+", | |
| 427 | + | m.dir.display(), | |
| 428 | + | ); | |
| 429 | + | } | |
| 430 | + | // A shared dir would run the same migrations twice; a shared | |
| 431 | + | // scratch_db (including two `None`s, which both mean scratch_db_url) | |
| 432 | + | // would have the second check drop the first one's restore. | |
| 433 | + | for prior in &self.migration_checks[..i] { | |
| 434 | + | anyhow::ensure!( | |
| 435 | + | prior.dir != m.dir, | |
| 436 | + | "two migration_check entries share dir {}", | |
| 437 | + | m.dir.display(), | |
| 438 | + | ); | |
| 439 | + | anyhow::ensure!( | |
| 440 | + | prior.scratch_db != m.scratch_db, | |
| 441 | + | "migration_check {} and {} share a scratch database ({}); each check drops \ | |
| 442 | + | and recreates its own, so they would clobber each other", | |
| 443 | + | prior.dir.display(), | |
| 444 | + | m.dir.display(), | |
| 445 | + | m.scratch_db | |
| 446 | + | .as_deref() | |
| 447 | + | .unwrap_or("the configured scratch_db_url"), | |
| 448 | + | ); | |
| 449 | + | } | |
| 450 | + | } | |
| 337 | 451 | for f in &self.frontend_builds { | |
| 338 | 452 | anyhow::ensure!( | |
| 339 | 453 | !f.script.is_empty(), | |
| @@ -368,6 +482,7 @@ | |||
| 368 | 482 | gate_timeout_secs: default_gate_timeout_secs(), | |
| 369 | 483 | companions: Vec::new(), | |
| 370 | 484 | test_targets: default_test_targets(), | |
| 485 | + | migration_checks: default_migration_checks(), | |
| 371 | 486 | frontend_builds: Vec::new(), | |
| 372 | 487 | backup_max_age_hours: default_backup_max_age_hours(), | |
| 373 | 488 | } | |
| @@ -400,6 +515,73 @@ | |||
| 400 | 515 | assert!(!t.all_features); | |
| 401 | 516 | } | |
| 402 | 517 | ||
| 518 | + | #[test] | |
| 519 | + | fn migration_checks_default_to_the_historical_server_entry() { | |
| 520 | + | // Same contract as test_targets: configure nothing, get exactly what the | |
| 521 | + | // gate did when the dir was hardcoded — server/migrations, the `server` | |
| 522 | + | // dump, and `scratch_db_url` itself (which cargo_test then reuses). | |
| 523 | + | let cfg: Config = toml::from_str(MINIMAL).unwrap(); | |
| 524 | + | assert_eq!(cfg.migration_checks.len(), 1); | |
| 525 | + | let m = &cfg.migration_checks[0]; | |
| 526 | + | assert_eq!(m.dir, PathBuf::from("server/migrations")); | |
| 527 | + | assert_eq!(m.backup, "server"); | |
| 528 | + | assert!(m.scratch_db.is_none()); | |
| 529 | + | assert!(m.owner_role.is_none()); | |
| 530 | + | } | |
| 531 | + | ||
| 532 | + | #[test] | |
| 533 | + | fn migration_checks_parse_as_a_list() { | |
| 534 | + | let raw = format!( | |
| 535 | + | "{MINIMAL}\n\ | |
| 536 | + | [[migration_check]]\ndir = \"server/migrations\"\n\ | |
| 537 | + | [[migration_check]]\ndir = \"multithreaded/migrations\"\n\ | |
| 538 | + | backup = \"multithreaded\"\nscratch_db = \"sando_scratch_mt\"\n\ | |
| 539 | + | owner_role = \"multithreaded\"\n" | |
| 540 | + | ); | |
| 541 | + | let cfg: Config = toml::from_str(&raw).unwrap(); | |
| 542 | + | cfg.validate().unwrap(); | |
| 543 | + | assert_eq!(cfg.migration_checks[0].backup, "server", "backup defaults"); | |
| 544 | + | let mt = &cfg.migration_checks[1]; | |
| 545 | + | assert_eq!(mt.scratch_db.as_deref(), Some("sando_scratch_mt")); | |
| 546 | + | assert_eq!(mt.owner_role.as_deref(), Some("multithreaded")); | |
| 547 | + | } | |
| 548 | + | ||
| 549 | + | #[test] | |
| 550 | + | fn two_migration_checks_sharing_a_scratch_db_are_rejected() { | |
| 551 | + | // Including the both-unset case, which is the easy one to write by | |
| 552 | + | // accident: each check drops and recreates its database, so the second | |
| 553 | + | // would destroy the first's restore and cargo_test would inherit | |
| 554 | + | // whichever ran last. | |
| 555 | + | let raw = format!( | |
| 556 | + | "{MINIMAL}\n\ | |
| 557 | + | [[migration_check]]\ndir = \"server/migrations\"\n\ | |
| 558 | + | [[migration_check]]\ndir = \"multithreaded/migrations\"\nbackup = \"multithreaded\"\n" | |
| 559 | + | ); | |
| 560 | + | let cfg: Config = toml::from_str(&raw).unwrap(); | |
| 561 | + | let err = cfg.validate().unwrap_err().to_string(); | |
| 562 | + | assert!(err.contains("share a scratch database"), "{err}"); | |
| 563 | + | } | |
| 564 | + | ||
| 565 | + | #[test] | |
| 566 | + | fn a_migration_check_scratch_db_that_is_not_an_identifier_is_rejected() { | |
| 567 | + | // It is interpolated into `CREATE DATABASE "..."`. | |
| 568 | + | let raw = | |
| 569 | + | format!("{MINIMAL}\n[[migration_check]]\ndir = \"m\"\nscratch_db = \"drop; --\"\n"); | |
| 570 | + | let cfg: Config = toml::from_str(&raw).unwrap(); | |
| 571 | + | let err = cfg.validate().unwrap_err().to_string(); | |
| 572 | + | assert!(err.contains("[A-Za-z0-9_]+"), "{err}"); | |
| 573 | + | } | |
| 574 | + | ||
| 575 | + | #[test] | |
| 576 | + | fn an_empty_migration_check_list_is_rejected() { | |
| 577 | + | // Same fail-closed rule as test_target: an empty list would make the | |
| 578 | + | // gate restore nothing and pass, which is worse than not having it. | |
| 579 | + | let raw = format!("{MINIMAL}\nmigration_check = []\n"); | |
| 580 | + | let cfg: Config = toml::from_str(&raw).unwrap(); | |
| 581 | + | let err = cfg.validate().unwrap_err().to_string(); | |
| 582 | + | assert!(err.contains("migration_check list is empty"), "{err}"); | |
| 583 | + | } | |
| 584 | + | ||
| 403 | 585 | #[test] | |
| 404 | 586 | fn test_targets_parse_as_a_list() { | |
| 405 | 587 | let raw = format!( |
| @@ -838,20 +838,82 @@ | |||
| 838 | 838 | /// The staged interior of [`migration_dry_run`], writing every step through the | |
| 839 | 839 | /// gate's live log. The caller owns the sink so it can flush it on every exit | |
| 840 | 840 | /// path, and attaches the `log_ref` once instead of at each return. | |
| 841 | + | /// Runs one configured check per database, in config order, and stops at the | |
| 842 | + | /// first that does not pass — a red gate is a red gate, and continuing would | |
| 843 | + | /// bury it under a second restore's output. | |
| 844 | + | /// | |
| 845 | + | /// The server's check runs against `scratch_db_url` itself and is deliberately | |
| 846 | + | /// last-writer for it: `cargo_test` reuses that database in migrated state, so | |
| 847 | + | /// every other check must name its own `scratch_db` (enforced at config load). | |
| 841 | 848 | async fn migration_dry_run_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> { | |
| 842 | - | let Some(db_url) = ctx.cfg.scratch_db_url.as_deref() else { | |
| 849 | + | let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else { | |
| 843 | 850 | log.line("scratch_db_url unset in daemon config\n").await; | |
| 844 | 851 | return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset)); | |
| 845 | 852 | }; | |
| 846 | 853 | ||
| 847 | - | let backup: Option<(String, String)> = | |
| 848 | - | sqlx::query_as("SELECT local_path, fetched_at FROM backups ORDER BY id DESC LIMIT 1") | |
| 849 | - | .fetch_optional(&ctx.pool) | |
| 850 | - | .await?; | |
| 851 | - | let Some((backup_path, fetched_at)) = backup else { | |
| 852 | - | log.line("no backup fetched; call /backup/fetch first\n") | |
| 854 | + | let mut checked = Vec::new(); | |
| 855 | + | let mut primary_backup_path = String::new(); | |
| 856 | + | for check in &ctx.cfg.migration_checks { | |
| 857 | + | let label = check.dir.display().to_string(); | |
| 858 | + | log.line(&format!("==== migration_check: {label} ====\n")) | |
| 853 | 859 | .await; | |
| 854 | - | return Ok(GateOutcome::blocked(GateBlocker::NoBackupAvailable)); | |
| 860 | + | match run_migration_check(ctx, log, scratch_url, check).await? { | |
| 861 | + | CheckResult::Passed { backup_path } => { | |
| 862 | + | if primary_backup_path.is_empty() { | |
| 863 | + | primary_backup_path = backup_path; | |
| 864 | + | } | |
| 865 | + | checked.push(label); | |
| 866 | + | } | |
| 867 | + | CheckResult::Stopped(outcome) => return Ok(outcome), | |
| 868 | + | } | |
| 869 | + | } | |
| 870 | + | ||
| 871 | + | log.line(&format!( | |
| 872 | + | "all {} migration check(s) passed: {}", | |
| 873 | + | checked.len(), | |
| 874 | + | checked.join(", ") | |
| 875 | + | )) | |
| 876 | + | .await; | |
| 877 | + | Ok(GateOutcome::passed(PassNote::Migrated { | |
| 878 | + | backup_path: primary_backup_path, | |
| 879 | + | checks: checked, | |
| 880 | + | })) | |
| 881 | + | } | |
| 882 | + | ||
| 883 | + | /// One check's verdict: it passed (against `backup_path`), or it produced the | |
| 884 | + | /// outcome the whole gate reports. | |
| 885 | + | enum CheckResult { | |
| 886 | + | Passed { backup_path: String }, | |
| 887 | + | Stopped(GateOutcome), | |
| 888 | + | } | |
| 889 | + | ||
| 890 | + | /// Restore one database's dump into its scratch DB and run its migrations on top. | |
| 891 | + | async fn run_migration_check( | |
| 892 | + | ctx: &GateCtx, | |
| 893 | + | log: &GateLog, | |
| 894 | + | scratch_url: &str, | |
| 895 | + | check: &crate::config::MigrationCheck, | |
| 896 | + | ) -> Result<CheckResult> { | |
| 897 | + | let label = check.dir.display().to_string(); | |
| 898 | + | ||
| 899 | + | let backup: Option<(String, String)> = sqlx::query_as( | |
| 900 | + | "SELECT local_path, fetched_at FROM backups WHERE name = ? ORDER BY id DESC LIMIT 1", | |
| 901 | + | ) | |
| 902 | + | .bind(&check.backup) | |
| 903 | + | .fetch_optional(&ctx.pool) | |
| 904 | + | .await?; | |
| 905 | + | let Some((backup_path, fetched_at)) = backup else { | |
| 906 | + | log.line(&format!( | |
| 907 | + | "no {} backup fetched; call /backup/fetch first\n", | |
| 908 | + | check.backup | |
| 909 | + | )) | |
| 910 | + | .await; | |
| 911 | + | return Ok(CheckResult::Stopped(GateOutcome::blocked( | |
| 912 | + | GateBlocker::NoBackupAvailable { | |
| 913 | + | check: label, | |
| 914 | + | backup: check.backup.clone(), | |
| 915 | + | }, | |
| 916 | + | ))); | |
| 855 | 917 | }; | |
| 856 | 918 | ||
| 857 | 919 | // Presence is not freshness. A fetch that quietly stopped working leaves this | |
| @@ -870,43 +932,73 @@ | |||
| 870 | 932 | {max_age_hours}h); re-run /backup/fetch\n" | |
| 871 | 933 | ); | |
| 872 | 934 | log.line(&msg).await; | |
| 873 | - | return Ok(GateOutcome::blocked(GateBlocker::BackupStale { | |
| 874 | - | age_hours, | |
| 875 | - | max_age_hours, | |
| 876 | - | })); | |
| 935 | + | return Ok(CheckResult::Stopped(GateOutcome::blocked( | |
| 936 | + | GateBlocker::BackupStale { | |
| 937 | + | age_hours, | |
| 938 | + | max_age_hours, | |
| 939 | + | check: label, | |
| 940 | + | }, | |
| 941 | + | ))); | |
| 877 | 942 | } | |
| 878 | 943 | ||
| 944 | + | // A check with its own `scratch_db` gets that database created here rather | |
| 945 | + | // than by a host bootstrap step: adding a `[[migration_check]]` should not | |
| 946 | + | // silently depend on someone having remembered to `createdb` on the Sando | |
| 947 | + | // host, which is exactly the class of footgun this gate exists to remove. | |
| 948 | + | // DROP + CREATE also makes the database sando-owned, so the PG15+ public | |
| 949 | + | // schema grants `reset_scratch` applies next are the owner's to give. | |
| 950 | + | let db_url = match check.scratch_db.as_deref() { | |
| 951 | + | None => scratch_url.to_string(), | |
| 952 | + | Some(dbname) => { | |
| 953 | + | let maintenance_url = pg_url_with_dbname(scratch_url, "postgres"); | |
| 954 | + | log.line(&format!("---- create scratch db {dbname} ----\n")) | |
| 955 | + | .await; | |
| 956 | + | if let Err(e) = pg_create_db(&maintenance_url, dbname).await { | |
| 957 | + | let msg = format!("{label}: creating scratch db {dbname}: {e}"); | |
| 958 | + | log.line(&msg).await; | |
| 959 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 960 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 961 | + | ))); | |
| 962 | + | } | |
| 963 | + | pg_url_with_dbname(scratch_url, dbname) | |
| 964 | + | } | |
| 965 | + | }; | |
| 966 | + | ||
| 967 | + | let owner_role = check | |
| 968 | + | .owner_role | |
| 969 | + | .as_deref() | |
| 970 | + | .unwrap_or(&ctx.cfg.scratch_owner_role); | |
| 879 | 971 | log.line("---- reset_scratch ----\n").await; | |
| 880 | - | if let Err(e) = reset_scratch(db_url, &ctx.cfg.scratch_owner_role).await { | |
| 881 | - | let msg = format!("scratch reset: {e}"); | |
| 972 | + | if let Err(e) = reset_scratch(&db_url, owner_role).await { | |
| 973 | + | let msg = format!("{label}: scratch reset: {e}"); | |
| 882 | 974 | log.line(&msg).await; | |
| 883 | - | return Ok(GateOutcome::failed(GateFailure::RestoreFailed { | |
| 884 | - | reason: msg, | |
| 885 | - | })); | |
| 975 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 976 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 977 | + | ))); | |
| 886 | 978 | } | |
| 887 | 979 | log.line(&format!("---- restore_dump ({backup_path}) ----\n")) | |
| 888 | 980 | .await; | |
| 889 | - | if let Err(e) = restore_dump(db_url, &backup_path, log).await { | |
| 890 | - | let msg = format!("restore: {e}"); | |
| 981 | + | if let Err(e) = restore_dump(&db_url, &backup_path, log).await { | |
| 982 | + | let msg = format!("{label}: restore: {e}"); | |
| 891 | 983 | log.line(&msg).await; | |
| 892 | - | return Ok(GateOutcome::failed(GateFailure::RestoreFailed { | |
| 893 | - | reason: msg, | |
| 894 | - | })); | |
| 984 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 985 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 986 | + | ))); | |
| 895 | 987 | } | |
| 896 | 988 | ||
| 897 | - | let migrations_dir = ctx.worktree.join("server").join("migrations"); | |
| 989 | + | let migrations_dir = ctx.worktree.join(&check.dir); | |
| 898 | 990 | log.line("---- run_migrator ----\n").await; | |
| 899 | - | match run_migrator(db_url, &migrations_dir).await { | |
| 991 | + | match run_migrator(&db_url, &migrations_dir).await { | |
| 900 | 992 | Ok(()) => { | |
| 901 | - | log.line(&format!("restored {backup_path} + migrated")) | |
| 993 | + | log.line(&format!("{label}: restored {backup_path} + migrated\n")) | |
| 902 | 994 | .await; | |
| 903 | - | Ok(GateOutcome::passed(PassNote::Migrated { backup_path })) | |
| 995 | + | Ok(CheckResult::Passed { backup_path }) | |
| 904 | 996 | } | |
| 905 | 997 | Err(e) => { | |
| 906 | - | let err_s = e.to_string(); | |
| 998 | + | let err_s = format!("{label}: {e}"); | |
| 907 | 999 | log.line(&err_s).await; | |
| 908 | - | Ok(GateOutcome::failed(classify::classify_migration_error( | |
| 909 | - | &err_s, None, | |
| 1000 | + | Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 1001 | + | classify::classify_migration_error(&err_s, None), | |
| 910 | 1002 | ))) | |
| 911 | 1003 | } | |
| 912 | 1004 | } | |
| @@ -2312,19 +2404,100 @@ | |||
| 2312 | 2404 | } | |
| 2313 | 2405 | } | |
| 2314 | 2406 | ||
| 2315 | - | /// Record a backup row fetched `hours_ago`, as `/backup/fetch` would. | |
| 2407 | + | /// Record a `server` backup row fetched `hours_ago`, as `/backup/fetch` would. | |
| 2316 | 2408 | async fn seed_backup(ctx: &GateCtx, hours_ago: i64) { | |
| 2409 | + | seed_named_backup(ctx, "server", hours_ago).await; | |
| 2410 | + | } | |
| 2411 | + | ||
| 2412 | + | /// Record a backup row for one named dump. | |
| 2413 | + | async fn seed_named_backup(ctx: &GateCtx, name: &str, hours_ago: i64) { | |
| 2317 | 2414 | let at = (Utc::now() - chrono::Duration::hours(hours_ago)).to_rfc3339(); | |
| 2318 | 2415 | sqlx::query( | |
| 2319 | - | "INSERT INTO backups (fetched_at, source, local_path, byte_size) | |
| 2320 | - | VALUES (?, 'file:///x.sql.gz', '/tmp/sando-test-backup.sql.gz', 1000000)", | |
| 2416 | + | "INSERT INTO backups (name, fetched_at, source, local_path, byte_size) | |
| 2417 | + | VALUES (?, ?, 'file:///x.sql.gz', '/tmp/sando-test-backup.sql.gz', 1000000)", | |
| 2321 | 2418 | ) | |
| 2419 | + | .bind(name) | |
| 2322 | 2420 | .bind(at) | |
| 2323 | 2421 | .execute(&ctx.pool) | |
| 2324 | 2422 | .await | |
| 2325 | 2423 | .unwrap(); | |
| 2326 | 2424 | } | |
| 2327 | 2425 | ||
| 2426 | + | /// The multithreaded check, as `sando-daemon.toml` configures it. | |
| 2427 | + | fn mt_check() -> crate::config::MigrationCheck { | |
| 2428 | + | crate::config::MigrationCheck { | |
| 2429 | + | dir: std::path::PathBuf::from("multithreaded/migrations"), | |
| 2430 | + | backup: "multithreaded".into(), | |
| 2431 | + | scratch_db: Some("sando_scratch_mt".into()), | |
| 2432 | + | owner_role: Some("multithreaded".into()), | |
| 2433 | + | } | |
| 2434 | + | } | |
| 2435 | + | ||
| 2436 | + | /// Re-point a `dry_run_ctx` at one check, keeping its pool and scratch URL. | |
| 2437 | + | fn with_check(ctx: &mut GateCtx, check: crate::config::MigrationCheck) { | |
| 2438 | + | let mut cfg = crate::config::Config::for_tests(); | |
| 2439 | + | cfg.scratch_db_url = ctx.cfg.scratch_db_url.clone(); | |
| 2440 | + | cfg.backup_max_age_hours = ctx.cfg.backup_max_age_hours; | |
| 2441 | + | cfg.logs_root = ctx.cfg.logs_root.clone(); | |
| 2442 | + | cfg.migration_checks = vec![check]; | |
| 2443 | + | ctx.cfg = std::sync::Arc::new(cfg); | |
| 2444 | + | } | |
| 2445 | + | ||
| 2446 | + | #[tokio::test] | |
| 2447 | + | async fn migration_dry_run_blocks_when_a_checks_own_dump_was_never_fetched() { | |
| 2448 | + | // The hazard the check list exists for: multithreaded applies its own | |
| 2449 | + | // migrations at boot against its own database, so the server's dump must | |
| 2450 | + | // never stand in for it. A fetched `server` row with no `multithreaded` | |
| 2451 | + | // row is exactly that substitution, and it has to block. | |
| 2452 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 2453 | + | let mut ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 2454 | + | with_check(&mut ctx, mt_check()); | |
| 2455 | + | seed_named_backup(&ctx, "server", 1).await; | |
| 2456 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 2457 | + | ||
| 2458 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 2459 | + | log.close().await; | |
| 2460 | + | ||
| 2461 | + | let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else { | |
| 2462 | + | panic!("a missing multithreaded dump must block"); | |
| 2463 | + | }; | |
| 2464 | + | let GateBlocker::NoBackupAvailable { check, backup } = blocker else { | |
| 2465 | + | panic!("expected NoBackupAvailable, got {blocker:?}"); | |
| 2466 | + | }; | |
| 2467 | + | assert_eq!(backup, "multithreaded", "names the dump that is missing"); | |
| 2468 | + | assert!( | |
| 2469 | + | check.contains("multithreaded/migrations"), | |
| 2470 | + | "names the check that wanted it, got {check}" | |
| 2471 | + | ); | |
| 2472 | + | } | |
| 2473 | + | ||
| 2474 | + | #[tokio::test] | |
| 2475 | + | async fn migration_dry_run_freshness_is_per_dump() { | |
| 2476 | + | // A fresh server dump must not make a 45-day-old multithreaded dump look | |
| 2477 | + | // current: the clock is per-database, or the second check inherits the | |
| 2478 | + | // first's freshness and the gate is theatre. | |
| 2479 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 2480 | + | let mut ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 2481 | + | with_check(&mut ctx, mt_check()); | |
| 2482 | + | seed_named_backup(&ctx, "server", 1).await; | |
| 2483 | + | seed_named_backup(&ctx, "multithreaded", 24 * 45).await; | |
| 2484 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 2485 | + | ||
| 2486 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 2487 | + | log.close().await; | |
| 2488 | + | ||
| 2489 | + | let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else { | |
| 2490 | + | panic!("a 45-day-old multithreaded dump must block"); | |
| 2491 | + | }; | |
| 2492 | + | let GateBlocker::BackupStale { check, .. } = blocker else { | |
| 2493 | + | panic!("expected BackupStale, got {blocker:?}"); | |
| 2494 | + | }; | |
| 2495 | + | assert!( | |
| 2496 | + | check.contains("multithreaded/migrations"), | |
| 2497 | + | "names the check whose dump is stale, got {check}" | |
| 2498 | + | ); | |
| 2499 | + | } | |
| 2500 | + | ||
| 2328 | 2501 | #[tokio::test] | |
| 2329 | 2502 | async fn migration_dry_run_blocks_on_a_stale_backup() { | |
| 2330 | 2503 | // The failure this closes: the gate used to check only that a backups row | |
| @@ -2344,6 +2517,7 @@ | |||
| 2344 | 2517 | let GateBlocker::BackupStale { | |
| 2345 | 2518 | age_hours, | |
| 2346 | 2519 | max_age_hours, | |
| 2520 | + | .. | |
| 2347 | 2521 | } = blocker | |
| 2348 | 2522 | else { | |
| 2349 | 2523 | panic!("expected BackupStale, got {blocker:?}"); |
| @@ -49,6 +49,7 @@ | |||
| 49 | 49 | cfg.validate()?; | |
| 50 | 50 | let topo = topology::Topology::load(&cfg.topology_path)?; | |
| 51 | 51 | topo.ensure_build_host_not_serving(&cfg.build_host)?; | |
| 52 | + | topo.ensure_migration_checks_have_backups(&cfg.migration_checks)?; | |
| 52 | 53 | Ok(topo) | |
| 53 | 54 | } | |
| 54 | 55 | ||
| @@ -71,6 +72,7 @@ | |||
| 71 | 72 | let cfg = Arc::new(config::Config::load()?); | |
| 72 | 73 | let topo = Arc::new(topology::Topology::load(&cfg.topology_path)?); | |
| 73 | 74 | topo.ensure_build_host_not_serving(&cfg.build_host)?; | |
| 75 | + | topo.ensure_migration_checks_have_backups(&cfg.migration_checks)?; | |
| 74 | 76 | tokio::fs::create_dir_all(&cfg.workdir).await?; | |
| 75 | 77 | tokio::fs::create_dir_all(&cfg.release_root).await?; | |
| 76 | 78 | git::ensure_bare_repo(Path::new(&topo.repo.bare_path)).await?; |
| @@ -98,8 +98,14 @@ | |||
| 98 | 98 | /// the gate's clock started. | |
| 99 | 99 | BurnInElapsed { hours: u32 }, | |
| 100 | 100 | /// `migration_dry_run` — scratch DB restored from `backup_path` and | |
| 101 | - | /// every migration ran without error. | |
| 102 | - | Migrated { backup_path: String }, | |
| 101 | + | /// every migration ran without error. `checks` names each configured | |
| 102 | + | /// migrations dir that passed; it is empty on rows written before the gate | |
| 103 | + | /// ran more than the server's, and `backup_path` is the first check's dump. | |
| 104 | + | Migrated { | |
| 105 | + | backup_path: String, | |
| 106 | + | #[serde(default)] | |
| 107 | + | checks: Vec<String>, | |
| 108 | + | }, | |
| 103 | 109 | /// `cargo_test` — `cargo test --release` exited 0. | |
| 104 | 110 | TestsPassed { duration_s: u32 }, | |
| 105 | 111 | /// `manual_confirm` — an operator inserted a passing row out-of-band. | |
| @@ -118,7 +124,15 @@ | |||
| 118 | 124 | match self { | |
| 119 | 125 | PassNote::HealthyProbe { after_ms } => format!("served /health in {after_ms}ms"), | |
| 120 | 126 | PassNote::BurnInElapsed { hours } => format!("{hours} hours elapsed"), | |
| 121 | - | PassNote::Migrated { backup_path } => format!("restored {backup_path} + migrated"), | |
| 127 | + | PassNote::Migrated { | |
| 128 | + | backup_path, | |
| 129 | + | checks, | |
| 130 | + | } => match checks.len() { | |
| 131 | + | // A pre-list row, or the single-check case: keep the wording the | |
| 132 | + | // operator surface has always shown. | |
| 133 | + | 0 | 1 => format!("restored {backup_path} + migrated"), | |
| 134 | + | n => format!("restored + migrated {n} databases: {}", checks.join(", ")), | |
| 135 | + | }, | |
| 122 | 136 | PassNote::TestsPassed { duration_s } => format!("tests passed in {duration_s}s"), | |
| 123 | 137 | PassNote::OperatorConfirmed { at } => format!("operator confirmed at {at}"), | |
| 124 | 138 | PassNote::NodesHealthy { nodes } => format!("{nodes} node(s) healthy"), | |
| @@ -140,13 +154,25 @@ | |||
| 140 | 154 | /// `manual_confirm`: no out-of-band passing row exists for this | |
| 141 | 155 | /// (tier, version). | |
| 142 | 156 | AwaitingOperatorConfirmation, | |
| 143 | - | /// `migration_dry_run`: no row in `backups` to restore from. | |
| 144 | - | NoBackupAvailable, | |
| 145 | - | /// `migration_dry_run`: the newest `backups` row is older than | |
| 146 | - | /// `cfg.backup_max_age_hours`. Restoring it would dry-run the migrations | |
| 147 | - | /// against a schema prod has since moved past, which passes green while | |
| 148 | - | /// proving nothing — so the gate blocks instead. | |
| 149 | - | BackupStale { age_hours: i64, max_age_hours: u32 }, | |
| 157 | + | /// `migration_dry_run`: no row in `backups` for the named dump to restore | |
| 158 | + | /// from. `check` is the migrations dir that wanted it. Both fields default | |
| 159 | + | /// to empty on rows written before the gate took a list of checks. | |
| 160 | + | NoBackupAvailable { | |
| 161 | + | #[serde(default)] | |
| 162 | + | check: String, | |
| 163 | + | #[serde(default)] | |
| 164 | + | backup: String, | |
| 165 | + | }, | |
| 166 | + | /// `migration_dry_run`: the newest `backups` row for this check's dump is | |
| 167 | + | /// older than `cfg.backup_max_age_hours`. Restoring it would dry-run the | |
| 168 | + | /// migrations against a schema prod has since moved past, which passes green | |
| 169 | + | /// while proving nothing — so the gate blocks instead. | |
| 170 | + | BackupStale { | |
| 171 | + | age_hours: i64, | |
| 172 | + | max_age_hours: u32, | |
| 173 | + | #[serde(default)] | |
| 174 | + | check: String, | |
| 175 | + | }, | |
| 150 | 176 | /// `migration_dry_run` / `boot_smoke` / `cargo_test`: daemon config | |
| 151 | 177 | /// has no `scratch_db_url`. | |
| 152 | 178 | ScratchDbUrlUnset, | |
| @@ -166,11 +192,29 @@ | |||
| 166 | 192 | hours_total, | |
| 167 | 193 | } => format!("{hours_remaining} hours remaining of {hours_total}"), | |
| 168 | 194 | GateBlocker::AwaitingOperatorConfirmation => "waiting on operator confirmation".into(), | |
| 169 | - | GateBlocker::NoBackupAvailable => "no backup fetched; call /backup/fetch first".into(), | |
| 195 | + | GateBlocker::NoBackupAvailable { backup, .. } => { | |
| 196 | + | let which = if backup.is_empty() { | |
| 197 | + | String::new() | |
| 198 | + | } else { | |
| 199 | + | format!("{backup} ") | |
| 200 | + | }; | |
| 201 | + | format!("no {which}backup fetched; call /backup/fetch first") | |
| 202 | + | } | |
| 170 | 203 | GateBlocker::BackupStale { | |
| 171 | 204 | age_hours, | |
| 172 | 205 | max_age_hours, | |
| 173 | - | } => format!("backup is {age_hours}h old (max {max_age_hours}h); re-run /backup/fetch"), | |
| 206 | + | check, | |
| 207 | + | } => { | |
| 208 | + | let which = if check.is_empty() { | |
| 209 | + | String::new() | |
| 210 | + | } else { | |
| 211 | + | format!(" for {check}") | |
| 212 | + | }; | |
| 213 | + | format!( | |
| 214 | + | "backup{which} is {age_hours}h old (max {max_age_hours}h); re-run \ | |
| 215 | + | /backup/fetch" | |
| 216 | + | ) | |
| 217 | + | } | |
| 174 | 218 | GateBlocker::ScratchDbUrlUnset => "scratch_db_url unset in daemon config".into(), | |
| 175 | 219 | GateBlocker::ArtifactMissing { version } => { | |
| 176 | 220 | format!("no artifact for version {version}") |
| @@ -135,10 +135,11 @@ | |||
| 135 | 135 | branch: "main".into(), | |
| 136 | 136 | upstream: None, | |
| 137 | 137 | }, | |
| 138 | - | backup: BackupConfig { | |
| 138 | + | backup: vec![BackupConfig { | |
| 139 | + | name: "server".into(), | |
| 139 | 140 | source: "file:///tmp/b".into(), | |
| 140 | 141 | local_path: "/tmp/b".into(), | |
| 141 | - | }, | |
| 142 | + | }], | |
| 142 | 143 | tiers, | |
| 143 | 144 | aux_repos: Vec::new(), | |
| 144 | 145 | } |
| @@ -6,7 +6,17 @@ | |||
| 6 | 6 | #[derive(Debug, Clone, Serialize, Deserialize)] | |
| 7 | 7 | pub struct Topology { | |
| 8 | 8 | pub repo: RepoConfig, | |
| 9 | - | pub backup: BackupConfig, | |
| 9 | + | /// Prod dumps `/backup/fetch` pulls, one per database `migration_dry_run` | |
| 10 | + | /// has a check for. A list because the repo ships more than one service | |
| 11 | + | /// with its own database and its own `sqlx::migrate!()` at boot: the server | |
| 12 | + | /// migrates `makenotwork`, multithreaded migrates `multithreaded`, and a | |
| 13 | + | /// gate that restores only the first proves nothing about the second. | |
| 14 | + | /// | |
| 15 | + | /// Accepts both the historical single `[backup]` table and a `[[backup]]` | |
| 16 | + | /// list, so a deployed `sando.toml` keeps working unedited (the single form | |
| 17 | + | /// deserializes to a one-entry list named `server`). | |
| 18 | + | #[serde(deserialize_with = "one_or_many_backup")] | |
| 19 | + | pub backup: Vec<BackupConfig>, | |
| 10 | 20 | #[serde(rename = "tier")] | |
| 11 | 21 | pub tiers: Vec<Tier>, | |
| 12 | 22 | /// Extra repos to fetch and check out beside the main worktree before a | |
| @@ -68,10 +78,41 @@ | |||
| 68 | 78 | ||
| 69 | 79 | #[derive(Debug, Clone, Serialize, Deserialize)] | |
| 70 | 80 | pub struct BackupConfig { | |
| 81 | + | /// Which database this dump is of, as referenced by a daemon-config | |
| 82 | + | /// `[[migration_check]]`'s `backup` key and recorded in the `backups` | |
| 83 | + | /// table's `name` column. Defaults to `server` so the historical single | |
| 84 | + | /// `[backup]` table needs no edit — and so the pre-existing rows, which the | |
| 85 | + | /// state-DB migration backfills to `server`, keep matching it. | |
| 86 | + | #[serde(default = "default_backup_name")] | |
| 87 | + | pub name: String, | |
| 71 | 88 | pub source: String, | |
| 72 | 89 | pub local_path: String, | |
| 73 | 90 | } | |
| 74 | 91 | ||
| 92 | + | fn default_backup_name() -> String { | |
| 93 | + | "server".into() | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | /// Accept `[backup]` (one table) or `[[backup]]` (a list) for the same key. | |
| 97 | + | /// Serde cannot express "table or sequence" on a `Vec` field on its own, and | |
| 98 | + | /// the alternative — renaming the key — would break every deployed | |
| 99 | + | /// `sando.toml` at startup, on the box whose whole job is deploying. | |
| 100 | + | fn one_or_many_backup<'de, D>(de: D) -> std::result::Result<Vec<BackupConfig>, D::Error> | |
| 101 | + | where | |
| 102 | + | D: serde::Deserializer<'de>, | |
| 103 | + | { | |
| 104 | + | #[derive(Deserialize)] | |
| 105 | + | #[serde(untagged)] | |
| 106 | + | enum OneOrMany { | |
| 107 | + | One(BackupConfig), | |
| 108 | + | Many(Vec<BackupConfig>), | |
| 109 | + | } | |
| 110 | + | Ok(match OneOrMany::deserialize(de)? { | |
| 111 | + | OneOrMany::One(b) => vec![b], | |
| 112 | + | OneOrMany::Many(v) => v, | |
| 113 | + | }) | |
| 114 | + | } | |
| 115 | + | ||
| 75 | 116 | #[derive(Debug, Clone, Serialize, Deserialize)] | |
| 76 | 117 | pub struct Tier { | |
| 77 | 118 | pub name: TierId, | |
| @@ -279,7 +320,80 @@ | |||
| 279 | 320 | self.validate() | |
| 280 | 321 | } | |
| 281 | 322 | ||
| 323 | + | /// Every `[[migration_check]]` in the daemon config must name a dump this | |
| 324 | + | /// topology declares. The two files are separate — daemon config is | |
| 325 | + | /// per-host, topology is per-project — so nothing but this catches a check | |
| 326 | + | /// pointing at a backup nobody fetches. Left uncaught it surfaces as a | |
| 327 | + | /// permanently `Blocked` gate the first time someone promotes, which reads | |
| 328 | + | /// like a missed fetch rather than a config typo. Called from `main` once | |
| 329 | + | /// both are loaded, and so under `--check-config`. | |
| 330 | + | pub fn ensure_migration_checks_have_backups( | |
| 331 | + | &self, | |
| 332 | + | checks: &[crate::config::MigrationCheck], | |
| 333 | + | ) -> Result<()> { | |
| 334 | + | for c in checks { | |
| 335 | + | anyhow::ensure!( | |
| 336 | + | self.backup_named(&c.backup).is_some(), | |
| 337 | + | "migration_check {} restores backup {:?}, which no [[backup]] in {} declares \ | |
| 338 | + | (have: {})", | |
| 339 | + | c.dir.display(), | |
| 340 | + | c.backup, | |
| 341 | + | "the topology", | |
| 342 | + | self.backup | |
| 343 | + | .iter() | |
| 344 | + | .map(|b| b.name.as_str()) | |
| 345 | + | .collect::<Vec<_>>() | |
| 346 | + | .join(", "), | |
| 347 | + | ); | |
| 348 | + | } | |
| 349 | + | Ok(()) | |
| 350 | + | } | |
| 351 | + | ||
| 352 | + | /// The configured dump for `name`, or `None` when nothing declares it. | |
| 353 | + | pub fn backup_named(&self, name: &str) -> Option<&BackupConfig> { | |
| 354 | + | self.backup.iter().find(|b| b.name == name) | |
| 355 | + | } | |
| 356 | + | ||
| 282 | 357 | fn validate(&self) -> Result<()> { | |
| 358 | + | anyhow::ensure!( | |
| 359 | + | !self.backup.is_empty(), | |
| 360 | + | "topology declares no [backup]; migration_dry_run would have nothing to restore" | |
| 361 | + | ); | |
| 362 | + | for (i, b) in self.backup.iter().enumerate() { | |
| 363 | + | anyhow::ensure!( | |
| 364 | + | !b.name.is_empty() | |
| 365 | + | && b.name | |
| 366 | + | .bytes() | |
| 367 | + | .all(|c| c.is_ascii_alphanumeric() || c == b'_' || c == b'-'), | |
| 368 | + | "backup name {:?} must be non-empty and match [A-Za-z0-9_-]+; it keys the \ | |
| 369 | + | `backups` table and a daemon-config migration_check", | |
| 370 | + | b.name, | |
| 371 | + | ); | |
| 372 | + | anyhow::ensure!( | |
| 373 | + | !b.source.is_empty() && !b.local_path.is_empty(), | |
| 374 | + | "backup {} has an empty source/local_path", | |
| 375 | + | b.name, | |
| 376 | + | ); | |
| 377 | + | // Two dumps sharing a name would interleave in `backups`, so the | |
| 378 | + | // freshness check and the plausibility floor would each read the | |
| 379 | + | // other's row. Two sharing a `local_path` would overwrite each | |
| 380 | + | // other on disk, and whichever fetched last would be restored for | |
| 381 | + | // both — green, and proving nothing about one of the databases. | |
| 382 | + | for prior in &self.backup[..i] { | |
| 383 | + | anyhow::ensure!( | |
| 384 | + | prior.name != b.name, | |
| 385 | + | "two backup entries share the name {:?}", | |
| 386 | + | b.name, | |
| 387 | + | ); | |
| 388 | + | anyhow::ensure!( | |
| 389 | + | prior.local_path != b.local_path, | |
| 390 | + | "backups {:?} and {:?} share local_path {:?}; they would overwrite each other", | |
| 391 | + | prior.name, | |
| 392 | + | b.name, | |
| 393 | + | b.local_path, | |
| 394 | + | ); | |
| 395 | + | } | |
| 396 | + | } | |
| 283 | 397 | anyhow::ensure!( | |
| 284 | 398 | !self.tiers.is_empty(), | |
| 285 | 399 | "topology must declare at least one tier" | |
| @@ -607,6 +721,139 @@ | |||
| 607 | 721 | assert!(err.contains("share or nest checkout_dir"), "{err}"); | |
| 608 | 722 | } | |
| 609 | 723 | ||
| 724 | + | /// A topology whose `[backup]`/`[[backup]]` section is `backup_block`. | |
| 725 | + | fn topo_with_backup_block(backup_block: &str) -> Result<Topology> { | |
| 726 | + | let raw = format!( | |
| 727 | + | r#" | |
| 728 | + | [repo] | |
| 729 | + | bare_path = "/tmp/repo.git" | |
| 730 | + | branch = "main" | |
| 731 | + | {backup_block} | |
| 732 | + | [[tier]] | |
| 733 | + | name = "b" | |
| 734 | + | provisioned = true | |
| 735 | + | gates = [{{ kind = "node_health" }}] | |
| 736 | + | [[tier.node]] | |
| 737 | + | name = "prod-1" | |
| 738 | + | ssh_target = "prod-1" | |
| 739 | + | release_root = "/srv/mnw" | |
| 740 | + | "# | |
| 741 | + | ); | |
| 742 | + | let topo: Topology = toml::from_str(&raw)?; | |
| 743 | + | topo.validate_for_test()?; | |
| 744 | + | Ok(topo) | |
| 745 | + | } | |
| 746 | + | ||
| 747 | + | #[test] | |
| 748 | + | fn a_single_backup_table_still_parses_as_one_named_server() { | |
| 749 | + | // Back-compat is the point: every deployed sando.toml uses the single | |
| 750 | + | // `[backup]` form, and the box this config lives on is the one whose job | |
| 751 | + | // is deploying — it must not need an edit to start. | |
| 752 | + | let topo = topo_with_backup_block( | |
| 753 | + | r#" | |
| 754 | + | [backup] | |
| 755 | + | source = "ssh://prod/dump.sql.gz" | |
| 756 | + | local_path = "/tmp/dump.sql.gz""#, | |
| 757 | + | ) | |
| 758 | + | .expect("the single-table form must still load"); | |
| 759 | + | assert_eq!(topo.backup.len(), 1); | |
| 760 | + | assert_eq!(topo.backup[0].name, "server"); | |
| 761 | + | assert!(topo.backup_named("server").is_some()); | |
| 762 | + | } | |
| 763 | + | ||
| 764 | + | #[test] | |
| 765 | + | fn a_backup_list_parses_and_keeps_its_names() { | |
| 766 | + | let topo = topo_with_backup_block( | |
| 767 | + | r#" | |
| 768 | + | [[backup]] | |
| 769 | + | name = "server" | |
| 770 | + | source = "ssh://prod/makenotwork/latest.sql.gz" | |
| 771 | + | local_path = "/tmp/server.sql.gz" | |
| 772 | + | [[backup]] | |
| 773 | + | name = "multithreaded" | |
| 774 | + | source = "ssh://prod/multithreaded/latest.sql.gz" | |
| 775 | + | local_path = "/tmp/mt.sql.gz""#, | |
| 776 | + | ) | |
| 777 | + | .expect("the list form must load"); | |
| 778 | + | assert_eq!(topo.backup.len(), 2); | |
| 779 | + | assert_eq!( | |
| 780 | + | topo.backup_named("multithreaded").unwrap().local_path, | |
| 781 | + | "/tmp/mt.sql.gz" | |
| 782 | + | ); | |
| 783 | + | assert!(topo.backup_named("nope").is_none()); | |
| 784 | + | } | |
| 785 | + | ||
| 786 | + | #[test] | |
| 787 | + | fn two_backups_sharing_a_name_are_rejected() { | |
| 788 | + | // They would interleave in `backups`, so the freshness check and the | |
| 789 | + | // plausibility floor would each read the other's row. | |
| 790 | + | let err = topo_with_backup_block( | |
| 791 | + | r#" | |
| 792 | + | [[backup]] | |
| 793 | + | name = "server" | |
| 794 | + | source = "a" | |
| 795 | + | local_path = "/tmp/a.sql.gz" | |
| 796 | + | [[backup]] | |
| 797 | + | name = "server" | |
| 798 | + | source = "b" | |
| 799 | + | local_path = "/tmp/b.sql.gz""#, | |
| 800 | + | ) | |
| 801 | + | .unwrap_err() | |
| 802 | + | .to_string(); | |
| 803 | + | assert!(err.contains("share the name"), "{err}"); | |
| 804 | + | } | |
| 805 | + | ||
| 806 | + | #[test] | |
| 807 | + | fn two_backups_sharing_a_local_path_are_rejected() { | |
| 808 | + | // Whichever fetched last would be restored for both checks — green, and | |
| 809 | + | // proving nothing about one of the two databases. | |
| 810 | + | let err = topo_with_backup_block( | |
| 811 | + | r#" | |
| 812 | + | [[backup]] | |
| 813 | + | name = "server" | |
| 814 | + | source = "a" | |
| 815 | + | local_path = "/tmp/same.sql.gz" | |
| 816 | + | [[backup]] | |
| 817 | + | name = "multithreaded" | |
| 818 | + | source = "b" | |
| 819 | + | local_path = "/tmp/same.sql.gz""#, | |
| 820 | + | ) | |
| 821 | + | .unwrap_err() | |
| 822 | + | .to_string(); | |
| 823 | + | assert!(err.contains("share local_path"), "{err}"); | |
| 824 | + | } | |
| 825 | + | ||
| 826 | + | #[test] | |
| 827 | + | fn a_migration_check_naming_an_undeclared_backup_is_rejected_at_startup() { | |
| 828 | + | // Daemon config and topology are separate files, so nothing but this | |
| 829 | + | // cross-check catches the typo. Uncaught it surfaces as a permanently | |
| 830 | + | // Blocked gate on the next promote, which reads like a missed fetch. | |
| 831 | + | let topo = topo_with_backup_block( | |
| 832 | + | r#" | |
| 833 | + | [backup] | |
| 834 | + | source = "s" | |
| 835 | + | local_path = "/tmp/d""#, | |
| 836 | + | ) | |
| 837 | + | .unwrap(); | |
| 838 | + | let checks = vec![crate::config::MigrationCheck { | |
| 839 | + | dir: std::path::PathBuf::from("multithreaded/migrations"), | |
| 840 | + | backup: "multithreaded".into(), | |
| 841 | + | scratch_db: Some("sando_scratch_mt".into()), | |
| 842 | + | owner_role: Some("multithreaded".into()), | |
| 843 | + | }]; | |
| 844 | + | let err = topo | |
| 845 | + | .ensure_migration_checks_have_backups(&checks) | |
| 846 | + | .unwrap_err() | |
| 847 | + | .to_string(); | |
| 848 | + | assert!(err.contains("which no [[backup]]"), "{err}"); | |
| 849 | + | ||
| 850 | + | // And the shipped pair agree, which is the case that actually ships. | |
| 851 | + | let shipped_checks = crate::config::default_migration_checks_for_test(); | |
| 852 | + | shipped() | |
| 853 | + | .ensure_migration_checks_have_backups(&shipped_checks) | |
| 854 | + | .expect("the default server check resolves against the shipped topology"); | |
| 855 | + | } | |
| 856 | + | ||
| 610 | 857 | #[test] | |
| 611 | 858 | fn real_sando_toml_loads_clean() { | |
| 612 | 859 | // The shipped topology must satisfy the invariant — guards against a |