max / makenotwork
- Co-Authored-By
- Claude Opus 5 (1M context) <noreply@anthropic.com>
- Claude-Session
- https://claude.ai/code/session_01EEmeiSJnmyL98QzA5Dwsvz
9 files changed,
+3170 insertions,
-500 deletions
| @@ -1,3972 +1,0 @@ | |||
| 1 | - | //! Gate execution. Each gate kind has a runner that produces a pass/fail | |
| 2 | - | //! outcome plus an optional detail string (typically a stderr tail or a | |
| 3 | - | //! human-readable reason). Outcomes are persisted to `gate_runs` so /state | |
| 4 | - | //! and the TUI can show them. | |
| 5 | - | ||
| 6 | - | use crate::classify; | |
| 7 | - | use crate::config::AppConfig; | |
| 8 | - | use crate::domain::{AppId, GateKind, GateRunId, TierId, Version}; | |
| 9 | - | use crate::events::{self, Event, EventTx}; | |
| 10 | - | use crate::outcome::{GateBlocker, GateFailure, GateOutcome, LogRef, PassNote}; | |
| 11 | - | use crate::topology::Gate; | |
| 12 | - | use anyhow::{Context, Result}; | |
| 13 | - | use chrono::Utc; | |
| 14 | - | use ops_core::live_log::LiveLog; | |
| 15 | - | use ops_core::remote::LogSink; // brings `LiveLog::write_chunk` (the sink trait) into scope | |
| 16 | - | use ops_exec::{DiscardSink, sh_quote}; | |
| 17 | - | use sqlx::SqlitePool; | |
| 18 | - | use std::collections::HashMap; | |
| 19 | - | use std::path::Path; | |
| 20 | - | use std::path::PathBuf; | |
| 21 | - | use std::sync::Arc; | |
| 22 | - | use tokio::io::AsyncReadExt; | |
| 23 | - | use tokio::process::Command; | |
| 24 | - | ||
| 25 | - | /// The gate live-log callback: emit each chunk as a `GateLogChunk` event so the | |
| 26 | - | /// TUI sees the tail stream in real time. `ops_core::live_log::LiveLog` owns the | |
| 27 | - | /// disk append and the per-run sequence counter; this closure is the one | |
| 28 | - | /// tool-specific bit. | |
| 29 | - | fn gate_chunk_cb(events: EventTx, run_id: GateRunId) -> ops_core::live_log::ChunkCallback { | |
| 30 | - | Box::new(move |seq, text| { | |
| 31 | - | events::emit( | |
| 32 | - | &events, | |
| 33 | - | Event::GateLogChunk { | |
| 34 | - | run_id, | |
| 35 | - | seq, | |
| 36 | - | text: text.to_owned(), | |
| 37 | - | }, | |
| 38 | - | ); | |
| 39 | - | }) | |
| 40 | - | } | |
| 41 | - | ||
| 42 | - | pub struct GateCtx { | |
| 43 | - | pub pool: SqlitePool, | |
| 44 | - | pub cfg: Arc<AppConfig>, | |
| 45 | - | pub tier: TierId, | |
| 46 | - | pub version: Version, | |
| 47 | - | /// The checkout this run's artifact was built from, when there is one. | |
| 48 | - | /// | |
| 49 | - | /// `None` for an accepted artifact: it was built elsewhere and Sando has no | |
| 50 | - | /// source tree for it. That is the boundary made visible (wiki | |
| 51 | - | /// [[sando-bento-boundary]]) — artifact-scoped gates belong to the builder, | |
| 52 | - | /// so a gate that reads source is one Sando should refuse to run here rather | |
| 53 | - | /// than resolve against a path that does not exist. | |
| 54 | - | pub worktree: Option<PathBuf>, | |
| 55 | - | /// The published, content-addressed bundle this run is about, when it has | |
| 56 | - | /// been published yet. `migration_dry_run` prefers it over the worktree, so | |
| 57 | - | /// what it proves is inside the digest rather than beside it. | |
| 58 | - | pub bundle: Option<PathBuf>, | |
| 59 | - | pub events: EventTx, | |
| 60 | - | /// Nodes the `node_health` post-deploy gate probes. Empty for build-time | |
| 61 | - | /// gate runs on the host (where `node_health` never appears); filled at | |
| 62 | - | /// promote time with each freshly-deployed node and its executor. | |
| 63 | - | pub nodes: Vec<NodeProbe>, | |
| 64 | - | /// The `build_runs.id` this gate run vouches for — the artifact identity | |
| 65 | - | /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so | |
| 66 | - | /// promote can resolve the artifact through the evidence for a specific build, | |
| 67 | - | /// not through a version string that a later rebuild can silently reuse. | |
| 68 | - | /// `None` for legacy/pre-identity runs and gate unit tests. | |
| 69 | - | pub build_id: Option<i64>, | |
| 70 | - | /// Where each `[[aux_repo]]` is checked out, by topology name. Aux repos sit | |
| 71 | - | /// beside the per-sha worktree rather than under it, so a `test_target` that | |
| 72 | - | /// names one cannot be resolved against `worktree` alone. Filled from the | |
| 73 | - | /// topology at build time; empty at promote time, where the only gate that | |
| 74 | - | /// runs is `node_health` and there is no checkout at all. | |
| 75 | - | pub aux_dirs: HashMap<String, PathBuf>, | |
| 76 | - | /// The tier's public URL, for [`Gate::PageSmoke`]. `None` on every | |
| 77 | - | /// build-time run and on any tier that declares none. | |
| 78 | - | /// | |
| 79 | - | /// [`Gate::PageSmoke`]: crate::topology::Gate::PageSmoke | |
| 80 | - | pub public_url: Option<String>, | |
| 81 | - | } | |
| 82 | - | ||
| 83 | - | impl GateCtx { | |
| 84 | - | /// The `logs_root` sub-directory this run's gate logs land in. | |
| 85 | - | /// | |
| 86 | - | /// The build id, so two runs of one version keep two sets of logs. Keying on | |
| 87 | - | /// the version would have a rebuild append to the previous attempt's file, | |
| 88 | - | /// with every run pointing at the same mixed log. | |
| 89 | - | /// | |
| 90 | - | /// Falls back to the version when there is no build identity (a | |
| 91 | - | /// pre-migration-008 run, or a gate unit test), which also keeps logs written | |
| 92 | - | /// under that scheme reachable: their rows record the version path, and | |
| 93 | - | /// nothing rewrites them. | |
| 94 | - | pub fn log_scope(&self) -> String { | |
| 95 | - | self.build_id | |
| 96 | - | .map_or_else(|| self.version.to_string(), |id| id.to_string()) | |
| 97 | - | } | |
| 98 | - | ||
| 99 | - | /// This run's log pointer for `gate`. Always paired with | |
| 100 | - | /// [`Self::log_path`], which resolves the same ref to an absolute path. | |
| 101 | - | pub fn log_ref(&self, gate: GateKind) -> LogRef { | |
| 102 | - | LogRef::new(&self.log_scope(), gate) | |
| 103 | - | } | |
| 104 | - | ||
| 105 | - | /// Where `gate`'s log is written on this host: `logs_root` joined to | |
| 106 | - | /// [`Self::log_ref`]. The two are derived from one scope so a row's | |
| 107 | - | /// `log_ref` can never name a file the gate did not write. | |
| 108 | - | pub fn log_path(&self, gate: GateKind) -> PathBuf { | |
| 109 | - | self.cfg | |
| 110 | - | .logs_root | |
| 111 | - | .join(self.log_scope()) | |
| 112 | - | .join(format!("{}.log", gate.as_str())) | |
| 113 | - | } | |
| 114 | - | ||
| 115 | - | /// Absolute directory a `test_target` runs in: under the worktree, or under | |
| 116 | - | /// the named aux repo's checkout. | |
| 117 | - | /// | |
| 118 | - | /// An `aux_repo` naming nothing this run knows about resolves to `None` | |
| 119 | - | /// rather than to a wrong path. Callers treat that as "not present in this | |
| 120 | - | /// run" and skip, the same as a target missing from an older sha — | |
| 121 | - | /// `--check-config` is what stops a genuine typo from reaching here | |
| 122 | - | /// (`Topology::ensure_test_target_aux_repos_exist`). | |
| 123 | - | pub fn target_dir(&self, target: &crate::config::TestTarget) -> Option<PathBuf> { | |
| 124 | - | match target.aux_repo.as_deref() { | |
| 125 | - | None => Some(self.worktree.as_ref()?.join(&target.dir)), | |
| 126 | - | Some(name) => Some(self.aux_dirs.get(name)?.join(&target.dir)), | |
| 127 | - | } | |
| 128 | - | } | |
| 129 | - | ||
| 130 | - | /// The checkout, or a typed refusal for a gate that cannot work without one. | |
| 131 | - | /// | |
| 132 | - | /// Every caller of this is a gate whose evidence is about the *artifact* | |
| 133 | - | /// rather than about the artifact in an environment, which the boundary | |
| 134 | - | /// assigns to the builder. Reaching this arm means a tier asked Sando to | |
| 135 | - | /// re-run a builder's gate against a bundle it was handed, and the honest | |
| 136 | - | /// answer is to say so rather than to pass on having run nothing. | |
| 137 | - | pub fn worktree_for(&self, gate: GateKind) -> std::result::Result<&Path, GateOutcome> { | |
| 138 | - | self.worktree.as_deref().ok_or_else(|| { | |
| 139 | - | GateOutcome::failed(GateFailure::NeedsSource { | |
| 140 | - | gate, | |
| 141 | - | artifact: self.bundle.as_ref().map_or_else( | |
| 142 | - | || "an artifact built elsewhere".into(), | |
| 143 | - | |b| b.display().to_string(), | |
| 144 | - | ), | |
| 145 | - | }) | |
| 146 | - | }) | |
| 147 | - | } | |
| 148 | - | ||
| 149 | - | /// Where a `migration_check` finds its migrations. | |
| 150 | - | /// | |
| 151 | - | /// The bundle wins when it carries them. That is the point of shipping | |
| 152 | - | /// migrations as a `release_contents` entry: it puts them inside the digest, | |
| 153 | - | /// so the dry run proves something about the bytes that ship rather than | |
| 154 | - | /// about a checkout that happens to sit next to them. The worktree is the | |
| 155 | - | /// fallback for a build whose config has not opted in yet, and for an | |
| 156 | - | /// accepted artifact there is no fallback at all — if the builder did not | |
| 157 | - | /// bundle its migrations, Sando cannot dry-run them and says so. | |
| 158 | - | pub fn migrations_dir(&self, dir: &Path) -> Option<PathBuf> { | |
| 159 | - | if let Some(bundle) = &self.bundle { | |
| 160 | - | let in_bundle = bundle.join(dir); | |
| 161 | - | if in_bundle.is_dir() { | |
| 162 | - | return Some(in_bundle); | |
| 163 | - | } | |
| 164 | - | } | |
| 165 | - | let in_worktree = self.worktree.as_ref()?.join(dir); | |
| 166 | - | in_worktree.is_dir().then_some(in_worktree) | |
| 167 | - | } | |
| 168 | - | } | |
| 169 | - | ||
| 170 | - | /// One node the `node_health` gate verifies: its id, the systemd unit to | |
| 171 | - | /// confirm active after the restart, an optional HTTP readiness URL, and the | |
| 172 | - | /// executor that reaches it (the same transport the deploy used). | |
| 173 | - | pub struct NodeProbe { | |
| 174 | - | pub node: crate::domain::NodeId, | |
| 175 | - | pub service: String, | |
| 176 | - | pub health_url: Option<String>, | |
| 177 | - | pub executor: Arc<dyn ops_exec::Executor>, | |
| 178 | - | } | |
| 179 | - | ||
| 180 | - | /// Run a single gate end-to-end: insert the in-flight row, execute the gate, | |
| 181 | - | /// update the row with the outcome. Returns the outcome for the caller. | |
| 182 | - | pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> { | |
| 183 | - | let kind = gate.kind(); | |
| 184 | - | let started_at = Utc::now().to_rfc3339(); | |
| 185 | - | ||
| 186 | - | let id: i64 = sqlx::query_scalar( | |
| 187 | - | "INSERT INTO gate_runs (app, version, tier, gate_kind, started_at, build_id) | |
| 188 | - | VALUES (?, ?, ?, ?, ?, ?) | |
| 189 | - | RETURNING id", | |
| 190 | - | ) | |
| 191 | - | .bind(&ctx.cfg.id) | |
| 192 | - | .bind(&ctx.version) | |
| 193 | - | .bind(&ctx.tier) | |
| 194 | - | .bind(kind) | |
| 195 | - | .bind(&started_at) | |
| 196 | - | .bind(ctx.build_id) | |
| 197 | - | .fetch_one(&ctx.pool) | |
| 198 | - | .await?; | |
| 199 | - | let run_id = GateRunId(id); | |
| 200 | - | ||
| 201 | - | tracing::info!( | |
| 202 | - | run_id = %run_id, tier = %ctx.tier, version = %ctx.version, gate = %kind, | |
| 203 | - | "gate start", | |
| 204 | - | ); | |
| 205 | - | events::emit( | |
| 206 | - | &ctx.events, | |
| 207 | - | Event::GateStart { | |
| 208 | - | run_id, | |
| 209 | - | tier: ctx.tier.clone(), | |
| 210 | - | version: ctx.version.clone(), | |
| 211 | - | gate: kind, | |
| 212 | - | }, | |
| 213 | - | ); | |
| 214 | - | ||
| 215 | - | let outcome = match gate { | |
| 216 | - | // cargo_test bounds its own run internally (it kills the specific child). | |
| 217 | - | Gate::CargoTest => cargo_test(ctx, run_id).await, | |
| 218 | - | // hardening_test bounds its own run internally, same as cargo_test. | |
| 219 | - | Gate::HardeningTest => hardening_test(ctx, run_id).await, | |
| 220 | - | // Each bounds itself the same way cargo_test does: one deadline across | |
| 221 | - | // every target, so N crates cannot multiply the ceiling by N. | |
| 222 | - | Gate::Clippy => clippy(ctx, run_id).await, | |
| 223 | - | Gate::Fmt => fmt_check(ctx, run_id).await, | |
| 224 | - | Gate::CargoAudit => supply_chain(ctx, run_id, GateKind::CargoAudit).await, | |
| 225 | - | Gate::CargoDeny => supply_chain(ctx, run_id, GateKind::CargoDeny).await, | |
| 226 | - | // migration_dry_run's psql restore + sqlx migrate could wedge; bound the | |
| 227 | - | // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop | |
| 228 | - | // doesn't orphan it. | |
| 229 | - | Gate::MigrationDryRun => { | |
| 230 | - | let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 231 | - | match tokio::time::timeout(ceiling, migration_dry_run(ctx, run_id)).await { | |
| 232 | - | Ok(res) => res, | |
| 233 | - | Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout { | |
| 234 | - | gate: GateKind::MigrationDryRun, | |
| 235 | - | after_s: ctx.cfg.gate_timeout_secs as u32, | |
| 236 | - | }) | |
| 237 | - | .with_log_ref(ctx.log_ref(GateKind::MigrationDryRun))), | |
| 238 | - | } | |
| 239 | - | } | |
| 240 | - | // code_smoke boots the real binary (migrate-from-scratch + seed + serve), | |
| 241 | - | // any step of which could wedge; bound the whole gate here. Both child | |
| 242 | - | // processes set kill_on_drop, so a timeout-drop can't orphan them. A | |
| 243 | - | // timeout leaves the throwaway DB behind; the next run's createdb drops | |
| 244 | - | // it first (DROP IF EXISTS), same as migration_dry_run's scratch reset. | |
| 245 | - | Gate::CodeSmoke => { | |
| 246 | - | let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 247 | - | match tokio::time::timeout(ceiling, code_smoke(ctx, run_id)).await { | |
| 248 | - | Ok(res) => res, | |
| 249 | - | Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout { | |
| 250 | - | gate: GateKind::CodeSmoke, | |
| 251 | - | after_s: ctx.cfg.gate_timeout_secs as u32, | |
| 252 | - | }) | |
| 253 | - | .with_log_ref(ctx.log_ref(GateKind::CodeSmoke))), | |
| 254 | - | } | |
| 255 | - | } | |
| 256 | - | Gate::BootSmoke => boot_smoke(ctx, run_id).await, | |
| 257 | - | Gate::NodeHealth => node_health(ctx).await, | |
| 258 | - | Gate::PageSmoke => page_smoke(ctx).await, | |
| 259 | - | Gate::BurnIn { hours } => burn_in(ctx, *hours).await, | |
| 260 | - | Gate::ManualConfirm => manual_confirm(ctx).await, | |
| 261 | - | }; | |
| 262 | - | ||
| 263 | - | let outcome = outcome.unwrap_or_else(|e| { | |
| 264 | - | GateOutcome::failed(GateFailure::Unclassified { | |
| 265 | - | legacy_detail: Some(format!("gate runner errored: {e}")), | |
| 266 | - | }) | |
| 267 | - | }); | |
| 268 | - | ||
| 269 | - | let outcome_json = serde_json::to_string(&outcome) | |
| 270 | - | .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}")); | |
| 271 | - | sqlx::query( | |
| 272 | - | "UPDATE gate_runs | |
| 273 | - | SET finished_at = ?, status = ?, outcome_json = ?, log_ref = ? | |
| 274 | - | WHERE id = ?", | |
| 275 | - | ) | |
| 276 | - | .bind(Utc::now().to_rfc3339()) | |
| 277 | - | .bind(outcome.status_str()) | |
| 278 | - | .bind(&outcome_json) | |
| 279 | - | .bind(outcome.log_ref.as_ref().map(super::outcome::LogRef::as_str)) | |
| 280 | - | .bind(id) | |
| 281 | - | .execute(&ctx.pool) | |
| 282 | - | .await?; | |
| 283 | - | ||
| 284 | - | tracing::info!( | |
| 285 | - | tier = %ctx.tier, version = %ctx.version, gate = %kind, | |
| 286 | - | status = outcome.status_str(), "gate done", | |
| 287 | - | ); | |
| 288 | - | events::emit( | |
| 289 | - | &ctx.events, | |
| 290 | - | Event::GateDone { | |
| 291 | - | run_id, | |
| 292 | - | tier: ctx.tier.clone(), | |
| 293 | - | version: ctx.version.clone(), | |
| 294 | - | gate: kind, | |
| 295 | - | outcome: outcome.clone(), | |
| 296 | - | }, | |
| 297 | - | ); | |
| 298 | - | ||
| 299 | - | Ok(outcome) | |
| 300 | - | } | |
| 301 | - | ||
| 302 | - | /// Run every gate in order and return the kinds that did not pass (empty means | |
| 303 | - | /// green). We deliberately do NOT short-circuit on first failure — every gate's | |
| 304 | - | /// outcome is recorded in `gate_runs`, which is the operator's only visibility | |
| 305 | - | /// into pipeline health. Hiding later gates because an earlier one failed makes | |
| 306 | - | /// diagnosis worse. | |
| 307 | - | /// | |
| 308 | - | /// Returning the failing kinds rather than a bare bool is what lets the promote | |
| 309 | - | /// path name them in the tier's `partial_reason` and in the error it returns to | |
| 310 | - | /// the operator, instead of a generic "something was red". | |
| 311 | - | pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result<Vec<GateKind>> { | |
| 312 | - | let mut failed = Vec::new(); | |
| 313 | - | for g in gates { | |
| 314 | - | let o = run(ctx, g).await?; | |
| 315 | - | if !o.is_passed() { | |
| 316 | - | failed.push(g.kind()); | |
| 317 | - | } | |
| 318 | - | } | |
| 319 | - | Ok(failed) | |
| 320 | - | } | |
| 321 | - | ||
| 322 | - | // ---- individual gate runners ---- | |
| 323 | - | ||
| 324 | - | /// Run every configured `test_target`'s suite, in order, under one gate. | |
| 325 | - | /// | |
| 326 | - | /// Targets are configured (`[[test_target]]` in the daemon config), defaulting | |
| 327 | - | /// to a single `server` entry. A crate with no target ships ungated, `mnw-cli` | |
| 328 | - | /// included, which is built as a companion and installed onto prod-1 in the same | |
| 329 | - | /// promote. | |
| 330 | - | /// | |
| 331 | - | /// The whole set shares one `gate_runs` row and one log file: from the | |
| 332 | - | /// pipeline's point of view "the tests" either pass or don't. The first failing | |
| 333 | - | /// target ends the gate, since a red suite blocks the promote regardless of what | |
| 334 | - | /// the remaining crates would have said, and running them would only delay the | |
| 335 | - | /// operator's answer. Its name is carried in the failure so the summary points | |
| 336 | - | /// at the crate, not just the test. | |
| 337 | - | async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 338 | - | let log_path = ctx.log_path(GateKind::CargoTest); | |
| 339 | - | let log_ref = ctx.log_ref(GateKind::CargoTest); | |
| 340 | - | ||
| 341 | - | // Best-effort: drop our own role's stale `mnw_test_*` databases (the | |
| 342 | - | // template + any per-test clones orphaned by a previously-killed run) | |
| 343 | - | // before the suite, so they can't accumulate or collide. Foreign-owned | |
| 344 | - | // leftovers are left alone — the harness now namespaces its template per | |
| 345 | - | // role, so they no longer wedge the gate. | |
| 346 | - | if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() { | |
| 347 | - | clean_stale_test_dbs(scratch_url).await; | |
| 348 | - | } | |
| 349 | - | ||
| 350 | - | let started = std::time::Instant::now(); | |
| 351 | - | // One ceiling for the whole gate, not per target: the point is to bound how | |
| 352 | - | // long a hung suite can block the pipeline, and N targets each allowed the | |
| 353 | - | // full timeout would multiply that by N. | |
| 354 | - | let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 355 | - | let mut ran = 0usize; | |
| 356 | - | ||
| 357 | - | for target in &ctx.cfg.test_targets { | |
| 358 | - | let label = target.label(); | |
| 359 | - | // A target absent from this sha is skipped, not fatal: sando has to be | |
| 360 | - | // able to build older shas (bisect, rollback rebuild) from a config that | |
| 361 | - | // describes the tip. The zero-targets-ran check below is what stops this | |
| 362 | - | // from quietly turning the gate into a no-op. | |
| 363 | - | let Some(dir) = ctx | |
| 364 | - | .target_dir(target) | |
| 365 | - | .filter(|d| d.join("Cargo.toml").is_file()) | |
| 366 | - | else { | |
| 367 | - | tracing::warn!( | |
| 368 | - | target = %label, version = %ctx.version, | |
| 369 | - | "test_target has no Cargo.toml in this run; skipping", | |
| 370 | - | ); | |
| 371 | - | continue; | |
| 372 | - | }; | |
| 373 | - | let features: Vec<&str> = target.features.iter().map(String::as_str).collect(); | |
| 374 | - | ||
| 375 | - | // Fast pre-gate: compile the test targets WITHOUT running them. This | |
| 376 | - | // builds the exact artifacts the full run needs (so the subsequent run | |
| 377 | - | // reuses the cache — no wasted work), but fails in ~minutes with the | |
| 378 | - | // real `error[Ennnn]: ...` on a test-only-target compile break. That | |
| 379 | - | // class (a field missing in a `#[cfg(test)]`-only binary like `load`) | |
| 380 | - | // otherwise compiles fine under the build step and only blows up here, | |
| 381 | - | // after a full build, as an opaque mass test failure. | |
| 382 | - | let banner = format!("\n==== test_target: {label} ====\n"); | |
| 383 | - | append_to_log(&log_path, banner.as_bytes()).await; | |
| 384 | - | let mut pre = match cargo_test_command(ctx, &dir, target, &features, &["--no-run"]).spawn() | |
| 385 | - | { | |
| 386 | - | Ok(c) => c, | |
| 387 | - | Err(e) => { | |
| 388 | - | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 389 | - | message: format!("{label}: {e}"), | |
| 390 | - | }) | |
| 391 | - | .with_log_ref(log_ref)); | |
| 392 | - | } | |
| 393 | - | }; | |
| 394 | - | let (pre_out, pre_err, pre_status) = match run_to_deadline_for( | |
| 395 | - | &mut pre, | |
| 396 | - | ctx, | |
| 397 | - | run_id, | |
| 398 | - | log_path.clone(), | |
| 399 | - | deadline, | |
| 400 | - | started, | |
| 401 | - | GateKind::CargoTest, | |
| 402 | - | ) | |
| 403 | - | .await? | |
| 404 | - | { | |
| 405 | - | Ok(v) => v, | |
| 406 | - | Err(timeout) => return Ok(timeout.with_log_ref(log_ref)), | |
| 407 | - | }; | |
| 408 | - | if !pre_status.success() { | |
| 409 | - | let failure = classify::classify_compile_error(&pre_out, &pre_err); | |
| 410 | - | return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref)); | |
| 411 | - | } | |
| 412 | - | ||
| 413 | - | // Full run: the test binaries are already built above, so cargo's | |
| 414 | - | // up-to-date check skips compilation and this just runs the tests. | |
| 415 | - | // | |
| 416 | - | // That claim is only true when the crate's build script is up to date | |
| 417 | - | // too, and for a long time it was not. server and multithreaded both | |
| 418 | - | // watched `.git/HEAD`, a path that does not exist at either package | |
| 419 | - | // root, and cargo reads a missing watch as changed: the build script | |
| 420 | - | // re-ran and the crate recompiled here, every time. It cost 347s a | |
| 421 | - | // pipeline, 35% of this gate, while this comment said it cost nothing. | |
| 422 | - | // Fixed 2026-08-20 in both build scripts; see `git_hash` in either. | |
| 423 | - | // | |
| 424 | - | // If this gate's duration ever climbs back toward the pre-pass's, look | |
| 425 | - | // for a new phantom watch before looking anywhere else. | |
| 426 | - | let mut child = match cargo_test_command(ctx, &dir, target, &features, &[]).spawn() { | |
| 427 | - | Ok(c) => c, | |
| 428 | - | Err(e) => { | |
| 429 | - | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 430 | - | message: format!("{label}: {e}"), | |
| 431 | - | }) | |
| 432 | - | .with_log_ref(log_ref)); | |
| 433 | - | } | |
| 434 | - | }; | |
| 435 | - | let (stdout_buf, stderr_buf, status) = match run_to_deadline_for( | |
| 436 | - | &mut child, | |
| 437 | - | ctx, | |
| 438 | - | run_id, | |
| 439 | - | log_path.clone(), | |
| 440 | - | deadline, | |
| 441 | - | started, | |
| 442 | - | GateKind::CargoTest, | |
| 443 | - | ) | |
| 444 | - | .await? | |
| 445 | - | { | |
| 446 | - | Ok(v) => v, | |
| 447 | - | Err(timeout) => return Ok(timeout.with_log_ref(log_ref)), | |
| 448 | - | }; | |
| 449 | - | if !status.success() { | |
| 450 | - | let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf); | |
| 451 | - | return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref)); | |
| 452 | - | } | |
| 453 | - | ran += 1; | |
| 454 | - | } | |
| 455 | - | ||
| 456 | - | // Every configured target was missing from the worktree. Exiting green here | |
| 457 | - | // would report "tests passed" having run none of them. | |
| 458 | - | if ran == 0 { | |
| 459 | - | return Ok(GateOutcome::failed(GateFailure::Unclassified { | |
| 460 | - | legacy_detail: Some(format!( | |
| 461 | - | "cargo_test ran no targets: none of the {} configured test_target dir(s) \ | |
| 462 | - | exist in this worktree", | |
| 463 | - | ctx.cfg.test_targets.len(), | |
| 464 | - | )), | |
| 465 | - | }) | |
| 466 | - | .with_log_ref(log_ref)); | |
| 467 | - | } | |
| 468 | - | ||
| 469 | - | let duration_s = started.elapsed().as_secs() as u32; | |
| 470 | - | Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref)) | |
| 471 | - | } | |
| 472 | - | ||
| 473 | - | /// Stream a child to the live log, bounded by the gate-wide `deadline`. `Ok(Err(_))` | |
| 474 | - | /// is the timeout outcome (child killed); `Err(_)` is an IO error on the stream. | |
| 475 | - | #[allow(clippy::type_complexity)] | |
| 476 | - | async fn run_to_deadline_for( | |
| 477 | - | child: &mut tokio::process::Child, | |
| 478 | - | ctx: &GateCtx, | |
| 479 | - | run_id: GateRunId, | |
| 480 | - | log_path: PathBuf, | |
| 481 | - | deadline: std::time::Instant, | |
| 482 | - | started: std::time::Instant, | |
| 483 | - | kind: GateKind, | |
| 484 | - | ) -> Result<std::result::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus), GateOutcome>> { | |
| 485 | - | let remaining = deadline.saturating_duration_since(std::time::Instant::now()); | |
| 486 | - | let stream = stream_child_to_live_log(child, ctx.events.clone(), run_id, log_path); | |
| 487 | - | match tokio::time::timeout(remaining, stream).await { | |
| 488 | - | Ok(res) => Ok(Ok(res?)), | |
| 489 | - | Err(_elapsed) => { | |
| 490 | - | child.start_kill().ok(); | |
| 491 | - | let _ = child.wait().await; | |
| 492 | - | Ok(Err(GateOutcome::failed(GateFailure::Timeout { | |
| 493 | - | gate: kind, | |
| 494 | - | after_s: started.elapsed().as_secs() as u32, | |
| 495 | - | }))) | |
| 496 | - | } | |
| 497 | - | } | |
| 498 | - | } | |
| 499 | - | ||
| 500 | - | /// Prefix a test/compile failure's headline with the crate it came from, so a |
Lines truncated
| @@ -1,0 +1,1003 @@ | |||
| 1 | + | //! The cargo-shaped gates: test, clippy, fmt, supply chain and the hardening | |
| 2 | + | //! build, plus the machinery that runs one command over a list of targets and | |
| 3 | + | //! turns its output into a failure note. | |
| 4 | + | ||
| 5 | + | use super::GateCtx; | |
| 6 | + | use super::log::{append_to_log, stream_child_to_live_log}; | |
| 7 | + | use super::pg::clean_stale_test_dbs; | |
| 8 | + | use crate::classify; | |
| 9 | + | use crate::domain::{GateKind, GateRunId}; | |
| 10 | + | use crate::outcome::{GateFailure, GateOutcome, PassNote}; | |
| 11 | + | use anyhow::Result; | |
| 12 | + | use std::path::PathBuf; | |
| 13 | + | use tokio::process::Command; | |
| 14 | + | ||
| 15 | + | /// Run every configured `test_target`'s suite, in order, under one gate. | |
| 16 | + | /// | |
| 17 | + | /// Targets are configured (`[[test_target]]` in the daemon config), defaulting | |
| 18 | + | /// to a single `server` entry. A crate with no target ships ungated, `mnw-cli` | |
| 19 | + | /// included, which is built as a companion and installed onto prod-1 in the same | |
| 20 | + | /// promote. | |
| 21 | + | /// | |
| 22 | + | /// The whole set shares one `gate_runs` row and one log file: from the | |
| 23 | + | /// pipeline's point of view "the tests" either pass or don't. The first failing | |
| 24 | + | /// target ends the gate, since a red suite blocks the promote regardless of what | |
| 25 | + | /// the remaining crates would have said, and running them would only delay the | |
| 26 | + | /// operator's answer. Its name is carried in the failure so the summary points | |
| 27 | + | /// at the crate, not just the test. | |
| 28 | + | pub(super) async fn cargo_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 29 | + | let log_path = ctx.log_path(GateKind::CargoTest); | |
| 30 | + | let log_ref = ctx.log_ref(GateKind::CargoTest); | |
| 31 | + | ||
| 32 | + | // Best-effort: drop our own role's stale `mnw_test_*` databases (the | |
| 33 | + | // template + any per-test clones orphaned by a previously-killed run) | |
| 34 | + | // before the suite, so they can't accumulate or collide. Foreign-owned | |
| 35 | + | // leftovers are left alone — the harness now namespaces its template per | |
| 36 | + | // role, so they no longer wedge the gate. | |
| 37 | + | if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() { | |
| 38 | + | clean_stale_test_dbs(scratch_url).await; | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | let started = std::time::Instant::now(); | |
| 42 | + | // One ceiling for the whole gate, not per target: the point is to bound how | |
| 43 | + | // long a hung suite can block the pipeline, and N targets each allowed the | |
| 44 | + | // full timeout would multiply that by N. | |
| 45 | + | let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 46 | + | let mut ran = 0usize; | |
| 47 | + | ||
| 48 | + | for target in &ctx.cfg.test_targets { | |
| 49 | + | let label = target.label(); | |
| 50 | + | // A target absent from this sha is skipped, not fatal: sando has to be | |
| 51 | + | // able to build older shas (bisect, rollback rebuild) from a config that | |
| 52 | + | // describes the tip. The zero-targets-ran check below is what stops this | |
| 53 | + | // from quietly turning the gate into a no-op. | |
| 54 | + | let Some(dir) = ctx | |
| 55 | + | .target_dir(target) | |
| 56 | + | .filter(|d| d.join("Cargo.toml").is_file()) | |
| 57 | + | else { | |
| 58 | + | tracing::warn!( | |
| 59 | + | target = %label, version = %ctx.version, | |
| 60 | + | "test_target has no Cargo.toml in this run; skipping", | |
| 61 | + | ); | |
| 62 | + | continue; | |
| 63 | + | }; | |
| 64 | + | let features: Vec<&str> = target.features.iter().map(String::as_str).collect(); | |
| 65 | + | ||
| 66 | + | // Fast pre-gate: compile the test targets WITHOUT running them. This | |
| 67 | + | // builds the exact artifacts the full run needs (so the subsequent run | |
| 68 | + | // reuses the cache — no wasted work), but fails in ~minutes with the | |
| 69 | + | // real `error[Ennnn]: ...` on a test-only-target compile break. That | |
| 70 | + | // class (a field missing in a `#[cfg(test)]`-only binary like `load`) | |
| 71 | + | // otherwise compiles fine under the build step and only blows up here, | |
| 72 | + | // after a full build, as an opaque mass test failure. | |
| 73 | + | let banner = format!("\n==== test_target: {label} ====\n"); | |
| 74 | + | append_to_log(&log_path, banner.as_bytes()).await; | |
| 75 | + | let mut pre = match cargo_test_command(ctx, &dir, target, &features, &["--no-run"]).spawn() | |
| 76 | + | { | |
| 77 | + | Ok(c) => c, | |
| 78 | + | Err(e) => { | |
| 79 | + | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 80 | + | message: format!("{label}: {e}"), | |
| 81 | + | }) | |
| 82 | + | .with_log_ref(log_ref)); | |
| 83 | + | } | |
| 84 | + | }; | |
| 85 | + | let (pre_out, pre_err, pre_status) = match run_to_deadline_for( | |
| 86 | + | &mut pre, | |
| 87 | + | ctx, | |
| 88 | + | run_id, | |
| 89 | + | log_path.clone(), | |
| 90 | + | deadline, | |
| 91 | + | started, | |
| 92 | + | GateKind::CargoTest, | |
| 93 | + | ) | |
| 94 | + | .await? | |
| 95 | + | { | |
| 96 | + | Ok(v) => v, | |
| 97 | + | Err(timeout) => return Ok(timeout.with_log_ref(log_ref)), | |
| 98 | + | }; | |
| 99 | + | if !pre_status.success() { | |
| 100 | + | let failure = classify::classify_compile_error(&pre_out, &pre_err); | |
| 101 | + | return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref)); | |
| 102 | + | } | |
| 103 | + | ||
| 104 | + | // Full run: the test binaries are already built above, so cargo's | |
| 105 | + | // up-to-date check skips compilation and this just runs the tests. | |
| 106 | + | // | |
| 107 | + | // That claim is only true when the crate's build script is up to date | |
| 108 | + | // too, and for a long time it was not. server and multithreaded both | |
| 109 | + | // watched `.git/HEAD`, a path that does not exist at either package | |
| 110 | + | // root, and cargo reads a missing watch as changed: the build script | |
| 111 | + | // re-ran and the crate recompiled here, every time. It cost 347s a | |
| 112 | + | // pipeline, 35% of this gate, while this comment said it cost nothing. | |
| 113 | + | // Fixed 2026-08-20 in both build scripts; see `git_hash` in either. | |
| 114 | + | // | |
| 115 | + | // If this gate's duration ever climbs back toward the pre-pass's, look | |
| 116 | + | // for a new phantom watch before looking anywhere else. | |
| 117 | + | let mut child = match cargo_test_command(ctx, &dir, target, &features, &[]).spawn() { | |
| 118 | + | Ok(c) => c, | |
| 119 | + | Err(e) => { | |
| 120 | + | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 121 | + | message: format!("{label}: {e}"), | |
| 122 | + | }) | |
| 123 | + | .with_log_ref(log_ref)); | |
| 124 | + | } | |
| 125 | + | }; | |
| 126 | + | let (stdout_buf, stderr_buf, status) = match run_to_deadline_for( | |
| 127 | + | &mut child, | |
| 128 | + | ctx, | |
| 129 | + | run_id, | |
| 130 | + | log_path.clone(), | |
| 131 | + | deadline, | |
| 132 | + | started, | |
| 133 | + | GateKind::CargoTest, | |
| 134 | + | ) | |
| 135 | + | .await? | |
| 136 | + | { | |
| 137 | + | Ok(v) => v, | |
| 138 | + | Err(timeout) => return Ok(timeout.with_log_ref(log_ref)), | |
| 139 | + | }; | |
| 140 | + | if !status.success() { | |
| 141 | + | let failure = classify::classify_cargo_test(&stdout_buf, &stderr_buf); | |
| 142 | + | return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref)); | |
| 143 | + | } | |
| 144 | + | ran += 1; | |
| 145 | + | } | |
| 146 | + | ||
| 147 | + | // Every configured target was missing from the worktree. Exiting green here | |
| 148 | + | // would report "tests passed" having run none of them. | |
| 149 | + | if ran == 0 { | |
| 150 | + | return Ok(GateOutcome::failed(GateFailure::Unclassified { | |
| 151 | + | legacy_detail: Some(format!( | |
| 152 | + | "cargo_test ran no targets: none of the {} configured test_target dir(s) \ | |
| 153 | + | exist in this worktree", | |
| 154 | + | ctx.cfg.test_targets.len(), | |
| 155 | + | )), | |
| 156 | + | }) | |
| 157 | + | .with_log_ref(log_ref)); | |
| 158 | + | } | |
| 159 | + | ||
| 160 | + | let duration_s = started.elapsed().as_secs() as u32; | |
| 161 | + | Ok(GateOutcome::passed(PassNote::TestsPassed { duration_s }).with_log_ref(log_ref)) | |
| 162 | + | } | |
| 163 | + | ||
| 164 | + | /// Stream a child to the live log, bounded by the gate-wide `deadline`. `Ok(Err(_))` | |
| 165 | + | /// is the timeout outcome (child killed); `Err(_)` is an IO error on the stream. | |
| 166 | + | #[allow(clippy::type_complexity)] | |
| 167 | + | async fn run_to_deadline_for( | |
| 168 | + | child: &mut tokio::process::Child, | |
| 169 | + | ctx: &GateCtx, | |
| 170 | + | run_id: GateRunId, | |
| 171 | + | log_path: PathBuf, | |
| 172 | + | deadline: std::time::Instant, | |
| 173 | + | started: std::time::Instant, | |
| 174 | + | kind: GateKind, | |
| 175 | + | ) -> Result<std::result::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus), GateOutcome>> { | |
| 176 | + | let remaining = deadline.saturating_duration_since(std::time::Instant::now()); | |
| 177 | + | let stream = stream_child_to_live_log(child, ctx.events.clone(), run_id, log_path); | |
| 178 | + | match tokio::time::timeout(remaining, stream).await { | |
| 179 | + | Ok(res) => Ok(Ok(res?)), | |
| 180 | + | Err(_elapsed) => { | |
| 181 | + | child.start_kill().ok(); | |
| 182 | + | let _ = child.wait().await; | |
| 183 | + | Ok(Err(GateOutcome::failed(GateFailure::Timeout { | |
| 184 | + | gate: kind, | |
| 185 | + | after_s: started.elapsed().as_secs() as u32, | |
| 186 | + | }))) | |
| 187 | + | } | |
| 188 | + | } | |
| 189 | + | } | |
| 190 | + | ||
| 191 | + | /// Prefix a test/compile failure's headline with the crate it came from, so a | |
| 192 | + | /// red gate across many targets says *which* crate broke. Other failure kinds | |
| 193 | + | /// are single-target by construction and pass through untouched. | |
| 194 | + | fn name_target(failure: GateFailure, dir: &std::path::Path) -> GateFailure { | |
| 195 | + | let at = dir.display(); | |
| 196 | + | match failure { | |
| 197 | + | GateFailure::CargoTest { | |
| 198 | + | failed_count, | |
| 199 | + | first_failed, | |
| 200 | + | first_panic, | |
| 201 | + | } => GateFailure::CargoTest { | |
| 202 | + | failed_count, | |
| 203 | + | first_failed: Some(match first_failed { | |
| 204 | + | Some(name) => format!("{at}: {name}"), | |
| 205 | + | None => at.to_string(), | |
| 206 | + | }), | |
| 207 | + | first_panic, | |
| 208 | + | }, | |
| 209 | + | GateFailure::CompileError { | |
| 210 | + | error_count, | |
| 211 | + | first_error, | |
| 212 | + | } => GateFailure::CompileError { | |
| 213 | + | error_count, | |
| 214 | + | first_error: Some(match first_error { | |
| 215 | + | Some(e) => format!("{at}: {e}"), | |
| 216 | + | None => at.to_string(), | |
| 217 | + | }), | |
| 218 | + | }, | |
| 219 | + | other => other, | |
| 220 | + | } | |
| 221 | + | } | |
| 222 | + | ||
| 223 | + | /// `cargo clippy --all-targets -- -D warnings` over every configured | |
| 224 | + | /// `test_target`. | |
| 225 | + | /// | |
| 226 | + | /// The only thing standing between lint drift and prod. | |
| 227 | + | pub(super) async fn clippy(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 228 | + | lint_over_targets(ctx, run_id, GateKind::Clippy, |target, features| { | |
| 229 | + | let mut args = vec!["clippy".to_string(), "--all-targets".to_string()]; | |
| 230 | + | if target.all_features { | |
| 231 | + | args.push("--all-features".to_string()); | |
| 232 | + | } else if !features.is_empty() { | |
| 233 | + | args.push("--features".to_string()); | |
| 234 | + | args.push(features.join(",")); | |
| 235 | + | } | |
| 236 | + | // Everything after `--` goes to rustc, which is where -D lives. | |
| 237 | + | args.push("--".to_string()); | |
| 238 | + | args.push("-D".to_string()); | |
| 239 | + | args.push("warnings".to_string()); | |
| 240 | + | args | |
| 241 | + | }) | |
| 242 | + | .await | |
| 243 | + | } | |
| 244 | + | ||
| 245 | + | /// `cargo fmt --check` over every configured `test_target`. | |
| 246 | + | /// | |
| 247 | + | /// No `rustfmt.toml` exists anywhere in the tree, so this is plain rustfmt | |
| 248 | + | /// defaults. Cheap: no compilation, just a parse. | |
| 249 | + | pub(super) async fn fmt_check(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 250 | + | lint_over_targets(ctx, run_id, GateKind::Fmt, |_target, _features| { | |
| 251 | + | vec!["fmt".to_string(), "--check".to_string()] | |
| 252 | + | }) | |
| 253 | + | .await | |
| 254 | + | } | |
| 255 | + | ||
| 256 | + | /// `cargo audit` / `cargo deny check`, over the crates that carry the matching | |
| 257 | + | /// config file. | |
| 258 | + | /// | |
| 259 | + | /// Config-gated on purpose. Both tools are only meaningful against a triaged | |
| 260 | + | /// posture: four crates in this repo fail `cargo audit` today purely because | |
| 261 | + | /// they have no `.cargo/audit.toml` recording which transitive advisories have | |
| 262 | + | /// been reviewed and accepted. Running them everywhere would make the gate | |
| 263 | + | /// permanently and uninformatively red. Dropping the config file into a crate | |
| 264 | + | /// is what opts it in. | |
| 265 | + | pub(super) async fn supply_chain( | |
| 266 | + | ctx: &GateCtx, | |
| 267 | + | run_id: GateRunId, | |
| 268 | + | kind: GateKind, | |
| 269 | + | ) -> Result<GateOutcome> { | |
| 270 | + | let (config_rel, args): (&str, Vec<String>) = match kind { | |
| 271 | + | GateKind::CargoAudit => (".cargo/audit.toml", vec!["audit".into()]), | |
| 272 | + | GateKind::CargoDeny => ("deny.toml", vec!["deny".into(), "check".into()]), | |
| 273 | + | other => unreachable!("supply_chain called for {other:?}"), | |
| 274 | + | }; | |
| 275 | + | run_over_targets(ctx, run_id, kind, |_target, target_dir| { | |
| 276 | + | target_dir.join(config_rel).is_file().then(|| args.clone()) | |
| 277 | + | }) | |
| 278 | + | .await | |
| 279 | + | } | |
| 280 | + | ||
| 281 | + | /// Shared driver for the lint gates: run `cargo <args>` in every configured | |
| 282 | + | /// `test_target` that exists in this worktree. | |
| 283 | + | async fn lint_over_targets( | |
| 284 | + | ctx: &GateCtx, | |
| 285 | + | run_id: GateRunId, | |
| 286 | + | kind: GateKind, | |
| 287 | + | build_args: impl Fn(&crate::config::TestTarget, &[String]) -> Vec<String>, | |
| 288 | + | ) -> Result<GateOutcome> { | |
| 289 | + | // The target is handed in directly. This used to reverse-look-it-up by | |
| 290 | + | // comparing `worktree.join(dir)` against the resolved path, which silently | |
| 291 | + | // stopped matching for anything resolved anywhere else — an aux repo, say. | |
| 292 | + | run_over_targets(ctx, run_id, kind, move |t, _dir| { | |
| 293 | + | Some(build_args(t, &t.features)) | |
| 294 | + | }) | |
| 295 | + | .await | |
| 296 | + | } | |
| 297 | + | ||
| 298 | + | /// Run one cargo invocation per configured `test_target`, under a single | |
| 299 | + | /// gate-wide deadline. `args_for` returns `None` to skip a target (used by the | |
| 300 | + | /// supply-chain gates, which only apply where their config file lives). | |
| 301 | + | /// | |
| 302 | + | /// Shares `cargo_test`'s conventions: per-target log banners, stop at the first | |
| 303 | + | /// failure with the crate named, skip targets absent from the worktree, and fail | |
| 304 | + | /// closed if that leaves nothing to run. | |
| 305 | + | async fn run_over_targets( | |
| 306 | + | ctx: &GateCtx, | |
| 307 | + | run_id: GateRunId, | |
| 308 | + | kind: GateKind, | |
| 309 | + | args_for: impl Fn(&crate::config::TestTarget, &std::path::Path) -> Option<Vec<String>>, | |
| 310 | + | ) -> Result<GateOutcome> { | |
| 311 | + | let log_path = ctx.log_path(kind); | |
| 312 | + | let log_ref = ctx.log_ref(kind); | |
| 313 | + | let started = std::time::Instant::now(); | |
| 314 | + | let deadline = started + std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 315 | + | let mut ran = 0usize; | |
| 316 | + | ||
| 317 | + | for target in &ctx.cfg.test_targets { | |
| 318 | + | let label = target.label(); | |
| 319 | + | let Some(dir) = ctx | |
| 320 | + | .target_dir(target) | |
| 321 | + | .filter(|d| d.join("Cargo.toml").is_file()) | |
| 322 | + | else { | |
| 323 | + | tracing::warn!( | |
| 324 | + | gate = kind.as_str(), target = %label, | |
| 325 | + | "target has no Cargo.toml in this run; skipping", | |
| 326 | + | ); | |
| 327 | + | continue; | |
| 328 | + | }; | |
| 329 | + | let Some(args) = args_for(target, &dir) else { | |
| 330 | + | continue; | |
| 331 | + | }; | |
| 332 | + | ||
| 333 | + | append_to_log( | |
| 334 | + | &log_path, | |
| 335 | + | format!("\n==== {}: {label} ====\n", kind.as_str()).as_bytes(), | |
| 336 | + | ) | |
| 337 | + | .await; | |
| 338 | + | ||
| 339 | + | let mut cmd = Command::new("cargo"); | |
| 340 | + | cmd.args(&args) | |
| 341 | + | .current_dir(&dir) | |
| 342 | + | .stdout(std::process::Stdio::piped()) | |
| 343 | + | .stderr(std::process::Stdio::piped()) | |
| 344 | + | .kill_on_drop(true); | |
| 345 | + | if let Some(t) = ctx.cfg.cargo_target_dir.as_deref() { | |
| 346 | + | cmd.env("CARGO_TARGET_DIR", t); | |
| 347 | + | } | |
| 348 | + | // clippy type-checks, so it needs the same sqlx online-mode env the | |
| 349 | + | // build and cargo_test steps get. fmt/audit/deny never touch the DB. | |
| 350 | + | if kind == GateKind::Clippy | |
| 351 | + | && let Some(url) = ctx | |
| 352 | + | .cfg | |
| 353 | + | .scratch_db_url | |
| 354 | + | .as_deref() | |
| 355 | + | .filter(|_| target.scratch_db) | |
| 356 | + | { | |
| 357 | + | cmd.env("DATABASE_URL", url); | |
| 358 | + | cmd.env( | |
| 359 | + | "TEST_DATABASE_URL", | |
| 360 | + | url.split_once('?').map_or(url, |(b, _)| b), | |
| 361 | + | ); | |
| 362 | + | } | |
| 363 | + | ||
| 364 | + | let mut child = match cmd.spawn() { | |
| 365 | + | Ok(c) => c, | |
| 366 | + | Err(e) => { | |
| 367 | + | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 368 | + | message: format!("{label}: {e}"), | |
| 369 | + | }) | |
| 370 | + | .with_log_ref(log_ref)); | |
| 371 | + | } | |
| 372 | + | }; | |
| 373 | + | let (stdout_buf, stderr_buf, status) = match run_to_deadline_for( | |
| 374 | + | &mut child, | |
| 375 | + | ctx, | |
| 376 | + | run_id, | |
| 377 | + | log_path.clone(), | |
| 378 | + | deadline, | |
| 379 | + | started, | |
| 380 | + | kind, | |
| 381 | + | ) | |
| 382 | + | .await? | |
| 383 | + | { | |
| 384 | + | Ok(v) => v, | |
| 385 | + | Err(timeout) => return Ok(timeout.with_log_ref(log_ref)), | |
| 386 | + | }; | |
| 387 | + | if !status.success() { | |
| 388 | + | let failure = match kind { | |
| 389 | + | // clippy speaks rustc diagnostics, so the compile-error | |
| 390 | + | // classifier extracts the real `error: ...` headline. | |
| 391 | + | GateKind::Clippy => classify::classify_compile_error(&stdout_buf, &stderr_buf), | |
| 392 | + | _ => GateFailure::Unclassified { | |
| 393 | + | legacy_detail: Some(first_meaningful_line(&stdout_buf, &stderr_buf)), | |
| 394 | + | }, | |
| 395 | + | }; | |
| 396 | + | return Ok(GateOutcome::failed(name_target(failure, &target.dir)).with_log_ref(log_ref)); | |
| 397 | + | } | |
| 398 | + | ran += 1; | |
| 399 | + | } | |
| 400 | + | ||
| 401 | + | if ran == 0 { | |
| 402 | + | return Ok(GateOutcome::failed(GateFailure::Unclassified { | |
| 403 | + | legacy_detail: Some(format!( | |
| 404 | + | "{} ran nothing: no configured target in this worktree qualified", | |
| 405 | + | kind.as_str(), | |
| 406 | + | )), | |
| 407 | + | }) | |
| 408 | + | .with_log_ref(log_ref)); | |
| 409 | + | } | |
| 410 | + | Ok(GateOutcome::passed(PassNote::TestsPassed { | |
| 411 | + | duration_s: started.elapsed().as_secs() as u32, | |
| 412 | + | }) | |
| 413 | + | .with_log_ref(log_ref)) | |
| 414 | + | } | |
| 415 | + | ||
| 416 | + | /// First line that looks like a diagnostic, for gates whose tools have no | |
| 417 | + | /// dedicated classifier (`cargo audit`, `cargo deny`). Falls back to a generic | |
| 418 | + | /// note rather than an empty string. | |
| 419 | + | fn first_meaningful_line(stdout: &[u8], stderr: &[u8]) -> String { | |
| 420 | + | for buf in [stderr, stdout] { | |
| 421 | + | let text = String::from_utf8_lossy(buf); | |
| 422 | + | if let Some(line) = text | |
| 423 | + | .lines() | |
| 424 | + | .map(str::trim) | |
| 425 | + | .find(|l| l.starts_with("error") || l.contains("vulnerabilit") || l.contains("FAILED")) | |
| 426 | + | { | |
| 427 | + | return line.chars().take(200).collect(); | |
| 428 | + | } | |
| 429 | + | } | |
| 430 | + | "tool reported failure; see the gate log".into() | |
| 431 | + | } | |
| 432 | + | ||
| 433 | + | /// The tests `cargo_test` cannot reach, run against production constants. | |
| 434 | + | /// | |
| 435 | + | /// `cargo_test` builds with `--features fast-tests`, which relaxes | |
| 436 | + | /// `AUTH_RATE_LIMIT_BURST` 5 → 20, `SANDBOX_RATE_LIMIT_MS` 30s → 10ms, and | |
| 437 | + | /// argon2 from 46 MiB/t=2 to 8 MiB/t=1 — and the rate-limiting suite is | |
| 438 | + | /// `#[cfg_attr(feature = "fast-tests", ignore)]`d on top of that, because a | |
| 439 | + | /// bucket refilling at 100/sec never depletes under parallel test threads. The | |
| 440 | + | /// net effect was that Sando's only code gate silently skipped every test of | |
| 441 | + | /// the auth hardening it most needs to protect. | |
| 442 | + | /// | |
| 443 | + | /// So: no features, `--test-threads=1` (these tests key on a shared per-IP | |
| 444 | + | /// bucket and must not interleave), and a name filter rather than the whole | |
| 445 | + | /// suite — the rest of the suite is tuned for `fast-tests` and would only go | |
| 446 | + | /// slow and flaky here. The filter is a substring match, so it catches | |
| 447 | + | /// `..._rate_limit_...` and `..._rate_limited` alike. | |
| 448 | + | /// | |
| 449 | + | /// This costs a second compile of the lib + integration binary (a different | |
| 450 | + | /// feature set is a different cfg, so no artifact sharing with `cargo_test`). | |
| 451 | + | /// That is the price of the coverage; the filter keeps the *run* to seconds. | |
| 452 | + | pub(super) async fn hardening_test(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 453 | + | let server_dir = match ctx.worktree_for(GateKind::HardeningTest) { | |
| 454 | + | Ok(w) => w.join("server"), | |
| 455 | + | Err(outcome) => return Ok(outcome), | |
| 456 | + | }; | |
| 457 | + | // No features is the whole point; the scratch DB is needed because the | |
| 458 | + | // server's sqlx macros type-check against it. Unlike cargo_test, this gate | |
| 459 | + | // is deliberately not driven by `test_targets`: it targets one specific | |
| 460 | + | // suite in one specific crate, not "the repo's tests". | |
| 461 | + | let target = crate::config::TestTarget { | |
| 462 | + | dir: std::path::PathBuf::from("server"), | |
| 463 | + | aux_repo: None, | |
| 464 | + | features: Vec::new(), | |
| 465 | + | all_features: false, | |
| 466 | + | scratch_db: true, | |
| 467 | + | }; | |
| 468 | + | let log_path = ctx.log_path(GateKind::HardeningTest); | |
| 469 | + | let log_ref = ctx.log_ref(GateKind::HardeningTest); | |
| 470 | + | ||
| 471 | + | if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() { | |
| 472 | + | clean_stale_test_dbs(scratch_url).await; | |
| 473 | + | } | |
| 474 | + | ||
| 475 | + | let started = std::time::Instant::now(); | |
| 476 | + | ||
| 477 | + | // Same two-step shape as cargo_test: compile first so a test-target break | |
| 478 | + | // reports as a compile error rather than an opaque mass test failure. | |
| 479 | + | let mut pre = match cargo_test_command( | |
| 480 | + | ctx, | |
| 481 | + | &server_dir, | |
| 482 | + | &target, | |
| 483 | + | &[], | |
| 484 | + | &["--no-run", "--test", "integration"], | |
| 485 | + | ) | |
| 486 | + | .spawn() | |
| 487 | + | { | |
| 488 | + | Ok(c) => c, | |
| 489 | + | Err(e) => { | |
| 490 | + | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 491 | + | message: e.to_string(), | |
| 492 | + | }) | |
| 493 | + | .with_log_ref(log_ref)); | |
| 494 | + | } | |
| 495 | + | }; | |
| 496 | + | let (pre_out, pre_err, pre_status) = | |
| 497 | + | stream_child_to_live_log(&mut pre, ctx.events.clone(), run_id, log_path.clone()).await?; | |
| 498 | + | if !pre_status.success() { | |
| 499 | + | let failure = classify::classify_compile_error(&pre_out, &pre_err); | |
| 500 | + | return Ok(GateOutcome::failed(failure).with_log_ref(log_ref)); |
Lines truncated
| @@ -1,0 +1,790 @@ | |||
| 1 | + | //! The code smoke gate: boot the real server against a throwaway database and | |
| 2 | + | //! prove it serves. | |
| 3 | + | //! | |
| 4 | + | //! This module holds the crate's concentration of MNW-specific knowledge, which | |
| 5 | + | //! is the reason it is named separately: the server crate's layout, its | |
| 6 | + | //! `--seed-examples` flag and `ALLOW_EXAMPLE_SEED` guard, which npm projects | |
| 7 | + | //! exist, and how `check-docs` reports a broken link. | |
| 8 | + | ||
| 9 | + | use super::GateCtx; | |
| 10 | + | use super::log::GateLog; | |
| 11 | + | use super::pg::{pg_create_db, pg_drop_db, pg_url_with_dbname}; | |
| 12 | + | use super::probes::probe_health; | |
| 13 | + | use crate::classify; | |
| 14 | + | use crate::domain::{GateKind, GateRunId, Version}; | |
| 15 | + | use crate::outcome::{GateBlocker, GateFailure, GateOutcome, PassNote}; | |
| 16 | + | use anyhow::Result; | |
| 17 | + | ||
| 18 | + | /// A 32+ char throwaway signing secret for the `code_smoke` boot. The server | |
| 19 | + | /// only enforces length (>= 32) outside production, and code_smoke's loopback | |
| 20 | + | /// `HOST_URL` keeps it in dev mode, so the value is irrelevant beyond that. | |
| 21 | + | const CODE_SMOKE_SIGNING_SECRET: &str = "sando-code-smoke-dummy-signing-secret-0000000000"; | |
| 22 | + | ||
| 23 | + | /// Seconds to wait for the real server to come up and serve `GET /health` | |
| 24 | + | /// during `code_smoke`. Longer than `boot_smoke`'s 3s: this boots the *full* | |
| 25 | + | /// server (config, pool, session store, webauthn, doc load, app build), not the | |
| 26 | + | /// minimal no-DB smoke server. The whole gate is also bounded by | |
| 27 | + | /// `gate_timeout_secs` at the dispatcher. | |
| 28 | + | const CODE_SMOKE_READY_SECS: u64 = 30; | |
| 29 | + | ||
| 30 | + | /// `code_smoke` — the first host gate. Boots the freshly-built binary against a | |
| 31 | + | /// throwaway *empty* DB it migrates from scratch and seeds the example catalog | |
| 32 | + | /// into, then proves the real server serves `GET /health` against that | |
| 33 | + | /// nonempty DB. Fast and infra-light (one local Postgres, no prod-dump restore, | |
| 34 | + | /// no scratch-role reset, no external services), so a green here proves the | |
| 35 | + | /// code is sound and isolates a later `cargo_test`/`migration_dry_run` red as an | |
| 36 | + | /// environment problem rather than a code one. | |
| 37 | + | /// | |
| 38 | + | /// Reuses existing binary entrypoints, so the server needs no smoke-specific | |
| 39 | + | /// mode: `<bin> --seed-examples` loads config, connects, migrates from scratch, | |
| 40 | + | /// seeds, and exits; a plain `<bin>` then serves the real app. Both run with CWD | |
| 41 | + | /// at the server crate root so `docs/business/assumptions.toml` + `site-docs/` | |
| 42 | + | /// resolve (a missing assumptions file panics real startup), and with a loopback | |
| 43 | + | /// `HOST_URL` so config stays in dev mode (no CDN/S3/signing-secret prod | |
| 44 | + | /// enforcement). The seed's host allowlist already admits `127.0.0.1`, and the | |
| 45 | + | /// fresh DB trivially satisfies its no-real-users guard. | |
| 46 | + | pub(super) async fn code_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 47 | + | let log = GateLog::open(ctx, run_id, GateKind::CodeSmoke).await; | |
| 48 | + | let outcome = code_smoke_inner(ctx, &log).await; | |
| 49 | + | log.close().await; | |
| 50 | + | outcome.map(|o| o.with_log_ref(ctx.log_ref(GateKind::CodeSmoke))) | |
| 51 | + | } | |
| 52 | + | ||
| 53 | + | /// The staged interior of [`code_smoke`], writing every step through the gate's | |
| 54 | + | /// live log. Same split as `migration_dry_run_inner`: the caller owns the sink | |
| 55 | + | /// and attaches the `log_ref`. | |
| 56 | + | async fn code_smoke_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> { | |
| 57 | + | let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else { | |
| 58 | + | log.line("scratch_db_url unset in daemon config\n").await; | |
| 59 | + | return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset)); | |
| 60 | + | }; | |
| 61 | + | ||
| 62 | + | // The staged binary (set by build_and_run_host before gating). code_smoke | |
| 63 | + | // runs first among the host gates, but staging precedes all gating, so the | |
| 64 | + | // artifact path is already recorded. | |
| 65 | + | let bin: Option<(String,)> = | |
| 66 | + | sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?") | |
| 67 | + | .bind(&ctx.cfg.id) | |
| 68 | + | .bind(&ctx.version) | |
| 69 | + | .fetch_optional(&ctx.pool) | |
| 70 | + | .await?; | |
| 71 | + | let Some((bin,)) = bin else { | |
| 72 | + | return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing { | |
| 73 | + | version: ctx.version.clone(), | |
| 74 | + | })); | |
| 75 | + | }; | |
| 76 | + | ||
| 77 | + | // Frontend builds, before anything else: they need no DB and no staged | |
| 78 | + | // binary, and a `tsc` error is the one failure the Rust build deliberately | |
| 79 | + | // swallows (both MNW build scripts emit `cargo::warning` and succeed against | |
| 80 | + | // a stale `static/dist/`). Failing here is what stops the deploy rsyncing | |
| 81 | + | // the previous build's bundle. | |
| 82 | + | if let Some(outcome) = code_smoke_frontends(ctx, log).await { | |
| 83 | + | return Ok(outcome); | |
| 84 | + | } | |
| 85 | + | ||
| 86 | + | // Docs integrity, first and cheapest: run the staged binary's DB-free | |
| 87 | + | // `MNW_CHECK_DOCS` mode before creating the throwaway DB. A broken internal | |
| 88 | + | // docs link (a `[..](x.md)` resolving to a slug no page serves) fails here | |
| 89 | + | // in well under a second instead of after a full migrate+seed+boot, and a | |
| 90 | + | // rotted link never reaches prod as a live 404. Collisions are reported by | |
| 91 | + | // the check but do not fail it; only broken links do. | |
| 92 | + | if let Some(outcome) = code_smoke_docs_check(ctx, &bin, log).await { | |
| 93 | + | return Ok(outcome); | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | let dbname = code_smoke_db_name(&ctx.version); | |
| 97 | + | let maintenance_url = pg_url_with_dbname(scratch_url, "postgres"); | |
| 98 | + | let throwaway_url = pg_url_with_dbname(scratch_url, &dbname); | |
| 99 | + | ||
| 100 | + | // Create the throwaway DB (dropping any stale one from a killed prior run). | |
| 101 | + | log.line(&format!("---- createdb {dbname} ----\n")).await; | |
| 102 | + | if let Err(e) = pg_create_db(&maintenance_url, &dbname).await { | |
| 103 | + | let reason = format!("createdb {dbname}: {e}"); | |
| 104 | + | log.line(&reason).await; | |
| 105 | + | return Ok(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason })); | |
| 106 | + | } | |
| 107 | + | ||
| 108 | + | // Everything past createdb must drop the DB on the way out, pass or fail. | |
| 109 | + | let outcome = code_smoke_body(ctx, &bin, &throwaway_url, log).await; | |
| 110 | + | ||
| 111 | + | log.line(&format!("\n---- dropdb {dbname} ----\n")).await; | |
| 112 | + | if let Err(e) = pg_drop_db(&maintenance_url, &dbname).await { | |
| 113 | + | // A teardown miss must not turn a passing gate red — log it and move on. | |
| 114 | + | // The next run's createdb drops it first anyway. | |
| 115 | + | tracing::warn!(error = %e, db = %dbname, "code_smoke: dropdb failed; next run will reclaim it"); | |
| 116 | + | log.line(&format!("dropdb warning (non-fatal): {e}")).await; | |
| 117 | + | } | |
| 118 | + | ||
| 119 | + | Ok(outcome) | |
| 120 | + | } | |
| 121 | + | ||
| 122 | + | /// Compile every configured `frontend_build` in the worktree. | |
| 123 | + | /// | |
| 124 | + | /// Returns `Some(failed)` on the first project that does not build; `None` when | |
| 125 | + | /// all of them do (or none are configured). Output streams to `log` either way. | |
| 126 | + | /// | |
| 127 | + | /// `npm ci` runs only when `node_modules` is absent. Usually it is not: the app | |
| 128 | + | /// build script installed it during the `cargo build` that produced the artifact | |
| 129 | + | /// this gate is about to smoke, so the common path here is just `npm run build` | |
| 130 | + | /// against a warm install — seconds. The install branch covers the gate running | |
| 131 | + | /// against a worktree whose build script was skipped or failed at `npm ci`, and | |
| 132 | + | /// it is as fatal as a compile failure, because the alternative is compiling | |
| 133 | + | /// against whatever some earlier sha installed. | |
| 134 | + | /// | |
| 135 | + | /// Unlike the app build scripts, nothing here is best-effort. That asymmetry is | |
| 136 | + | /// the point of the gate. | |
| 137 | + | async fn code_smoke_frontends(ctx: &GateCtx, log: &GateLog) -> Option<GateOutcome> { | |
| 138 | + | if ctx.cfg.frontend_builds.is_empty() { | |
| 139 | + | return None; | |
| 140 | + | } | |
| 141 | + | let worktree = match ctx.worktree_for(GateKind::CodeSmoke) { | |
| 142 | + | Ok(w) => w.to_path_buf(), | |
| 143 | + | Err(outcome) => return Some(outcome), | |
| 144 | + | }; | |
| 145 | + | for fe in &ctx.cfg.frontend_builds { | |
| 146 | + | let dir = worktree.join(&fe.dir); | |
| 147 | + | let label = fe.dir.display().to_string(); | |
| 148 | + | log.line(&format!("---- frontend build ({label}) ----\n")) | |
| 149 | + | .await; | |
| 150 | + | ||
| 151 | + | if !dir.is_dir() { | |
| 152 | + | // An older sha predating the frontend, mid-bisect. Skipping keeps | |
| 153 | + | // sando able to rebuild history; the log says so out loud. | |
| 154 | + | log.line(&format!("{label} absent from this worktree; skipping\n")) | |
| 155 | + | .await; | |
| 156 | + | continue; | |
| 157 | + | } | |
| 158 | + | ||
| 159 | + | if !dir.join("node_modules").is_dir() | |
| 160 | + | && let Some(outcome) = run_npm(&dir, &label, &["ci"], "npm ci", ctx, log).await | |
| 161 | + | { | |
| 162 | + | return Some(outcome); | |
| 163 | + | } | |
| 164 | + | ||
| 165 | + | if let Some(outcome) = run_npm( | |
| 166 | + | &dir, | |
| 167 | + | &label, | |
| 168 | + | &["run", &fe.script], | |
| 169 | + | &format!("npm run {}", fe.script), | |
| 170 | + | ctx, | |
| 171 | + | log, | |
| 172 | + | ) | |
| 173 | + | .await | |
| 174 | + | { | |
| 175 | + | return Some(outcome); | |
| 176 | + | } | |
| 177 | + | } | |
| 178 | + | None | |
| 179 | + | } | |
| 180 | + | ||
| 181 | + | /// One `npm` invocation for [`code_smoke_frontends`], bounded by the gate | |
| 182 | + | /// timeout so a wedged install cannot hold the whole pipeline (the enclosing | |
| 183 | + | /// `code_smoke` ceiling would catch it eventually, but this attributes the | |
| 184 | + | /// failure to the project that hung). | |
| 185 | + | async fn run_npm( | |
| 186 | + | dir: &std::path::Path, | |
| 187 | + | label: &str, | |
| 188 | + | args: &[&str], | |
| 189 | + | what: &str, | |
| 190 | + | ctx: &GateCtx, | |
| 191 | + | log: &GateLog, | |
| 192 | + | ) -> Option<GateOutcome> { | |
| 193 | + | log.line(&format!("$ {what}\n")).await; | |
| 194 | + | let mut cmd = tokio::process::Command::new("npm"); | |
| 195 | + | cmd.args(args).current_dir(dir).kill_on_drop(true); | |
| 196 | + | let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 197 | + | // On the timeout branch the whole `run` future is dropped, which drops the | |
| 198 | + | // child; `kill_on_drop` is what turns that into an actual kill. | |
| 199 | + | let status = match tokio::time::timeout(ceiling, log.run(&mut cmd)).await { | |
| 200 | + | Ok(Ok((_stdout, _stderr, status))) => status, | |
| 201 | + | Ok(Err(e)) => { | |
| 202 | + | // A missing `npm` lands here. Fatal, not skipped: a build host | |
| 203 | + | // without Node cannot produce the bundle the release serves, and | |
| 204 | + | // silently passing is how the stale bundle shipped in the first place. | |
| 205 | + | log.line(&format!("{what} could not be spawned: {e}\n")) | |
| 206 | + | .await; | |
| 207 | + | return Some(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 208 | + | message: format!("{what} in {label}: {e}"), | |
| 209 | + | })); | |
| 210 | + | } | |
| 211 | + | Err(_elapsed) => { | |
| 212 | + | log.line(&format!( | |
| 213 | + | "{what} timed out after {}s\n", | |
| 214 | + | ctx.cfg.gate_timeout_secs | |
| 215 | + | )) | |
| 216 | + | .await; | |
| 217 | + | return Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend { | |
| 218 | + | dir: label.to_string(), | |
| 219 | + | exit_code: None, | |
| 220 | + | })); | |
| 221 | + | } | |
| 222 | + | }; | |
| 223 | + | if status.success() { | |
| 224 | + | return None; | |
| 225 | + | } | |
| 226 | + | Some(GateOutcome::failed(GateFailure::CodeSmokeFrontend { | |
| 227 | + | dir: label.to_string(), | |
| 228 | + | exit_code: status.code(), | |
| 229 | + | })) | |
| 230 | + | } | |
| 231 | + | ||
| 232 | + | /// Run the staged binary's DB-free docs integrity check (`MNW_CHECK_DOCS=1`). | |
| 233 | + | /// | |
| 234 | + | /// Returns `Some(failed)` if the check reports broken links, cannot be spawned, | |
| 235 | + | /// or overruns its ceiling; `None` when the docs are clean. Output streams to | |
| 236 | + | /// `log` either way. The 60s ceiling backstops the case where the staged binary | |
| 237 | + | /// predates the flag and would fall through to a normal (DB-needing) boot and | |
| 238 | + | /// hang. | |
| 239 | + | async fn code_smoke_docs_check(ctx: &GateCtx, bin: &str, log: &GateLog) -> Option<GateOutcome> { | |
| 240 | + | let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) { | |
| 241 | + | Ok(w) => w.join("server"), | |
| 242 | + | Err(outcome) => return Some(outcome), | |
| 243 | + | }; | |
| 244 | + | log.line("---- docs check (MNW_CHECK_DOCS) ----\n").await; | |
| 245 | + | let mut cmd = tokio::process::Command::new(bin); | |
| 246 | + | cmd.env("MNW_CHECK_DOCS", "1") | |
| 247 | + | .current_dir(&server_dir) | |
| 248 | + | .kill_on_drop(true); | |
| 249 | + | let (stdout, _stderr, status) = | |
| 250 | + | match tokio::time::timeout(std::time::Duration::from_mins(1), log.run(&mut cmd)).await { | |
| 251 | + | Ok(Ok(out)) => out, | |
| 252 | + | Ok(Err(e)) => { | |
| 253 | + | log.line(&format!("docs check spawn failed: {e}\n")).await; | |
| 254 | + | return Some(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 255 | + | message: e.to_string(), | |
| 256 | + | })); | |
| 257 | + | } | |
| 258 | + | Err(_elapsed) => { | |
| 259 | + | let reason = | |
| 260 | + | "docs check timed out after 60s (staged binary may predate MNW_CHECK_DOCS)" | |
| 261 | + | .to_string(); | |
| 262 | + | log.line(&format!("{reason}\n")).await; | |
| 263 | + | return Some(GateOutcome::failed(GateFailure::CodeSmokeSetup { reason })); | |
| 264 | + | } | |
| 265 | + | }; | |
| 266 | + | if status.success() { | |
| 267 | + | return None; | |
| 268 | + | } | |
| 269 | + | Some(GateOutcome::failed(GateFailure::CodeSmokeDocs { | |
| 270 | + | broken: parse_check_docs_broken_count(&stdout), | |
| 271 | + | })) | |
| 272 | + | } | |
| 273 | + | ||
| 274 | + | /// Best-effort parse of the broken-link count from the `MNW_CHECK_DOCS` sentinel | |
| 275 | + | /// line (`MNW_CHECK_DOCS: N broken link(s)`). Returns 0 if absent — the failure | |
| 276 | + | /// still stands, only the summary count is unknown. | |
| 277 | + | fn parse_check_docs_broken_count(stdout: &[u8]) -> u32 { | |
| 278 | + | let text = String::from_utf8_lossy(stdout); | |
| 279 | + | for line in text.lines() { | |
| 280 | + | if let Some(rest) = line.strip_prefix("MNW_CHECK_DOCS:") { | |
| 281 | + | for tok in rest.split_whitespace() { | |
| 282 | + | if let Ok(n) = tok.parse::<u32>() { | |
| 283 | + | return n; | |
| 284 | + | } | |
| 285 | + | } | |
| 286 | + | } | |
| 287 | + | } | |
| 288 | + | 0 | |
| 289 | + | } | |
| 290 | + | ||
| 291 | + | /// The createdb-to-dropdb interior of `code_smoke`: migrate+seed, then boot and | |
| 292 | + | /// probe. Returns the outcome without a `log_ref` (the caller attaches it after | |
| 293 | + | /// teardown). Never returns `Err` — spawn/child failures map to typed outcomes. | |
| 294 | + | async fn code_smoke_body(ctx: &GateCtx, bin: &str, db_url: &str, log: &GateLog) -> GateOutcome { | |
| 295 | + | let server_dir = match ctx.worktree_for(GateKind::CodeSmoke) { | |
| 296 | + | Ok(w) => w.join("server"), | |
| 297 | + | Err(outcome) => return outcome, | |
| 298 | + | }; | |
| 299 | + | ||
| 300 | + | // Phase 1: migrate-from-scratch + seed. `--seed-examples` loads config, | |
| 301 | + | // connects, runs migrations against the empty DB, seeds the catalog, exits. | |
| 302 | + | // A non-zero exit here is the "code is unsound" signal (broken migration, | |
| 303 | + | // seed error, or config-load failure). | |
| 304 | + | log.line("---- migrate + seed (--seed-examples) ----\n") | |
| 305 | + | .await; | |
| 306 | + | let mut seed_cmd = tokio::process::Command::new(bin); | |
| 307 | + | seed_cmd.arg("--seed-examples").current_dir(&server_dir); | |
| 308 | + | code_smoke_env(&mut seed_cmd, ctx, db_url); | |
| 309 | + | seed_cmd.env("ALLOW_EXAMPLE_SEED", "1").kill_on_drop(true); | |
| 310 | + | let seed_status = match log.run(&mut seed_cmd).await { | |
| 311 | + | Ok((_stdout, _stderr, status)) => status, | |
| 312 | + | Err(e) => { | |
| 313 | + | return GateOutcome::failed(GateFailure::SpawnFailed { | |
| 314 | + | message: e.to_string(), | |
| 315 | + | }); | |
| 316 | + | } | |
| 317 | + | }; | |
| 318 | + | if !seed_status.success() { | |
| 319 | + | return GateOutcome::failed(GateFailure::CodeSmokeSeed { | |
| 320 | + | exit_code: seed_status.code(), | |
| 321 | + | }); | |
| 322 | + | } | |
| 323 | + | ||
| 324 | + | // Phase 2: boot the real server against the now-migrated + seeded DB and | |
| 325 | + | // assert both startup signals: it logs `listening` (emitted just before the | |
| 326 | + | // socket bind) AND serves GET /health with a 200. The full stdout/stderr is | |
| 327 | + | // persisted for the operator either way. | |
| 328 | + | log.line("\n---- boot + probe /health ----\n").await; | |
| 329 | + | let mut serve_cmd = tokio::process::Command::new(bin); | |
| 330 | + | serve_cmd.current_dir(&server_dir); | |
| 331 | + | code_smoke_env(&mut serve_cmd, ctx, db_url); | |
| 332 | + | serve_cmd | |
| 333 | + | .stdout(std::process::Stdio::piped()) | |
| 334 | + | .stderr(std::process::Stdio::piped()) | |
| 335 | + | .kill_on_drop(true); | |
| 336 | + | let mut child = match serve_cmd.spawn() { | |
| 337 | + | Ok(c) => c, | |
| 338 | + | Err(e) => { | |
| 339 | + | return GateOutcome::failed(GateFailure::SpawnFailed { | |
| 340 | + | message: e.to_string(), | |
| 341 | + | }); | |
| 342 | + | } | |
| 343 | + | }; | |
| 344 | + | ||
| 345 | + | // Stream stdout/stderr into the gate log (and out as chunk events) while | |
| 346 | + | // the probe loop runs below; the tasks finish when the pipes close (child | |
| 347 | + | // exits or is killed). The buffers they return are what the `listening` | |
| 348 | + | // assertion reads. | |
| 349 | + | let (stdout_task, stderr_task) = log.drain_pipes(&mut child); | |
| 350 | + | ||
| 351 | + | let probe_timeout = std::time::Duration::from_millis(500); | |
| 352 | + | let started = std::time::Instant::now(); | |
| 353 | + | let window = std::time::Duration::from_secs(CODE_SMOKE_READY_SECS); | |
| 354 | + | let mut probe_ok_after: Option<u32> = None; | |
| 355 | + | let mut last_probe_err = "never responded".to_string(); | |
| 356 | + | let mut early_exit = None; | |
| 357 | + | while started.elapsed() < window { | |
| 358 | + | if let Ok(Some(status)) = child.try_wait() { | |
| 359 | + | early_exit = Some(status); | |
| 360 | + | break; | |
| 361 | + | } | |
| 362 | + | match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.code_smoke_port)).await { | |
| 363 | + | Ok(Ok(())) => { | |
| 364 | + | probe_ok_after = Some(started.elapsed().as_millis() as u32); | |
| 365 | + | break; | |
| 366 | + | } | |
| 367 | + | Ok(Err(e)) => last_probe_err = e, | |
| 368 | + | Err(_) => last_probe_err = "probe timed out".to_string(), | |
| 369 | + | } | |
| 370 | + | tokio::time::sleep(std::time::Duration::from_millis(250)).await; | |
| 371 | + | } | |
| 372 | + | ||
| 373 | + | let exit = match early_exit { | |
| 374 | + | Some(status) => Some(status), | |
| 375 | + | None => { | |
| 376 | + | let e = child.try_wait().ok().flatten(); | |
| 377 | + | if e.is_none() { | |
| 378 | + | let _ = child.kill().await; | |
| 379 | + | } | |
| 380 | + | e | |
| 381 | + | } | |
| 382 | + | }; | |
| 383 | + | // Keep the serve run's own output so the `listening`-log assertion checks it | |
| 384 | + | // (not the seed run's, which exits before binding). The bytes are already in | |
| 385 | + | // the gate log; these buffers exist only for the assertion. | |
| 386 | + | let serve_stdout = stdout_task.await.unwrap_or_default(); | |
| 387 | + | let serve_stderr = stderr_task.await.unwrap_or_default(); | |
| 388 | + | let logged_listening = | |
| 389 | + | bytes_contain(&serve_stdout, b"listening") || bytes_contain(&serve_stderr, b"listening"); | |
| 390 | + | ||
| 391 | + | match (exit, probe_ok_after) { | |
| 392 | + | // Exited on its own within the window — panic / config error / bind fail. | |
| 393 | + | (Some(status), _) => GateOutcome::failed(classify::classify_boot_smoke(status.code())), | |
| 394 | + | // Stayed up and served /health. Assert both required startup signals: | |
| 395 | + | // the `listening` bind log AND the /health 200. | |
| 396 | + | (None, Some(after_ms)) if logged_listening => { | |
| 397 | + | GateOutcome::passed(PassNote::HealthyProbe { after_ms }) | |
| 398 | + | } | |
| 399 | + | // Served /health but the `listening` log never appeared. | |
| 400 | + | (None, Some(_)) => GateOutcome::failed(GateFailure::CodeSmokeNoListeningLog), | |
| 401 | + | // Stayed up but never served /health — started, not ready. | |
| 402 | + | (None, None) => GateOutcome::failed(GateFailure::BootHealthProbeFailed { | |
| 403 | + | last_error: last_probe_err, | |
| 404 | + | }), | |
| 405 | + | } | |
| 406 | + | } | |
| 407 | + | ||
| 408 | + | /// Substring search over raw bytes (the server's log output), for the | |
| 409 | + | /// `code_smoke` startup-log assertion. Avoids a lossy UTF-8 conversion of the | |
| 410 | + | /// whole buffer just to run `str::contains`. | |
| 411 | + | fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { | |
| 412 | + | if needle.is_empty() || haystack.len() < needle.len() { | |
| 413 | + | return needle.is_empty(); | |
| 414 | + | } | |
| 415 | + | haystack.windows(needle.len()).any(|w| w == needle) | |
| 416 | + | } | |
| 417 | + | ||
| 418 | + | /// Apply the minimal env every `code_smoke` invocation shares: point the binary | |
| 419 | + | /// at the throwaway DB, force loopback (dev-mode config, no prod enforcement), | |
| 420 | + | /// hand it a dummy signing secret, and disable file scanning (no AV/YARA on the | |
| 421 | + | /// build host). The worktree is a clean git checkout, so no stray `.env` shadows | |
| 422 | + | /// these (and dotenvy never overrides already-set vars). | |
| 423 | + | /// | |
| 424 | + | /// This list has to carry EVERY var the server's `Config::from_env` treats as | |
| 425 | + | /// mandatory, because `code_smoke` is the only gate that reaches that function | |
| 426 | + | /// at all: `boot_smoke` and the docs check both short-circuit in `main` before | |
| 427 | + | /// config is loaded. So when the server makes a new var required, this is where | |
| 428 | + | /// it has to be answered, and nothing connects the two lists automatically. | |
| 429 | + | /// | |
| 430 | + | /// The values are deliberately throwaway. The gate asks whether this code can | |
| 431 | + | /// migrate, seed, boot and serve; whether a given deployment's env is complete | |
| 432 | + | /// is the `config_check_env_file` guard's job, on the node, against that node's | |
| 433 | + | /// real env file. | |
| 434 | + | /// | |
| 435 | + | /// `app.code_smoke_env` is prepended to all of this, for vars a product needs | |
| 436 | + | /// that sando has no business knowing about. The fixed set overwrites it on a | |
| 437 | + | /// collision, so nothing in a config file can redirect the gate off its own | |
| 438 | + | /// throwaway DB. | |
| 439 | + | fn code_smoke_env(cmd: &mut tokio::process::Command, ctx: &GateCtx, db_url: &str) { | |
| 440 | + | // `localhost`, not `127.0.0.1`, and the distinction is load-bearing. The | |
| 441 | + | // server derives its WebAuthn relying-party id from HOST_URL's host, and | |
| 442 | + | // `WebauthnBuilder::new` validates that id against `Url::domain()` — which | |
| 443 | + | // is `None` for an IP literal, so an origin of `http://127.0.0.1:<port>` | |
| 444 | + | // fails with WebauthnError::Configuration before the server ever binds. It | |
| 445 | + | // is a real ceiling on the gate, not a preference: no IP-literal origin can | |
| 446 | + | // boot this binary. `localhost` is a domain, and matches the derived rp_id. | |
| 447 | + | // | |
| 448 | + | // HOST stays 127.0.0.1: that is the bind address, and the gate probes the | |
| 449 | + | // loopback address directly, so only the advertised origin changes. | |
| 450 | + | let origin = format!("http://localhost:{}", ctx.cfg.code_smoke_port); | |
| 451 | + | // Project-supplied extras go on first so the fixed set below overwrites any | |
| 452 | + | // key they collide on: what points this run at its throwaway DB and its | |
| 453 | + | // loopback port is not negotiable from a config file. | |
| 454 | + | cmd.envs(&ctx.cfg.code_smoke_env); | |
| 455 | + | cmd.env("DATABASE_URL", db_url) | |
| 456 | + | .env("HOST", "127.0.0.1") | |
| 457 | + | .env("PORT", ctx.cfg.code_smoke_port.to_string()) | |
| 458 | + | .env("HOST_URL", &origin) | |
| 459 | + | // Required unconditionally by Config::from_env. Pointing it at the smoke | |
| 460 | + | // server's own origin keeps every rendered media URL resolvable within | |
| 461 | + | // the gate; no request is ever made to it. | |
| 462 | + | .env("CDN_BASE_URL", &origin) | |
| 463 | + | .env("SIGNING_SECRET", CODE_SMOKE_SIGNING_SECRET) | |
| 464 | + | .env("SCAN_ENABLED", "false") | |
| 465 | + | .env("INSECURE_COOKIES", "1"); | |
| 466 | + | } | |
| 467 | + | ||
| 468 | + | /// The throwaway smoke DB name for a version: `sando_code_smoke_<version>` with | |
| 469 | + | /// every non-alphanumeric char folded to `_` and lowercased, capped at Postgres' | |
| 470 | + | /// 63-byte identifier limit. Sanitized to `[a-z0-9_]` so it's safe to quote into | |
| 471 | + | /// DDL. Deterministic per version, so a stale DB from a killed run is reclaimed | |
| 472 | + | /// by the next run's `DROP DATABASE IF EXISTS` rather than accumulating. | |
| 473 | + | fn code_smoke_db_name(version: &Version) -> String { | |
| 474 | + | let mut name = String::from("sando_code_smoke_"); | |
| 475 | + | for c in version.to_string().chars() { | |
| 476 | + | name.push(if c.is_ascii_alphanumeric() { | |
| 477 | + | c.to_ascii_lowercase() | |
| 478 | + | } else { | |
| 479 | + | '_' | |
| 480 | + | }); | |
| 481 | + | } | |
| 482 | + | name.truncate(63); | |
| 483 | + | name | |
| 484 | + | } | |
| 485 | + | ||
| 486 | + | #[cfg(test)] | |
| 487 | + | mod tests { | |
| 488 | + | use super::super::run; | |
| 489 | + | use super::*; | |
| 490 | + | use crate::domain::TierId; | |
| 491 | + | use crate::events::{self, Event}; | |
| 492 | + | use crate::gates::testkit::{ | |
| 493 | + | frontend_ctx, read_gate_log, resolving_ctx, test_gate_log, url_host_is_a_domain, | |
| 494 | + | }; | |
| 495 | + | use crate::topology::Gate; | |
| 496 | + | use sqlx::sqlite::SqlitePoolOptions; | |
| 497 | + | use std::collections::HashMap; | |
| 498 | + | ||
| 499 | + | #[test] | |
| 500 | + | fn parse_check_docs_broken_count_reads_the_sentinel() { |
Lines truncated
| @@ -1,0 +1,200 @@ | |||
| 1 | + | //! The gate live log: where a running gate's output goes. | |
| 2 | + | //! | |
| 3 | + | //! Every gate that shells out streams its bytes through here, so the TUI sees | |
| 4 | + | //! the tail as it happens and the finished log is on disk under | |
| 5 | + | //! [`super::GateCtx::log_path`] for the failure note to point at. | |
| 6 | + | ||
| 7 | + | use super::GateCtx; | |
| 8 | + | use crate::domain::{GateKind, GateRunId}; | |
| 9 | + | use crate::events::{self, Event, EventTx}; | |
| 10 | + | use anyhow::Result; | |
| 11 | + | use ops_core::live_log::LiveLog; | |
| 12 | + | use ops_core::remote::LogSink; | |
| 13 | + | use std::path::PathBuf; | |
| 14 | + | use std::sync::Arc; | |
| 15 | + | use tokio::io::AsyncReadExt; | |
| 16 | + | use tokio::process::Command; | |
| 17 | + | ||
| 18 | + | /// The gate live-log callback: emit each chunk as a `GateLogChunk` event so the | |
| 19 | + | /// TUI sees the tail stream in real time. `ops_core::live_log::LiveLog` owns the | |
| 20 | + | /// disk append and the per-run sequence counter; this closure is the one | |
| 21 | + | /// tool-specific bit. | |
| 22 | + | pub(super) fn gate_chunk_cb( | |
| 23 | + | events: EventTx, | |
| 24 | + | run_id: GateRunId, | |
| 25 | + | ) -> ops_core::live_log::ChunkCallback { | |
| 26 | + | Box::new(move |seq, text| { | |
| 27 | + | events::emit( | |
| 28 | + | &events, | |
| 29 | + | Event::GateLogChunk { | |
| 30 | + | run_id, | |
| 31 | + | seq, | |
| 32 | + | text: text.to_owned(), | |
| 33 | + | }, | |
| 34 | + | ); | |
| 35 | + | }) | |
| 36 | + | } | |
| 37 | + | ||
| 38 | + | /// Append raw bytes to a gate log outside the child-streaming path (target | |
| 39 | + | /// banners). Best-effort, same as `LiveLog`: a broken log dir never turns a | |
| 40 | + | /// passing gate red. | |
| 41 | + | pub(super) async fn append_to_log(path: &std::path::Path, bytes: &[u8]) { | |
| 42 | + | use tokio::io::AsyncWriteExt; | |
| 43 | + | if let Some(parent) = path.parent() | |
| 44 | + | && tokio::fs::create_dir_all(parent).await.is_err() | |
| 45 | + | { | |
| 46 | + | return; | |
| 47 | + | } | |
| 48 | + | if let Ok(mut f) = tokio::fs::OpenOptions::new() | |
| 49 | + | .create(true) | |
| 50 | + | .append(true) | |
| 51 | + | .open(path) | |
| 52 | + | .await | |
| 53 | + | { | |
| 54 | + | let _ = f.write_all(bytes).await; | |
| 55 | + | } | |
| 56 | + | } | |
| 57 | + | ||
| 58 | + | /// Drain `stream` into the shared `LiveLog` (which forwards each chunk to | |
| 59 | + | /// the on-disk log file AND broadcasts a `GateLogChunk` event), and return | |
| 60 | + | /// the concatenated bytes so the classifier can still operate on the full | |
| 61 | + | /// output post-hoc. | |
| 62 | + | pub(super) async fn stream_into_log<R>( | |
| 63 | + | stream: Option<R>, | |
| 64 | + | log: std::sync::Arc<tokio::sync::Mutex<LiveLog>>, | |
| 65 | + | ) -> Vec<u8> | |
| 66 | + | where | |
| 67 | + | R: tokio::io::AsyncRead + Unpin + Send + 'static, | |
| 68 | + | { | |
| 69 | + | let mut total = Vec::new(); | |
| 70 | + | let Some(mut s) = stream else { return total }; | |
| 71 | + | let mut buf = [0u8; 4096]; | |
| 72 | + | loop { | |
| 73 | + | match s.read(&mut buf).await { | |
| 74 | + | Ok(0) => break, | |
| 75 | + | Err(_) => break, | |
| 76 | + | Ok(n) => { | |
| 77 | + | total.extend_from_slice(&buf[..n]); | |
| 78 | + | log.lock().await.write_chunk(&buf[..n]).await; | |
| 79 | + | } | |
| 80 | + | } | |
| 81 | + | } | |
| 82 | + | total | |
| 83 | + | } | |
| 84 | + | ||
| 85 | + | /// Spawn a child, drain its stdout/stderr through a `LiveLog`, return the | |
| 86 | + | /// combined buffers and exit status. Shared by `cargo_test` (no deadline) | |
| 87 | + | /// and ad-hoc callers — `boot_smoke` rolls its own variant because of its | |
| 88 | + | /// 3s kill window. | |
| 89 | + | pub(super) async fn stream_child_to_live_log( | |
| 90 | + | child: &mut tokio::process::Child, | |
| 91 | + | events: EventTx, | |
| 92 | + | run_id: GateRunId, | |
| 93 | + | log_path: PathBuf, | |
| 94 | + | ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> { | |
| 95 | + | let log = GateLog::new(LiveLog::open(log_path, gate_chunk_cb(events, run_id)).await); | |
| 96 | + | let out = log.stream_child(child).await; | |
| 97 | + | log.close().await; | |
| 98 | + | out | |
| 99 | + | } | |
| 100 | + | ||
| 101 | + | /// One gate's live log, held across every step of a multi-step gate. | |
| 102 | + | /// | |
| 103 | + | /// Single-child gates call [`stream_child_to_live_log`] and are done. The | |
| 104 | + | /// staged gates (`code_smoke`, `migration_dry_run`) instead run several | |
| 105 | + | /// children plus banner lines between them, and they hold one `GateLog` across | |
| 106 | + | /// the lot: a single sink means chunk sequence numbers stay monotonic for the | |
| 107 | + | /// whole gate, and the on-disk log reads in the order things actually happened | |
| 108 | + | /// rather than as stdout-then-stderr assembled at the end. | |
| 109 | + | /// | |
| 110 | + | /// Every write is best-effort in the same way `LiveLog` is: a log directory | |
| 111 | + | /// that cannot be written degrades to callback-only and never turns a passing | |
| 112 | + | /// gate red. | |
| 113 | + | pub(super) struct GateLog { | |
| 114 | + | sink: Arc<tokio::sync::Mutex<LiveLog>>, | |
| 115 | + | } | |
| 116 | + | ||
| 117 | + | impl GateLog { | |
| 118 | + | pub(super) fn new(sink: LiveLog) -> Self { | |
| 119 | + | Self { | |
| 120 | + | sink: Arc::new(tokio::sync::Mutex::new(sink)), | |
| 121 | + | } | |
| 122 | + | } | |
| 123 | + | ||
| 124 | + | /// Open the sink for `gate`'s log file, streaming to the TUI under `run_id`. | |
| 125 | + | pub(super) async fn open(ctx: &GateCtx, run_id: GateRunId, gate: GateKind) -> Self { | |
| 126 | + | Self::new( | |
| 127 | + | LiveLog::open( | |
| 128 | + | ctx.log_path(gate), | |
| 129 | + | gate_chunk_cb(ctx.events.clone(), run_id), | |
| 130 | + | ) | |
| 131 | + | .await, | |
| 132 | + | ) | |
| 133 | + | } | |
| 134 | + | ||
| 135 | + | /// Emit a banner (or any line the gate itself produces) through the same | |
| 136 | + | /// sink the children stream to, so it lands in sequence with their output. | |
| 137 | + | pub(super) async fn write(&self, bytes: &[u8]) { | |
| 138 | + | self.sink.lock().await.write_chunk(bytes).await; | |
| 139 | + | } | |
| 140 | + | ||
| 141 | + | /// Same, for the common `format!`-a-line case. | |
| 142 | + | pub(super) async fn line(&self, s: &str) { | |
| 143 | + | self.write(s.as_bytes()).await; | |
| 144 | + | } | |
| 145 | + | ||
| 146 | + | /// Spawn `cmd` with both pipes captured, stream them into the sink as they | |
| 147 | + | /// arrive, and return the buffers plus the exit status. The buffers are | |
| 148 | + | /// what the classifiers still operate on post-hoc. | |
| 149 | + | pub(super) async fn run( | |
| 150 | + | &self, | |
| 151 | + | cmd: &mut Command, | |
| 152 | + | ) -> std::io::Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> { | |
| 153 | + | cmd.stdout(std::process::Stdio::piped()) | |
| 154 | + | .stderr(std::process::Stdio::piped()); | |
| 155 | + | let mut child = cmd.spawn()?; | |
| 156 | + | self.stream_child(&mut child) | |
| 157 | + | .await | |
| 158 | + | .map_err(std::io::Error::other) | |
| 159 | + | } | |
| 160 | + | ||
| 161 | + | /// Drain an already-spawned child's pipes into the sink and wait for it. | |
| 162 | + | pub(super) async fn stream_child( | |
| 163 | + | &self, | |
| 164 | + | child: &mut tokio::process::Child, | |
| 165 | + | ) -> Result<(Vec<u8>, Vec<u8>, std::process::ExitStatus)> { | |
| 166 | + | let (stdout_task, stderr_task) = self.drain_pipes(child); | |
| 167 | + | let status = child.wait().await?; | |
| 168 | + | let stdout_buf = stdout_task.await.unwrap_or_default(); | |
| 169 | + | let stderr_buf = stderr_task.await.unwrap_or_default(); | |
| 170 | + | Ok((stdout_buf, stderr_buf, status)) | |
| 171 | + | } | |
| 172 | + | ||
| 173 | + | /// Start draining a child's pipes into the sink *without* waiting on the | |
| 174 | + | /// child. For `code_smoke`'s serve phase, which probes `/health` while the | |
| 175 | + | /// server is still up. The caller must await both handles for the buffers | |
| 176 | + | /// (and before [`Self::close`], so the flush isn't skipped). | |
| 177 | + | #[allow(clippy::type_complexity)] | |
| 178 | + | pub(super) fn drain_pipes( | |
| 179 | + | &self, | |
| 180 | + | child: &mut tokio::process::Child, | |
| 181 | + | ) -> ( | |
| 182 | + | tokio::task::JoinHandle<Vec<u8>>, | |
| 183 | + | tokio::task::JoinHandle<Vec<u8>>, | |
| 184 | + | ) { | |
| 185 | + | ( | |
| 186 | + | tokio::spawn(stream_into_log(child.stdout.take(), self.sink.clone())), | |
| 187 | + | tokio::spawn(stream_into_log(child.stderr.take(), self.sink.clone())), | |
| 188 | + | ) | |
| 189 | + | } | |
| 190 | + | ||
| 191 | + | /// Flush the file. A still-outstanding streaming task (only possible if the | |
| 192 | + | /// gate returned without awaiting it) leaves the `Arc` shared, in which case | |
| 193 | + | /// the flush is skipped — `LiveLog` writes unbuffered to the OS either way, | |
| 194 | + | /// so nothing already written is lost. | |
| 195 | + | pub(super) async fn close(self) { | |
| 196 | + | if let Ok(mutex) = Arc::try_unwrap(self.sink) { | |
| 197 | + | mutex.into_inner().close().await; | |
| 198 | + | } | |
| 199 | + | } | |
| 200 | + | } |
| @@ -1,0 +1,352 @@ | |||
| 1 | + | //! The migration dry run: prove every pending migration applies to a restored | |
| 2 | + | //! copy of what production is actually holding, before a deploy makes that | |
| 3 | + | //! irreversible. | |
| 4 | + | ||
| 5 | + | use super::GateCtx; | |
| 6 | + | use super::log::GateLog; | |
| 7 | + | use super::pg::{pg_create_db, pg_url_with_dbname, reset_scratch, restore_dump, run_migrator}; | |
| 8 | + | use crate::classify; | |
| 9 | + | use crate::domain::{GateKind, GateRunId}; | |
| 10 | + | use crate::outcome::{GateBlocker, GateFailure, GateOutcome, PassNote}; | |
| 11 | + | use anyhow::Result; | |
| 12 | + | use chrono::Utc; | |
| 13 | + | ||
| 14 | + | pub(super) async fn migration_dry_run(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 15 | + | let log = GateLog::open(ctx, run_id, GateKind::MigrationDryRun).await; | |
| 16 | + | let outcome = migration_dry_run_inner(ctx, &log).await; | |
| 17 | + | log.close().await; | |
| 18 | + | outcome.map(|o| o.with_log_ref(ctx.log_ref(GateKind::MigrationDryRun))) | |
| 19 | + | } | |
| 20 | + | ||
| 21 | + | /// The staged interior of [`migration_dry_run`], writing every step through the | |
| 22 | + | /// gate's live log. The caller owns the sink so it can flush it on every exit | |
| 23 | + | /// path, and attaches the `log_ref` once instead of at each return. | |
| 24 | + | /// Runs one configured check per database, in config order, and stops at the | |
| 25 | + | /// first that does not pass — a red gate is a red gate, and continuing would | |
| 26 | + | /// bury it under a second restore's output. | |
| 27 | + | /// | |
| 28 | + | /// The server's check runs against `scratch_db_url` itself and is deliberately | |
| 29 | + | /// last-writer for it: `cargo_test` reuses that database in migrated state, so | |
| 30 | + | /// every other check must name its own `scratch_db` (enforced at config load). | |
| 31 | + | async fn migration_dry_run_inner(ctx: &GateCtx, log: &GateLog) -> Result<GateOutcome> { | |
| 32 | + | let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() else { | |
| 33 | + | log.line("scratch_db_url unset in daemon config\n").await; | |
| 34 | + | return Ok(GateOutcome::blocked(GateBlocker::ScratchDbUrlUnset)); | |
| 35 | + | }; | |
| 36 | + | ||
| 37 | + | let mut checked = Vec::new(); | |
| 38 | + | let mut primary_backup_path = String::new(); | |
| 39 | + | for check in &ctx.cfg.migration_checks { | |
| 40 | + | let label = check.dir.display().to_string(); | |
| 41 | + | log.line(&format!("==== migration_check: {label} ====\n")) | |
| 42 | + | .await; | |
| 43 | + | match run_migration_check(ctx, log, scratch_url, check).await? { | |
| 44 | + | CheckResult::Passed { backup_path } => { | |
| 45 | + | if primary_backup_path.is_empty() { | |
| 46 | + | primary_backup_path = backup_path; | |
| 47 | + | } | |
| 48 | + | checked.push(label); | |
| 49 | + | } | |
| 50 | + | CheckResult::Stopped(outcome) => return Ok(outcome), | |
| 51 | + | } | |
| 52 | + | } | |
| 53 | + | ||
| 54 | + | log.line(&format!( | |
| 55 | + | "all {} migration check(s) passed: {}", | |
| 56 | + | checked.len(), | |
| 57 | + | checked.join(", ") | |
| 58 | + | )) | |
| 59 | + | .await; | |
| 60 | + | Ok(GateOutcome::passed(PassNote::Migrated { | |
| 61 | + | backup_path: primary_backup_path, | |
| 62 | + | checks: checked, | |
| 63 | + | })) | |
| 64 | + | } | |
| 65 | + | ||
| 66 | + | /// One check's verdict: it passed (against `backup_path`), or it produced the | |
| 67 | + | /// outcome the whole gate reports. | |
| 68 | + | enum CheckResult { | |
| 69 | + | Passed { backup_path: String }, | |
| 70 | + | Stopped(GateOutcome), | |
| 71 | + | } | |
| 72 | + | ||
| 73 | + | /// Restore one database's dump into its scratch DB and run its migrations on top. | |
| 74 | + | async fn run_migration_check( | |
| 75 | + | ctx: &GateCtx, | |
| 76 | + | log: &GateLog, | |
| 77 | + | scratch_url: &str, | |
| 78 | + | check: &crate::config::MigrationCheck, | |
| 79 | + | ) -> Result<CheckResult> { | |
| 80 | + | let label = check.dir.display().to_string(); | |
| 81 | + | ||
| 82 | + | let backup: Option<(String, String)> = sqlx::query_as( | |
| 83 | + | "SELECT local_path, fetched_at FROM backups | |
| 84 | + | WHERE app = ? AND name = ? ORDER BY id DESC LIMIT 1", | |
| 85 | + | ) | |
| 86 | + | .bind(&ctx.cfg.id) | |
| 87 | + | .bind(&check.backup) | |
| 88 | + | .fetch_optional(&ctx.pool) | |
| 89 | + | .await?; | |
| 90 | + | let Some((backup_path, fetched_at)) = backup else { | |
| 91 | + | log.line(&format!( | |
| 92 | + | "no {} backup fetched; call /backup/fetch first\n", | |
| 93 | + | check.backup | |
| 94 | + | )) | |
| 95 | + | .await; | |
| 96 | + | return Ok(CheckResult::Stopped(GateOutcome::blocked( | |
| 97 | + | GateBlocker::NoBackupAvailable { | |
| 98 | + | check: label, | |
| 99 | + | backup: check.backup.clone(), | |
| 100 | + | }, | |
| 101 | + | ))); | |
| 102 | + | }; | |
| 103 | + | ||
| 104 | + | // Presence is not freshness. A fetch that quietly stopped working leaves this | |
| 105 | + | // row in place, and restoring it dry-runs the migrations against a schema prod | |
| 106 | + | // has moved past — green, and worthless. Block on age instead. An unparsable | |
| 107 | + | // timestamp is treated as stale: this row is daemon-written RFC 3339, so a | |
| 108 | + | // value that will not parse means something is wrong, and failing closed on a | |
| 109 | + | // freshness check is the whole point. | |
| 110 | + | let age_hours = chrono::DateTime::parse_from_rfc3339(&fetched_at).map_or(i64::MAX, |t| { | |
| 111 | + | (Utc::now() - t.with_timezone(&Utc)).num_hours() | |
| 112 | + | }); | |
| 113 | + | let max_age_hours = ctx.cfg.backup_max_age_hours; | |
| 114 | + | if age_hours > i64::from(max_age_hours) { | |
| 115 | + | let msg = format!( | |
| 116 | + | "backup {backup_path} was fetched {fetched_at} ({age_hours}h ago, max \ | |
| 117 | + | {max_age_hours}h); re-run /backup/fetch\n" | |
| 118 | + | ); | |
| 119 | + | log.line(&msg).await; | |
| 120 | + | return Ok(CheckResult::Stopped(GateOutcome::blocked( | |
| 121 | + | GateBlocker::BackupStale { | |
| 122 | + | age_hours, | |
| 123 | + | max_age_hours, | |
| 124 | + | check: label, | |
| 125 | + | }, | |
| 126 | + | ))); | |
| 127 | + | } | |
| 128 | + | ||
| 129 | + | // A check with its own `scratch_db` gets that database created here rather | |
| 130 | + | // than by a host bootstrap step: adding a `[[migration_check]]` should not | |
| 131 | + | // silently depend on someone having remembered to `createdb` on the Sando | |
| 132 | + | // host, which is exactly the class of footgun this gate exists to remove. | |
| 133 | + | // DROP + CREATE also makes the database sando-owned, so the PG15+ public | |
| 134 | + | // schema grants `reset_scratch` applies next are the owner's to give. | |
| 135 | + | let db_url = match check.scratch_db.as_deref() { | |
| 136 | + | None => scratch_url.to_string(), | |
| 137 | + | Some(dbname) => { | |
| 138 | + | let maintenance_url = pg_url_with_dbname(scratch_url, "postgres"); | |
| 139 | + | log.line(&format!("---- create scratch db {dbname} ----\n")) | |
| 140 | + | .await; | |
| 141 | + | if let Err(e) = pg_create_db(&maintenance_url, dbname).await { | |
| 142 | + | let msg = format!("{label}: creating scratch db {dbname}: {e}"); | |
| 143 | + | log.line(&msg).await; | |
| 144 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 145 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 146 | + | ))); | |
| 147 | + | } | |
| 148 | + | pg_url_with_dbname(scratch_url, dbname) | |
| 149 | + | } | |
| 150 | + | }; | |
| 151 | + | ||
| 152 | + | let owner_role = check | |
| 153 | + | .owner_role | |
| 154 | + | .as_deref() | |
| 155 | + | .unwrap_or(&ctx.cfg.scratch_owner_role); | |
| 156 | + | log.line("---- reset_scratch ----\n").await; | |
| 157 | + | if let Err(e) = reset_scratch(&db_url, owner_role).await { | |
| 158 | + | let msg = format!("{label}: scratch reset: {e}"); | |
| 159 | + | log.line(&msg).await; | |
| 160 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 161 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 162 | + | ))); | |
| 163 | + | } | |
| 164 | + | log.line(&format!("---- restore_dump ({backup_path}) ----\n")) | |
| 165 | + | .await; | |
| 166 | + | if let Err(e) = restore_dump(&db_url, &backup_path, log).await { | |
| 167 | + | let msg = format!("{label}: restore: {e}"); | |
| 168 | + | log.line(&msg).await; | |
| 169 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 170 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 171 | + | ))); | |
| 172 | + | } | |
| 173 | + | ||
| 174 | + | let Some(migrations_dir) = ctx.migrations_dir(&check.dir) else { | |
| 175 | + | // Neither the bundle nor a checkout holds them. For an accepted | |
| 176 | + | // artifact that means the builder did not ship its migrations, and a | |
| 177 | + | // dry run over nothing would report green having proved nothing. | |
| 178 | + | let msg = format!( | |
| 179 | + | "{label}: no migrations at {} in the bundle or a checkout", | |
| 180 | + | check.dir.display() | |
| 181 | + | ); | |
| 182 | + | log.line(&msg).await; | |
| 183 | + | return Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 184 | + | GateFailure::RestoreFailed { reason: msg }, | |
| 185 | + | ))); | |
| 186 | + | }; | |
| 187 | + | log.line("---- run_migrator ----\n").await; | |
| 188 | + | match run_migrator(&db_url, &migrations_dir).await { | |
| 189 | + | Ok(()) => { | |
| 190 | + | log.line(&format!("{label}: restored {backup_path} + migrated\n")) | |
| 191 | + | .await; | |
| 192 | + | Ok(CheckResult::Passed { backup_path }) | |
| 193 | + | } | |
| 194 | + | Err(e) => { | |
| 195 | + | let err_s = format!("{label}: {e}"); | |
| 196 | + | log.line(&err_s).await; | |
| 197 | + | Ok(CheckResult::Stopped(GateOutcome::failed( | |
| 198 | + | classify::classify_migration_error(&err_s, None), | |
| 199 | + | ))) | |
| 200 | + | } | |
| 201 | + | } | |
| 202 | + | } | |
| 203 | + | ||
| 204 | + | #[cfg(test)] | |
| 205 | + | mod tests { | |
| 206 | + | use super::*; | |
| 207 | + | use crate::gates::testkit::{ | |
| 208 | + | dry_run_ctx, mt_check, seed_backup, seed_named_backup, with_check, | |
| 209 | + | }; | |
| 210 | + | ||
| 211 | + | #[tokio::test] | |
| 212 | + | async fn migration_dry_run_blocks_when_a_checks_own_dump_was_never_fetched() { | |
| 213 | + | // The hazard the check list exists for: multithreaded applies its own | |
| 214 | + | // migrations at boot against its own database, so the server's dump must | |
| 215 | + | // never stand in for it. A fetched `server` row with no `multithreaded` | |
| 216 | + | // row is exactly that substitution, and it has to block. | |
| 217 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 218 | + | let mut ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 219 | + | with_check(&mut ctx, mt_check()); | |
| 220 | + | seed_named_backup(&ctx, "server", 1).await; | |
| 221 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 222 | + | ||
| 223 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 224 | + | log.close().await; | |
| 225 | + | ||
| 226 | + | let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else { | |
| 227 | + | panic!("a missing multithreaded dump must block"); | |
| 228 | + | }; | |
| 229 | + | let GateBlocker::NoBackupAvailable { check, backup } = blocker else { | |
| 230 | + | panic!("expected NoBackupAvailable, got {blocker:?}"); | |
| 231 | + | }; | |
| 232 | + | assert_eq!(backup, "multithreaded", "names the dump that is missing"); | |
| 233 | + | assert!( | |
| 234 | + | check.contains("multithreaded/migrations"), | |
| 235 | + | "names the check that wanted it, got {check}" | |
| 236 | + | ); | |
| 237 | + | } | |
| 238 | + | ||
| 239 | + | #[tokio::test] | |
| 240 | + | async fn migration_dry_run_freshness_is_per_dump() { | |
| 241 | + | // A fresh server dump must not make a 45-day-old multithreaded dump look | |
| 242 | + | // current: the clock is per-database, or the second check inherits the | |
| 243 | + | // first's freshness and the gate is theatre. | |
| 244 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 245 | + | let mut ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 246 | + | with_check(&mut ctx, mt_check()); | |
| 247 | + | seed_named_backup(&ctx, "server", 1).await; | |
| 248 | + | seed_named_backup(&ctx, "multithreaded", 24 * 45).await; | |
| 249 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 250 | + | ||
| 251 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 252 | + | log.close().await; | |
| 253 | + | ||
| 254 | + | let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else { | |
| 255 | + | panic!("a 45-day-old multithreaded dump must block"); | |
| 256 | + | }; | |
| 257 | + | let GateBlocker::BackupStale { check, .. } = blocker else { | |
| 258 | + | panic!("expected BackupStale, got {blocker:?}"); | |
| 259 | + | }; | |
| 260 | + | assert!( | |
| 261 | + | check.contains("multithreaded/migrations"), | |
| 262 | + | "names the check whose dump is stale, got {check}" | |
| 263 | + | ); | |
| 264 | + | } | |
| 265 | + | ||
| 266 | + | #[tokio::test] | |
| 267 | + | async fn migration_dry_run_blocks_on_a_stale_backup() { | |
| 268 | + | // The failure this closes: the gate used to check only that a backups row | |
| 269 | + | // existed, so a fetch that silently stopped working left it green against | |
| 270 | + | // an ever-older schema. Sando ran 45 days that way. | |
| 271 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 272 | + | let ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 273 | + | seed_backup(&ctx, 24 * 45).await; | |
| 274 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 275 | + | ||
| 276 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 277 | + | log.close().await; | |
| 278 | + | ||
| 279 | + | let crate::outcome::GateStatus::Blocked { blocker } = outcome.status else { | |
| 280 | + | panic!("a 45-day-old backup must block"); | |
| 281 | + | }; | |
| 282 | + | let GateBlocker::BackupStale { | |
| 283 | + | age_hours, | |
| 284 | + | max_age_hours, | |
| 285 | + | .. | |
| 286 | + | } = blocker | |
| 287 | + | else { | |
| 288 | + | panic!("expected BackupStale, got {blocker:?}"); | |
| 289 | + | }; | |
| 290 | + | assert_eq!(max_age_hours, 48); | |
| 291 | + | assert!( | |
| 292 | + | age_hours >= 24 * 45, | |
| 293 | + | "reports the real age, got {age_hours}" | |
| 294 | + | ); | |
| 295 | + | } | |
| 296 | + | ||
| 297 | + | #[tokio::test] | |
| 298 | + | async fn migration_dry_run_accepts_a_fresh_backup() { | |
| 299 | + | // The other side of the boundary: a backup inside the window must not be | |
| 300 | + | // blocked on freshness. It fails later (there is no such dump on disk), | |
| 301 | + | // which is exactly the proof the age check let it through. | |
| 302 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 303 | + | let ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 304 | + | seed_backup(&ctx, 6).await; | |
| 305 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 306 | + | ||
| 307 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 308 | + | log.close().await; | |
| 309 | + | ||
| 310 | + | assert!( | |
| 311 | + | !matches!( | |
| 312 | + | outcome.status, | |
| 313 | + | crate::outcome::GateStatus::Blocked { | |
| 314 | + | blocker: GateBlocker::BackupStale { .. } | |
| 315 | + | } | |
| 316 | + | ), | |
| 317 | + | "a 6h-old backup is fresh, got {:?}", | |
| 318 | + | outcome.status, | |
| 319 | + | ); | |
| 320 | + | } | |
| 321 | + | ||
| 322 | + | #[tokio::test] | |
| 323 | + | async fn migration_dry_run_treats_an_unparsable_fetched_at_as_stale() { | |
| 324 | + | // Fail closed: `fetched_at` is daemon-written RFC 3339, so a value that | |
| 325 | + | // will not parse means the row is untrustworthy — and a freshness check | |
| 326 | + | // that shrugs at a timestamp it cannot read is not a freshness check. | |
| 327 | + | let tmp = tempfile::tempdir().unwrap(); | |
| 328 | + | let ctx = dry_run_ctx(tmp.path(), 48).await; | |
| 329 | + | sqlx::query( | |
| 330 | + | "INSERT INTO backups (fetched_at, source, local_path, byte_size) | |
| 331 | + | VALUES ('not-a-timestamp', 'file:///x.sql.gz', '/tmp/x.sql.gz', 1000000)", | |
| 332 | + | ) | |
| 333 | + | .execute(&ctx.pool) | |
| 334 | + | .await | |
| 335 | + | .unwrap(); | |
| 336 | + | let log = GateLog::open(&ctx, GateRunId(0), GateKind::MigrationDryRun).await; | |
| 337 | + | ||
| 338 | + | let outcome = migration_dry_run_inner(&ctx, &log).await.unwrap(); | |
| 339 | + | log.close().await; | |
| 340 | + | ||
| 341 | + | assert!( | |
| 342 | + | matches!( | |
| 343 | + | outcome.status, | |
| 344 | + | crate::outcome::GateStatus::Blocked { | |
| 345 | + | blocker: GateBlocker::BackupStale { .. } | |
| 346 | + | } | |
| 347 | + | ), | |
| 348 | + | "an unreadable fetched_at must block, got {:?}", | |
| 349 | + | outcome.status, | |
| 350 | + | ); | |
| 351 | + | } | |
| 352 | + | } |
| @@ -1,0 +1,545 @@ | |||
| 1 | + | //! Gate execution. Each gate kind has a runner that produces a pass/fail | |
| 2 | + | //! outcome plus an optional detail string (typically a stderr tail or a | |
| 3 | + | //! human-readable reason). Outcomes are persisted to `gate_runs` so /state | |
| 4 | + | //! and the TUI can show them. | |
| 5 | + | //! | |
| 6 | + | //! This module holds the context every runner reads ([`GateCtx`]), the | |
| 7 | + | //! dispatcher ([`run`]) and the two gates that are pure table reads (burn-in | |
| 8 | + | //! and manual confirmation). The runners themselves sit in siblings, one per | |
| 9 | + | //! tool family: `cargo`, `pg`, `migration`, `code_smoke`, `probes`, with `log` | |
| 10 | + | //! under all of them. | |
| 11 | + | //! | |
| 12 | + | //! Nothing here shares mutable state. `GateCtx` is a plain struct every runner | |
| 13 | + | //! reads and none writes, which is what lets the families separate. | |
| 14 | + | //! | |
| 15 | + | //! # Design | |
| 16 | + | //! | |
| 17 | + | //! The tier/gate/deploy architecture, the typed-observability redesign, and the | |
| 18 | + | //! deploy.sh-parity and account-permission notes live in the maintainer wiki. | |
| 19 | + | //! <!-- wiki: sando-overview --> | |
| 20 | + | ||
| 21 | + | use self::cargo::{cargo_test, clippy, fmt_check, hardening_test, supply_chain}; | |
| 22 | + | use self::code_smoke::code_smoke; | |
| 23 | + | use self::migration::migration_dry_run; | |
| 24 | + | use self::probes::{boot_smoke, node_health, page_smoke}; | |
| 25 | + | use crate::config::AppConfig; | |
| 26 | + | use crate::domain::{AppId, GateKind, GateRunId, TierId, Version}; | |
| 27 | + | use crate::events::{self, Event, EventTx}; | |
| 28 | + | use crate::outcome::{GateBlocker, GateFailure, GateOutcome, LogRef, PassNote}; | |
| 29 | + | use crate::topology::Gate; | |
| 30 | + | use anyhow::Result; | |
| 31 | + | use chrono::Utc; | |
| 32 | + | use sqlx::SqlitePool; | |
| 33 | + | use std::collections::HashMap; | |
| 34 | + | use std::path::Path; | |
| 35 | + | use std::path::PathBuf; | |
| 36 | + | use std::sync::Arc; | |
| 37 | + | ||
| 38 | + | mod cargo; | |
| 39 | + | mod code_smoke; | |
| 40 | + | mod log; | |
| 41 | + | mod migration; | |
| 42 | + | mod pg; | |
| 43 | + | mod probes; | |
| 44 | + | #[cfg(test)] | |
| 45 | + | mod testkit; | |
| 46 | + | ||
| 47 | + | pub use pg::preflight_scratch_privileges; | |
| 48 | + | pub(crate) use pg::{reset_scratch, run_migrator}; | |
| 49 | + | ||
| 50 | + | pub struct GateCtx { | |
| 51 | + | pub pool: SqlitePool, | |
| 52 | + | pub cfg: Arc<AppConfig>, | |
| 53 | + | pub tier: TierId, | |
| 54 | + | pub version: Version, | |
| 55 | + | /// The checkout this run's artifact was built from, when there is one. | |
| 56 | + | /// | |
| 57 | + | /// `None` for an accepted artifact: it was built elsewhere and Sando has no | |
| 58 | + | /// source tree for it. That is the boundary made visible (wiki | |
| 59 | + | /// [[sando-bento-boundary]]) — artifact-scoped gates belong to the builder, | |
| 60 | + | /// so a gate that reads source is one Sando should refuse to run here rather | |
| 61 | + | /// than resolve against a path that does not exist. | |
| 62 | + | pub worktree: Option<PathBuf>, | |
| 63 | + | /// The published, content-addressed bundle this run is about, when it has | |
| 64 | + | /// been published yet. `migration_dry_run` prefers it over the worktree, so | |
| 65 | + | /// what it proves is inside the digest rather than beside it. | |
| 66 | + | pub bundle: Option<PathBuf>, | |
| 67 | + | pub events: EventTx, | |
| 68 | + | /// Nodes the `node_health` post-deploy gate probes. Empty for build-time | |
| 69 | + | /// gate runs on the host (where `node_health` never appears); filled at | |
| 70 | + | /// promote time with each freshly-deployed node and its executor. | |
| 71 | + | pub nodes: Vec<NodeProbe>, | |
| 72 | + | /// The `build_runs.id` this gate run vouches for — the artifact identity | |
| 73 | + | /// (wiki [[release-artifact-identity]]). Recorded on every `gate_runs` row so | |
| 74 | + | /// promote can resolve the artifact through the evidence for a specific build, | |
| 75 | + | /// not through a version string that a later rebuild can silently reuse. | |
| 76 | + | /// `None` for legacy/pre-identity runs and gate unit tests. | |
| 77 | + | pub build_id: Option<i64>, | |
| 78 | + | /// Where each `[[aux_repo]]` is checked out, by topology name. Aux repos sit | |
| 79 | + | /// beside the per-sha worktree rather than under it, so a `test_target` that | |
| 80 | + | /// names one cannot be resolved against `worktree` alone. Filled from the | |
| 81 | + | /// topology at build time; empty at promote time, where the only gate that | |
| 82 | + | /// runs is `node_health` and there is no checkout at all. | |
| 83 | + | pub aux_dirs: HashMap<String, PathBuf>, | |
| 84 | + | /// The tier's public URL, for [`Gate::PageSmoke`]. `None` on every | |
| 85 | + | /// build-time run and on any tier that declares none. | |
| 86 | + | /// | |
| 87 | + | /// [`Gate::PageSmoke`]: crate::topology::Gate::PageSmoke | |
| 88 | + | pub public_url: Option<String>, | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | impl GateCtx { | |
| 92 | + | /// The `logs_root` sub-directory this run's gate logs land in. | |
| 93 | + | /// | |
| 94 | + | /// The build id, so two runs of one version keep two sets of logs. Keying on | |
| 95 | + | /// the version would have a rebuild append to the previous attempt's file, | |
| 96 | + | /// with every run pointing at the same mixed log. | |
| 97 | + | /// | |
| 98 | + | /// Falls back to the version when there is no build identity (a | |
| 99 | + | /// pre-migration-008 run, or a gate unit test), which also keeps logs written | |
| 100 | + | /// under that scheme reachable: their rows record the version path, and | |
| 101 | + | /// nothing rewrites them. | |
| 102 | + | pub fn log_scope(&self) -> String { | |
| 103 | + | self.build_id | |
| 104 | + | .map_or_else(|| self.version.to_string(), |id| id.to_string()) | |
| 105 | + | } | |
| 106 | + | ||
| 107 | + | /// This run's log pointer for `gate`. Always paired with | |
| 108 | + | /// [`Self::log_path`], which resolves the same ref to an absolute path. | |
| 109 | + | pub fn log_ref(&self, gate: GateKind) -> LogRef { | |
| 110 | + | LogRef::new(&self.log_scope(), gate) | |
| 111 | + | } | |
| 112 | + | ||
| 113 | + | /// Where `gate`'s log is written on this host: `logs_root` joined to | |
| 114 | + | /// [`Self::log_ref`]. The two are derived from one scope so a row's | |
| 115 | + | /// `log_ref` can never name a file the gate did not write. | |
| 116 | + | pub fn log_path(&self, gate: GateKind) -> PathBuf { | |
| 117 | + | self.cfg | |
| 118 | + | .logs_root | |
| 119 | + | .join(self.log_scope()) | |
| 120 | + | .join(format!("{}.log", gate.as_str())) | |
| 121 | + | } | |
| 122 | + | ||
| 123 | + | /// Absolute directory a `test_target` runs in: under the worktree, or under | |
| 124 | + | /// the named aux repo's checkout. | |
| 125 | + | /// | |
| 126 | + | /// An `aux_repo` naming nothing this run knows about resolves to `None` | |
| 127 | + | /// rather than to a wrong path. Callers treat that as "not present in this | |
| 128 | + | /// run" and skip, the same as a target missing from an older sha — | |
| 129 | + | /// `--check-config` is what stops a genuine typo from reaching here | |
| 130 | + | /// (`Topology::ensure_test_target_aux_repos_exist`). | |
| 131 | + | pub fn target_dir(&self, target: &crate::config::TestTarget) -> Option<PathBuf> { | |
| 132 | + | match target.aux_repo.as_deref() { | |
| 133 | + | None => Some(self.worktree.as_ref()?.join(&target.dir)), | |
| 134 | + | Some(name) => Some(self.aux_dirs.get(name)?.join(&target.dir)), | |
| 135 | + | } | |
| 136 | + | } | |
| 137 | + | ||
| 138 | + | /// The checkout, or a typed refusal for a gate that cannot work without one. | |
| 139 | + | /// | |
| 140 | + | /// Every caller of this is a gate whose evidence is about the *artifact* | |
| 141 | + | /// rather than about the artifact in an environment, which the boundary | |
| 142 | + | /// assigns to the builder. Reaching this arm means a tier asked Sando to | |
| 143 | + | /// re-run a builder's gate against a bundle it was handed, and the honest | |
| 144 | + | /// answer is to say so rather than to pass on having run nothing. | |
| 145 | + | pub fn worktree_for(&self, gate: GateKind) -> std::result::Result<&Path, GateOutcome> { | |
| 146 | + | self.worktree.as_deref().ok_or_else(|| { | |
| 147 | + | GateOutcome::failed(GateFailure::NeedsSource { | |
| 148 | + | gate, | |
| 149 | + | artifact: self.bundle.as_ref().map_or_else( | |
| 150 | + | || "an artifact built elsewhere".into(), | |
| 151 | + | |b| b.display().to_string(), | |
| 152 | + | ), | |
| 153 | + | }) | |
| 154 | + | }) | |
| 155 | + | } | |
| 156 | + | ||
| 157 | + | /// Where a `migration_check` finds its migrations. | |
| 158 | + | /// | |
| 159 | + | /// The bundle wins when it carries them. That is the point of shipping | |
| 160 | + | /// migrations as a `release_contents` entry: it puts them inside the digest, | |
| 161 | + | /// so the dry run proves something about the bytes that ship rather than | |
| 162 | + | /// about a checkout that happens to sit next to them. The worktree is the | |
| 163 | + | /// fallback for a build whose config has not opted in yet, and for an | |
| 164 | + | /// accepted artifact there is no fallback at all — if the builder did not | |
| 165 | + | /// bundle its migrations, Sando cannot dry-run them and says so. | |
| 166 | + | pub fn migrations_dir(&self, dir: &Path) -> Option<PathBuf> { | |
| 167 | + | if let Some(bundle) = &self.bundle { | |
| 168 | + | let in_bundle = bundle.join(dir); | |
| 169 | + | if in_bundle.is_dir() { | |
| 170 | + | return Some(in_bundle); | |
| 171 | + | } | |
| 172 | + | } | |
| 173 | + | let in_worktree = self.worktree.as_ref()?.join(dir); | |
| 174 | + | in_worktree.is_dir().then_some(in_worktree) | |
| 175 | + | } | |
| 176 | + | } | |
| 177 | + | ||
| 178 | + | /// One node the `node_health` gate verifies: its id, the systemd unit to | |
| 179 | + | /// confirm active after the restart, an optional HTTP readiness URL, and the | |
| 180 | + | /// executor that reaches it (the same transport the deploy used). | |
| 181 | + | pub struct NodeProbe { | |
| 182 | + | pub node: crate::domain::NodeId, | |
| 183 | + | pub service: String, | |
| 184 | + | pub health_url: Option<String>, | |
| 185 | + | pub executor: Arc<dyn ops_exec::Executor>, | |
| 186 | + | } | |
| 187 | + | ||
| 188 | + | /// Run a single gate end-to-end: insert the in-flight row, execute the gate, | |
| 189 | + | /// update the row with the outcome. Returns the outcome for the caller. | |
| 190 | + | pub async fn run(ctx: &GateCtx, gate: &Gate) -> Result<GateOutcome> { | |
| 191 | + | let kind = gate.kind(); | |
| 192 | + | let started_at = Utc::now().to_rfc3339(); | |
| 193 | + | ||
| 194 | + | let id: i64 = sqlx::query_scalar( | |
| 195 | + | "INSERT INTO gate_runs (app, version, tier, gate_kind, started_at, build_id) | |
| 196 | + | VALUES (?, ?, ?, ?, ?, ?) | |
| 197 | + | RETURNING id", | |
| 198 | + | ) | |
| 199 | + | .bind(&ctx.cfg.id) | |
| 200 | + | .bind(&ctx.version) | |
| 201 | + | .bind(&ctx.tier) | |
| 202 | + | .bind(kind) | |
| 203 | + | .bind(&started_at) | |
| 204 | + | .bind(ctx.build_id) | |
| 205 | + | .fetch_one(&ctx.pool) | |
| 206 | + | .await?; | |
| 207 | + | let run_id = GateRunId(id); | |
| 208 | + | ||
| 209 | + | tracing::info!( | |
| 210 | + | run_id = %run_id, tier = %ctx.tier, version = %ctx.version, gate = %kind, | |
| 211 | + | "gate start", | |
| 212 | + | ); | |
| 213 | + | events::emit( | |
| 214 | + | &ctx.events, | |
| 215 | + | Event::GateStart { | |
| 216 | + | run_id, | |
| 217 | + | tier: ctx.tier.clone(), | |
| 218 | + | version: ctx.version.clone(), | |
| 219 | + | gate: kind, | |
| 220 | + | }, | |
| 221 | + | ); | |
| 222 | + | ||
| 223 | + | let outcome = match gate { | |
| 224 | + | // cargo_test bounds its own run internally (it kills the specific child). | |
| 225 | + | Gate::CargoTest => cargo_test(ctx, run_id).await, | |
| 226 | + | // hardening_test bounds its own run internally, same as cargo_test. | |
| 227 | + | Gate::HardeningTest => hardening_test(ctx, run_id).await, | |
| 228 | + | // Each bounds itself the same way cargo_test does: one deadline across | |
| 229 | + | // every target, so N crates cannot multiply the ceiling by N. | |
| 230 | + | Gate::Clippy => clippy(ctx, run_id).await, | |
| 231 | + | Gate::Fmt => fmt_check(ctx, run_id).await, | |
| 232 | + | Gate::CargoAudit => supply_chain(ctx, run_id, GateKind::CargoAudit).await, | |
| 233 | + | Gate::CargoDeny => supply_chain(ctx, run_id, GateKind::CargoDeny).await, | |
| 234 | + | // migration_dry_run's psql restore + sqlx migrate could wedge; bound the | |
| 235 | + | // whole gate here. Its bash restore sets kill_on_drop, so a timeout-drop | |
| 236 | + | // doesn't orphan it. | |
| 237 | + | Gate::MigrationDryRun => { | |
| 238 | + | let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 239 | + | match tokio::time::timeout(ceiling, migration_dry_run(ctx, run_id)).await { | |
| 240 | + | Ok(res) => res, | |
| 241 | + | Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout { | |
| 242 | + | gate: GateKind::MigrationDryRun, | |
| 243 | + | after_s: ctx.cfg.gate_timeout_secs as u32, | |
| 244 | + | }) | |
| 245 | + | .with_log_ref(ctx.log_ref(GateKind::MigrationDryRun))), | |
| 246 | + | } | |
| 247 | + | } | |
| 248 | + | // code_smoke boots the real binary (migrate-from-scratch + seed + serve), | |
| 249 | + | // any step of which could wedge; bound the whole gate here. Both child | |
| 250 | + | // processes set kill_on_drop, so a timeout-drop can't orphan them. A | |
| 251 | + | // timeout leaves the throwaway DB behind; the next run's createdb drops | |
| 252 | + | // it first (DROP IF EXISTS), same as migration_dry_run's scratch reset. | |
| 253 | + | Gate::CodeSmoke => { | |
| 254 | + | let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 255 | + | match tokio::time::timeout(ceiling, code_smoke(ctx, run_id)).await { | |
| 256 | + | Ok(res) => res, | |
| 257 | + | Err(_elapsed) => Ok(GateOutcome::failed(GateFailure::Timeout { | |
| 258 | + | gate: GateKind::CodeSmoke, | |
| 259 | + | after_s: ctx.cfg.gate_timeout_secs as u32, | |
| 260 | + | }) | |
| 261 | + | .with_log_ref(ctx.log_ref(GateKind::CodeSmoke))), | |
| 262 | + | } | |
| 263 | + | } | |
| 264 | + | Gate::BootSmoke => boot_smoke(ctx, run_id).await, | |
| 265 | + | Gate::NodeHealth => node_health(ctx).await, | |
| 266 | + | Gate::PageSmoke => page_smoke(ctx).await, | |
| 267 | + | Gate::BurnIn { hours } => burn_in(ctx, *hours).await, | |
| 268 | + | Gate::ManualConfirm => manual_confirm(ctx).await, | |
| 269 | + | }; | |
| 270 | + | ||
| 271 | + | let outcome = outcome.unwrap_or_else(|e| { | |
| 272 | + | GateOutcome::failed(GateFailure::Unclassified { | |
| 273 | + | legacy_detail: Some(format!("gate runner errored: {e}")), | |
| 274 | + | }) | |
| 275 | + | }); | |
| 276 | + | ||
| 277 | + | let outcome_json = serde_json::to_string(&outcome) | |
| 278 | + | .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}")); | |
| 279 | + | sqlx::query( | |
| 280 | + | "UPDATE gate_runs | |
| 281 | + | SET finished_at = ?, status = ?, outcome_json = ?, log_ref = ? | |
| 282 | + | WHERE id = ?", | |
| 283 | + | ) | |
| 284 | + | .bind(Utc::now().to_rfc3339()) | |
| 285 | + | .bind(outcome.status_str()) | |
| 286 | + | .bind(&outcome_json) | |
| 287 | + | .bind(outcome.log_ref.as_ref().map(super::outcome::LogRef::as_str)) | |
| 288 | + | .bind(id) | |
| 289 | + | .execute(&ctx.pool) | |
| 290 | + | .await?; | |
| 291 | + | ||
| 292 | + | tracing::info!( | |
| 293 | + | tier = %ctx.tier, version = %ctx.version, gate = %kind, | |
| 294 | + | status = outcome.status_str(), "gate done", | |
| 295 | + | ); | |
| 296 | + | events::emit( | |
| 297 | + | &ctx.events, | |
| 298 | + | Event::GateDone { | |
| 299 | + | run_id, | |
| 300 | + | tier: ctx.tier.clone(), | |
| 301 | + | version: ctx.version.clone(), | |
| 302 | + | gate: kind, | |
| 303 | + | outcome: outcome.clone(), | |
| 304 | + | }, | |
| 305 | + | ); | |
| 306 | + | ||
| 307 | + | Ok(outcome) | |
| 308 | + | } | |
| 309 | + | ||
| 310 | + | /// Run every gate in order and return the kinds that did not pass (empty means | |
| 311 | + | /// green). We deliberately do NOT short-circuit on first failure — every gate's | |
| 312 | + | /// outcome is recorded in `gate_runs`, which is the operator's only visibility | |
| 313 | + | /// into pipeline health. Hiding later gates because an earlier one failed makes | |
| 314 | + | /// diagnosis worse. | |
| 315 | + | /// | |
| 316 | + | /// Returning the failing kinds rather than a bare bool is what lets the promote | |
| 317 | + | /// path name them in the tier's `partial_reason` and in the error it returns to | |
| 318 | + | /// the operator, instead of a generic "something was red". | |
| 319 | + | pub async fn run_all(ctx: &GateCtx, gates: &[Gate]) -> Result<Vec<GateKind>> { | |
| 320 | + | let mut failed = Vec::new(); | |
| 321 | + | for g in gates { | |
| 322 | + | let o = run(ctx, g).await?; | |
| 323 | + | if !o.is_passed() { | |
| 324 | + | failed.push(g.kind()); | |
| 325 | + | } | |
| 326 | + | } | |
| 327 | + | Ok(failed) | |
| 328 | + | } | |
| 329 | + | ||
| 330 | + | /// Live check: has `tier`'s burn-in window of `hours` elapsed since its clock | |
| 331 | + | /// (`tier_state.burn_in_started_at`, started by a promote onto the tier)? Used | |
| 332 | + | /// by the promote-time gate check (`unsatisfied_gates`) so a stale `blocked` | |
| 333 | + | /// row never masks an elapsed — or not-yet-elapsed — window. The `burn_in` gate | |
| 334 | + | /// runner below wraps the same state with a richer outcome for `/state`. | |
| 335 | + | pub async fn burn_in_satisfied( | |
| 336 | + | pool: &SqlitePool, | |
| 337 | + | app: &AppId, | |
| 338 | + | tier: &TierId, | |
| 339 | + | hours: u32, | |
| 340 | + | ) -> Result<bool> { | |
| 341 | + | let started: Option<String> = | |
| 342 | + | sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?") | |
| 343 | + | .bind(app) | |
| 344 | + | .bind(tier) | |
| 345 | + | .fetch_optional(pool) | |
| 346 | + | .await? | |
| 347 | + | .flatten(); | |
| 348 | + | let Some(started) = started else { | |
| 349 | + | return Ok(false); | |
| 350 | + | }; | |
| 351 | + | let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc); | |
| 352 | + | Ok(Utc::now() - started >= chrono::Duration::hours(hours as i64)) | |
| 353 | + | } | |
| 354 | + | ||
| 355 | + | async fn burn_in(ctx: &GateCtx, hours: u32) -> Result<GateOutcome> { | |
| 356 | + | // Check tier_state.burn_in_started_at on this tier; pass if enough time | |
| 357 | + | // has elapsed. The clock is started by /promote when a version lands on | |
| 358 | + | // the burn-in tier. | |
| 359 | + | let started: Option<String> = | |
| 360 | + | sqlx::query_scalar("SELECT burn_in_started_at FROM tier_state WHERE app = ? AND tier = ?") | |
| 361 | + | .bind(&ctx.cfg.id) | |
| 362 | + | .bind(&ctx.tier) | |
| 363 | + | .fetch_optional(&ctx.pool) | |
| 364 | + | .await? | |
| 365 | + | .flatten(); | |
| 366 | + | let Some(started) = started else { | |
| 367 | + | return Ok(GateOutcome::blocked(GateBlocker::BurnInClockNotStarted)); | |
| 368 | + | }; | |
| 369 | + | let started = chrono::DateTime::parse_from_rfc3339(&started)?.with_timezone(&Utc); | |
| 370 | + | let elapsed = Utc::now() - started; | |
| 371 | + | let needed = chrono::Duration::hours(hours as i64); | |
| 372 | + | if elapsed >= needed { | |
| 373 | + | Ok(GateOutcome::passed(PassNote::BurnInElapsed { | |
| 374 | + | hours: elapsed.num_hours() as u32, | |
| 375 | + | })) | |
| 376 | + | } else { | |
| 377 | + | let remaining = (needed - elapsed).num_hours().max(0) as u32; | |
| 378 | + | Ok(GateOutcome::blocked(GateBlocker::BurnInRemaining { | |
| 379 | + | hours_remaining: remaining, | |
| 380 | + | hours_total: hours, | |
| 381 | + | })) | |
| 382 | + | } | |
| 383 | + | } | |
| 384 | + | ||
| 385 | + | async fn manual_confirm(ctx: &GateCtx) -> Result<GateOutcome> { | |
| 386 | + | // Pass iff a row in gate_runs exists with status='passed' for this | |
| 387 | + | // (tier, version, manual_confirm) that was inserted out-of-band by an | |
| 388 | + | // operator action. Since the harness inserts the in-flight row itself, | |
| 389 | + | // look for a prior confirmation row. | |
| 390 | + | let prior_at: Option<String> = sqlx::query_scalar( | |
| 391 | + | "SELECT finished_at FROM gate_runs | |
| 392 | + | WHERE app = ? AND tier = ? AND version = ? AND gate_kind = 'manual_confirm' | |
| 393 | + | AND status = 'passed' | |
| 394 | + | ORDER BY id DESC LIMIT 1", | |
| 395 | + | ) | |
| 396 | + | .bind(&ctx.cfg.id) | |
| 397 | + | .bind(&ctx.tier) | |
| 398 | + | .bind(&ctx.version) | |
| 399 | + | .fetch_optional(&ctx.pool) | |
| 400 | + | .await?; | |
| 401 | + | match prior_at { | |
| 402 | + | Some(at_str) => { | |
| 403 | + | let at = chrono::DateTime::parse_from_rfc3339(&at_str) | |
| 404 | + | .map_or_else(|_| Utc::now(), |d| d.with_timezone(&Utc)); | |
| 405 | + | Ok(GateOutcome::passed(PassNote::OperatorConfirmed { at })) | |
| 406 | + | } | |
| 407 | + | None => Ok(GateOutcome::blocked( | |
| 408 | + | GateBlocker::AwaitingOperatorConfirmation, | |
| 409 | + | )), | |
| 410 | + | } | |
| 411 | + | } | |
| 412 | + | ||
| 413 | + | #[cfg(test)] | |
| 414 | + | mod tests { | |
| 415 | + | use super::*; | |
| 416 | + | use crate::gates::testkit::{aux_target, resolving_ctx, target}; | |
| 417 | + | use sqlx::sqlite::SqlitePoolOptions; | |
| 418 | + | ||
| 419 | + | #[tokio::test] | |
| 420 | + | async fn a_plain_target_resolves_under_the_worktree() { | |
| 421 | + | let ctx = resolving_ctx("/w/abc123", &[]); | |
| 422 | + | assert_eq!( | |
| 423 | + | ctx.target_dir(&target("shared/tagtree")), | |
| 424 | + | Some(PathBuf::from("/w/abc123/shared/tagtree")), | |
| 425 | + | ); | |
| 426 | + | } | |
| 427 | + | ||
| 428 | + | #[tokio::test] | |
| 429 | + | async fn an_aux_target_resolves_beside_the_worktree_not_under_it() { | |
| 430 | + | // The whole point: `Libraries/docengine` is a sibling of the per-sha | |
| 431 | + | // worktree, so a worktree-relative path can never reach it. | |
| 432 | + | let ctx = resolving_ctx("/w/abc123", &[("docengine", "/w/Libraries/docengine")]); | |
| 433 | + | assert_eq!( | |
| 434 | + | ctx.target_dir(&aux_target("", "docengine")), | |
| 435 | + | Some(PathBuf::from("/w/Libraries/docengine")), | |
| 436 | + | ); | |
| 437 | + | // A subdirectory of an aux repo resolves under its checkout. | |
| 438 | + | assert_eq!( | |
| 439 | + | ctx.target_dir(&aux_target("crates/inner", "docengine")), | |
| 440 | + | Some(PathBuf::from("/w/Libraries/docengine/crates/inner")), | |
| 441 | + | ); | |
| 442 | + | } | |
| 443 | + | ||
| 444 | + | #[tokio::test] | |
| 445 | + | async fn an_aux_target_with_no_checkout_this_run_resolves_to_nothing() { | |
| 446 | + | // Promote-time gates carry no aux dirs. Resolving to a wrong path (say, | |
| 447 | + | // the worktree) would run the gate against whatever happened to sit | |
| 448 | + | // there; `None` makes the caller skip, and --check-config is what | |
| 449 | + | // catches a real typo. | |
| 450 | + | let ctx = resolving_ctx("/w/abc123", &[]); | |
| 451 | + | assert_eq!(ctx.target_dir(&aux_target("", "docengine")), None); | |
| 452 | + | } | |
| 453 | + | ||
| 454 | + | #[test] | |
| 455 | + | fn labels_name_the_repo_an_aux_target_lives_in() { | |
| 456 | + | assert_eq!(target("server").label(), "server"); | |
| 457 | + | assert_eq!(aux_target("", "docengine").label(), "docengine (aux)"); | |
| 458 | + | assert_eq!( | |
| 459 | + | aux_target("crates/inner", "docengine").label(), | |
| 460 | + | "docengine/crates/inner (aux)", | |
| 461 | + | ); | |
| 462 | + | } | |
| 463 | + | ||
| 464 | + | #[test] | |
| 465 | + | fn every_gate_kind_round_trips_through_its_wire_string() { | |
| 466 | + | // gate_kind is a TEXT column and a WS event field; as_str and FromStr | |
| 467 | + | // disagreeing would make a gate's evidence unreadable by | |
| 468 | + | // unsatisfied_gates, which fails the promote closed with no explanation. | |
| 469 | + | for k in [ | |
| 470 | + | GateKind::CargoTest, | |
| 471 | + | GateKind::HardeningTest, | |
| 472 | + | GateKind::Clippy, | |
| 473 | + | GateKind::Fmt, | |
| 474 | + | GateKind::CargoAudit, | |
| 475 | + | GateKind::CargoDeny, | |
| 476 | + | GateKind::MigrationDryRun, | |
| 477 | + | GateKind::CodeSmoke, | |
| 478 | + | GateKind::BootSmoke, | |
| 479 | + | GateKind::NodeHealth, | |
| 480 | + | GateKind::BurnIn, | |
| 481 | + | GateKind::ManualConfirm, | |
| 482 | + | ] { | |
| 483 | + | assert_eq!( | |
| 484 | + | k.as_str().parse::<GateKind>().unwrap(), | |
| 485 | + | k, | |
| 486 | + | "round trip for {k:?}" | |
| 487 | + | ); | |
| 488 | + | } | |
| 489 | + | } | |
| 490 | + | ||
| 491 | + | /// burn_in returns a typed Blocked when the clock isn't started; the | |
| 492 | + | /// runner persists status='blocked' + outcome_json (the json carries | |
| 493 | + | /// blocker.kind = 'burn_in_clock_not_started'). | |
| 494 | + | #[tokio::test] | |
| 495 | + | async fn burn_in_blocked_persists_typed_outcome() { | |
| 496 | + | let pool = SqlitePoolOptions::new() | |
| 497 | + | .max_connections(1) | |
| 498 | + | .connect("sqlite::memory:") | |
| 499 | + | .await | |
| 500 | + | .unwrap(); |
Lines truncated
| @@ -1,0 +1,637 @@ | |||
| 1 | + | //! Postgres work the gates need: the scratch database they run against, the | |
| 2 | + | //! backup restore that seeds it, and the URL surgery both require. | |
| 3 | + | //! | |
| 4 | + | //! A peer of the gate families rather than one family's helper. | |
| 5 | + | //! [`clean_stale_test_dbs`] is called from both `cargo_test` and | |
| 6 | + | //! `hardening_test`, and [`pg_url_with_dbname`] from the migration check as | |
| 7 | + | //! well as from code_smoke. | |
| 8 | + | ||
| 9 | + | use super::log::GateLog; | |
| 10 | + | use anyhow::{Context, Result}; | |
| 11 | + | use ops_exec::sh_quote; | |
| 12 | + | use tokio::process::Command; | |
| 13 | + | ||
| 14 | + | pub(crate) async fn reset_scratch(db_url: &str, owner_role: &str) -> Result<()> { | |
| 15 | + | use sqlx::Executor; | |
| 16 | + | use sqlx::postgres::PgPoolOptions; | |
| 17 | + | let pool = PgPoolOptions::new() | |
| 18 | + | .max_connections(1) | |
| 19 | + | .connect(db_url) | |
| 20 | + | .await?; | |
| 21 | + | // `owner_role` is validated `[A-Za-z0-9_]+` at config load, so interpolating | |
| 22 | + | // it into DDL is sound. It still goes through `format('%I')` inside the DO | |
| 23 | + | // block for the quoting Postgres expects on an identifier. | |
| 24 | + | let sql = format!( | |
| 25 | + | r#" | |
| 26 | + | DO $$ | |
| 27 | + | DECLARE s text; | |
| 28 | + | BEGIN | |
| 29 | + | -- The dump restores objects owned by the prod role and re-grants to | |
| 30 | + | -- it (`ALTER ... OWNER TO {owner_role}`), which errors if the role | |
| 31 | + | -- is absent — superuser does not imply the role exists. Create it | |
| 32 | + | -- NOLOGIN: the scratch DB needs the role as an *owner* only, never | |
| 33 | + | -- as a connecting identity. Idempotent, so a re-reset is a no-op. | |
| 34 | + | IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{owner_role}') THEN | |
| 35 | + | EXECUTE format('CREATE ROLE %I NOLOGIN', '{owner_role}'); | |
| 36 | + | END IF; | |
| 37 | + | ||
| 38 | + | -- Drop every non-system schema, not just public — migrations create | |
| 39 | + | -- custom schemas (e.g. tower_sessions) that survive `DROP SCHEMA | |
| 40 | + | -- public CASCADE` and then collide on the next migration run. | |
| 41 | + | FOR s IN | |
| 42 | + | SELECT nspname FROM pg_namespace | |
| 43 | + | WHERE nspname NOT LIKE 'pg_%' | |
| 44 | + | AND nspname NOT IN ('information_schema') | |
| 45 | + | LOOP | |
| 46 | + | EXECUTE format('DROP SCHEMA IF EXISTS %I CASCADE', s); | |
| 47 | + | END LOOP; | |
| 48 | + | EXECUTE 'CREATE SCHEMA public'; | |
| 49 | + | -- Restore the pre-PG15 public-schema default on the throwaway | |
| 50 | + | -- scratch DB. Without this, the freshly-created public is owned by | |
| 51 | + | -- the connecting role (sando) with no grant to anyone else, so a | |
| 52 | + | -- migration's FK/trigger check that Postgres runs as a *restored* | |
| 53 | + | -- prod-owned table's owner ({owner_role} from the backup dump) | |
| 54 | + | -- fails with "permission denied for schema public". Granting to | |
| 55 | + | -- PUBLIC is role-agnostic and safe here — this DB is disposable and | |
| 56 | + | -- exists only to dry-run migrations. | |
| 57 | + | EXECUTE 'GRANT USAGE, CREATE ON SCHEMA public TO PUBLIC'; | |
| 58 | + | -- PG15+: the new owner needs CREATE on public in its own right, not | |
| 59 | + | -- only via PUBLIC, for the restore's owner-scoped DDL. | |
| 60 | + | EXECUTE format('GRANT USAGE, CREATE ON SCHEMA public TO %I', '{owner_role}'); | |
| 61 | + | END $$; | |
| 62 | + | "# | |
| 63 | + | ); | |
| 64 | + | pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(sql))) | |
| 65 | + | .await?; | |
| 66 | + | pool.close().await; | |
| 67 | + | Ok(()) | |
| 68 | + | } | |
| 69 | + | ||
| 70 | + | /// Startup assertion for the scratch cluster: the gates reset it, seed an owner | |
| 71 | + | /// role into it, and drop leftover test databases in it, none of which a plain | |
| 72 | + | /// unprivileged role can do. Satisfied by hand on the build host | |
| 73 | + | /// (`ALTER ROLE sando SUPERUSER`, a created `makenotwork` role); unasserted, a | |
| 74 | + | /// rebuild elsewhere fails one gate at a time with an opaque permissions error. | |
| 75 | + | /// Assert once, at boot, loudly. | |
| 76 | + | /// | |
| 77 | + | /// Not part of `--check-config`: that path is pure by design (no DB, no | |
| 78 | + | /// network), and a green there must mean "this build understands its config", | |
| 79 | + | /// not "the cluster is reachable". | |
| 80 | + | pub async fn preflight_scratch_privileges(db_url: &str) -> Result<()> { | |
| 81 | + | use sqlx::postgres::PgPoolOptions; | |
| 82 | + | let pool = PgPoolOptions::new() | |
| 83 | + | .max_connections(1) | |
| 84 | + | .connect(db_url) | |
| 85 | + | .await | |
| 86 | + | .context("connecting to scratch_db_url for the startup privilege check")?; | |
| 87 | + | let (is_super, can_signal): (bool, bool) = sqlx::query_as( | |
| 88 | + | "SELECT rolsuper, pg_catalog.pg_has_role(current_user, 'pg_signal_backend', 'USAGE') | |
| 89 | + | FROM pg_roles WHERE rolname = current_user", | |
| 90 | + | ) | |
| 91 | + | .fetch_one(&pool) | |
| 92 | + | .await?; | |
| 93 | + | pool.close().await; | |
| 94 | + | anyhow::ensure!( | |
| 95 | + | is_super || can_signal, | |
| 96 | + | "the scratch_db_url role has neither SUPERUSER nor pg_signal_backend; migration_dry_run \ | |
| 97 | + | and cargo_test cannot reset the scratch DB or clear stale test databases. Grant one:\n \ | |
| 98 | + | ALTER ROLE <role> SUPERUSER; -- what fw13 uses\n \ | |
| 99 | + | GRANT pg_signal_backend TO <role>; -- narrower: terminate only, cannot drop \ | |
| 100 | + | foreign-owned databases", | |
| 101 | + | ); | |
| 102 | + | if !is_super { | |
| 103 | + | tracing::warn!( | |
| 104 | + | "scratch role has pg_signal_backend but not SUPERUSER: stale test databases owned by \ | |
| 105 | + | another role cannot be dropped, and the scratch owner role cannot be created if absent" | |
| 106 | + | ); | |
| 107 | + | } | |
| 108 | + | Ok(()) | |
| 109 | + | } | |
| 110 | + | ||
| 111 | + | /// Best-effort cleanup of stale per-test database clones (`mnw_test_<uuid>`) | |
| 112 | + | /// left behind by a killed `cargo_test` run. | |
| 113 | + | /// | |
| 114 | + | /// Drops **foreign-owned leftovers too**, which is why the daemon asserts | |
| 115 | + | /// SUPERUSER at startup (`preflight_scratch_privileges`): `DROP DATABASE` | |
| 116 | + | /// requires ownership or superuser, and the `WITH (FORCE)` terminate requires | |
| 117 | + | /// superuser or `pg_signal_backend`. Without both, orphans from a run under a | |
| 118 | + | /// different role accumulate and degrade the gate — the failure this cleanup | |
| 119 | + | /// exists to prevent. | |
| 120 | + | /// | |
| 121 | + | /// OPERATIONAL HAZARD: fw13 runs one Postgres cluster shared with local `cargo | |
| 122 | + | /// test` as `max`, so a gate firing mid-local-test will force-drop that run's | |
| 123 | + | /// databases out from under it. That collision is known and tracked separately | |
| 124 | + | /// (give the gate its own cluster); until then, do not run local tests on fw13 | |
| 125 | + | /// while a Sando gate is live. | |
| 126 | + | /// | |
| 127 | + | /// Deliberately **excludes the template** (`mnw_test_template_*`): the harness | |
| 128 | + | /// reuses it across runs when it's migration-current (skipping a full | |
| 129 | + | /// drop+migrate), so dropping it here would force a needless rebuild every | |
| 130 | + | /// gate run. Templates are bounded (one per role) and never accumulate, so | |
| 131 | + | /// leaving them is free. Never returns an error: a cleanup miss must not turn a | |
| 132 | + | /// deploy red. | |
| 133 | + | pub(super) async fn clean_stale_test_dbs(db_url: &str) { | |
| 134 | + | use sqlx::Executor; | |
| 135 | + | use sqlx::postgres::PgPoolOptions; | |
| 136 | + | let pool = match PgPoolOptions::new() | |
| 137 | + | .max_connections(1) | |
| 138 | + | .connect(db_url) | |
| 139 | + | .await | |
| 140 | + | { | |
| 141 | + | Ok(p) => p, | |
| 142 | + | Err(e) => { | |
| 143 | + | tracing::warn!(error = %e, "stale test-db cleanup: could not connect; skipping"); | |
| 144 | + | return; | |
| 145 | + | } | |
| 146 | + | }; | |
| 147 | + | // Every per-test clone, whoever owns it. The ownership filter this used to | |
| 148 | + | // carry is what let foreign-owned orphans pile up; superuser (asserted at | |
| 149 | + | // startup) makes them droppable. | |
| 150 | + | let names: Vec<(String,)> = sqlx::query_as( | |
| 151 | + | "SELECT datname FROM pg_database | |
| 152 | + | WHERE datname LIKE 'mnw_test_%' | |
| 153 | + | AND datname NOT LIKE '%template%'", | |
| 154 | + | ) | |
| 155 | + | .fetch_all(&pool) | |
| 156 | + | .await | |
| 157 | + | .unwrap_or_default(); | |
| 158 | + | let count = names.len(); | |
| 159 | + | for (name,) in names { | |
| 160 | + | // `name` comes straight from pg_database; quoting it is sufficient. | |
| 161 | + | if let Err(e) = pool | |
| 162 | + | .execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!( | |
| 163 | + | "DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)" | |
| 164 | + | )))) | |
| 165 | + | .await | |
| 166 | + | { | |
| 167 | + | tracing::warn!(error = %e, db = %name, "stale test-db cleanup: drop failed"); | |
| 168 | + | } | |
| 169 | + | } | |
| 170 | + | if count > 0 { | |
| 171 | + | tracing::info!( | |
| 172 | + | count, | |
| 173 | + | "stale test-db cleanup: dropped leftover mnw_test_* databases" | |
| 174 | + | ); | |
| 175 | + | } | |
| 176 | + | pool.close().await; | |
| 177 | + | } | |
| 178 | + | ||
| 179 | + | /// Build the restore shell line. Two pipelines we accept: | |
| 180 | + | /// *.sql -> psql -v ON_ERROR_STOP=1 $url < dump | |
| 181 | + | /// *.sql.gz -> set -o pipefail; gunzip -c dump | psql -v ON_ERROR_STOP=1 $url | |
| 182 | + | /// | |
| 183 | + | /// Two safety flags are load-bearing (CF4): | |
| 184 | + | /// - `ON_ERROR_STOP=1`: without it, psql exits 0 even when individual statements | |
| 185 | + | /// error, so a partial/corrupt restore would *pass* the gate. | |
| 186 | + | /// - `set -o pipefail`: without it a shell pipeline reports only the last | |
| 187 | + | /// command's status, so a `gunzip` failure on a truncated archive is masked by | |
| 188 | + | /// psql's exit. pipefail is a bash builtin (not POSIX sh), so the runner uses | |
| 189 | + | /// `bash -c`. | |
| 190 | + | pub(super) fn restore_shell(db_url: &str, dump: &str) -> String { | |
| 191 | + | if std::path::Path::new(dump) | |
| 192 | + | .extension() | |
| 193 | + | .is_some_and(|ext| ext.eq_ignore_ascii_case("gz")) | |
| 194 | + | { | |
| 195 | + | format!( | |
| 196 | + | "set -o pipefail; gunzip -c {q} | psql -v ON_ERROR_STOP=1 {url}", | |
| 197 | + | q = sh_quote(dump), | |
| 198 | + | url = sh_quote(db_url), | |
| 199 | + | ) | |
| 200 | + | } else { | |
| 201 | + | format!( | |
| 202 | + | "psql -v ON_ERROR_STOP=1 {url} < {q}", | |
| 203 | + | url = sh_quote(db_url), | |
| 204 | + | q = sh_quote(dump), | |
| 205 | + | ) | |
| 206 | + | } | |
| 207 | + | } | |
| 208 | + | ||
| 209 | + | pub(super) async fn restore_dump(db_url: &str, dump: &str, log: &GateLog) -> Result<()> { | |
| 210 | + | // Split the password out of the URL and hand it to psql via PGPASSWORD, so it | |
| 211 | + | // never lands in argv (visible in /proc/<pid>/cmdline to any local user). | |
| 212 | + | // The sanitized URL — user/host/db, no secret — goes on the command line. | |
| 213 | + | let (sanitized, password) = split_pg_password(db_url); | |
| 214 | + | let shell = restore_shell(&sanitized, dump); | |
| 215 | + | // `bash` (not `sh`): `set -o pipefail` is a bash builtin. The restore runs | |
| 216 | + | // locally on the Sando host (fw13), which has bash. | |
| 217 | + | let mut cmd = Command::new("bash"); | |
| 218 | + | cmd.arg("-c").arg(&shell); | |
| 219 | + | // kill_on_drop so the gate's wall-clock ceiling (dispatcher-level timeout on | |
| 220 | + | // migration_dry_run) can't orphan a wedged psql restore. | |
| 221 | + | cmd.kill_on_drop(true); | |
| 222 | + | if let Some(pw) = password { | |
| 223 | + | cmd.env("PGPASSWORD", pw); | |
| 224 | + | } | |
| 225 | + | // Streamed, not `.output()`: a prod-sized restore runs for minutes, and | |
| 226 | + | // psql's progress is the only thing an operator has to watch during it. | |
| 227 | + | let (_stdout, stderr, status) = log.run(&mut cmd).await?; | |
| 228 | + | anyhow::ensure!( | |
| 229 | + | status.success(), | |
| 230 | + | "restore failed: {}", | |
| 231 | + | String::from_utf8_lossy(&stderr), | |
| 232 | + | ); | |
| 233 | + | Ok(()) | |
| 234 | + | } | |
| 235 | + | ||
| 236 | + | /// Split a `postgres://user:password@host/db` URL into its password-free form and | |
| 237 | + | /// the (percent-decoded) password. Returns the URL unchanged with `None` when | |
| 238 | + | /// there is no userinfo password. psql reads the password from `PGPASSWORD`, so | |
| 239 | + | /// keeping it off the command line removes the /proc exposure. | |
| 240 | + | pub(super) fn split_pg_password(db_url: &str) -> (String, Option<String>) { | |
| 241 | + | let Some(after) = db_url.find("://").map(|i| i + 3) else { | |
| 242 | + | return (db_url.to_string(), None); | |
| 243 | + | }; | |
| 244 | + | // The authority ends at the first '/', '?' or '#'; the password (if any) is | |
| 245 | + | // between the first ':' and the '@' within the userinfo of that authority. | |
| 246 | + | let authority_end = db_url[after..] | |
| 247 | + | .find(['/', '?', '#']) | |
| 248 | + | .map_or(db_url.len(), |i| after + i); | |
| 249 | + | let Some(at) = db_url[after..authority_end].find('@').map(|i| after + i) else { | |
| 250 | + | return (db_url.to_string(), None); | |
| 251 | + | }; | |
| 252 | + | let userinfo = &db_url[after..at]; | |
| 253 | + | let Some(colon) = userinfo.find(':') else { | |
| 254 | + | return (db_url.to_string(), None); | |
| 255 | + | }; | |
| 256 | + | let password = percent_decode(&userinfo[colon + 1..]); | |
| 257 | + | let sanitized = format!( | |
| 258 | + | "{}{}{}", | |
| 259 | + | &db_url[..after], | |
| 260 | + | &userinfo[..colon], | |
| 261 | + | &db_url[at..] | |
| 262 | + | ); | |
| 263 | + | (sanitized, Some(password)) | |
| 264 | + | } | |
| 265 | + | ||
| 266 | + | /// Minimal `%XX` percent-decode for a URL userinfo component. Non-escape bytes | |
| 267 | + | /// pass through; a malformed escape is left literal. | |
| 268 | + | pub(super) fn percent_decode(s: &str) -> String { | |
| 269 | + | let b = s.as_bytes(); | |
| 270 | + | let mut out = Vec::with_capacity(b.len()); | |
| 271 | + | let mut i = 0; | |
| 272 | + | while i < b.len() { | |
| 273 | + | if b[i] == b'%' | |
| 274 | + | && i + 2 < b.len() | |
| 275 | + | && let (Some(h), Some(l)) = (hex_val(b[i + 1]), hex_val(b[i + 2])) | |
| 276 | + | { | |
| 277 | + | out.push((h << 4) | l); | |
| 278 | + | i += 3; | |
| 279 | + | } else { | |
| 280 | + | out.push(b[i]); | |
| 281 | + | i += 1; | |
| 282 | + | } | |
| 283 | + | } | |
| 284 | + | String::from_utf8_lossy(&out).into_owned() | |
| 285 | + | } | |
| 286 | + | ||
| 287 | + | pub(super) fn hex_val(c: u8) -> Option<u8> { | |
| 288 | + | match c { | |
| 289 | + | b'0'..=b'9' => Some(c - b'0'), | |
| 290 | + | b'a'..=b'f' => Some(c - b'a' + 10), | |
| 291 | + | b'A'..=b'F' => Some(c - b'A' + 10), | |
| 292 | + | _ => None, | |
| 293 | + | } | |
| 294 | + | } | |
| 295 | + | ||
| 296 | + | pub(crate) async fn run_migrator(db_url: &str, dir: &std::path::Path) -> Result<()> { | |
| 297 | + | use sqlx::postgres::PgPoolOptions; | |
| 298 | + | let pool = PgPoolOptions::new() | |
| 299 | + | .max_connections(1) | |
| 300 | + | .connect(db_url) | |
| 301 | + | .await?; | |
| 302 | + | let migrator = sqlx::migrate::Migrator::new(dir).await?; | |
| 303 | + | migrator.run(&pool).await?; | |
| 304 | + | pool.close().await; | |
| 305 | + | Ok(()) | |
| 306 | + | } | |
| 307 | + | ||
| 308 | + | /// Rewrite a `postgres://` URL to point at database `dbname`, preserving scheme, | |
| 309 | + | /// userinfo, host/port, and any query (e.g. the socket `?host=/var/run/postgresql` | |
| 310 | + | /// form) + fragment. Used to derive the maintenance connection (`postgres`) and | |
| 311 | + | /// the throwaway smoke DB URL from the configured `scratch_db_url`. | |
| 312 | + | pub(super) fn pg_url_with_dbname(url: &str, dbname: &str) -> String { | |
| 313 | + | let Some(after_scheme) = url.find("://").map(|i| i + 3) else { | |
| 314 | + | return url.to_string(); | |
| 315 | + | }; | |
| 316 | + | let rest = &url[after_scheme..]; | |
| 317 | + | // Authority ends at the first '/', '?' or '#'; whatever follows is the | |
| 318 | + | // path (the old dbname) plus an optional query/fragment we must keep. | |
| 319 | + | let auth_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); | |
| 320 | + | let authority = &rest[..auth_end]; | |
| 321 | + | let tail = &rest[auth_end..]; | |
| 322 | + | let query_and_frag = match tail.find(['?', '#']) { | |
| 323 | + | Some(i) => &tail[i..], | |
| 324 | + | None => "", | |
| 325 | + | }; | |
| 326 | + | format!( | |
| 327 | + | "{}{}/{}{}", | |
| 328 | + | &url[..after_scheme], | |
| 329 | + | authority, | |
| 330 | + | dbname, | |
| 331 | + | query_and_frag | |
| 332 | + | ) | |
| 333 | + | } | |
| 334 | + | ||
| 335 | + | /// Create the throwaway smoke DB on the cluster `maintenance_url` points at, | |
| 336 | + | /// dropping any stale one first. `dbname` is sanitized to `[a-z0-9_]` by | |
| 337 | + | /// `code_smoke_db_name`, so quoting it is sufficient. `CREATE DATABASE` cannot | |
| 338 | + | /// run inside a transaction, so these go through the simple-query protocol (a | |
| 339 | + | /// raw `&str` execute), matching `reset_scratch`. | |
| 340 | + | pub(super) async fn pg_create_db(maintenance_url: &str, dbname: &str) -> Result<()> { | |
| 341 | + | use sqlx::Executor; | |
| 342 | + | use sqlx::postgres::PgPoolOptions; | |
| 343 | + | let pool = PgPoolOptions::new() | |
| 344 | + | .max_connections(1) | |
| 345 | + | .connect(maintenance_url) | |
| 346 | + | .await?; | |
| 347 | + | pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!( | |
| 348 | + | "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)" | |
| 349 | + | )))) | |
| 350 | + | .await?; | |
| 351 | + | pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!( | |
| 352 | + | "CREATE DATABASE \"{dbname}\"" | |
| 353 | + | )))) | |
| 354 | + | .await?; | |
| 355 | + | pool.close().await; | |
| 356 | + | Ok(()) | |
| 357 | + | } | |
| 358 | + | ||
| 359 | + | /// Drop the throwaway smoke DB, forcing off any lingering connection (the killed | |
| 360 | + | /// server's pool). Best-effort at the call site — a failure is logged, not fatal. | |
| 361 | + | pub(super) async fn pg_drop_db(maintenance_url: &str, dbname: &str) -> Result<()> { | |
| 362 | + | use sqlx::Executor; | |
| 363 | + | use sqlx::postgres::PgPoolOptions; | |
| 364 | + | let pool = PgPoolOptions::new() | |
| 365 | + | .max_connections(1) | |
| 366 | + | .connect(maintenance_url) | |
| 367 | + | .await?; | |
| 368 | + | pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!( | |
| 369 | + | "DROP DATABASE IF EXISTS \"{dbname}\" WITH (FORCE)" | |
| 370 | + | )))) | |
| 371 | + | .await?; | |
| 372 | + | pool.close().await; | |
| 373 | + | Ok(()) | |
| 374 | + | } | |
| 375 | + | ||
| 376 | + | #[cfg(test)] | |
| 377 | + | mod tests { | |
| 378 | + | use super::*; | |
| 379 | + | ||
| 380 | + | /// reset_scratch must drop every non-system schema, not just `public` — | |
| 381 | + | /// otherwise migrations that create custom schemas (e.g. tower_sessions) | |
| 382 | + | /// collide on the next run. This regressed once (Phase 0) and the fix is | |
| 383 | + | /// load-bearing for migration_dry_run. | |
| 384 | + | /// | |
| 385 | + | /// Gated on `SANDO_TEST_PG_URL` so it only runs where postgres is | |
| 386 | + | /// available. Set `SANDO_TEST_PG_URL=postgres:///sando_scratch?host=/var/run/postgresql` | |
| 387 | + | /// (or similar) before `cargo test`. | |
| 388 | + | #[tokio::test] | |
| 389 | + | async fn reset_scratch_drops_all_non_system_schemas() { | |
| 390 | + | let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else { | |
| 391 | + | eprintln!("skipping: SANDO_TEST_PG_URL not set"); | |
| 392 | + | return; | |
| 393 | + | }; | |
| 394 | + | use sqlx::Executor; | |
| 395 | + | use sqlx::postgres::PgPoolOptions; | |
| 396 | + | ||
| 397 | + | let pool = PgPoolOptions::new() | |
| 398 | + | .max_connections(1) | |
| 399 | + | .connect(&url) | |
| 400 | + | .await | |
| 401 | + | .unwrap(); | |
| 402 | + | // Plant two non-system schemas + a table in each. | |
| 403 | + | pool.execute( | |
| 404 | + | "DROP SCHEMA IF EXISTS foo CASCADE; CREATE SCHEMA foo; CREATE TABLE foo.t (i int);", | |
| 405 | + | ) | |
| 406 | + | .await | |
| 407 | + | .unwrap(); | |
| 408 | + | pool.execute("DROP SCHEMA IF EXISTS tower_sessions CASCADE; CREATE SCHEMA tower_sessions; CREATE TABLE tower_sessions.session (id text);") | |
| 409 | + | .await.unwrap(); | |
| 410 | + | pool.close().await; | |
| 411 | + | ||
| 412 | + | reset_scratch(&url, "makenotwork") | |
| 413 | + | .await | |
| 414 | + | .expect("reset_scratch"); | |
| 415 | + | ||
| 416 | + | let pool = PgPoolOptions::new() | |
| 417 | + | .max_connections(1) | |
| 418 | + | .connect(&url) | |
| 419 | + | .await | |
| 420 | + | .unwrap(); | |
| 421 | + | let rows: Vec<(String,)> = sqlx::query_as( | |
| 422 | + | "SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname <> 'information_schema'", | |
| 423 | + | ) | |
| 424 | + | .fetch_all(&pool) | |
| 425 | + | .await | |
| 426 | + | .unwrap(); | |
| 427 | + | let names: Vec<String> = rows.into_iter().map(|(s,)| s).collect(); | |
| 428 | + | // After reset, only `public` should remain among non-system schemas. | |
| 429 | + | assert_eq!(names, vec!["public".to_string()], "got: {names:?}"); | |
| 430 | + | pool.close().await; | |
| 431 | + | } | |
| 432 | + | ||
| 433 | + | /// reset_scratch must leave the dump's owner role existing and able to | |
| 434 | + | /// create in `public`, because a prod `pg_dump` carries `ALTER ... OWNER TO | |
| 435 | + | /// <role>` for every object. This was satisfied by a hand-created NOLOGIN | |
| 436 | + | /// role on fw13; nothing recorded it, so any other box failed | |
| 437 | + | /// migration_dry_run at the restore with "role does not exist". | |
| 438 | + | /// | |
| 439 | + | /// Uses a throwaway role name so it can prove the *creation* path rather | |
| 440 | + | /// than passing on fw13's pre-existing `makenotwork`. Same | |
| 441 | + | /// `SANDO_TEST_PG_URL` gate as above; needs a superuser connection. | |
| 442 | + | #[tokio::test] | |
| 443 | + | async fn reset_scratch_seeds_the_dump_owner_role_when_absent() { | |
| 444 | + | let Ok(url) = std::env::var("SANDO_TEST_PG_URL") else { | |
| 445 | + | eprintln!("skipping: SANDO_TEST_PG_URL not set"); | |
| 446 | + | return; | |
| 447 | + | }; | |
| 448 | + | use sqlx::Executor; | |
| 449 | + | use sqlx::postgres::PgPoolOptions; | |
| 450 | + | ||
| 451 | + | let role = "sando_test_owner_probe"; | |
| 452 | + | // `DROP ROLE` refuses while the role still holds the grants reset_scratch | |
| 453 | + | // gave it, so drop what it owns first. Idempotent, and a no-op when the | |
| 454 | + | // role is absent (the usual case on a first run). | |
| 455 | + | let drop_role = format!( | |
| 456 | + | "DO $$ BEGIN | |
| 457 | + | IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{role}') THEN | |
| 458 | + | EXECUTE 'DROP OWNED BY {role}'; | |
| 459 | + | EXECUTE 'DROP ROLE {role}'; | |
| 460 | + | END IF; | |
| 461 | + | END $$;" | |
| 462 | + | ); | |
| 463 | + | ||
| 464 | + | let pool = PgPoolOptions::new() | |
| 465 | + | .max_connections(1) | |
| 466 | + | .connect(&url) | |
| 467 | + | .await | |
| 468 | + | .unwrap(); | |
| 469 | + | pool.execute(sqlx::raw_sql(sqlx::AssertSqlSafe(drop_role.clone()))) | |
| 470 | + | .await | |
| 471 | + | .unwrap(); | |
| 472 | + | pool.close().await; | |
| 473 | + | ||
| 474 | + | reset_scratch(&url, role) | |
| 475 | + | .await | |
| 476 | + | .expect("reset_scratch creates the owner role"); | |
| 477 | + | ||
| 478 | + | let pool = PgPoolOptions::new() | |
| 479 | + | .max_connections(1) | |
| 480 | + | .connect(&url) | |
| 481 | + | .await | |
| 482 | + | .unwrap(); | |
| 483 | + | let (exists, can_login): (bool, bool) = | |
| 484 | + | sqlx::query_as("SELECT true, rolcanlogin FROM pg_roles WHERE rolname = $1") | |
| 485 | + | .bind(role) | |
| 486 | + | .fetch_one(&pool) | |
| 487 | + | .await | |
| 488 | + | .expect("owner role exists after reset"); | |
| 489 | + | assert!(exists); | |
| 490 | + | assert!( | |
| 491 | + | !can_login, | |
| 492 | + | "the owner role is an owner only, never a login identity" | |
| 493 | + | ); | |
| 494 | + | ||
| 495 | + | // The restore's owner-scoped DDL needs CREATE on public in the role's | |
| 496 | + | // own right (PG15+ dropped the implicit grant). | |
| 497 | + | let (has_create,): (bool,) = | |
| 498 | + | sqlx::query_as("SELECT pg_catalog.has_schema_privilege($1, 'public', 'CREATE')") | |
| 499 | + | .bind(role) | |
| 500 | + | .fetch_one(&pool) |
Lines truncated
| @@ -1,0 +1,437 @@ | |||
| 1 | + | //! The gates that ask a running thing whether it is serving: the staged | |
| 2 | + | //! artifact on the build host, the deployed nodes, and the public pages through | |
| 3 | + | //! the CDN. | |
| 4 | + | ||
| 5 | + | use super::log::{append_to_log, gate_chunk_cb, stream_into_log}; | |
| 6 | + | use super::{GateCtx, NodeProbe}; | |
| 7 | + | use crate::classify; | |
| 8 | + | use crate::domain::{GateKind, GateRunId}; | |
| 9 | + | use crate::outcome::{GateBlocker, GateFailure, GateOutcome, PassNote}; | |
| 10 | + | use anyhow::Result; | |
| 11 | + | use ops_core::live_log::LiveLog; | |
| 12 | + | use ops_core::remote::LogSink; | |
| 13 | + | use ops_exec::{DiscardSink, sh_quote}; | |
| 14 | + | ||
| 15 | + | pub(super) async fn boot_smoke(ctx: &GateCtx, run_id: GateRunId) -> Result<GateOutcome> { | |
| 16 | + | let bin: Option<(String,)> = | |
| 17 | + | sqlx::query_as("SELECT artifact_path FROM versions WHERE app = ? AND version = ?") | |
| 18 | + | .bind(&ctx.cfg.id) | |
| 19 | + | .bind(&ctx.version) | |
| 20 | + | .fetch_optional(&ctx.pool) | |
| 21 | + | .await?; | |
| 22 | + | let Some((bin,)) = bin else { | |
| 23 | + | return Ok(GateOutcome::blocked(GateBlocker::ArtifactMissing { | |
| 24 | + | version: ctx.version.clone(), | |
| 25 | + | })); | |
| 26 | + | }; | |
| 27 | + | ||
| 28 | + | // Readiness smoke: start the binary and confirm it serves `GET /health` | |
| 29 | + | // within the window, not merely that the process stays up. Panics in main, | |
| 30 | + | // missing config, and port-bind failures still surface as an early exit; a | |
| 31 | + | // process that comes up but never serves /health is now its own failure. | |
| 32 | + | // | |
| 33 | + | // The server requires DATABASE_URL or it panics on config load before | |
| 34 | + | // we can observe anything. We point it at the scratch DB (already | |
| 35 | + | // migrated by the build step and refreshed by migration_dry_run if | |
| 36 | + | // that gate ran first). SCAN_ENABLED=false skips loading YARA rules | |
| 37 | + | // from /opt/makenotwork/yara-rules which doesn't exist on the build | |
| 38 | + | // host. SANDO_BOOT_SMOKE_PORT tells the smoke server which loopback port | |
| 39 | + | // to bind so we know where to probe. Other config has sane optional defaults. | |
| 40 | + | let mut cmd = tokio::process::Command::new(&bin); | |
| 41 | + | cmd.env("SANDO_BOOT_SMOKE", "1") | |
| 42 | + | .env("SANDO_BOOT_SMOKE_PORT", ctx.cfg.boot_smoke_port.to_string()) | |
| 43 | + | .env("SCAN_ENABLED", "false") | |
| 44 | + | .stdout(std::process::Stdio::piped()) | |
| 45 | + | .stderr(std::process::Stdio::piped()) | |
| 46 | + | .kill_on_drop(true); | |
| 47 | + | if let Some(scratch_url) = ctx.cfg.scratch_db_url.as_deref() { | |
| 48 | + | cmd.env("DATABASE_URL", scratch_url); | |
| 49 | + | } | |
| 50 | + | let log_path = ctx.log_path(GateKind::BootSmoke); | |
| 51 | + | let log_ref = ctx.log_ref(GateKind::BootSmoke); | |
| 52 | + | let mut child = match cmd.spawn() { | |
| 53 | + | Ok(c) => c, | |
| 54 | + | Err(e) => { | |
| 55 | + | // Spawn failures get a one-off log line via LiveLog so the | |
| 56 | + | // on-disk file still exists for `GET /logs/...`. | |
| 57 | + | let mut log = LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await; | |
| 58 | + | log.write_chunk(format!("spawn: {e}\n").as_bytes()).await; | |
| 59 | + | log.close().await; | |
| 60 | + | return Ok(GateOutcome::failed(GateFailure::SpawnFailed { | |
| 61 | + | message: e.to_string(), | |
| 62 | + | }) | |
| 63 | + | .with_log_ref(log_ref)); | |
| 64 | + | } | |
| 65 | + | }; | |
| 66 | + | ||
| 67 | + | // The boot smoke window is 3s. Drain stdout/stderr concurrently through | |
| 68 | + | // a shared LiveLog sink so the operator sees panics/log lines stream in | |
| 69 | + | // real time before the kill, AND the on-disk log gets the full byte | |
| 70 | + | // stream for post-mortem reads. The drainers exit when their pipe | |
| 71 | + | // closes — which happens when the child exits naturally or after kill. | |
| 72 | + | let log = std::sync::Arc::new(tokio::sync::Mutex::new( | |
| 73 | + | LiveLog::open(log_path, gate_chunk_cb(ctx.events.clone(), run_id)).await, | |
| 74 | + | )); | |
| 75 | + | let stdout_task = tokio::spawn(stream_into_log(child.stdout.take(), log.clone())); | |
| 76 | + | let stderr_task = tokio::spawn(stream_into_log(child.stderr.take(), log.clone())); | |
| 77 | + | ||
| 78 | + | // Poll readiness across the 3s window instead of a flat sleep: GET /health | |
| 79 | + | // must return 2xx. A crash mid-window short-circuits to the exit-code | |
| 80 | + | // failure path (try_wait below); a process that stays up but never serves | |
| 81 | + | // /health is a distinct readiness failure. | |
| 82 | + | let probe_timeout = std::time::Duration::from_millis(500); | |
| 83 | + | let started = std::time::Instant::now(); | |
| 84 | + | let window = std::time::Duration::from_secs(3); | |
| 85 | + | let mut probe_ok_after: Option<u32> = None; | |
| 86 | + | let mut last_probe_err = "never responded".to_string(); | |
| 87 | + | let mut early_exit = None; | |
| 88 | + | while started.elapsed() < window { | |
| 89 | + | if let Some(status) = child.try_wait()? { | |
| 90 | + | early_exit = Some(status); | |
| 91 | + | break; | |
| 92 | + | } | |
| 93 | + | match tokio::time::timeout(probe_timeout, probe_health(ctx.cfg.boot_smoke_port)).await { | |
| 94 | + | Ok(Ok(())) => { | |
| 95 | + | probe_ok_after = Some(started.elapsed().as_millis() as u32); | |
| 96 | + | break; | |
| 97 | + | } | |
| 98 | + | Ok(Err(e)) => last_probe_err = e, | |
| 99 | + | Err(_) => last_probe_err = "probe timed out".to_string(), | |
| 100 | + | } | |
| 101 | + | tokio::time::sleep(std::time::Duration::from_millis(150)).await; | |
| 102 | + | } | |
| 103 | + | ||
| 104 | + | // Stop the child unless it already exited, then drain the log tasks. | |
| 105 | + | let exit = match early_exit { | |
| 106 | + | Some(status) => Some(status), | |
| 107 | + | None => { | |
| 108 | + | let e = child.try_wait()?; | |
| 109 | + | if e.is_none() { | |
| 110 | + | let _ = child.kill().await; | |
| 111 | + | } | |
| 112 | + | e | |
| 113 | + | } | |
| 114 | + | }; | |
| 115 | + | // The streamed bytes already landed in the live log and the on-disk file for | |
| 116 | + | // the post-mortem reader. Drain the join handles to avoid hangs. | |
| 117 | + | let _ = stdout_task.await; | |
| 118 | + | let _ = stderr_task.await; | |
| 119 | + | // Unique owner of the Arc at this point (both tasks dropped their clones). | |
| 120 | + | if let Ok(mutex) = std::sync::Arc::try_unwrap(log) { | |
| 121 | + | mutex.into_inner().close().await; | |
| 122 | + | } | |
| 123 | + | ||
| 124 | + | match (exit, probe_ok_after) { | |
| 125 | + | // Exited on its own within the window — a crash/panic/bind failure. | |
| 126 | + | (Some(status), _) => { | |
| 127 | + | let failure = classify::classify_boot_smoke(status.code()); | |
| 128 | + | Ok(GateOutcome::failed(failure).with_log_ref(log_ref)) | |
| 129 | + | } | |
| 130 | + | // Stayed up and served /health — readiness proven. | |
| 131 | + | (None, Some(after_ms)) => { | |
| 132 | + | Ok(GateOutcome::passed(PassNote::HealthyProbe { after_ms }).with_log_ref(log_ref)) | |
| 133 | + | } | |
| 134 | + | // Stayed up but never served /health — started, not ready. | |
| 135 | + | (None, None) => Ok(GateOutcome::failed(GateFailure::BootHealthProbeFailed { | |
| 136 | + | last_error: last_probe_err, | |
| 137 | + | }) | |
| 138 | + | .with_log_ref(log_ref)), | |
| 139 | + | } | |
| 140 | + | } | |
| 141 | + | ||
| 142 | + | /// One readiness probe of the boot-smoke server: connect to `127.0.0.1:port` | |
| 143 | + | /// and `GET /health`, returning `Ok(())` only on a `200`. A hand-rolled HTTP/1.0 | |
| 144 | + | /// request over a raw `TcpStream` keeps the outbound probe dependency-free | |
| 145 | + | /// (reqwest is dev-only); the smoke server serves the one route over axum, which | |
| 146 | + | /// speaks 1.0. `Err` carries a short reason for the operator's failure note. The | |
| 147 | + | /// caller wraps each call in a timeout. | |
| 148 | + | pub(super) async fn probe_health(port: u16) -> std::result::Result<(), String> { | |
| 149 | + | use tokio::io::{AsyncReadExt, AsyncWriteExt}; | |
| 150 | + | let mut stream = tokio::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, port)) | |
| 151 | + | .await | |
| 152 | + | .map_err(|e| format!("connect: {e}"))?; | |
| 153 | + | stream | |
| 154 | + | .write_all(b"GET /health HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n") | |
| 155 | + | .await | |
| 156 | + | .map_err(|e| format!("write: {e}"))?; | |
| 157 | + | let mut buf = Vec::new(); | |
| 158 | + | stream | |
| 159 | + | .read_to_end(&mut buf) | |
| 160 | + | .await | |
| 161 | + | .map_err(|e| format!("read: {e}"))?; | |
| 162 | + | let text = String::from_utf8_lossy(&buf); | |
| 163 | + | let status_line = text.lines().next().unwrap_or(""); | |
| 164 | + | if status_line.contains(" 200 ") { | |
| 165 | + | Ok(()) | |
| 166 | + | } else { | |
| 167 | + | Err(format!("unexpected status line: {status_line:?}")) | |
| 168 | + | } | |
| 169 | + | } | |
| 170 | + | ||
| 171 | + | /// `node_health` — the post-deploy gate that proves the *deployed nodes* are | |
| 172 | + | /// serving, recording one outcome per (tier, version) that the next promote | |
| 173 | + | /// checks. Distinct from `boot_smoke`, which boots the staged artifact on the | |
| 174 | + | /// build host: this probes each node over the same executor the deploy used, so | |
| 175 | + | /// a node that took a corrupt artifact, wrong-arch binary, or failed restart is | |
| 176 | + | /// caught here rather than waved through. Fails closed: any | |
| 177 | + | /// unhealthy node fails the gate, and an empty node set is `Blocked`. | |
| 178 | + | /// Load the tier's public pages in a real browser and fail if the JavaScript | |
| 179 | + | /// did not run. | |
| 180 | + | /// | |
| 181 | + | /// The only gate here that crosses the CDN. `boot_smoke` runs on the build host | |
| 182 | + | /// and `node_health` reaches a node over its executor, so between them nothing | |
| 183 | + | /// requests the site the way a visitor does. A CDN holding a module from an | |
| 184 | + | /// earlier deploy can fail to link against the fresh one beside it, and since | |
| 185 | + | /// the bundle's entry point side-effect-imports every island, one bad link takes | |
| 186 | + | /// all of them down together while every artifact is individually correct: right | |
| 187 | + | /// markup, right stylesheets, every module answering 200 with current bytes. | |
| 188 | + | /// Composition is observable in a browser and nowhere else. | |
| 189 | + | /// | |
| 190 | + | /// Runs on the daemon host rather than on a node, because it is a *client*: it | |
| 191 | + | /// should reach the site through whatever the public reaches it through, and a | |
| 192 | + | /// probe that ran on the origin would inherit the blind spot this exists to | |
| 193 | + | /// close. | |
| 194 | + | pub(super) async fn page_smoke(ctx: &GateCtx) -> Result<GateOutcome> { | |
| 195 | + | let Some(cmd) = ctx.cfg.page_smoke_cmd.as_deref() else { | |
| 196 | + | // A service with no pages says so by configuring no command. Blocked | |
| 197 | + | // rather than passed: a gate that proves nothing must not read green. | |
| 198 | + | return Ok(GateOutcome::blocked(GateBlocker::NotConfigured { | |
| 199 | + | what: "page_smoke_cmd".into(), | |
| 200 | + | })); | |
| 201 | + | }; | |
| 202 | + | let Some(base) = ctx.public_url.as_deref() else { | |
| 203 | + | return Ok(GateOutcome::blocked(GateBlocker::NotConfigured { | |
| 204 | + | what: "public_url".into(), | |
| 205 | + | })); | |
| 206 | + | }; | |
| 207 | + | ||
| 208 | + | let ceiling = std::time::Duration::from_secs(ctx.cfg.gate_timeout_secs); | |
| 209 | + | let child = tokio::process::Command::new("sh") | |
| 210 | + | .arg("-c") | |
| 211 | + | .arg(cmd) | |
| 212 | + | .env("BASE", base) | |
| 213 | + | .stdout(std::process::Stdio::piped()) | |
| 214 | + | .stderr(std::process::Stdio::piped()) | |
| 215 | + | .kill_on_drop(true) | |
| 216 | + | .spawn()?; | |
| 217 | + | ||
| 218 | + | let out = match tokio::time::timeout(ceiling, child.wait_with_output()).await { | |
| 219 | + | Ok(res) => res?, | |
| 220 | + | Err(_elapsed) => { | |
| 221 | + | return Ok(GateOutcome::failed(GateFailure::Timeout { | |
| 222 | + | gate: GateKind::PageSmoke, | |
| 223 | + | after_s: ctx.cfg.gate_timeout_secs as u32, | |
| 224 | + | }) | |
| 225 | + | .with_log_ref(ctx.log_ref(GateKind::PageSmoke))); | |
| 226 | + | } | |
| 227 | + | }; | |
| 228 | + | ||
| 229 | + | let log = format!( | |
| 230 | + | "{}{}", | |
| 231 | + | String::from_utf8_lossy(&out.stdout), | |
| 232 | + | String::from_utf8_lossy(&out.stderr) | |
| 233 | + | ); | |
| 234 | + | append_to_log(&ctx.log_path(GateKind::PageSmoke), log.as_bytes()).await; | |
| 235 | + | ||
| 236 | + | if out.status.success() { | |
| 237 | + | return Ok( | |
| 238 | + | GateOutcome::passed(PassNote::PagesClean { base: base.into() }) | |
| 239 | + | .with_log_ref(ctx.log_ref(GateKind::PageSmoke)), | |
| 240 | + | ); | |
| 241 | + | } | |
| 242 | + | ||
| 243 | + | // The script prints one `FAIL <url>` line per bad page and indents the | |
| 244 | + | // reasons under it. Lift the first reason into the summary so a red gate | |
| 245 | + | // says what broke without anyone opening the log. | |
| 246 | + | let first = log | |
| 247 | + | .lines() | |
| 248 | + | .skip_while(|l| !l.starts_with("FAIL")) | |
| 249 | + | .nth(1) | |
| 250 | + | .map(str::trim) | |
| 251 | + | .filter(|l| !l.is_empty()) | |
| 252 | + | .unwrap_or("see log"); | |
| 253 | + | Ok(GateOutcome::failed(GateFailure::PagesBroken { | |
| 254 | + | base: base.into(), | |
| 255 | + | detail: first.to_string(), | |
| 256 | + | }) | |
| 257 | + | .with_log_ref(ctx.log_ref(GateKind::PageSmoke))) | |
| 258 | + | } | |
| 259 | + | ||
| 260 | + | pub(super) async fn node_health(ctx: &GateCtx) -> Result<GateOutcome> { | |
| 261 | + | if ctx.nodes.is_empty() { | |
| 262 | + | return Ok(GateOutcome::blocked(GateBlocker::NoNodesToProbe)); | |
| 263 | + | } | |
| 264 | + | for probe in &ctx.nodes { | |
| 265 | + | if let Err(detail) = probe_node(probe).await { | |
| 266 | + | return Ok(GateOutcome::failed(GateFailure::NodeUnhealthy { | |
| 267 | + | node: probe.node.to_string(), | |
| 268 | + | detail, | |
| 269 | + | })); | |
| 270 | + | } | |
| 271 | + | } | |
| 272 | + | Ok(GateOutcome::passed(PassNote::NodesHealthy { | |
| 273 | + | nodes: ctx.nodes.len() as u32, | |
| 274 | + | })) | |
| 275 | + | } | |
| 276 | + | ||
| 277 | + | /// Probe one node over its executor: confirm the unit is active post-restart | |
| 278 | + | /// and, when a `health_url` is configured, that it serves a 2xx. Retries across | |
| 279 | + | /// ~10s because the service may still be restarting / warming. Runs under the | |
| 280 | + | /// read-only `Observe(Health)` capability (every Sando node grants it), so the | |
| 281 | + | /// probe needs no deploy authority. `Ok(())` = healthy; `Err(detail)` carries a | |
| 282 | + | /// short reason for the gate's failure note. | |
| 283 | + | async fn probe_node(probe: &NodeProbe) -> std::result::Result<(), String> { | |
| 284 | + | use ops_exec::{Action, ObserveKind, Step}; | |
| 285 | + | let svc = sh_quote(&probe.service); | |
| 286 | + | let url = probe | |
| 287 | + | .health_url | |
| 288 | + | .as_deref() | |
| 289 | + | .map_or_else(|| "''".to_string(), sh_quote); | |
| 290 | + | // One executor round-trip with the retry loop on the node: is-active, then | |
| 291 | + | // (if a url is set) curl it for a 2xx. Exit 0 only when both hold. | |
| 292 | + | let script = format!( | |
| 293 | + | "svc={svc}; url={url}; \ | |
| 294 | + | for _ in $(seq 1 10); do \ | |
| 295 | + | if systemctl is-active --quiet \"$svc\"; then \ | |
| 296 | + | if [ -z \"$url\" ] || curl -fsS --max-time 5 \"$url\" >/dev/null 2>&1; then exit 0; fi; \ | |
| 297 | + | fi; \ | |
| 298 | + | sleep 1; \ | |
| 299 | + | done; \ | |
| 300 | + | echo 'service not active or health url not 2xx after retries' >&2; exit 1" | |
| 301 | + | ); | |
| 302 | + | let step = Step::shell(Action::Observe(ObserveKind::Health), script); | |
| 303 | + | let mut sink = DiscardSink; | |
| 304 | + | let out = probe | |
| 305 | + | .executor | |
| 306 | + | .run_streaming(&step, &mut sink) | |
| 307 | + | .await | |
| 308 | + | .map_err(|e| format!("probe spawn: {e}"))?; | |
| 309 | + | if out.status.success() { | |
| 310 | + | Ok(()) | |
| 311 | + | } else { | |
| 312 | + | let code = out | |
| 313 | + | .status | |
| 314 | + | .code() | |
| 315 | + | .map_or_else(|| "signal".to_string(), |c| c.to_string()); | |
| 316 | + | let stderr: String = String::from_utf8_lossy(&out.stderr) | |
| 317 | + | .chars() | |
| 318 | + | .take(200) | |
| 319 | + | .collect(); | |
| 320 | + | Err(format!("exit {code}: {stderr}")) | |
| 321 | + | } | |
| 322 | + | } | |
| 323 | + | ||
| 324 | + | #[cfg(test)] | |
| 325 | + | mod tests { | |
| 326 | + | use super::super::run; | |
| 327 | + | use super::*; | |
| 328 | + | use crate::domain::TierId; | |
| 329 | + | use crate::events; | |
| 330 | + | use crate::topology::Gate; | |
| 331 | + | use sqlx::sqlite::SqlitePoolOptions; | |
| 332 | + | use std::collections::HashMap; | |
| 333 | + | ||
| 334 | + | /// Spawn a one-shot loopback server that answers the first connection with | |
| 335 | + | /// `status_line` + a tiny body, then closes. Returns the bound port. | |
| 336 | + | async fn oneshot_http(status_line: &'static str) -> u16 { | |
| 337 | + | use tokio::io::{AsyncReadExt, AsyncWriteExt}; | |
| 338 | + | let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) | |
| 339 | + | .await | |
| 340 | + | .unwrap(); | |
| 341 | + | let port = listener.local_addr().unwrap().port(); | |
| 342 | + | tokio::spawn(async move { | |
| 343 | + | if let Ok((mut sock, _)) = listener.accept().await { | |
| 344 | + | let mut scratch = [0u8; 1024]; | |
| 345 | + | let _ = sock.read(&mut scratch).await; // drain the request line | |
| 346 | + | let resp = | |
| 347 | + | format!("{status_line}\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"); | |
| 348 | + | let _ = sock.write_all(resp.as_bytes()).await; | |
| 349 | + | } | |
| 350 | + | }); | |
| 351 | + | port | |
| 352 | + | } | |
| 353 | + | ||
| 354 | + | #[tokio::test] | |
| 355 | + | async fn probe_health_ok_on_200() { | |
| 356 | + | let port = oneshot_http("HTTP/1.1 200 OK").await; | |
| 357 | + | assert!(probe_health(port).await.is_ok()); | |
| 358 | + | } | |
| 359 | + | ||
| 360 | + | #[tokio::test] | |
| 361 | + | async fn probe_health_err_on_non_200() { | |
| 362 | + | let port = oneshot_http("HTTP/1.1 503 Service Unavailable").await; | |
| 363 | + | let err = probe_health(port).await.unwrap_err(); | |
| 364 | + | assert!(err.contains("status line"), "{err}"); | |
| 365 | + | } | |
| 366 | + | ||
| 367 | + | #[tokio::test] | |
| 368 | + | async fn probe_health_err_on_connection_refused() { | |
| 369 | + | // Bind then drop to get an almost-certainly-free port nothing listens on. | |
| 370 | + | let port = { | |
| 371 | + | let l = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) | |
| 372 | + | .await | |
| 373 | + | .unwrap(); | |
| 374 | + | l.local_addr().unwrap().port() | |
| 375 | + | }; | |
| 376 | + | let err = probe_health(port).await.unwrap_err(); | |
| 377 | + | // Under load the kernel does not always refuse the connect: a socket | |
| 378 | + | // left in TIME_WAIT on that port completes the handshake and then | |
| 379 | + | // resets, so the failure surfaces on the write or the read instead. | |
| 380 | + | // Refused-on-connect and reset-on-write/read are the same fact, that | |
| 381 | + | // nothing is serving the port, and no other outcome counts as a pass. | |
| 382 | + | let refused = err.starts_with("connect: ") && err.contains("refused"); | |
| 383 | + | let reset = | |
| 384 | + | (err.starts_with("write: ") || err.starts_with("read: ")) && err.contains("reset"); | |
| 385 | + | assert!(refused || reset, "{err}"); | |
| 386 | + | } | |
| 387 | + | ||
| 388 | + | /// node_health fails closed when there are no nodes to probe: a serving tier | |
| 389 | + | /// should always carry nodes, so an empty set is a misconfiguration that must | |
| 390 | + | /// block promotion, not pass it. | |
| 391 | + | #[tokio::test] | |
| 392 | + | async fn node_health_blocks_with_no_nodes() { | |
| 393 | + | let pool = SqlitePoolOptions::new() | |
| 394 | + | .max_connections(1) | |
| 395 | + | .connect("sqlite::memory:") | |
| 396 | + | .await | |
| 397 | + | .unwrap(); | |
| 398 | + | crate::db::migrate(&pool).await.unwrap(); | |
| 399 | + | sqlx::query( | |
| 400 | + | "INSERT INTO tiers (name, ord, provisioned, canary) VALUES ('b', 2, 1, 'sequential')", | |
| 401 | + | ) | |
| 402 | + | .execute(&pool) | |
| 403 | + | .await | |
| 404 | + | .unwrap(); | |
| 405 | + | sqlx::query("INSERT INTO tier_state (tier) VALUES ('b')") | |
| 406 | + | .execute(&pool) | |
| 407 | + | .await | |
| 408 | + | .unwrap(); | |
| 409 | + | sqlx::query("INSERT INTO versions (version, git_sha, built_at, artifact_path) VALUES ('0.1.0', 'abc1234', '2026-01-01T00:00:00Z', '/tmp/x')") | |
| 410 | + | .execute(&pool).await.unwrap(); | |
| 411 | + | ||
| 412 | + | let cfg = std::sync::Arc::new(crate::config::AppConfig::for_tests()); | |
| 413 | + | let ctx = GateCtx { | |
| 414 | + | public_url: None, | |
| 415 | + | pool: pool.clone(), | |
| 416 | + | cfg, | |
| 417 | + | tier: TierId::new("b"), | |
| 418 | + | version: "0.1.0".parse().unwrap(), | |
| 419 | + | worktree: None, | |
| 420 | + | bundle: None, | |
| 421 | + | events: events::channel(), | |
| 422 | + | nodes: Vec::new(), // no nodes -> fail closed | |
| 423 | + | build_id: None, | |
| 424 | + | aux_dirs: HashMap::new(), | |
| 425 | + | }; | |
| 426 | + | let out = run(&ctx, &Gate::NodeHealth).await.unwrap(); | |
| 427 | + | assert_eq!(out.status_str(), "blocked"); | |
| 428 | + | assert!(!out.is_passed()); | |
| 429 | + | let row: (Option<String>, Option<String>) = | |
| 430 | + | sqlx::query_as("SELECT status, outcome_json FROM gate_runs ORDER BY id DESC LIMIT 1") | |
| 431 | + | .fetch_one(&pool) | |
| 432 | + | .await | |
| 433 | + | .unwrap(); | |
| 434 | + | let json: serde_json::Value = serde_json::from_str(row.1.as_deref().unwrap()).unwrap(); | |
| 435 | + | assert_eq!(json["status"]["blocker"]["kind"], "no_nodes_to_probe"); | |
| 436 | + | } | |
| 437 | + | } |
| @@ -1,0 +1,181 @@ | |||
| 1 | + | //! Fixtures shared by the gate modules' test suites. | |
| 2 | + | //! | |
| 3 | + | //! The three `*_ctx` builders each construct a full [`GateCtx`], and each is | |
| 4 | + | //! used from more than one sibling suite, which is what makes this a shared | |
| 5 | + | //! module rather than three private copies. | |
| 6 | + | ||
| 7 | + | use super::GateCtx; | |
| 8 | + | use super::log::GateLog; | |
| 9 | + | use crate::domain::{GateKind, GateRunId, TierId}; | |
| 10 | + | use crate::events; | |
| 11 | + | use chrono::Utc; | |
| 12 | + | use sqlx::SqlitePool; | |
| 13 | + | use sqlx::sqlite::SqlitePoolOptions; | |
| 14 | + | use std::collections::HashMap; | |
| 15 | + | use std::path::PathBuf; | |
| 16 | + | ||
| 17 | + | pub(super) fn target(dir: &str) -> crate::config::TestTarget { | |
| 18 | + | crate::config::TestTarget { | |
| 19 | + | dir: std::path::PathBuf::from(dir), | |
| 20 | + | aux_repo: None, | |
| 21 | + | features: Vec::new(), | |
| 22 | + | all_features: false, | |
| 23 | + | scratch_db: false, | |
| 24 | + | } | |
| 25 | + | } | |
| 26 | + | ||
| 27 | + | /// `target()` above, but resolved against an aux repo's checkout. | |
| 28 | + | pub(super) fn aux_target(dir: &str, repo: &str) -> crate::config::TestTarget { | |
| 29 | + | crate::config::TestTarget { | |
| 30 | + | aux_repo: Some(repo.to_string()), | |
| 31 | + | ..target(dir) | |
| 32 | + | } | |
| 33 | + | } | |
| 34 | + | ||
| 35 | + | /// True when the URL's host parses as a domain rather than an IP literal, | |
| 36 | + | /// which is the distinction `Url::domain()` draws and WebAuthn depends on. | |
| 37 | + | pub(super) fn url_host_is_a_domain(url: &str) -> bool { | |
| 38 | + | let after = url.split("://").nth(1).unwrap_or(""); | |
| 39 | + | let host = after.split(['/', '?', '#']).next().unwrap_or(""); | |
| 40 | + | let host = host.rsplit('@').next().unwrap_or(host); | |
| 41 | + | let host = if let Some(rest) = host.strip_prefix('[') { | |
| 42 | + | rest.split(']').next().unwrap_or("") | |
| 43 | + | } else { | |
| 44 | + | host.split(':').next().unwrap_or("") | |
| 45 | + | }; | |
| 46 | + | !host.is_empty() && host.parse::<std::net::IpAddr>().is_err() | |
| 47 | + | } | |
| 48 | + | ||
| 49 | + | pub(super) fn resolving_ctx(worktree: &str, aux: &[(&str, &str)]) -> GateCtx { | |
| 50 | + | GateCtx { | |
| 51 | + | public_url: None, | |
| 52 | + | pool: SqlitePool::connect_lazy("sqlite::memory:").unwrap(), | |
| 53 | + | cfg: std::sync::Arc::new(crate::config::AppConfig::for_tests()), | |
| 54 | + | tier: TierId::new("host"), | |
| 55 | + | version: "0.1.0".parse().unwrap(), | |
| 56 | + | worktree: Some(PathBuf::from(worktree)), | |
| 57 | + | bundle: None, | |
| 58 | + | events: events::channel(), | |
| 59 | + | nodes: Vec::new(), | |
| 60 | + | build_id: None, | |
| 61 | + | aux_dirs: aux | |
| 62 | + | .iter() | |
| 63 | + | .map(|(n, d)| ((*n).to_string(), PathBuf::from(d))) | |
| 64 | + | .collect(), | |
| 65 | + | } | |
| 66 | + | } | |
| 67 | + | ||
| 68 | + | /// A `GateCtx` over `worktree` with the given frontend projects configured. | |
| 69 | + | /// No DB, no artifact — `code_smoke_frontends` touches neither. | |
| 70 | + | pub(super) async fn frontend_ctx(worktree: &std::path::Path, dirs: &[&str]) -> GateCtx { | |
| 71 | + | let mut cfg = crate::config::AppConfig::for_tests(); | |
| 72 | + | cfg.frontend_builds = dirs | |
| 73 | + | .iter() | |
| 74 | + | .map(|d| crate::config::FrontendBuild { | |
| 75 | + | dir: PathBuf::from(d), | |
| 76 | + | script: "build".into(), | |
| 77 | + | }) | |
| 78 | + | .collect(); | |
| 79 | + | cfg.logs_root = worktree.join("logs"); | |
| 80 | + | GateCtx { | |
| 81 | + | public_url: None, | |
| 82 | + | pool: SqlitePoolOptions::new() | |
| 83 | + | .max_connections(1) | |
| 84 | + | .connect("sqlite::memory:") | |
| 85 | + | .await | |
| 86 | + | .unwrap(), | |
| 87 | + | cfg: std::sync::Arc::new(cfg), | |
| 88 | + | tier: TierId::new("host"), | |
| 89 | + | version: "0.1.0".parse().unwrap(), | |
| 90 | + | worktree: Some(worktree.to_path_buf()), | |
| 91 | + | bundle: None, | |
| 92 | + | events: events::channel(), | |
| 93 | + | nodes: Vec::new(), | |
| 94 | + | build_id: None, | |
| 95 | + | aux_dirs: HashMap::new(), | |
| 96 | + | } | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | /// A `GateCtx` for the `migration_dry_run` freshness checks: a migrated | |
| 100 | + | /// in-memory pool (so `backups` exists) and a scratch URL set, so the gate | |
| 101 | + | /// reaches the backup lookup instead of bailing on config. Nothing here | |
| 102 | + | /// touches postgres — every assertion below blocks before `reset_scratch`. | |
| 103 | + | pub(super) async fn dry_run_ctx(worktree: &std::path::Path, max_age_hours: u32) -> GateCtx { | |
| 104 | + | let mut cfg = crate::config::AppConfig::for_tests(); | |
| 105 | + | cfg.scratch_db_url = Some("postgres:///sando_scratch".into()); | |
| 106 | + | cfg.backup_max_age_hours = max_age_hours; | |
| 107 | + | cfg.logs_root = worktree.join("logs"); | |
| 108 | + | let pool = SqlitePoolOptions::new() | |
| 109 | + | .max_connections(1) | |
| 110 | + | .connect("sqlite::memory:") | |
| 111 | + | .await | |
| 112 | + | .unwrap(); | |
| 113 | + | crate::db::migrate(&pool).await.unwrap(); | |
| 114 | + | GateCtx { | |
| 115 | + | public_url: None, | |
| 116 | + | pool, | |
| 117 | + | cfg: std::sync::Arc::new(cfg), | |
| 118 | + | tier: TierId::new("host"), | |
| 119 | + | version: "0.1.0".parse().unwrap(), | |
| 120 | + | worktree: Some(worktree.to_path_buf()), | |
| 121 | + | bundle: None, | |
| 122 | + | events: events::channel(), | |
| 123 | + | nodes: Vec::new(), | |
| 124 | + | build_id: None, | |
| 125 | + | aux_dirs: HashMap::new(), | |
| 126 | + | } | |
| 127 | + | } | |
| 128 | + | ||
| 129 | + | /// Record a `server` backup row fetched `hours_ago`, as `/backup/fetch` would. | |
| 130 | + | pub(super) async fn seed_backup(ctx: &GateCtx, hours_ago: i64) { | |
| 131 | + | seed_named_backup(ctx, "server", hours_ago).await; | |
| 132 | + | } | |
| 133 | + | ||
| 134 | + | /// Record a backup row for one named dump. | |
| 135 | + | pub(super) async fn seed_named_backup(ctx: &GateCtx, name: &str, hours_ago: i64) { | |
| 136 | + | let at = (Utc::now() - chrono::Duration::hours(hours_ago)).to_rfc3339(); | |
| 137 | + | sqlx::query( | |
| 138 | + | "INSERT INTO backups (name, fetched_at, source, local_path, byte_size) | |
| 139 | + | VALUES (?, ?, 'file:///x.sql.gz', '/tmp/sando-test-backup.sql.gz', 1000000)", | |
| 140 | + | ) | |
| 141 | + | .bind(name) | |
| 142 | + | .bind(at) | |
| 143 | + | .execute(&ctx.pool) | |
| 144 | + | .await | |
| 145 | + | .unwrap(); | |
| 146 | + | } | |
| 147 | + | ||
| 148 | + | /// The multithreaded check, as `sando-daemon.toml` configures it. | |
| 149 | + | pub(super) fn mt_check() -> crate::config::MigrationCheck { | |
| 150 | + | crate::config::MigrationCheck { | |
| 151 | + | dir: std::path::PathBuf::from("multithreaded/migrations"), | |
| 152 | + | backup: "multithreaded".into(), | |
| 153 | + | scratch_db: Some("sando_scratch_mt".into()), | |
| 154 | + | owner_role: Some("multithreaded".into()), | |
| 155 | + | } | |
| 156 | + | } | |
| 157 | + | ||
| 158 | + | /// Re-point a `dry_run_ctx` at one check, keeping its pool and scratch URL. | |
| 159 | + | pub(super) fn with_check(ctx: &mut GateCtx, check: crate::config::MigrationCheck) { | |
| 160 | + | let mut cfg = crate::config::AppConfig::for_tests(); | |
| 161 | + | cfg.scratch_db_url = ctx.cfg.scratch_db_url.clone(); | |
| 162 | + | cfg.backup_max_age_hours = ctx.cfg.backup_max_age_hours; | |
| 163 | + | cfg.logs_root = ctx.cfg.logs_root.clone(); | |
| 164 | + | cfg.migration_checks = vec![check]; | |
| 165 | + | ctx.cfg = std::sync::Arc::new(cfg); | |
| 166 | + | } | |
| 167 | + | ||
| 168 | + | /// A `code_smoke` live log over `ctx.cfg.logs_root`, for the helpers that | |
| 169 | + | /// take one. `GateRunId(0)` never matches a real row; nothing reads the | |
| 170 | + | /// chunk events in these tests. | |
| 171 | + | pub(super) async fn test_gate_log(ctx: &GateCtx) -> GateLog { | |
| 172 | + | GateLog::open(ctx, GateRunId(0), GateKind::CodeSmoke).await | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | /// Close `log` (flushing it) and read back what it wrote on disk. | |
| 176 | + | pub(super) async fn read_gate_log(ctx: &GateCtx, log: GateLog) -> String { | |
| 177 | + | log.close().await; | |
| 178 | + | tokio::fs::read_to_string(ctx.log_path(GateKind::CodeSmoke)) | |
| 179 | + | .await | |
| 180 | + | .expect("the gate log must exist on disk") | |
| 181 | + | } |