Skip to main content

max / makenotwork

22.6 KB · 535 lines History Blame Raw
1 //! Typed gate outcomes.
2 //!
3 //! Replaces the `(passed: bool, detail: Option<String>)` pair on
4 //! `GateOutcome`. The point is to push failure classification into the
5 //! type itself: a `GateFailure::MigrationDrift { migration }` is what it
6 //! says, not a string the operator has to parse. See
7 //! `plans/observability.md` for the full argument.
8 //!
9 //! The variants here describe what the gate runner actually observed.
10 //! Mapping raw process output (stderr tails, exit codes) to these
11 //! variants is the classifier's job — `classify.rs`.
12
13 use crate::domain::{GateKind, Version};
14 use chrono::{DateTime, Utc};
15 use serde::{Deserialize, Serialize};
16
17 /// A gate's result, persisted to `gate_runs.outcome_json` and emitted
18 /// over WS in `GateDone`.
19 #[derive(Debug, Clone, Serialize, Deserialize)]
20 pub struct GateOutcome {
21 pub status: GateStatus,
22 /// Relative path under `cfg.logs_root` to the persisted stdout/stderr
23 /// for this run. `None` for gates that don't produce process output
24 /// (burn_in, manual_confirm).
25 #[serde(skip_serializing_if = "Option::is_none", default)]
26 pub log_ref: Option<LogRef>,
27 }
28
29 impl GateOutcome {
30 pub fn passed(note: PassNote) -> Self {
31 Self {
32 status: GateStatus::Passed { note },
33 log_ref: None,
34 }
35 }
36 pub fn failed(failure: GateFailure) -> Self {
37 Self {
38 status: GateStatus::Failed { failure },
39 log_ref: None,
40 }
41 }
42 pub fn blocked(blocker: GateBlocker) -> Self {
43 Self {
44 status: GateStatus::Blocked { blocker },
45 log_ref: None,
46 }
47 }
48 #[must_use]
49 pub fn with_log_ref(mut self, log_ref: LogRef) -> Self {
50 self.log_ref = Some(log_ref);
51 self
52 }
53
54 /// True iff the gate ran and succeeded. `Blocked` is not passing:
55 /// the gate has not satisfied the pipeline, the operator just owes
56 /// it a precondition.
57 pub fn is_passed(&self) -> bool {
58 matches!(self.status, GateStatus::Passed { .. })
59 }
60
61 /// The high-level status word for the `gate_runs.status` column.
62 pub fn status_str(&self) -> &'static str {
63 match self.status {
64 GateStatus::Passed { .. } => "passed",
65 GateStatus::Failed { .. } => "failed",
66 GateStatus::Blocked { .. } => "blocked",
67 }
68 }
69 }
70
71 #[derive(Debug, Clone, Serialize, Deserialize)]
72 #[serde(tag = "kind", rename_all = "snake_case")]
73 pub enum GateStatus {
74 /// Gate ran and succeeded. The note carries gate-specific evidence
75 /// (e.g. `TestsPassed { duration_s }`).
76 Passed { note: PassNote },
77 /// Gate ran and failed. Two-layer tag: outer `kind = "failed"`, inner
78 /// `failure.kind` names the classified variant. If no classifier
79 /// matched, that's `unclassified`.
80 Failed { failure: GateFailure },
81 /// Gate cannot run yet. Burn-in clock not started, scratch DB not
82 /// configured, backup missing — pre-conditions the operator can fix
83 /// out of band. Distinguished from `Failed` so the TUI can render
84 /// these yellow rather than red.
85 Blocked { blocker: GateBlocker },
86 }
87
88 #[derive(Debug, Clone, Serialize, Deserialize)]
89 #[serde(tag = "kind", rename_all = "snake_case")]
90 pub enum PassNote {
91 /// `boot_smoke` / `code_smoke` — the binary came up and served `GET /health`
92 /// (readiness, not just liveness). `after_ms` is how long until the first
93 /// good probe. For `code_smoke` this is a probe of the real server booted
94 /// against a freshly migrated + seeded throwaway DB; for `boot_smoke` it is
95 /// the minimal no-DB smoke server.
96 HealthyProbe { after_ms: u32 },
97 /// `burn_in` — the configured number of hours have elapsed since
98 /// the gate's clock started.
99 BurnInElapsed { hours: u32 },
100 /// `migration_dry_run` — scratch DB restored from `backup_path` and
101 /// every migration ran without error.
102 Migrated { backup_path: String },
103 /// `cargo_test` — `cargo test --release` exited 0.
104 TestsPassed { duration_s: u32 },
105 /// `manual_confirm` — an operator inserted a passing row out-of-band.
106 OperatorConfirmed { at: DateTime<Utc> },
107 /// `node_health` — every deployed node's service was active and (where a
108 /// `health_url` is configured) served its readiness probe. `nodes` is how
109 /// many nodes were verified.
110 NodesHealthy { nodes: u32 },
111 /// Legacy rows backfilled from the pre-typed schema. Carries the
112 /// original `detail` string so nothing is lost.
113 Legacy { text: String },
114 }
115
116 impl PassNote {
117 pub fn summary(&self) -> String {
118 match self {
119 PassNote::HealthyProbe { after_ms } => format!("served /health in {after_ms}ms"),
120 PassNote::BurnInElapsed { hours } => format!("{hours} hours elapsed"),
121 PassNote::Migrated { backup_path } => format!("restored {backup_path} + migrated"),
122 PassNote::TestsPassed { duration_s } => format!("tests passed in {duration_s}s"),
123 PassNote::OperatorConfirmed { at } => format!("operator confirmed at {at}"),
124 PassNote::NodesHealthy { nodes } => format!("{nodes} node(s) healthy"),
125 PassNote::Legacy { text } => text.clone(),
126 }
127 }
128 }
129
130 #[derive(Debug, Clone, Serialize, Deserialize)]
131 #[serde(tag = "kind", rename_all = "snake_case")]
132 pub enum GateBlocker {
133 /// `burn_in`: the tier's `tier_state.burn_in_started_at` is NULL.
134 BurnInClockNotStarted,
135 /// `burn_in`: clock running but not enough time elapsed yet.
136 BurnInRemaining {
137 hours_remaining: u32,
138 hours_total: u32,
139 },
140 /// `manual_confirm`: no out-of-band passing row exists for this
141 /// (tier, version).
142 AwaitingOperatorConfirmation,
143 /// `migration_dry_run`: no row in `backups` to restore from.
144 NoBackupAvailable,
145 /// `migration_dry_run`: the newest `backups` row is older than
146 /// `cfg.backup_max_age_hours`. Restoring it would dry-run the migrations
147 /// against a schema prod has since moved past, which passes green while
148 /// proving nothing — so the gate blocks instead.
149 BackupStale { age_hours: i64, max_age_hours: u32 },
150 /// `migration_dry_run` / `boot_smoke` / `cargo_test`: daemon config
151 /// has no `scratch_db_url`.
152 ScratchDbUrlUnset,
153 /// `boot_smoke`: no `artifact_path` in `versions` for this version.
154 ArtifactMissing { version: Version },
155 /// `node_health`: the gate ran with no nodes to probe (a serving tier should
156 /// always have nodes; this fails closed so a misconfigured tier can't pass).
157 NoNodesToProbe,
158 }
159
160 impl GateBlocker {
161 pub fn summary(&self) -> String {
162 match self {
163 GateBlocker::BurnInClockNotStarted => "burn-in clock not started".into(),
164 GateBlocker::BurnInRemaining {
165 hours_remaining,
166 hours_total,
167 } => format!("{hours_remaining} hours remaining of {hours_total}"),
168 GateBlocker::AwaitingOperatorConfirmation => "waiting on operator confirmation".into(),
169 GateBlocker::NoBackupAvailable => "no backup fetched; call /backup/fetch first".into(),
170 GateBlocker::BackupStale {
171 age_hours,
172 max_age_hours,
173 } => format!("backup is {age_hours}h old (max {max_age_hours}h); re-run /backup/fetch"),
174 GateBlocker::ScratchDbUrlUnset => "scratch_db_url unset in daemon config".into(),
175 GateBlocker::ArtifactMissing { version } => {
176 format!("no artifact for version {version}")
177 }
178 GateBlocker::NoNodesToProbe => "node_health has no nodes to probe".into(),
179 }
180 }
181 }
182
183 #[derive(Debug, Clone, Serialize, Deserialize)]
184 #[serde(tag = "kind", rename_all = "snake_case")]
185 pub enum GateFailure {
186 /// `cargo_test` exited non-zero. `failed_count` may be 0 if the
187 /// classifier couldn't parse the count (e.g. compile error).
188 /// `first_failed` is the first failing test's name; `first_panic` is the
189 /// first panic *message* (root cause), chosen to skip the "Once instance
190 /// has previously been poisoned" cascade so 800 poisoned tests don't bury
191 /// the one real panic that poisoned them.
192 CargoTest {
193 failed_count: u32,
194 first_failed: Option<String>,
195 #[serde(default, skip_serializing_if = "Option::is_none")]
196 first_panic: Option<String>,
197 },
198 /// `cargo_test` fast pre-gate (`cargo test --no-run`): the test
199 /// targets failed to compile, so no tests ran. `first_error` is the
200 /// headline diagnostic (e.g. `error[E0063]: missing field
201 /// user_pages_host`) and `error_count` is cargo's "N previous errors".
202 /// Distinct from `CargoTest` so a test-only-target compile break reads
203 /// as a build error, not "0 tests failed".
204 CompileError {
205 error_count: u32,
206 first_error: Option<String>,
207 },
208 /// `migration_dry_run`: a migration that was previously applied is
209 /// no longer present in the resolved migrations directory.
210 MigrationDrift { migration: String },
211 /// `migration_dry_run`: a migration that was previously applied has
212 /// been modified (checksum mismatch).
213 MigrationModified { migration: String },
214 /// `migration_dry_run`: postgres rejected a migration's SQL.
215 MigrationSqlError {
216 migration: String,
217 sqlstate: Option<String>,
218 },
219 /// `migration_dry_run`: scratch DB reset or dump restore failed.
220 RestoreFailed { reason: String },
221 /// `boot_smoke`: binary exited with a non-zero status during the
222 /// smoke window. Most likely a panic; `exit_code` carries the OS
223 /// status when one is available.
224 BootPanic { exit_code: Option<i32> },
225 /// `boot_smoke`: binary exited 0 before the smoke window elapsed.
226 BootExitedEarly { exit_code: Option<i32> },
227 /// `boot_smoke`: binary stayed up but never served `GET /health` 200 within
228 /// the smoke window — started but not ready. `last_error` is the final probe
229 /// error (connection refused, non-200, timeout).
230 BootHealthProbeFailed { last_error: String },
231 /// `node_health`: a deployed node's service was not active, or did not serve
232 /// its `health_url`, within the probe window. `node` is the failing node;
233 /// `detail` carries the probe's stderr/reason.
234 NodeUnhealthy { node: String, detail: String },
235 /// `code_smoke`: creating or dropping the throwaway smoke DB failed (a
236 /// postgres/cluster problem, not the built code). `reason` carries the DB
237 /// error. Kept distinct from the boot/seed failures so an environment issue
238 /// here isn't misread as unsound code.
239 CodeSmokeSetup { reason: String },
240 /// `code_smoke`: the `--seed-examples` run (which migrates the empty DB from
241 /// scratch, then seeds the example catalog, then exits) returned non-zero —
242 /// a broken migration, a seed error, or a config-load failure. The persisted
243 /// gate log carries the process output; `exit_code` is the OS status.
244 CodeSmokeSeed { exit_code: Option<i32> },
245 /// `code_smoke`: the DB-free `MNW_CHECK_DOCS` run found broken internal docs
246 /// links (an internal `[..](x.md)` resolving to a slug no page serves — a
247 /// live 404 waiting in the public docs). Runs first, before the throwaway
248 /// DB is even created, so a rotted link fails cheaply. `broken` is the
249 /// reported count (0 if the sentinel could not be parsed); the gate log
250 /// carries the offending source→target lines.
251 CodeSmokeDocs { broken: u32 },
252 /// `code_smoke`: the booted server answered `GET /health` but never emitted
253 /// its `listening` startup log. The gate asserts both signals — the bind
254 /// readiness log and the health probe — so a missing log line fails even
255 /// when the probe passes.
256 CodeSmokeNoListeningLog,
257 /// `code_smoke`: a configured `frontend_build` failed to compile. `dir` is
258 /// the npm project under the worktree; `exit_code` is npm's status. This is
259 /// the failure both MNW build scripts downgrade to a `cargo::warning` on
260 /// purpose (so a broken chat widget cannot stop the forum compiling) — which
261 /// means this gate is the only place it is ever fatal, and the only thing
262 /// standing between a `tsc` error and a deploy that rsyncs the previous
263 /// build's `static/dist/`.
264 CodeSmokeFrontend { dir: String, exit_code: Option<i32> },
265 /// `cargo_test` / `boot_smoke`: tokio could not spawn the child.
266 SpawnFailed { message: String },
267 /// Gate took longer than the configured ceiling.
268 Timeout { gate: GateKind, after_s: u32 },
269 /// Classifier could not match the output to any known variant. The
270 /// `log_ref` on the enclosing `GateOutcome` is the diagnostic path.
271 Unclassified { legacy_detail: Option<String> },
272 }
273
274 impl GateFailure {
275 pub fn summary(&self) -> String {
276 match self {
277 // The panic message is the diagnostic; prefer it over the test name.
278 GateFailure::CargoTest {
279 failed_count,
280 first_panic: Some(p),
281 ..
282 } => format!("{failed_count} test(s) failed; first panic: {p}"),
283 GateFailure::CargoTest {
284 failed_count,
285 first_failed: Some(name),
286 first_panic: None,
287 } => format!("{failed_count} test(s) failed; first: {name}"),
288 GateFailure::CargoTest {
289 failed_count,
290 first_failed: None,
291 first_panic: None,
292 } => format!("{failed_count} test(s) failed"),
293 GateFailure::CompileError {
294 error_count,
295 first_error: Some(e),
296 } => format!("compile failed ({error_count} error(s)); first: {e}"),
297 GateFailure::CompileError {
298 error_count,
299 first_error: None,
300 } => format!("compile failed ({error_count} error(s))"),
301 GateFailure::MigrationDrift { migration } => {
302 format!("migration {migration} previously applied but missing")
303 }
304 GateFailure::MigrationModified { migration } => {
305 format!("migration {migration} previously applied but modified")
306 }
307 GateFailure::MigrationSqlError {
308 migration,
309 sqlstate: Some(s),
310 } => format!("migration {migration} sql error ({s})"),
311 GateFailure::MigrationSqlError {
312 migration,
313 sqlstate: None,
314 } => format!("migration {migration} sql error"),
315 GateFailure::RestoreFailed { reason } => format!("restore: {reason}"),
316 GateFailure::BootPanic { exit_code: Some(c) } => format!("binary panicked: exit {c}"),
317 GateFailure::BootPanic { exit_code: None } => "binary panicked".into(),
318 GateFailure::BootExitedEarly { exit_code: Some(c) } => {
319 format!("binary exited early: exit {c}")
320 }
321 GateFailure::BootExitedEarly { exit_code: None } => "binary exited early".into(),
322 GateFailure::BootHealthProbeFailed { last_error } => {
323 format!("started but never served /health: {last_error}")
324 }
325 GateFailure::NodeUnhealthy { node, detail } => {
326 format!("node {node} unhealthy: {detail}")
327 }
328 GateFailure::CodeSmokeSetup { reason } => format!("code smoke db setup: {reason}"),
329 GateFailure::CodeSmokeSeed { exit_code: Some(c) } => {
330 format!("migrate+seed run failed: exit {c}")
331 }
332 GateFailure::CodeSmokeSeed { exit_code: None } => "migrate+seed run failed".into(),
333 GateFailure::CodeSmokeDocs { broken: 0 } => "broken internal docs link(s)".into(),
334 GateFailure::CodeSmokeDocs { broken } => {
335 format!("{broken} broken internal docs link(s)")
336 }
337 GateFailure::CodeSmokeNoListeningLog => {
338 "served /health but never logged 'listening'".into()
339 }
340 GateFailure::CodeSmokeFrontend { dir, exit_code } => match exit_code {
341 Some(c) => format!("frontend build failed in {dir}: exit {c}"),
342 None => format!("frontend build failed in {dir}"),
343 },
344 GateFailure::SpawnFailed { message } => format!("spawn: {message}"),
345 GateFailure::Timeout { gate, after_s } => format!("{gate} timed out after {after_s}s"),
346 GateFailure::Unclassified {
347 legacy_detail: Some(d),
348 } => d.clone(),
349 GateFailure::Unclassified {
350 legacy_detail: None,
351 } => "unclassified failure".into(),
352 }
353 }
354 }
355
356 // ---------------------------------------------------------------------
357 // Deploy outcomes (step 7)
358 // ---------------------------------------------------------------------
359
360 /// Typed outcome of one node-deploy attempt. Stored as `outcome_json` in
361 /// the `deploys` table and emitted in `Event::DeployFailed` so consumers
362 /// can distinguish a node-unreachable error (operator: check the box)
363 /// from rsync mid-transfer corruption (operator: check disk/network).
364 #[derive(Debug, Clone, Serialize, Deserialize)]
365 pub struct DeployOutcome {
366 pub status: DeployStatus,
367 }
368
369 impl DeployOutcome {
370 pub fn ok() -> Self {
371 Self {
372 status: DeployStatus::Ok,
373 }
374 }
375 pub fn failed(failure: DeployFailureKind) -> Self {
376 Self {
377 status: DeployStatus::Failed { failure },
378 }
379 }
380 pub fn in_progress() -> Self {
381 Self {
382 status: DeployStatus::InProgress,
383 }
384 }
385
386 /// `'in_progress' | 'ok' | 'failed'` — the value of the legacy
387 /// `deploys.outcome` column.
388 pub fn status_str(&self) -> &'static str {
389 match self.status {
390 DeployStatus::InProgress => "in_progress",
391 DeployStatus::Ok => "ok",
392 DeployStatus::Failed { .. } => "failed",
393 }
394 }
395 }
396
397 #[derive(Debug, Clone, Serialize, Deserialize)]
398 #[serde(tag = "kind", rename_all = "snake_case")]
399 pub enum DeployStatus {
400 InProgress,
401 Ok,
402 Failed { failure: DeployFailureKind },
403 }
404
405 #[derive(Debug, Clone, Serialize, Deserialize)]
406 #[serde(tag = "kind", rename_all = "snake_case")]
407 pub enum DeployFailureKind {
408 /// SSH to the node failed before any state changed. Typically a dead
409 /// host, network partition, or stale known_hosts.
410 NodeUnreachable { detail: String },
411 /// rsync exited non-zero mid-transfer. The on-target release dir may
412 /// be partially populated, but the `current` symlink is untouched.
413 RsyncFailed { detail: String },
414 /// Files copied successfully but the atomic symlink swap step
415 /// failed. The new release is on disk; the service is still running
416 /// the old one.
417 SymlinkSwapFailed { detail: String },
418 /// Symlink swapped but `systemctl reload-or-restart` returned
419 /// non-zero. The new code is current but the service may have
420 /// crashed on startup.
421 ServiceRestartFailed { detail: String },
422 /// Classifier couldn't match the error to a known variant. The full
423 /// anyhow chain is in `detail`.
424 Unclassified { detail: String },
425 }
426
427 impl DeployFailureKind {
428 pub fn summary(&self) -> String {
429 match self {
430 DeployFailureKind::NodeUnreachable { detail } => format!("node unreachable: {detail}"),
431 DeployFailureKind::RsyncFailed { detail } => format!("rsync: {detail}"),
432 DeployFailureKind::SymlinkSwapFailed { detail } => format!("symlink swap: {detail}"),
433 DeployFailureKind::ServiceRestartFailed { detail } => {
434 format!("service restart: {detail}")
435 }
436 DeployFailureKind::Unclassified { detail } => detail.chars().take(200).collect(),
437 }
438 }
439 }
440
441 /// Pointer to the on-disk gate log: a path relative to `cfg.logs_root`
442 /// of the form `<version>/<gate_kind>.log`. Stored in `gate_runs.log_ref`
443 /// and surfaced in `/state` so the TUI/operator can request the full
444 /// tail via `GET /logs/<version>/<gate>` only when needed.
445 #[derive(Debug, Clone, Serialize, Deserialize)]
446 #[serde(transparent)]
447 pub struct LogRef(pub String);
448
449 impl LogRef {
450 pub fn new(version: &Version, gate: GateKind) -> Self {
451 Self(format!("{}/{}.log", version, gate.as_str()))
452 }
453 pub fn as_str(&self) -> &str {
454 &self.0
455 }
456 }
457
458 impl std::fmt::Display for LogRef {
459 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
460 self.0.fmt(f)
461 }
462 }
463
464 #[cfg(test)]
465 mod tests {
466 use super::*;
467
468 #[test]
469 fn outcome_serialization_is_two_layer_tagged() {
470 let o = GateOutcome::failed(GateFailure::MigrationDrift {
471 migration: "0047_widgets".into(),
472 });
473 let v: serde_json::Value = serde_json::to_value(&o).unwrap();
474 assert_eq!(v["status"]["kind"], "failed");
475 assert_eq!(v["status"]["failure"]["kind"], "migration_drift");
476 assert_eq!(v["status"]["failure"]["migration"], "0047_widgets");
477 }
478
479 #[test]
480 fn outcome_round_trips_through_json() {
481 let o = GateOutcome::passed(PassNote::TestsPassed { duration_s: 42 });
482 let s = serde_json::to_string(&o).unwrap();
483 let back: GateOutcome = serde_json::from_str(&s).unwrap();
484 assert!(back.is_passed());
485 assert_eq!(back.status_str(), "passed");
486 }
487
488 #[test]
489 fn timeout_failure_renders_and_is_not_passed() {
490 // Timeout is now a live failure (cargo_test / migration_dry_run ceilings),
491 // not an aspirational variant — keep it mapped in summary() and serde.
492 let o = GateOutcome::failed(GateFailure::Timeout {
493 gate: GateKind::CargoTest,
494 after_s: 2400,
495 });
496 assert!(!o.is_passed());
497 let v: serde_json::Value = serde_json::to_value(&o).unwrap();
498 assert_eq!(v["status"]["failure"]["kind"], "timeout");
499 assert!(
500 matches!(&o.status, GateStatus::Failed { failure } if failure.summary().contains("timed out"))
501 );
502 }
503
504 #[test]
505 fn blocked_is_not_passed() {
506 let o = GateOutcome::blocked(GateBlocker::BurnInClockNotStarted);
507 assert!(!o.is_passed());
508 assert_eq!(o.status_str(), "blocked");
509 }
510
511 #[test]
512 fn log_ref_construction_matches_disk_layout() {
513 let v: Version = "0.9.6".parse().unwrap();
514 let lr = LogRef::new(&v, GateKind::CargoTest);
515 assert_eq!(lr.as_str(), "0.9.6/cargo_test.log");
516 }
517
518 #[test]
519 fn unclassified_preserves_legacy_detail() {
520 let o = GateOutcome::failed(GateFailure::Unclassified {
521 legacy_detail: Some(
522 "binary exited early: exit status: 101\n==== stdout ====\n...".into(),
523 ),
524 });
525 let v: serde_json::Value = serde_json::to_value(&o).unwrap();
526 assert_eq!(v["status"]["failure"]["kind"], "unclassified");
527 assert!(
528 v["status"]["failure"]["legacy_detail"]
529 .as_str()
530 .unwrap()
531 .contains("exit status: 101")
532 );
533 }
534 }
535