Skip to main content

max / makenotwork

pom 0.4.4: an incident left open by a restart is closed at boot An incident closes on a confirmed transition back to Operational. If the process restarts across that recovery, the transition gate seeds itself from the last stored health, observes the same status, reports no transition, and nothing ever reopens the question: the record stays open forever and the instance reads `failed` while the target is up. Measured on astra. Incident 307 (mt) opened 2026-08-16 16:57 and still had a NULL ended_at a week later. The two incidents opened by the same blip in the same minute, mnw 308 and htpy 309, both closed an hour on. mt's health was Operational at 100% uptime throughout, so astra's pom had been reporting its own bookkeeping as an outage for seven days -- and it is the top line of that instance's status table, which is exactly where a real outage would appear. The health task now reconciles at seed time: if the seeded status is Operational, close any open incident before the first tick. One query per target at boot and nothing afterwards. Deliberately silent -- it closes a stale record, and a recovery notice for an outage that ended a week ago is noise.
Co-Authored-By
Claude Opus 5 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-08-23 18:34 UTC
Signed with PGP, not checked
Commit: 4f7d8bcf030b6f0615cc6695d8295e1b03f5d74a
Parent: d76122a
4 files changed, +112 insertions, -36 deletions
M pom/Cargo.lock +43 -35
@@ -1651,7 +1651,7 @@
1651 1651
1652 1652 [[package]]
1653 1653 name = "pom"
1654 - version = "0.4.3"
1654 + version = "0.4.4"
1655 1655 dependencies = [
1656 1656 "axum",
1657 1657 "chrono",
@@ -3601,6 +3601,46 @@
3601 3601 source = "registry+https://github.com/rust-lang/crates.io-index"
3602 3602 checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
3603 3603
3604 + [[patch.unused]]
3605 + name = "quasi-axum"
3606 + version = "0.56.0"
3607 +
3608 + [[patch.unused]]
3609 + name = "quasi-basics"
3610 + version = "0.56.0"
3611 +
3612 + [[patch.unused]]
3613 + name = "quasi-http"
3614 + version = "0.56.0"
3615 +
3616 + [[patch.unused]]
3617 + name = "quasi-immediate"
3618 + version = "0.56.0"
3619 +
3620 + [[patch.unused]]
3621 + name = "quasi-notifs"
3622 + version = "0.56.0"
3623 +
3624 + [[patch.unused]]
3625 + name = "quasi-router"
3626 + version = "0.56.0"
3627 +
3628 + [[patch.unused]]
3629 + name = "quasi-store"
3630 + version = "0.1.0"
3631 +
3632 + [[patch.unused]]
3633 + name = "quasi-tauri"
3634 + version = "0.56.0"
3635 +
3636 + [[patch.unused]]
3637 + name = "quasi-webview"
3638 + version = "0.56.0"
3639 +
3640 + [[patch.unused]]
3641 + name = "quasi-type"
3642 + version = "0.1.0"
3643 +
3604 3644 [[patch.unused]]
3605 3645 name = "docengine"
3606 3646 version = "0.7.0"
@@ -3615,44 +3655,12 @@
3615 3655
3616 3656 [[patch.unused]]
3617 3657 name = "tagtree"
3618 - version = "0.4.0"
3658 + version = "0.4.1"
3619 3659
3620 3660 [[patch.unused]]
3621 3661 name = "synckit-client"
3622 - version = "0.8.0"
3662 + version = "0.8.1"
3623 3663
3624 3664 [[patch.unused]]
3625 3665 name = "synckit-config"
3626 3666 version = "0.2.0"
3627 -
3628 - [[patch.unused]]
3629 - name = "quasi-axum"
3630 - version = "0.11.0"
3631 -
3632 - [[patch.unused]]
3633 - name = "quasi-basics"
3634 - version = "0.11.0"
3635 -
3636 - [[patch.unused]]
3637 - name = "quasi-http"
3638 - version = "0.11.0"
3639 -
3640 - [[patch.unused]]
3641 - name = "quasi-immediate"
3642 - version = "0.11.0"
3643 -
3644 - [[patch.unused]]
3645 - name = "quasi-router"
3646 - version = "0.11.0"
3647 -
3648 - [[patch.unused]]
3649 - name = "quasi-store"
3650 - version = "0.1.0"
3651 -
3652 - [[patch.unused]]
3653 - name = "quasi-tauri"
3654 - version = "0.11.0"
3655 -
3656 - [[patch.unused]]
3657 - name = "quasi-webview"
3658 - version = "0.11.0"
M pom/Cargo.toml +1 -1
@@ -1,6 +1,6 @@
1 1 [package]
2 2 name = "pom"
3 - version = "0.4.3"
3 + version = "0.4.4"
4 4 edition = "2024"
5 5 license = "LicenseRef-PolyForm-Noncommercial-1.0.0"
6 6
M pom/tests/integration.rs +34
@@ -1230,6 +1230,40 @@
1230 1230 assert!(recent[0].duration_secs.is_some());
1231 1231 }
1232 1232
1233 + /// The restart hole, in miniature.
1234 + ///
1235 + /// An incident closes on a *confirmed* transition back to Operational. If the
1236 + /// process restarts across that recovery, the transition gate seeds itself from
1237 + /// the last stored health, observes the same status, reports no transition, and
1238 + /// nothing ever closes the record: the instance reads `failed` forever while the
1239 + /// target is up.
1240 + ///
1241 + /// Measured on astra 2026-08-23. Incident 307 (`mt`) opened 2026-08-16 16:57 and
1242 + /// still had a NULL `ended_at` a week later, while the two incidents opened by
1243 + /// the same blip in the same minute (`mnw`, `htpy`) both closed an hour on and
1244 + /// mt's health sat at 100% uptime throughout.
1245 + ///
1246 + /// The reconciliation is `close_open_incidents` at seed time when the seeded
1247 + /// status is Operational. This pins the property it relies on: closing is safe
1248 + /// to run when there is nothing to close, so the boot path can be unconditional
1249 + /// rather than having to ask first.
1250 + #[tokio::test]
1251 + async fn closing_incidents_is_idempotent_so_a_boot_time_reconcile_is_safe() {
1252 + let pool = db::connect_in_memory().await.unwrap();
1253 +
1254 + // Nothing open: the reconcile a healthy target performs on every start.
1255 + assert_eq!(db::close_open_incidents(&pool, "mt").await.unwrap(), 0);
1256 +
1257 + db::insert_incident(&pool, "mt", "operational", "unreachable")
1258 + .await
1259 + .unwrap();
1260 + // The recovery the restart swallowed, applied late.
1261 + assert_eq!(db::close_open_incidents(&pool, "mt").await.unwrap(), 1);
1262 + assert!(db::get_open_incident(&pool, "mt").await.unwrap().is_none());
1263 + // And again on the next start, with nothing left to do.
1264 + assert_eq!(db::close_open_incidents(&pool, "mt").await.unwrap(), 0);
1265 + }
1266 +
1233 1267 #[tokio::test]
1234 1268 async fn incident_close_only_affects_target() {
1235 1269 let pool = db::connect_in_memory().await.unwrap();
M pom/src/cli/tasks/health.rs +34
@@ -49,6 +49,40 @@
49 49 .flatten()
50 50 .map(|s| s.status);
51 51 let mut gate = TransitionGate::seeded(seed, confirmations);
52 +
53 + // RECONCILE THE INCIDENT LEDGER AGAINST THE SEED, before the
54 + // first tick.
55 + //
56 + // Seeding is what stops a restart re-firing a transition, and it
57 + // is also how a restart can LOSE one. An incident closes on a
58 + // confirmed transition back to Operational; if the process
59 + // restarts across that recovery, the gate seeds Operational,
60 + // observes Operational, reports no transition, and the incident
61 + // stays open forever with nothing to reopen the question.
62 + //
63 + // Measured on astra 2026-08-23: incident 307, `mt`, opened
64 + // 2026-08-16 16:57 with `ended_at` still NULL a week later. Two
65 + // sibling incidents opened in the same minute by the same blip
66 + // (`mnw` 308, `htpy` 309) both closed an hour later, and mt's
67 + // health had been Operational at 100% uptime throughout. The
68 + // instance had read `failed` ever since, which is a monitor
69 + // reporting its own bookkeeping as an outage.
70 + //
71 + // Idempotent and once per target per start, so it costs one
72 + // query at boot and nothing afterwards. Deliberately does NOT
73 + // alert: this closes a stale record, and a recovery notice for
74 + // an outage that ended a week ago is noise.
75 + if seed == Some(HealthStatus::Operational) {
76 + match db::close_open_incidents(&pool, &name).await {
77 + Ok(0) => {}
78 + Ok(n) => info!(
79 + "{name}: closed {n} incident(s) left open across a restart; health reads operational"
80 + ),
81 + Err(e) => {
82 + tracing::error!("{name}: failed to reconcile open incidents: {e}");
83 + }
84 + }
85 + }
52 86 while ticks.next().await {
53 87 let snapshot = http::check_health(&name, &health_config, expect).await;
54 88 info!(