Skip to main content

max / makenotwork

sando: reconcile deploys against tier_state on startup A promote writes the per-node deploys row and then advances tier_state. A SIGKILL between those two writes left the node serving the new version while tier_state still recorded the old one, with partial_reason NULL: /state looked clean on the wrong version and a later /rollback targeted previous_version, now two versions behind reality (audit 2026-07-21, sando-h5-crash-mid-promote). - Add tier_state.advanced_at (migration 009): the instant the deployed identity last changed, set on every advance and rollback and never nulled, unlike burn_in_started_at. Backfilled to trust existing state. - Write the deploys row as in_progress BEFORE touching the node and finalize it to ok/failed after, so a crash after the swap always leaves a durable trace. - New reconcile module, run once at startup: flag any tier whose latest deploy postdates advanced_at but disagrees with tier_state (landed but not advanced) or is still in-flight (interrupted mid-node), via partial_reason; settle orphaned in_progress rows to terminal failed. The only uncovered sliver is a crash between the pre-write and the swap, where the node was never changed and there is nothing to reconcile. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-25 02:47 UTC
Commit: 73c460f6077002e4393b4ce800267eb94fdedbbf
Parent: 14eacfb
7 files changed, +600 insertions, -12 deletions
A sando/daemon/migrations/009_tier_advanced_at.sql +33
@@ -0,0 +1,33 @@
1 + -- Record when a tier's deployed identity (current_version / current_build_id)
2 + -- last changed, so a startup reconcile can catch a promote that died between
3 + -- landing on the nodes and advancing tier_state.
4 + --
5 + -- The gap this closes (audit 2026-07-21, sando-h5-crash-mid-promote): a promote
6 + -- inserts the per-node `deploys` row (outcome 'ok') and then calls advance_tier.
7 + -- If sandod is SIGKILLed between those two writes (OOM, systemctl restart, a
8 + -- self-update), the node is running the new version but tier_state still records
9 + -- the old one, and partial_reason is NULL because the failure path never ran.
10 + -- /state reports a clean tier on the wrong version, and a later /rollback then
11 + -- targets previous_version -- now two versions behind what the node is running.
12 + --
13 + -- The reconcile needs a trustworthy "tier_state last changed at" instant to
14 + -- decide whether a `deploys` row postdates the recorded state. burn_in_started_at
15 + -- cannot serve that role: /rollback and reset_burn_in both null it while the
16 + -- deployed identity is real, so a NULL there is ambiguous. advanced_at is set on
17 + -- every genuine advance (runs::advance_tier) and every rollback (routes) and is
18 + -- never nulled, so `deploy.finished_at > advanced_at` cleanly means "a deploy
19 + -- landed after the last state change we recorded".
20 + ALTER TABLE tier_state ADD COLUMN advanced_at TEXT;
21 +
22 + -- Backfill existing rows to trust current state: seed advanced_at from the
23 + -- burn-in clock when present, else from the tier's most recent deploy. This
24 + -- makes the first post-migration boot quiet across every existing tier
25 + -- (clean, rolled-back, or burn-in-reset) -- the reconcile only fires on a
26 + -- crash that happens after this column starts being maintained live. A tier
27 + -- with neither signal (never deployed) keeps NULL and has no deploys row for
28 + -- the reconcile to weigh, so it is skipped anyway.
29 + UPDATE tier_state
30 + SET advanced_at = COALESCE(
31 + burn_in_started_at,
32 + (SELECT MAX(d.finished_at) FROM deploys d WHERE d.tier = tier_state.tier)
33 + );
M sando/daemon/src/lib.rs +1
@@ -26,6 +26,7 @@ pub mod events;
26 26 pub mod gates;
27 27 pub mod git;
28 28 pub mod outcome;
29 + pub mod reconcile;
29 30 pub mod routes;
30 31 pub mod runs;
31 32 pub mod state;
M sando/daemon/src/main.rs +17 -1
@@ -1,5 +1,5 @@
1 1 use anyhow::Result;
2 - use sando_daemon::{config, db, events, git, routes, runs, state, sync, topology};
2 + use sando_daemon::{config, db, events, git, reconcile, routes, runs, state, sync, topology};
3 3 use std::net::SocketAddr;
4 4 use std::path::Path;
5 5 use std::sync::Arc;
@@ -91,6 +91,22 @@ async fn run() -> Result<()> {
91 91 sync::sync(&pool, &topo).await?;
92 92 tracing::info!(tiers = topo.tiers.len(), bare = %topo.repo.bare_path, "topology synced");
93 93
94 + // Reconcile deploys against tier_state (after sync, so every tier_state row
95 + // exists): catch a promote that landed on the nodes but died before advancing
96 + // tier_state, so a stale-green tier on the wrong version is flagged partial
97 + // instead of misleading /state and a later /rollback (audit
98 + // sando-h5-crash-mid-promote).
99 + match reconcile::recover_unrecorded_deploys(&pool).await {
100 + Ok(0) => {}
101 + Ok(n) => tracing::error!(
102 + flagged = n,
103 + "flagged tier(s) with an unrecorded deploy from a prior daemon; see /state partial_reason"
104 + ),
105 + Err(e) => {
106 + tracing::error!(error = %e, "failed to reconcile deploys against tier_state at startup");
107 + }
108 + }
109 +
94 110 // Fail closed on the scratch cluster's privileges before any gate can hit
95 111 // them, rather than letting the first migration_dry_run discover it as an
96 112 // opaque "permission denied". Unset scratch_db_url is already a per-gate
A sando/daemon/src/reconcile.rs +500
@@ -0,0 +1,574 @@
1 + //! Startup crash-recovery reconcile of `deploys` against `tier_state`.
2 + //!
3 + //! A promote lands a build on a tier's nodes and *then* advances tier_state:
4 + //! [`crate::routes`] inserts the per-node `deploys` row (outcome `ok`) inside the
5 + //! canary loop, and only after every node succeeds does [`crate::runs::advance_tier`]
6 + //! record the new version. If sandod dies between those two writes — OOM, a
7 + //! `systemctl restart`, a self-update SIGKILL — the node is serving the new
8 + //! version but tier_state still names the old one, and `partial_reason` is NULL
9 + //! because no failure path executed. `/state` then shows a clean tier on the
10 + //! wrong version, and a later `/rollback` walks back to `previous_version`, which
11 + //! is now *two* versions behind what the node actually runs (audit 2026-07-21,
12 + //! sando-h5-crash-mid-promote).
13 + //!
14 + //! [`recover_unrecorded_deploys`] runs once at startup, next to
15 + //! [`crate::runs::recover_orphaned_running`], and flags any tier whose latest
16 + //! successful deploy postdates the last recorded state change yet names a
17 + //! different version/build than tier_state holds. The flag is a `partial_reason`,
18 + //! the same channel the promote/rollback failure paths use, so `/state` and the
19 + //! TUI go loud and force the operator to reconcile by hand rather than trusting a
20 + //! stale-green tier.
21 + //!
22 + //! The comparison instant is `tier_state.advanced_at` (migration 009), set on
23 + //! every advance and rollback and never nulled — unlike `burn_in_started_at`,
24 + //! which rollback and `reset_burn_in` clear while the deployed identity stays
25 + //! real. A clean promote sets `advanced_at` *after* its `deploys` row, so its own
26 + //! landing never postdates it; a rollback sets it after the roll-off row, so a
27 + //! cleanly rolled-back tier is not mistaken for an unrecorded deploy.
28 + //!
29 + //! The `deploys` row is written `in_progress` *before* the node is touched and
30 + //! finalized to ok/failed right after, so the only sliver still uncovered is a
31 + //! crash between the pre-write and the swap — where the node was never changed,
32 + //! so there is nothing to reconcile. Any row still `in_progress` at startup is an
33 + //! orphaned in-flight deploy: it is flagged (the swap may or may not have landed)
34 + //! and settled to a terminal `failed` so the audit trail carries no eternal
35 + //! in-flight row.
36 +
37 + use anyhow::Result;
38 + use chrono::{DateTime, Utc};
39 + use sqlx::{Row, SqlitePool};
40 +
41 + /// One tier's most recent deploy row, whatever its outcome.
42 + struct LastDeploy {
43 + version: String,
44 + build_id: Option<i64>,
45 + /// `'ok' | 'failed' | 'in_progress'`.
46 + outcome: String,
47 + /// `finished_at` when the row is terminal, else `started_at` — the instant to
48 + /// weigh against the tier's last state change. `started_at` is NOT NULL.
49 + instant: String,
50 + }
51 +
52 + /// Reconcile `deploys` against `tier_state` after a crash. Flags any tier whose
53 + /// most recent deploy postdates the last recorded state change but disagrees with
54 + /// what tier_state holds (a promote that died before advancing) or is still
55 + /// in-flight (a promote interrupted mid-node). Sets `partial_reason` — only when
56 + /// it is currently NULL, so a more specific promote/rollback failure reason
57 + /// stands — and settles orphaned `in_progress` rows. Returns how many tiers were
58 + /// flagged. Run once at startup, before serving.
59 + pub async fn recover_unrecorded_deploys(pool: &SqlitePool) -> Result<u64> {
60 + let tiers = sqlx::query(
61 + "SELECT tier, current_version, current_build_id, advanced_at, partial_reason
62 + FROM tier_state",
63 + )
64 + .fetch_all(pool)
65 + .await?;
66 +
67 + let mut flagged = 0u64;
68 + for row in tiers {
69 + let tier: String = row.get("tier");
70 + // A tier already flagged partial carries a more specific reason from the
71 + // path that flagged it; don't overwrite it.
72 + if row.get::<Option<String>, _>("partial_reason").is_some() {
73 + continue;
74 + }
75 + let Some(landing) = latest_deploy(pool, &tier).await? else {
76 + continue;
77 + };
78 + let current_version: Option<String> = row.get("current_version");
79 + let current_build_id: Option<i64> = row.get("current_build_id");
80 + let advanced_at: Option<String> = row.get("advanced_at");
81 +
82 + // Only a deploy that postdates the last recorded state change is suspect:
83 + // a clean promote/rollback stamps `advanced_at` after its own rows.
84 + if !postdates(&landing.instant, advanced_at.as_deref()) {
85 + continue;
86 + }
87 + let reason = match landing.outcome.as_str() {
88 + // Interrupted mid-node: the swap may or may not have landed, and no
89 + // outcome was ever recorded. Force a look regardless of identity.
90 + "in_progress" => Some(format!(
91 + "crash-recovery: a deploy of {deployed} to this tier was in flight when the \
92 + daemon last stopped and never recorded an outcome. The node's binary may or \
93 + may not have been swapped. Verify what the tier is actually running, then \
94 + re-promote or roll back deliberately.",
95 + deployed = describe(&landing.version, landing.build_id),
96 + )),
97 + // Landed successfully but tier_state names something else — the promote
98 + // died after the deploy but before `advance_tier`.
99 + "ok" if !identity_matches(&landing, current_version.as_deref(), current_build_id) => {
100 + Some(format!(
101 + "crash-recovery: node(s) on this tier were deployed {deployed} at {when} but \
102 + tier_state still records {recorded}. A promote most likely died after the \
103 + deploy landed but before tier_state advanced. Verify what the tier is \
104 + actually running, then re-promote or roll back deliberately — a blind \
105 + /rollback would target the wrong version.",
106 + deployed = describe(&landing.version, landing.build_id),
107 + when = landing.instant,
108 + recorded = current_version.as_deref().unwrap_or("no version"),
109 + ))
110 + }
111 + // A terminal 'failed' row was handled by the deploy's own failure path
112 + // (which sets its own partial_reason); nothing to add here.
113 + _ => None,
114 + };
115 + let Some(reason) = reason else { continue };
116 +
117 + // `AND partial_reason IS NULL` keeps the write a no-op if anything set a
118 + // reason since the read above (startup is single-threaded, but the guard
119 + // costs nothing and states the intent).
120 + let res = sqlx::query(
121 + "UPDATE tier_state SET partial_reason = ? WHERE tier = ? AND partial_reason IS NULL",
122 + )
123 + .bind(&reason)
124 + .bind(&tier)
125 + .execute(pool)
126 + .await?;
127 + if res.rows_affected() > 0 {
128 + tracing::error!(tier = %tier, %reason, "startup reconcile: interrupted/unrecorded deploy on tier");
129 + flagged += 1;
130 + }
131 + }
132 +
133 + // Settle any orphaned in-flight deploy rows so the trail carries no eternal
134 + // 'in_progress'. Done after the flag decisions above, which read that state.
135 + // At startup (single-threaded, pre-serving) any 'in_progress' row is an orphan.
136 + settle_orphaned_in_progress(pool).await?;
137 + Ok(flagged)
138 + }
139 +
140 + /// The most recent deploy on `tier`, any outcome. Rows are ordered by `id`
141 + /// (monotonic autoincrement), so this is the last one regardless of clock skew.
142 + async fn latest_deploy(pool: &SqlitePool, tier: &str) -> Result<Option<LastDeploy>> {
143 + let row = sqlx::query(
144 + "SELECT version, build_id, outcome, started_at, finished_at FROM deploys
145 + WHERE tier = ? ORDER BY id DESC LIMIT 1",
146 + )
147 + .bind(tier)
148 + .fetch_optional(pool)
149 + .await?;
150 + Ok(row.map(|r| {
151 + let started_at: String = r.get("started_at");
152 + let finished_at: Option<String> = r.get("finished_at");
153 + LastDeploy {
154 + version: r.get("version"),
155 + build_id: r.get("build_id"),
156 + outcome: r.get("outcome"),
157 + instant: finished_at.unwrap_or(started_at),
158 + }
159 + }))
160 + }
161 +
162 + /// Mark every `in_progress` deploy row terminal-failed with an interruption note,
163 + /// mirroring [`crate::runs::recover_orphaned_running`] for build runs. Returns the
164 + /// number of rows settled.
165 + async fn settle_orphaned_in_progress(pool: &SqlitePool) -> Result<u64> {
166 + let outcome =
167 + crate::outcome::DeployOutcome::failed(crate::outcome::DeployFailureKind::Unclassified {
168 + detail: "daemon restarted mid-deploy".into(),
169 + });
170 + let outcome_json = serde_json::to_string(&outcome)?;
171 + let res = sqlx::query(
172 + "UPDATE deploys SET outcome = 'failed', finished_at = ?, outcome_json = ?
173 + WHERE outcome = 'in_progress'",
174 + )
175 + .bind(Utc::now().to_rfc3339())
176 + .bind(&outcome_json)
177 + .execute(pool)
178 + .await?;
179 + let n = res.rows_affected();
180 + if n > 0 {
181 + tracing::warn!(
182 + settled = n,
183 + "startup reconcile: settled orphaned in-flight deploy row(s)"
184 + );
185 + }
186 + Ok(n)
187 + }
188 +
189 + /// Whether `deploy_finished_at` is later than the tier's last recorded state
190 + /// change. A NULL `advanced_at` (state never advanced, yet a deploy landed) and
191 + /// any unparseable timestamp both count as postdating — fail toward flagging, so
192 + /// a corrupt or missing instant forces operator attention rather than hiding.
193 + fn postdates(deploy_finished_at: &str, advanced_at: Option<&str>) -> bool {
194 + let Some(advanced_at) = advanced_at else {
195 + return true;
196 + };
197 + match (
198 + DateTime::parse_from_rfc3339(deploy_finished_at),
199 + DateTime::parse_from_rfc3339(advanced_at),
200 + ) {
201 + (Ok(d), Ok(a)) => d > a,
202 + _ => true,
203 + }
204 + }
205 +
206 + /// Whether the landed artifact is the one tier_state records. Build identity wins
207 + /// when both sides carry it (a re-deploy of the same version from a different
208 + /// build is still a mismatch); otherwise fall back to the version string. A NULL
209 + /// `current_version` with a real landing is a mismatch.
210 + fn identity_matches(
211 + landing: &LastDeploy,
212 + current_version: Option<&str>,
213 + current_build_id: Option<i64>,
214 + ) -> bool {
215 + match (landing.build_id, current_build_id) {
216 + (Some(landed), Some(current)) => landed == current,
217 + _ => current_version == Some(landing.version.as_str()),
218 + }
219 + }
220 +
221 + /// Human label for a landed artifact: the version, plus the build id when known.
222 + fn describe(version: &str, build_id: Option<i64>) -> String {
223 + match build_id {
224 + Some(id) => format!("{version} (build {id})"),
225 + None => version.to_string(),
226 + }
227 + }
228 +
229 + #[cfg(test)]
230 + mod tests {
231 + use super::*;
232 + use chrono::{Duration, Utc};
233 + use sqlx::sqlite::SqlitePoolOptions;
234 +
235 + async fn pool() -> SqlitePool {
236 + let pool = SqlitePoolOptions::new()
237 + .max_connections(1)
238 + .connect("sqlite::memory:")
239 + .await
240 + .unwrap();
241 + crate::db::migrate(&pool).await.unwrap();
242 + pool
243 + }
244 +
245 + async fn seed_tier(pool: &SqlitePool, tier: &str) {
246 + sqlx::query(
247 + "INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, 0, 1, 'sequential')",
248 + )
249 + .bind(tier)
250 + .execute(pool)
251 + .await
252 + .unwrap();
253 + sqlx::query("INSERT INTO tier_state (tier) VALUES (?)")
254 + .bind(tier)
255 + .execute(pool)
256 + .await
257 + .unwrap();
258 + }
259 +
260 + /// Set the tier's recorded state the way an advance/rollback would.
261 + async fn set_state(
262 + pool: &SqlitePool,
263 + tier: &str,
264 + current_version: Option<&str>,
265 + advanced_at: Option<&str>,
266 + ) {
267 + if let Some(v) = current_version {
268 + sqlx::query(
269 + "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
270 + VALUES (?, 'sha', '2026-07-21T00:00:00Z', '/r/x')",
271 + )
272 + .bind(v)
273 + .execute(pool)
274 + .await
275 + .unwrap();
276 + }
277 + sqlx::query("UPDATE tier_state SET current_version = ?, advanced_at = ? WHERE tier = ?")
278 + .bind(current_version)
279 + .bind(advanced_at)
280 + .bind(tier)
281 + .execute(pool)
282 + .await
283 + .unwrap();
284 + }
285 +
286 + /// FK parents `deploys` needs: a `versions` row and a `nodes` row. sqlx
287 + /// enables `foreign_keys` by default, so these must exist first.
288 + async fn seed_deploy_fks(pool: &SqlitePool, tier: &str, version: &str) {
289 + sqlx::query(
290 + "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
291 + VALUES (?, 'sha', '2026-07-21T00:00:00Z', '/r/x')",
292 + )
293 + .bind(version)
294 + .execute(pool)
295 + .await
296 + .unwrap();
297 + sqlx::query(
298 + "INSERT OR IGNORE INTO nodes (name, tier, ssh_target, release_root)
299 + VALUES ('n1', ?, 'local', '/opt/mnw')",
300 + )
301 + .bind(tier)
302 + .execute(pool)
303 + .await
304 + .unwrap();
305 + }
306 +
307 + /// Insert a `deploys` row, seeding its FK parents first.
308 + async fn insert_deploy(
309 + pool: &SqlitePool,
310 + tier: &str,
311 + version: &str,
312 + outcome: &str,
313 + finished_at: &str,
314 + ) {
315 + seed_deploy_fks(pool, tier, version).await;
316 + sqlx::query(
317 + "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome)
318 + VALUES (?, ?, 'n1', ?, ?, ?)",
319 + )
320 + .bind(version)
321 + .bind(tier)
322 + .bind(finished_at)
323 + .bind(finished_at)
324 + .bind(outcome)
325 + .execute(pool)
326 + .await
327 + .unwrap();
328 + }
329 +
330 + async fn partial_reason(pool: &SqlitePool, tier: &str) -> Option<String> {
331 + sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = ?")
332 + .bind(tier)
333 + .fetch_one(pool)
334 + .await
335 + .unwrap()
336 + }
337 +
338 + #[tokio::test]
339 + async fn clean_promote_is_not_flagged() {
340 + // deploy lands, THEN tier_state advances: advanced_at postdates the row.
341 + let pool = pool().await;
342 + seed_tier(&pool, "b").await;
343 + let deployed = Utc::now();
344 + insert_deploy(&pool, "b", "0.10.15", "ok", &deployed.to_rfc3339()).await;
345 + set_state(
346 + &pool,
347 + "b",
348 + Some("0.10.15"),
349 + Some(&(deployed + Duration::seconds(1)).to_rfc3339()),
350 + )
351 + .await;
352 +
353 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
354 + assert!(partial_reason(&pool, "b").await.is_none());
355 + }
356 +
357 + #[tokio::test]
358 + async fn crash_mid_promote_is_flagged() {
359 + // tier_state still on the OLD version; an 'ok' deploy of the NEW version
360 + // postdates the last advance — the exact SIGKILL-before-advance window.
361 + let pool = pool().await;
362 + seed_tier(&pool, "b").await;
363 + let advanced = Utc::now() - Duration::hours(1);
364 + set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
365 + insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
366 +
367 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
368 + let reason = partial_reason(&pool, "b").await.expect("flagged");
369 + assert!(reason.contains("0.10.15"), "names the deployed version");
370 + assert!(reason.contains("0.10.14"), "names the recorded version");
371 + assert!(reason.contains("crash-recovery"));
372 + }
373 +
374 + #[tokio::test]
375 + async fn rolled_back_tier_is_not_flagged() {
376 + // Promote to 0.10.15 landed (ok deploy), then a rollback moved tier_state
377 + // to 0.10.14 and stamped advanced_at AFTER the roll-off deploy. The stale
378 + // deploy row must not read as unrecorded.
379 + let pool = pool().await;
380 + seed_tier(&pool, "b").await;
381 + let deployed = Utc::now() - Duration::minutes(5);
382 + insert_deploy(&pool, "b", "0.10.15", "ok", &deployed.to_rfc3339()).await;
383 + set_state(&pool, "b", Some("0.10.14"), Some(&Utc::now().to_rfc3339())).await;
384 +
385 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
386 + assert!(partial_reason(&pool, "b").await.is_none());
387 + }
388 +
389 + #[tokio::test]
390 + async fn an_already_partial_tier_keeps_its_reason() {
391 + let pool = pool().await;
392 + seed_tier(&pool, "b").await;
393 + let advanced = Utc::now() - Duration::hours(1);
394 + set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
395 + insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
396 + sqlx::query(
397 + "UPDATE tier_state SET partial_reason = 'canary rollback incomplete' WHERE tier = 'b'",
398 + )
399 + .execute(&pool)
400 + .await
401 + .unwrap();
402 +
403 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
404 + assert_eq!(
405 + partial_reason(&pool, "b").await.as_deref(),
406 + Some("canary rollback incomplete"),
407 + "the specific reason is not clobbered",
408 + );
409 + }
410 +
411 + #[tokio::test]
412 + async fn a_failed_deploy_does_not_flag() {
413 + // The latest deploy is a failure (its own path handles partial state);
414 + // there is no successful landing that tier_state failed to record.
415 + let pool = pool().await;
416 + seed_tier(&pool, "b").await;
417 + let advanced = Utc::now() - Duration::hours(1);
418 + set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
419 + insert_deploy(&pool, "b", "0.10.15", "failed", &Utc::now().to_rfc3339()).await;
420 +
421 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
422 + assert!(partial_reason(&pool, "b").await.is_none());
423 + }
424 +
425 + #[tokio::test]
426 + async fn a_tier_with_no_deploys_is_skipped() {
427 + let pool = pool().await;
428 + seed_tier(&pool, "b").await;
429 + set_state(&pool, "b", None, None).await;
430 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
431 + }
432 +
433 + #[tokio::test]
434 + async fn crash_on_first_ever_promote_is_flagged() {
435 + // Never advanced (advanced_at NULL, current_version NULL) but a node was
436 + // deployed: prod is running something tier_state has no record of.
437 + let pool = pool().await;
438 + seed_tier(&pool, "b").await;
439 + insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
440 +
441 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
442 + assert!(
443 + partial_reason(&pool, "b")
444 + .await
445 + .unwrap()
446 + .contains("0.10.15")
447 + );
448 + }
449 +
450 + #[tokio::test]
451 + async fn build_id_mismatch_is_flagged_even_when_version_matches() {
452 + // Same version string, different build — a re-deploy landed a build the
453 + // tier never recorded. Build identity is the tie-breaker.
454 + let pool = pool().await;
455 + seed_tier(&pool, "b").await;
456 + seed_deploy_fks(&pool, "b", "0.10.15").await;
457 + // build_runs 7 and 9, referenced by tier_state.current_build_id and
458 + // deploys.build_id respectively.
459 + for id in [7, 9] {
460 + sqlx::query(
461 + "INSERT INTO build_runs (id, sha, phase, result, started_at)
462 + VALUES (?, 'sha', 'done', 'passed', '2026-07-21T00:00:00Z')",
463 + )
464 + .bind(id)
465 + .execute(&pool)
466 + .await
467 + .unwrap();
468 + }
469 + let advanced = Utc::now() - Duration::hours(1);
470 + // tier_state: version 0.10.15, build 7.
471 + sqlx::query(
472 + "UPDATE tier_state SET current_version = '0.10.15', current_build_id = 7, advanced_at = ? WHERE tier = 'b'",
473 + )
474 + .bind(advanced.to_rfc3339())
475 + .execute(&pool)
476 + .await
477 + .unwrap();
478 + // deploy: same version 0.10.15 but build 9, postdating the advance.
479 + sqlx::query(
480 + "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, build_id)
481 + VALUES ('0.10.15', 'b', 'n1', ?, ?, 'ok', 9)",
482 + )
483 + .bind(Utc::now().to_rfc3339())
484 + .bind(Utc::now().to_rfc3339())
485 + .execute(&pool)
486 + .await
487 + .unwrap();
488 +
489 + assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
490 + }
491 +
492 + /// Insert an in-flight deploy row: `outcome = 'in_progress'`, `finished_at`
493 + /// NULL, `started_at` set — the shape the promote loop writes before touching
494 + /// a node.
495 + async fn insert_in_progress(pool: &SqlitePool, tier: &str, version: &str, started_at: &str) {
496 + seed_deploy_fks(pool, tier, version).await;
497 + sqlx::query(
498 + "INSERT INTO deploys (version, tier, node, started_at, outcome)
499 + VALUES (?, ?, 'n1', ?, 'in_progress')",
500 + )
Lines truncated
M sando/daemon/src/routes/mod.rs +14 -2
@@ -367,13 +367,25 @@ async fn rollback(
367 367 // operator was escaping. NULL makes the second call fail loudly with "no
368 368 // previous_version to roll back to" — honest about the depth we keep.
369 369 // Stepping back further is the deploys-table walk we deliberately don't do.
370 + //
371 + // `advanced_at = now` even though burn_in_started_at is nulled: a rollback IS
372 + // a change to the deployed identity, so the startup reconcile must treat the
373 + // roll-off `deploys` row as predating this state — otherwise a cleanly
374 + // rolled-back tier would look like an unrecorded deploy (migration 009 /
375 + // [`crate::reconcile`]).
376 + let now = chrono::Utc::now().to_rfc3339();
370 377 sqlx::query(
371 - "UPDATE tier_state SET current_version = ?, previous_version = NULL, burn_in_started_at = NULL
378 + "UPDATE tier_state
379 + SET current_version = ?, previous_version = NULL,
380 + burn_in_started_at = NULL, advanced_at = ?
372 381 WHERE tier = ?",
373 382 )
374 383 .bind(&previous)
384 + .bind(&now)
375 385 .bind(&tier)
376 - .execute(&s.pool).await.map_err(crate::error::Error::Db)?;
386 + .execute(&s.pool)
387 + .await
388 + .map_err(crate::error::Error::Db)?;
377 389
378 390 // Full rollback succeeded on every node: the tier is consistent again.
379 391 clear_partial(&s, &tier).await;
M sando/daemon/src/routes/promotion.rs +24 -7
@@ -195,6 +195,23 @@ pub(super) async fn promote_inner(
195 195 .get(&node.name)
196 196 .cloned()
197 197 .unwrap_or_else(|| crate::state::build_executor(node));
198 + // Record the deploy as `in_progress` BEFORE touching the node, so a crash
199 + // between the node's symlink swap and this promote's `advance_tier` leaves
200 + // a durable trace. Without the pre-write, a SIGKILL after the swap but
201 + // before the row is inserted would land the new binary on the node with no
202 + // DB evidence at all, and the startup reconcile — which reads `deploys` —
203 + // could not see it. The row is finalized to ok/failed immediately below;
204 + // any row still `in_progress` at startup is an orphan the reconcile settles
205 + // and flags ([`crate::reconcile`]).
206 + let deploy_id: i64 = sqlx::query_scalar(
207 + "INSERT INTO deploys (version, tier, node, started_at, outcome, hotfix, reset_burn_in, build_id)
208 + VALUES (?, ?, ?, ?, 'in_progress', ?, ?, ?) RETURNING id",
209 + )
210 + .bind(&version).bind(&target.name).bind(&node.name)
211 + .bind(&started)
212 + .bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
213 + .bind(build_id)
214 + .fetch_one(&s.pool).await.map_err(crate::error::Error::Db)?;
198 215 let result = crate::deploy::deploy_node(
199 216 executor.as_ref(),
200 217 node,
@@ -215,15 +232,15 @@ pub(super) async fn promote_inner(
215 232 let outcome_json = serde_json::to_string(&outcome_obj)
216 233 .unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
217 234 sqlx::query(
218 - "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, outcome_json, hotfix, reset_burn_in, build_id)
219 - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
235 + "UPDATE deploys SET finished_at = ?, outcome = ?, outcome_json = ? WHERE id = ?",
220 236 )
221 - .bind(&version).bind(&target.name).bind(&node.name)
222 - .bind(&started).bind(&finished).bind(outcome_obj.status_str())
237 + .bind(&finished)
238 + .bind(outcome_obj.status_str())
223 239 .bind(&outcome_json)
224 - .bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
225 - .bind(build_id)
226 - .execute(&s.pool).await.map_err(crate::error::Error::Db)?;
240 + .bind(deploy_id)
241 + .execute(&s.pool)
242 + .await
243 + .map_err(crate::error::Error::Db)?;
227 244 if let Some(e) = err_for_propagation {
228 245 let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
229 246 unreachable!("err_for_propagation is Some iff status is Failed");
M sando/daemon/src/runs.rs +11 -2
@@ -91,24 +91,33 @@ pub async fn set_phase(pool: &SqlitePool, run_id: RunId, phase: Phase) -> Result
91 91 /// clock resets on every advance and the clock is what burn-in reads, the clock
92 92 /// always belongs to the build now current on the tier — this is what stops a
93 93 /// promote from crediting one build with another build's elapsed burn-in.
94 + ///
95 + /// `advanced_at = now` records when the tier's deployed identity last changed.
96 + /// It tracks the burn-in clock on an advance but, unlike it, is never nulled by
97 + /// rollback or reset_burn_in — so the startup reconcile can trust it as the
98 + /// instant to compare a `deploys` row against (wiki [[sando-overview]], migration
99 + /// 009 / [`crate::reconcile`]).
94 100 pub async fn advance_tier(
95 101 pool: &SqlitePool,
96 102 tier: &str,
97 103 version: &Version,
98 104 build_id: Option<i64>,
99 105 ) -> Result<(), sqlx::Error> {
106 + let now = Utc::now().to_rfc3339();
100 107 sqlx::query(
101 108 "UPDATE tier_state
102 109 SET previous_version = current_version,
103 110 current_version = ?,
104 111 previous_build_id = current_build_id,
105 112 current_build_id = ?,
106 - burn_in_started_at = ?
113 + burn_in_started_at = ?,
114 + advanced_at = ?
107 115 WHERE tier = ?",
108 116 )
109 117 .bind(version)
110 118 .bind(build_id)
111 - .bind(Utc::now().to_rfc3339())
119 + .bind(&now)
120 + .bind(&now)
112 121 .bind(tier)
113 122 .execute(pool)
114 123 .await?;