Skip to main content

max / makenotwork

23.9 KB · 587 lines History Blame Raw
1 //! Startup crash-recovery reconcile of `deploys` against `tier_state`.
2 //!
3 //! A promote lands a build on a tier's nodes and *then* advances tier_state:
4 //! [`crate::routes`] inserts the per-node `deploys` row (outcome `ok`) inside the
5 //! canary loop, and only after every node succeeds does [`crate::runs::advance_tier`]
6 //! record the new version. If sandod dies between those two writes — OOM, a
7 //! `systemctl restart`, a self-update SIGKILL — the node is serving the new
8 //! version but tier_state still names the old one, and `partial_reason` is NULL
9 //! because no failure path executed. `/state` then shows a clean tier on the
10 //! wrong version, and a later `/rollback` walks back to `previous_version`, which
11 //! is now *two* versions behind what the node actually runs (audit 2026-07-21,
12 //! sando-h5-crash-mid-promote).
13 //!
14 //! [`recover_unrecorded_deploys`] runs once at startup, next to
15 //! [`crate::runs::recover_orphaned_running`], and flags any tier whose latest
16 //! successful deploy postdates the last recorded state change yet names a
17 //! different version/build than tier_state holds. The flag is a `partial_reason`,
18 //! the same channel the promote/rollback failure paths use, so `/state` and the
19 //! TUI go loud and force the operator to reconcile by hand rather than trusting a
20 //! stale-green tier.
21 //!
22 //! The comparison instant is `tier_state.advanced_at` (migration 009), set on
23 //! every advance and rollback and never nulled — unlike `burn_in_started_at`,
24 //! which rollback and `reset_burn_in` clear while the deployed identity stays
25 //! real. A clean promote sets `advanced_at` *after* its `deploys` row, so its own
26 //! landing never postdates it; a rollback sets it after the roll-off row, so a
27 //! cleanly rolled-back tier is not mistaken for an unrecorded deploy.
28 //!
29 //! The `deploys` row is written `in_progress` *before* the node is touched and
30 //! finalized to ok/failed right after, so the only sliver still uncovered is a
31 //! crash between the pre-write and the swap — where the node was never changed,
32 //! so there is nothing to reconcile. Any row still `in_progress` at startup is an
33 //! orphaned in-flight deploy: it is flagged (the swap may or may not have landed)
34 //! and settled to a terminal `failed` so the audit trail carries no eternal
35 //! in-flight row.
36
37 use crate::domain::AppId;
38 use anyhow::Result;
39 use chrono::{DateTime, Utc};
40 use sqlx::{Row, SqlitePool};
41
42 /// One tier's most recent deploy row, whatever its outcome.
43 struct LastDeploy {
44 version: String,
45 build_id: Option<i64>,
46 /// `'ok' | 'failed' | 'in_progress'`.
47 outcome: String,
48 /// `finished_at` when the row is terminal, else `started_at` — the instant to
49 /// weigh against the tier's last state change. `started_at` is NOT NULL.
50 instant: String,
51 }
52
53 /// Reconcile `deploys` against `tier_state` after a crash. Flags any tier whose
54 /// most recent deploy postdates the last recorded state change but disagrees with
55 /// what tier_state holds (a promote that died before advancing) or is still
56 /// in-flight (a promote interrupted mid-node). Sets `partial_reason` — only when
57 /// it is currently NULL, so a more specific promote/rollback failure reason
58 /// stands — and settles orphaned `in_progress` rows. Returns how many tiers were
59 /// flagged. Run once at startup, before serving.
60 pub async fn recover_unrecorded_deploys(pool: &SqlitePool) -> Result<u64> {
61 // Every product's tiers, deliberately: startup reconciles the whole daemon,
62 // and a promote interrupted by the restart is no less interrupted for
63 // belonging to another product. Each tier is then compared only against its
64 // own product's deploy trail.
65 let tiers = sqlx::query(
66 "SELECT app, tier, current_version, current_build_id, advanced_at, partial_reason
67 FROM tier_state",
68 )
69 .fetch_all(pool)
70 .await?;
71
72 let mut flagged = 0u64;
73 for row in tiers {
74 let app: AppId = AppId::new(row.get::<String, _>("app"));
75 let tier: String = row.get("tier");
76 // A tier already flagged partial carries a more specific reason from the
77 // path that flagged it; don't overwrite it.
78 if row.get::<Option<String>, _>("partial_reason").is_some() {
79 continue;
80 }
81 let Some(landing) = latest_deploy(pool, &app, &tier).await? else {
82 continue;
83 };
84 let current_version: Option<String> = row.get("current_version");
85 let current_build_id: Option<i64> = row.get("current_build_id");
86 let advanced_at: Option<String> = row.get("advanced_at");
87
88 // Only a deploy that postdates the last recorded state change is suspect:
89 // a clean promote/rollback stamps `advanced_at` after its own rows.
90 if !postdates(&landing.instant, advanced_at.as_deref()) {
91 continue;
92 }
93 let reason = match landing.outcome.as_str() {
94 // Interrupted mid-node: the swap may or may not have landed, and no
95 // outcome was ever recorded. Force a look regardless of identity.
96 "in_progress" => Some(format!(
97 "crash-recovery: a deploy of {deployed} to this tier was in flight when the \
98 daemon last stopped and never recorded an outcome. The node's binary may or \
99 may not have been swapped. Verify what the tier is actually running, then \
100 re-promote or roll back deliberately.",
101 deployed = describe(&landing.version, landing.build_id),
102 )),
103 // Landed successfully but tier_state names something else — the promote
104 // died after the deploy but before `advance_tier`.
105 "ok" if !identity_matches(&landing, current_version.as_deref(), current_build_id) => {
106 Some(format!(
107 "crash-recovery: node(s) on this tier were deployed {deployed} at {when} but \
108 tier_state still records {recorded}. A promote most likely died after the \
109 deploy landed but before tier_state advanced. Verify what the tier is \
110 actually running, then re-promote or roll back deliberately — a blind \
111 /rollback would target the wrong version.",
112 deployed = describe(&landing.version, landing.build_id),
113 when = landing.instant,
114 recorded = current_version.as_deref().unwrap_or("no version"),
115 ))
116 }
117 // A terminal 'failed' row was handled by the deploy's own failure path
118 // (which sets its own partial_reason); nothing to add here.
119 _ => None,
120 };
121 let Some(reason) = reason else { continue };
122
123 // `AND partial_reason IS NULL` keeps the write a no-op if anything set a
124 // reason since the read above (startup is single-threaded, but the guard
125 // costs nothing and states the intent).
126 let res = sqlx::query(
127 "UPDATE tier_state SET partial_reason = ?
128 WHERE app = ? AND tier = ? AND partial_reason IS NULL",
129 )
130 .bind(&reason)
131 .bind(&app)
132 .bind(&tier)
133 .execute(pool)
134 .await?;
135 if res.rows_affected() > 0 {
136 tracing::error!(%app, tier = %tier, %reason, "startup reconcile: interrupted/unrecorded deploy on tier");
137 flagged += 1;
138 }
139 }
140
141 // Settle any orphaned in-flight deploy rows so the trail carries no eternal
142 // 'in_progress'. Done after the flag decisions above, which read that state.
143 // At startup (single-threaded, pre-serving) any 'in_progress' row is an orphan.
144 settle_orphaned_in_progress(pool).await?;
145 Ok(flagged)
146 }
147
148 /// The most recent deploy on `tier`, any outcome. Rows are ordered by `id`
149 /// (monotonic autoincrement), so this is the last one regardless of clock skew.
150 async fn latest_deploy(pool: &SqlitePool, app: &AppId, tier: &str) -> Result<Option<LastDeploy>> {
151 let row = sqlx::query(
152 "SELECT version, build_id, outcome, started_at, finished_at FROM deploys
153 WHERE app = ? AND tier = ? ORDER BY id DESC LIMIT 1",
154 )
155 .bind(app)
156 .bind(tier)
157 .fetch_optional(pool)
158 .await?;
159 Ok(row.map(|r| {
160 let started_at: String = r.get("started_at");
161 let finished_at: Option<String> = r.get("finished_at");
162 LastDeploy {
163 version: r.get("version"),
164 build_id: r.get("build_id"),
165 outcome: r.get("outcome"),
166 instant: finished_at.unwrap_or(started_at),
167 }
168 }))
169 }
170
171 /// Mark every `in_progress` deploy row terminal-failed with an interruption note,
172 /// across every product: the daemon that was interrupted was running all of
173 /// them, so an unscoped sweep is the accurate one here.
174 ///
175 /// mirroring [`crate::runs::recover_orphaned_running`] for build runs. Returns the
176 /// number of rows settled.
177 async fn settle_orphaned_in_progress(pool: &SqlitePool) -> Result<u64> {
178 let outcome =
179 crate::outcome::DeployOutcome::failed(crate::outcome::DeployFailureKind::Unclassified {
180 detail: "daemon restarted mid-deploy".into(),
181 });
182 let outcome_json = serde_json::to_string(&outcome)?;
183 let res = sqlx::query(
184 "UPDATE deploys SET outcome = 'failed', finished_at = ?, outcome_json = ?
185 WHERE outcome = 'in_progress'",
186 )
187 .bind(Utc::now().to_rfc3339())
188 .bind(&outcome_json)
189 .execute(pool)
190 .await?;
191 let n = res.rows_affected();
192 if n > 0 {
193 tracing::warn!(
194 settled = n,
195 "startup reconcile: settled orphaned in-flight deploy row(s)"
196 );
197 }
198 Ok(n)
199 }
200
201 /// Whether `deploy_finished_at` is later than the tier's last recorded state
202 /// change. A NULL `advanced_at` (state never advanced, yet a deploy landed) and
203 /// any unparseable timestamp both count as postdating — fail toward flagging, so
204 /// a corrupt or missing instant forces operator attention rather than hiding.
205 fn postdates(deploy_finished_at: &str, advanced_at: Option<&str>) -> bool {
206 let Some(advanced_at) = advanced_at else {
207 return true;
208 };
209 match (
210 DateTime::parse_from_rfc3339(deploy_finished_at),
211 DateTime::parse_from_rfc3339(advanced_at),
212 ) {
213 (Ok(d), Ok(a)) => d > a,
214 _ => true,
215 }
216 }
217
218 /// Whether the landed artifact is the one tier_state records. Build identity wins
219 /// when both sides carry it (a re-deploy of the same version from a different
220 /// build is still a mismatch); otherwise fall back to the version string. A NULL
221 /// `current_version` with a real landing is a mismatch.
222 fn identity_matches(
223 landing: &LastDeploy,
224 current_version: Option<&str>,
225 current_build_id: Option<i64>,
226 ) -> bool {
227 match (landing.build_id, current_build_id) {
228 (Some(landed), Some(current)) => landed == current,
229 _ => current_version == Some(landing.version.as_str()),
230 }
231 }
232
233 /// Human label for a landed artifact: the version, plus the build id when known.
234 fn describe(version: &str, build_id: Option<i64>) -> String {
235 match build_id {
236 Some(id) => format!("{version} (build {id})"),
237 None => version.to_string(),
238 }
239 }
240
241 #[cfg(test)]
242 mod tests {
243 use super::*;
244 use chrono::{Duration, Utc};
245 use sqlx::sqlite::SqlitePoolOptions;
246
247 async fn pool() -> SqlitePool {
248 let pool = SqlitePoolOptions::new()
249 .max_connections(1)
250 .connect("sqlite::memory:")
251 .await
252 .unwrap();
253 crate::db::migrate(&pool).await.unwrap();
254 pool
255 }
256
257 async fn seed_tier(pool: &SqlitePool, tier: &str) {
258 sqlx::query(
259 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, 0, 1, 'sequential')",
260 )
261 .bind(tier)
262 .execute(pool)
263 .await
264 .unwrap();
265 sqlx::query("INSERT INTO tier_state (tier) VALUES (?)")
266 .bind(tier)
267 .execute(pool)
268 .await
269 .unwrap();
270 }
271
272 /// Set the tier's recorded state the way an advance/rollback would.
273 async fn set_state(
274 pool: &SqlitePool,
275 tier: &str,
276 current_version: Option<&str>,
277 advanced_at: Option<&str>,
278 ) {
279 if let Some(v) = current_version {
280 sqlx::query(
281 "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
282 VALUES (?, 'sha', '2026-07-21T00:00:00Z', '/r/x')",
283 )
284 .bind(v)
285 .execute(pool)
286 .await
287 .unwrap();
288 }
289 sqlx::query("UPDATE tier_state SET current_version = ?, advanced_at = ? WHERE tier = ?")
290 .bind(current_version)
291 .bind(advanced_at)
292 .bind(tier)
293 .execute(pool)
294 .await
295 .unwrap();
296 }
297
298 /// FK parents `deploys` needs: a `versions` row and a `nodes` row. sqlx
299 /// enables `foreign_keys` by default, so these must exist first.
300 async fn seed_deploy_fks(pool: &SqlitePool, tier: &str, version: &str) {
301 sqlx::query(
302 "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
303 VALUES (?, 'sha', '2026-07-21T00:00:00Z', '/r/x')",
304 )
305 .bind(version)
306 .execute(pool)
307 .await
308 .unwrap();
309 sqlx::query(
310 "INSERT OR IGNORE INTO nodes (name, tier, ssh_target, release_root)
311 VALUES ('n1', ?, 'local', '/opt/mnw')",
312 )
313 .bind(tier)
314 .execute(pool)
315 .await
316 .unwrap();
317 }
318
319 /// Insert a `deploys` row, seeding its FK parents first.
320 async fn insert_deploy(
321 pool: &SqlitePool,
322 tier: &str,
323 version: &str,
324 outcome: &str,
325 finished_at: &str,
326 ) {
327 seed_deploy_fks(pool, tier, version).await;
328 sqlx::query(
329 "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome)
330 VALUES (?, ?, 'n1', ?, ?, ?)",
331 )
332 .bind(version)
333 .bind(tier)
334 .bind(finished_at)
335 .bind(finished_at)
336 .bind(outcome)
337 .execute(pool)
338 .await
339 .unwrap();
340 }
341
342 async fn partial_reason(pool: &SqlitePool, tier: &str) -> Option<String> {
343 sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = ?")
344 .bind(tier)
345 .fetch_one(pool)
346 .await
347 .unwrap()
348 }
349
350 #[tokio::test]
351 async fn clean_promote_is_not_flagged() {
352 // deploy lands, THEN tier_state advances: advanced_at postdates the row.
353 let pool = pool().await;
354 seed_tier(&pool, "b").await;
355 let deployed = Utc::now();
356 insert_deploy(&pool, "b", "0.10.15", "ok", &deployed.to_rfc3339()).await;
357 set_state(
358 &pool,
359 "b",
360 Some("0.10.15"),
361 Some(&(deployed + Duration::seconds(1)).to_rfc3339()),
362 )
363 .await;
364
365 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
366 assert!(partial_reason(&pool, "b").await.is_none());
367 }
368
369 #[tokio::test]
370 async fn crash_mid_promote_is_flagged() {
371 // tier_state still on the OLD version; an 'ok' deploy of the NEW version
372 // postdates the last advance — the exact SIGKILL-before-advance window.
373 let pool = pool().await;
374 seed_tier(&pool, "b").await;
375 let advanced = Utc::now() - Duration::hours(1);
376 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
377 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
378
379 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
380 let reason = partial_reason(&pool, "b").await.expect("flagged");
381 assert!(reason.contains("0.10.15"), "names the deployed version");
382 assert!(reason.contains("0.10.14"), "names the recorded version");
383 assert!(reason.contains("crash-recovery"));
384 }
385
386 #[tokio::test]
387 async fn rolled_back_tier_is_not_flagged() {
388 // Promote to 0.10.15 landed (ok deploy), then a rollback moved tier_state
389 // to 0.10.14 and stamped advanced_at AFTER the roll-off deploy. The stale
390 // deploy row must not read as unrecorded.
391 let pool = pool().await;
392 seed_tier(&pool, "b").await;
393 let deployed = Utc::now() - Duration::minutes(5);
394 insert_deploy(&pool, "b", "0.10.15", "ok", &deployed.to_rfc3339()).await;
395 set_state(&pool, "b", Some("0.10.14"), Some(&Utc::now().to_rfc3339())).await;
396
397 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
398 assert!(partial_reason(&pool, "b").await.is_none());
399 }
400
401 #[tokio::test]
402 async fn an_already_partial_tier_keeps_its_reason() {
403 let pool = pool().await;
404 seed_tier(&pool, "b").await;
405 let advanced = Utc::now() - Duration::hours(1);
406 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
407 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
408 sqlx::query(
409 "UPDATE tier_state SET partial_reason = 'canary rollback incomplete' WHERE tier = 'b'",
410 )
411 .execute(&pool)
412 .await
413 .unwrap();
414
415 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
416 assert_eq!(
417 partial_reason(&pool, "b").await.as_deref(),
418 Some("canary rollback incomplete"),
419 "the specific reason is not clobbered",
420 );
421 }
422
423 #[tokio::test]
424 async fn a_failed_deploy_does_not_flag() {
425 // The latest deploy is a failure (its own path handles partial state);
426 // there is no successful landing that tier_state failed to record.
427 let pool = pool().await;
428 seed_tier(&pool, "b").await;
429 let advanced = Utc::now() - Duration::hours(1);
430 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
431 insert_deploy(&pool, "b", "0.10.15", "failed", &Utc::now().to_rfc3339()).await;
432
433 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
434 assert!(partial_reason(&pool, "b").await.is_none());
435 }
436
437 #[tokio::test]
438 async fn a_tier_with_no_deploys_is_skipped() {
439 let pool = pool().await;
440 seed_tier(&pool, "b").await;
441 set_state(&pool, "b", None, None).await;
442 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
443 }
444
445 #[tokio::test]
446 async fn crash_on_first_ever_promote_is_flagged() {
447 // Never advanced (advanced_at NULL, current_version NULL) but a node was
448 // deployed: prod is running something tier_state has no record of.
449 let pool = pool().await;
450 seed_tier(&pool, "b").await;
451 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
452
453 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
454 assert!(
455 partial_reason(&pool, "b")
456 .await
457 .unwrap()
458 .contains("0.10.15")
459 );
460 }
461
462 #[tokio::test]
463 async fn build_id_mismatch_is_flagged_even_when_version_matches() {
464 // Same version string, different build — a re-deploy landed a build the
465 // tier never recorded. Build identity is the tie-breaker.
466 let pool = pool().await;
467 seed_tier(&pool, "b").await;
468 seed_deploy_fks(&pool, "b", "0.10.15").await;
469 // build_runs 7 and 9, referenced by tier_state.current_build_id and
470 // deploys.build_id respectively.
471 for id in [7, 9] {
472 sqlx::query(
473 "INSERT INTO build_runs (id, sha, phase, result, started_at)
474 VALUES (?, 'sha', 'done', 'passed', '2026-07-21T00:00:00Z')",
475 )
476 .bind(id)
477 .execute(&pool)
478 .await
479 .unwrap();
480 }
481 let advanced = Utc::now() - Duration::hours(1);
482 // tier_state: version 0.10.15, build 7.
483 sqlx::query(
484 "UPDATE tier_state SET current_version = '0.10.15', current_build_id = 7, advanced_at = ? WHERE tier = 'b'",
485 )
486 .bind(advanced.to_rfc3339())
487 .execute(&pool)
488 .await
489 .unwrap();
490 // deploy: same version 0.10.15 but build 9, postdating the advance.
491 sqlx::query(
492 "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, build_id)
493 VALUES ('0.10.15', 'b', 'n1', ?, ?, 'ok', 9)",
494 )
495 .bind(Utc::now().to_rfc3339())
496 .bind(Utc::now().to_rfc3339())
497 .execute(&pool)
498 .await
499 .unwrap();
500
501 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
502 }
503
504 /// Insert an in-flight deploy row: `outcome = 'in_progress'`, `finished_at`
505 /// NULL, `started_at` set — the shape the promote loop writes before touching
506 /// a node.
507 async fn insert_in_progress(pool: &SqlitePool, tier: &str, version: &str, started_at: &str) {
508 seed_deploy_fks(pool, tier, version).await;
509 sqlx::query(
510 "INSERT INTO deploys (version, tier, node, started_at, outcome)
511 VALUES (?, ?, 'n1', ?, 'in_progress')",
512 )
513 .bind(version)
514 .bind(tier)
515 .bind(started_at)
516 .execute(pool)
517 .await
518 .unwrap();
519 }
520
521 async fn outcome_of_latest(pool: &SqlitePool, tier: &str) -> String {
522 sqlx::query_scalar("SELECT outcome FROM deploys WHERE tier = ? ORDER BY id DESC LIMIT 1")
523 .bind(tier)
524 .fetch_one(pool)
525 .await
526 .unwrap()
527 }
528
529 #[tokio::test]
530 async fn orphaned_in_progress_deploy_is_flagged_and_settled() {
531 // A deploy was in flight (row written before the swap) when the daemon
532 // died: flag the tier and settle the row so it isn't in_progress forever.
533 let pool = pool().await;
534 seed_tier(&pool, "b").await;
535 let advanced = Utc::now() - Duration::hours(1);
536 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
537 insert_in_progress(&pool, "b", "0.10.15", &Utc::now().to_rfc3339()).await;
538
539 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
540 let reason = partial_reason(&pool, "b").await.expect("flagged");
541 assert!(
542 reason.contains("in flight"),
543 "names the interruption: {reason}"
544 );
545 assert!(reason.contains("0.10.15"));
546 // The orphan row is settled to a terminal outcome.
547 assert_eq!(outcome_of_latest(&pool, "b").await, "failed");
548 }
549
550 #[tokio::test]
551 async fn in_progress_row_predating_last_advance_is_only_settled_not_flagged() {
552 // An old in_progress row from before the tier's last clean advance is
553 // stale bookkeeping, not an unrecorded deploy: settle it, don't flag.
554 let pool = pool().await;
555 seed_tier(&pool, "b").await;
556 let old = Utc::now() - Duration::hours(2);
557 insert_in_progress(&pool, "b", "0.10.13", &old.to_rfc3339()).await;
558 // A later clean promote advanced the tier past that row.
559 let deployed = Utc::now() - Duration::minutes(10);
560 insert_deploy(&pool, "b", "0.10.14", "ok", &deployed.to_rfc3339()).await;
561 set_state(&pool, "b", Some("0.10.14"), Some(&Utc::now().to_rfc3339())).await;
562
563 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
564 assert!(partial_reason(&pool, "b").await.is_none());
565 // But the stale in_progress row is still settled.
566 let leftover: i64 =
567 sqlx::query_scalar("SELECT COUNT(*) FROM deploys WHERE outcome = 'in_progress'")
568 .fetch_one(&pool)
569 .await
570 .unwrap();
571 assert_eq!(leftover, 0, "no in_progress rows survive");
572 }
573
574 #[tokio::test]
575 async fn is_idempotent_after_flagging() {
576 let pool = pool().await;
577 seed_tier(&pool, "b").await;
578 let advanced = Utc::now() - Duration::hours(1);
579 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
580 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
581
582 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
583 // Second pass: the reason is already set, so nothing new is flagged.
584 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
585 }
586 }
587