Skip to main content

max / makenotwork

23.8 KB · 586 lines History Blame Raw
1 //! Startup crash-recovery reconcile of `deploys` against `tier_state`.
2 //!
3 //! A promote lands a build on a tier's nodes and *then* advances tier_state:
4 //! [`crate::routes`] inserts the per-node `deploys` row (outcome `ok`) inside the
5 //! canary loop, and only after every node succeeds does [`crate::runs::advance_tier`]
6 //! record the new version. If sandod dies between those two writes (OOM, a
7 //! `systemctl restart`, a self-update SIGKILL) the node is serving the new
8 //! version but tier_state still names the old one, and `partial_reason` is NULL
9 //! because no failure path executed. `/state` then shows a clean tier on the
10 //! wrong version, and a later `/rollback` walks back to `previous_version`, which
11 //! is now *two* versions behind what the node actually runs.
12 //!
13 //! [`recover_unrecorded_deploys`] runs once at startup, next to
14 //! [`crate::runs::recover_orphaned_running`], and flags any tier whose latest
15 //! successful deploy postdates the last recorded state change yet names a
16 //! different version/build than tier_state holds. The flag is a `partial_reason`,
17 //! the same channel the promote/rollback failure paths use, so `/state` and the
18 //! TUI go loud and force the operator to reconcile by hand rather than trusting a
19 //! stale-green tier.
20 //!
21 //! The comparison instant is `tier_state.advanced_at` (migration 009), set on
22 //! every advance and rollback and never nulled, unlike `burn_in_started_at`,
23 //! which rollback and `reset_burn_in` clear while the deployed identity stays
24 //! real. A clean promote sets `advanced_at` *after* its `deploys` row, so its own
25 //! landing never postdates it; a rollback sets it after the roll-off row, so a
26 //! cleanly rolled-back tier is not mistaken for an unrecorded deploy.
27 //!
28 //! The `deploys` row is written `in_progress` *before* the node is touched and
29 //! finalized to ok/failed right after, so the only sliver still uncovered is a
30 //! crash between the pre-write and the swap — where the node was never changed,
31 //! so there is nothing to reconcile. Any row still `in_progress` at startup is an
32 //! orphaned in-flight deploy: it is flagged (the swap may or may not have landed)
33 //! and settled to a terminal `failed` so the audit trail carries no eternal
34 //! in-flight row.
35
36 use crate::domain::AppId;
37 use anyhow::Result;
38 use chrono::{DateTime, Utc};
39 use sqlx::{Row, SqlitePool};
40
41 /// One tier's most recent deploy row, whatever its outcome.
42 struct LastDeploy {
43 version: String,
44 build_id: Option<i64>,
45 /// `'ok' | 'failed' | 'in_progress'`.
46 outcome: String,
47 /// `finished_at` when the row is terminal, else `started_at` — the instant to
48 /// weigh against the tier's last state change. `started_at` is NOT NULL.
49 instant: String,
50 }
51
52 /// Reconcile `deploys` against `tier_state` after a crash. Flags any tier whose
53 /// most recent deploy postdates the last recorded state change but disagrees with
54 /// what tier_state holds (a promote that died before advancing) or is still
55 /// in-flight (a promote interrupted mid-node). Sets `partial_reason` — only when
56 /// it is currently NULL, so a more specific promote/rollback failure reason
57 /// stands — and settles orphaned `in_progress` rows. Returns how many tiers were
58 /// flagged. Run once at startup, before serving.
59 pub async fn recover_unrecorded_deploys(pool: &SqlitePool) -> Result<u64> {
60 // Every product's tiers, deliberately: startup reconciles the whole daemon,
61 // and a promote interrupted by the restart is no less interrupted for
62 // belonging to another product. Each tier is then compared only against its
63 // own product's deploy trail.
64 let tiers = sqlx::query(
65 "SELECT app, tier, current_version, current_build_id, advanced_at, partial_reason
66 FROM tier_state",
67 )
68 .fetch_all(pool)
69 .await?;
70
71 let mut flagged = 0u64;
72 for row in tiers {
73 let app: AppId = AppId::new(row.get::<String, _>("app"));
74 let tier: String = row.get("tier");
75 // A tier already flagged partial carries a more specific reason from the
76 // path that flagged it; don't overwrite it.
77 if row.get::<Option<String>, _>("partial_reason").is_some() {
78 continue;
79 }
80 let Some(landing) = latest_deploy(pool, &app, &tier).await? else {
81 continue;
82 };
83 let current_version: Option<String> = row.get("current_version");
84 let current_build_id: Option<i64> = row.get("current_build_id");
85 let advanced_at: Option<String> = row.get("advanced_at");
86
87 // Only a deploy that postdates the last recorded state change is suspect:
88 // a clean promote/rollback stamps `advanced_at` after its own rows.
89 if !postdates(&landing.instant, advanced_at.as_deref()) {
90 continue;
91 }
92 let reason = match landing.outcome.as_str() {
93 // Interrupted mid-node: the swap may or may not have landed, and no
94 // outcome was ever recorded. Force a look regardless of identity.
95 "in_progress" => Some(format!(
96 "crash-recovery: a deploy of {deployed} to this tier was in flight when the \
97 daemon last stopped and never recorded an outcome. The node's binary may or \
98 may not have been swapped. Verify what the tier is actually running, then \
99 re-promote or roll back deliberately.",
100 deployed = describe(&landing.version, landing.build_id),
101 )),
102 // Landed successfully but tier_state names something else — the promote
103 // died after the deploy but before `advance_tier`.
104 "ok" if !identity_matches(&landing, current_version.as_deref(), current_build_id) => {
105 Some(format!(
106 "crash-recovery: node(s) on this tier were deployed {deployed} at {when} but \
107 tier_state still records {recorded}. A promote most likely died after the \
108 deploy landed but before tier_state advanced. Verify what the tier is \
109 actually running, then re-promote or roll back deliberately — a blind \
110 /rollback would target the wrong version.",
111 deployed = describe(&landing.version, landing.build_id),
112 when = landing.instant,
113 recorded = current_version.as_deref().unwrap_or("no version"),
114 ))
115 }
116 // A terminal 'failed' row was handled by the deploy's own failure path
117 // (which sets its own partial_reason); nothing to add here.
118 _ => None,
119 };
120 let Some(reason) = reason else { continue };
121
122 // `AND partial_reason IS NULL` keeps the write a no-op if anything set a
123 // reason since the read above (startup is single-threaded, but the guard
124 // costs nothing and states the intent).
125 let res = sqlx::query(
126 "UPDATE tier_state SET partial_reason = ?
127 WHERE app = ? AND tier = ? AND partial_reason IS NULL",
128 )
129 .bind(&reason)
130 .bind(&app)
131 .bind(&tier)
132 .execute(pool)
133 .await?;
134 if res.rows_affected() > 0 {
135 tracing::error!(%app, tier = %tier, %reason, "startup reconcile: interrupted/unrecorded deploy on tier");
136 flagged += 1;
137 }
138 }
139
140 // Settle any orphaned in-flight deploy rows so the trail carries no eternal
141 // 'in_progress'. Done after the flag decisions above, which read that state.
142 // At startup (single-threaded, pre-serving) any 'in_progress' row is an orphan.
143 settle_orphaned_in_progress(pool).await?;
144 Ok(flagged)
145 }
146
147 /// The most recent deploy on `tier`, any outcome. Rows are ordered by `id`
148 /// (monotonic autoincrement), so this is the last one regardless of clock skew.
149 async fn latest_deploy(pool: &SqlitePool, app: &AppId, tier: &str) -> Result<Option<LastDeploy>> {
150 let row = sqlx::query(
151 "SELECT version, build_id, outcome, started_at, finished_at FROM deploys
152 WHERE app = ? AND tier = ? ORDER BY id DESC LIMIT 1",
153 )
154 .bind(app)
155 .bind(tier)
156 .fetch_optional(pool)
157 .await?;
158 Ok(row.map(|r| {
159 let started_at: String = r.get("started_at");
160 let finished_at: Option<String> = r.get("finished_at");
161 LastDeploy {
162 version: r.get("version"),
163 build_id: r.get("build_id"),
164 outcome: r.get("outcome"),
165 instant: finished_at.unwrap_or(started_at),
166 }
167 }))
168 }
169
170 /// Mark every `in_progress` deploy row terminal-failed with an interruption note,
171 /// across every product: the daemon that was interrupted was running all of
172 /// them, so an unscoped sweep is the accurate one here.
173 ///
174 /// mirroring [`crate::runs::recover_orphaned_running`] for build runs. Returns the
175 /// number of rows settled.
176 async fn settle_orphaned_in_progress(pool: &SqlitePool) -> Result<u64> {
177 let outcome =
178 crate::outcome::DeployOutcome::failed(crate::outcome::DeployFailureKind::Unclassified {
179 detail: "daemon restarted mid-deploy".into(),
180 });
181 let outcome_json = serde_json::to_string(&outcome)?;
182 let res = sqlx::query(
183 "UPDATE deploys SET outcome = 'failed', finished_at = ?, outcome_json = ?
184 WHERE outcome = 'in_progress'",
185 )
186 .bind(Utc::now().to_rfc3339())
187 .bind(&outcome_json)
188 .execute(pool)
189 .await?;
190 let n = res.rows_affected();
191 if n > 0 {
192 tracing::warn!(
193 settled = n,
194 "startup reconcile: settled orphaned in-flight deploy row(s)"
195 );
196 }
197 Ok(n)
198 }
199
200 /// Whether `deploy_finished_at` is later than the tier's last recorded state
201 /// change. A NULL `advanced_at` (state never advanced, yet a deploy landed) and
202 /// any unparseable timestamp both count as postdating — fail toward flagging, so
203 /// a corrupt or missing instant forces operator attention rather than hiding.
204 fn postdates(deploy_finished_at: &str, advanced_at: Option<&str>) -> bool {
205 let Some(advanced_at) = advanced_at else {
206 return true;
207 };
208 match (
209 DateTime::parse_from_rfc3339(deploy_finished_at),
210 DateTime::parse_from_rfc3339(advanced_at),
211 ) {
212 (Ok(d), Ok(a)) => d > a,
213 _ => true,
214 }
215 }
216
217 /// Whether the landed artifact is the one tier_state records. Build identity wins
218 /// when both sides carry it (a re-deploy of the same version from a different
219 /// build is still a mismatch); otherwise fall back to the version string. A NULL
220 /// `current_version` with a real landing is a mismatch.
221 fn identity_matches(
222 landing: &LastDeploy,
223 current_version: Option<&str>,
224 current_build_id: Option<i64>,
225 ) -> bool {
226 match (landing.build_id, current_build_id) {
227 (Some(landed), Some(current)) => landed == current,
228 _ => current_version == Some(landing.version.as_str()),
229 }
230 }
231
232 /// Human label for a landed artifact: the version, plus the build id when known.
233 fn describe(version: &str, build_id: Option<i64>) -> String {
234 match build_id {
235 Some(id) => format!("{version} (build {id})"),
236 None => version.to_string(),
237 }
238 }
239
240 #[cfg(test)]
241 mod tests {
242 use super::*;
243 use chrono::{Duration, Utc};
244 use sqlx::sqlite::SqlitePoolOptions;
245
246 async fn pool() -> SqlitePool {
247 let pool = SqlitePoolOptions::new()
248 .max_connections(1)
249 .connect("sqlite::memory:")
250 .await
251 .unwrap();
252 crate::db::migrate(&pool).await.unwrap();
253 pool
254 }
255
256 async fn seed_tier(pool: &SqlitePool, tier: &str) {
257 sqlx::query(
258 "INSERT INTO tiers (name, ord, provisioned, canary) VALUES (?, 0, 1, 'sequential')",
259 )
260 .bind(tier)
261 .execute(pool)
262 .await
263 .unwrap();
264 sqlx::query("INSERT INTO tier_state (tier) VALUES (?)")
265 .bind(tier)
266 .execute(pool)
267 .await
268 .unwrap();
269 }
270
271 /// Set the tier's recorded state the way an advance/rollback would.
272 async fn set_state(
273 pool: &SqlitePool,
274 tier: &str,
275 current_version: Option<&str>,
276 advanced_at: Option<&str>,
277 ) {
278 if let Some(v) = current_version {
279 sqlx::query(
280 "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
281 VALUES (?, 'sha', '2026-07-21T00:00:00Z', '/r/x')",
282 )
283 .bind(v)
284 .execute(pool)
285 .await
286 .unwrap();
287 }
288 sqlx::query("UPDATE tier_state SET current_version = ?, advanced_at = ? WHERE tier = ?")
289 .bind(current_version)
290 .bind(advanced_at)
291 .bind(tier)
292 .execute(pool)
293 .await
294 .unwrap();
295 }
296
297 /// FK parents `deploys` needs: a `versions` row and a `nodes` row. sqlx
298 /// enables `foreign_keys` by default, so these must exist first.
299 async fn seed_deploy_fks(pool: &SqlitePool, tier: &str, version: &str) {
300 sqlx::query(
301 "INSERT OR IGNORE INTO versions (version, git_sha, built_at, artifact_path)
302 VALUES (?, 'sha', '2026-07-21T00:00:00Z', '/r/x')",
303 )
304 .bind(version)
305 .execute(pool)
306 .await
307 .unwrap();
308 sqlx::query(
309 "INSERT OR IGNORE INTO nodes (name, tier, ssh_target, release_root)
310 VALUES ('n1', ?, 'local', '/opt/mnw')",
311 )
312 .bind(tier)
313 .execute(pool)
314 .await
315 .unwrap();
316 }
317
318 /// Insert a `deploys` row, seeding its FK parents first.
319 async fn insert_deploy(
320 pool: &SqlitePool,
321 tier: &str,
322 version: &str,
323 outcome: &str,
324 finished_at: &str,
325 ) {
326 seed_deploy_fks(pool, tier, version).await;
327 sqlx::query(
328 "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome)
329 VALUES (?, ?, 'n1', ?, ?, ?)",
330 )
331 .bind(version)
332 .bind(tier)
333 .bind(finished_at)
334 .bind(finished_at)
335 .bind(outcome)
336 .execute(pool)
337 .await
338 .unwrap();
339 }
340
341 async fn partial_reason(pool: &SqlitePool, tier: &str) -> Option<String> {
342 sqlx::query_scalar("SELECT partial_reason FROM tier_state WHERE tier = ?")
343 .bind(tier)
344 .fetch_one(pool)
345 .await
346 .unwrap()
347 }
348
349 #[tokio::test]
350 async fn clean_promote_is_not_flagged() {
351 // deploy lands, THEN tier_state advances: advanced_at postdates the row.
352 let pool = pool().await;
353 seed_tier(&pool, "b").await;
354 let deployed = Utc::now();
355 insert_deploy(&pool, "b", "0.10.15", "ok", &deployed.to_rfc3339()).await;
356 set_state(
357 &pool,
358 "b",
359 Some("0.10.15"),
360 Some(&(deployed + Duration::seconds(1)).to_rfc3339()),
361 )
362 .await;
363
364 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
365 assert!(partial_reason(&pool, "b").await.is_none());
366 }
367
368 #[tokio::test]
369 async fn crash_mid_promote_is_flagged() {
370 // tier_state still on the OLD version; an 'ok' deploy of the NEW version
371 // postdates the last advance — the exact SIGKILL-before-advance window.
372 let pool = pool().await;
373 seed_tier(&pool, "b").await;
374 let advanced = Utc::now() - Duration::hours(1);
375 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
376 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
377
378 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
379 let reason = partial_reason(&pool, "b").await.expect("flagged");
380 assert!(reason.contains("0.10.15"), "names the deployed version");
381 assert!(reason.contains("0.10.14"), "names the recorded version");
382 assert!(reason.contains("crash-recovery"));
383 }
384
385 #[tokio::test]
386 async fn rolled_back_tier_is_not_flagged() {
387 // Promote to 0.10.15 landed (ok deploy), then a rollback moved tier_state
388 // to 0.10.14 and stamped advanced_at AFTER the roll-off deploy. The stale
389 // deploy row must not read as unrecorded.
390 let pool = pool().await;
391 seed_tier(&pool, "b").await;
392 let deployed = Utc::now() - Duration::minutes(5);
393 insert_deploy(&pool, "b", "0.10.15", "ok", &deployed.to_rfc3339()).await;
394 set_state(&pool, "b", Some("0.10.14"), Some(&Utc::now().to_rfc3339())).await;
395
396 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
397 assert!(partial_reason(&pool, "b").await.is_none());
398 }
399
400 #[tokio::test]
401 async fn an_already_partial_tier_keeps_its_reason() {
402 let pool = pool().await;
403 seed_tier(&pool, "b").await;
404 let advanced = Utc::now() - Duration::hours(1);
405 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
406 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
407 sqlx::query(
408 "UPDATE tier_state SET partial_reason = 'canary rollback incomplete' WHERE tier = 'b'",
409 )
410 .execute(&pool)
411 .await
412 .unwrap();
413
414 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
415 assert_eq!(
416 partial_reason(&pool, "b").await.as_deref(),
417 Some("canary rollback incomplete"),
418 "the specific reason is not clobbered",
419 );
420 }
421
422 #[tokio::test]
423 async fn a_failed_deploy_does_not_flag() {
424 // The latest deploy is a failure (its own path handles partial state);
425 // there is no successful landing that tier_state failed to record.
426 let pool = pool().await;
427 seed_tier(&pool, "b").await;
428 let advanced = Utc::now() - Duration::hours(1);
429 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
430 insert_deploy(&pool, "b", "0.10.15", "failed", &Utc::now().to_rfc3339()).await;
431
432 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
433 assert!(partial_reason(&pool, "b").await.is_none());
434 }
435
436 #[tokio::test]
437 async fn a_tier_with_no_deploys_is_skipped() {
438 let pool = pool().await;
439 seed_tier(&pool, "b").await;
440 set_state(&pool, "b", None, None).await;
441 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
442 }
443
444 #[tokio::test]
445 async fn crash_on_first_ever_promote_is_flagged() {
446 // Never advanced (advanced_at NULL, current_version NULL) but a node was
447 // deployed: prod is running something tier_state has no record of.
448 let pool = pool().await;
449 seed_tier(&pool, "b").await;
450 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
451
452 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
453 assert!(
454 partial_reason(&pool, "b")
455 .await
456 .unwrap()
457 .contains("0.10.15")
458 );
459 }
460
461 #[tokio::test]
462 async fn build_id_mismatch_is_flagged_even_when_version_matches() {
463 // Same version string, different build — a re-deploy landed a build the
464 // tier never recorded. Build identity is the tie-breaker.
465 let pool = pool().await;
466 seed_tier(&pool, "b").await;
467 seed_deploy_fks(&pool, "b", "0.10.15").await;
468 // build_runs 7 and 9, referenced by tier_state.current_build_id and
469 // deploys.build_id respectively.
470 for id in [7, 9] {
471 sqlx::query(
472 "INSERT INTO build_runs (id, sha, phase, result, started_at)
473 VALUES (?, 'sha', 'done', 'passed', '2026-07-21T00:00:00Z')",
474 )
475 .bind(id)
476 .execute(&pool)
477 .await
478 .unwrap();
479 }
480 let advanced = Utc::now() - Duration::hours(1);
481 // tier_state: version 0.10.15, build 7.
482 sqlx::query(
483 "UPDATE tier_state SET current_version = '0.10.15', current_build_id = 7, advanced_at = ? WHERE tier = 'b'",
484 )
485 .bind(advanced.to_rfc3339())
486 .execute(&pool)
487 .await
488 .unwrap();
489 // deploy: same version 0.10.15 but build 9, postdating the advance.
490 sqlx::query(
491 "INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, build_id)
492 VALUES ('0.10.15', 'b', 'n1', ?, ?, 'ok', 9)",
493 )
494 .bind(Utc::now().to_rfc3339())
495 .bind(Utc::now().to_rfc3339())
496 .execute(&pool)
497 .await
498 .unwrap();
499
500 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
501 }
502
503 /// Insert an in-flight deploy row: `outcome = 'in_progress'`, `finished_at`
504 /// NULL, `started_at` set — the shape the promote loop writes before touching
505 /// a node.
506 async fn insert_in_progress(pool: &SqlitePool, tier: &str, version: &str, started_at: &str) {
507 seed_deploy_fks(pool, tier, version).await;
508 sqlx::query(
509 "INSERT INTO deploys (version, tier, node, started_at, outcome)
510 VALUES (?, ?, 'n1', ?, 'in_progress')",
511 )
512 .bind(version)
513 .bind(tier)
514 .bind(started_at)
515 .execute(pool)
516 .await
517 .unwrap();
518 }
519
520 async fn outcome_of_latest(pool: &SqlitePool, tier: &str) -> String {
521 sqlx::query_scalar("SELECT outcome FROM deploys WHERE tier = ? ORDER BY id DESC LIMIT 1")
522 .bind(tier)
523 .fetch_one(pool)
524 .await
525 .unwrap()
526 }
527
528 #[tokio::test]
529 async fn orphaned_in_progress_deploy_is_flagged_and_settled() {
530 // A deploy was in flight (row written before the swap) when the daemon
531 // died: flag the tier and settle the row so it isn't in_progress forever.
532 let pool = pool().await;
533 seed_tier(&pool, "b").await;
534 let advanced = Utc::now() - Duration::hours(1);
535 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
536 insert_in_progress(&pool, "b", "0.10.15", &Utc::now().to_rfc3339()).await;
537
538 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
539 let reason = partial_reason(&pool, "b").await.expect("flagged");
540 assert!(
541 reason.contains("in flight"),
542 "names the interruption: {reason}"
543 );
544 assert!(reason.contains("0.10.15"));
545 // The orphan row is settled to a terminal outcome.
546 assert_eq!(outcome_of_latest(&pool, "b").await, "failed");
547 }
548
549 #[tokio::test]
550 async fn in_progress_row_predating_last_advance_is_only_settled_not_flagged() {
551 // An old in_progress row from before the tier's last clean advance is
552 // stale bookkeeping, not an unrecorded deploy: settle it, don't flag.
553 let pool = pool().await;
554 seed_tier(&pool, "b").await;
555 let old = Utc::now() - Duration::hours(2);
556 insert_in_progress(&pool, "b", "0.10.13", &old.to_rfc3339()).await;
557 // A later clean promote advanced the tier past that row.
558 let deployed = Utc::now() - Duration::minutes(10);
559 insert_deploy(&pool, "b", "0.10.14", "ok", &deployed.to_rfc3339()).await;
560 set_state(&pool, "b", Some("0.10.14"), Some(&Utc::now().to_rfc3339())).await;
561
562 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
563 assert!(partial_reason(&pool, "b").await.is_none());
564 // But the stale in_progress row is still settled.
565 let leftover: i64 =
566 sqlx::query_scalar("SELECT COUNT(*) FROM deploys WHERE outcome = 'in_progress'")
567 .fetch_one(&pool)
568 .await
569 .unwrap();
570 assert_eq!(leftover, 0, "no in_progress rows survive");
571 }
572
573 #[tokio::test]
574 async fn is_idempotent_after_flagging() {
575 let pool = pool().await;
576 seed_tier(&pool, "b").await;
577 let advanced = Utc::now() - Duration::hours(1);
578 set_state(&pool, "b", Some("0.10.14"), Some(&advanced.to_rfc3339())).await;
579 insert_deploy(&pool, "b", "0.10.15", "ok", &Utc::now().to_rfc3339()).await;
580
581 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 1);
582 // Second pass: the reason is already set, so nothing new is flagged.
583 assert_eq!(recover_unrecorded_deploys(&pool).await.unwrap(), 0);
584 }
585 }
586