Skip to main content

max / makenotwork

22.1 KB · 598 lines History Blame Raw
1 //! Which release directories the deployed state still points at.
2 //!
3 //! <!-- wiki: release-artifact-identity -->
4 //!
5 //! Release dirs are content-addressed (`releases/<digest16>`) and trimmed after
6 //! every publish. Trimming used to be a count alone — keep the 5 newest by
7 //! mtime — which cannot express "this one is still in production". Three
8 //! rebuilds of a single version fill three of the five slots, and the artifacts
9 //! a tier is running and would roll back to fall off the end. That happened
10 //! twice (2026-08-19, 2026-08-25); the second time neither surviving directory
11 //! held a runnable binary and prod had nothing to roll back to.
12 //!
13 //! So the set below is computed first and set aside, and the count applies only
14 //! to what is left. The count is a floor on how much history to keep, not a
15 //! ceiling on what may be retained.
16 //!
17 //! # What counts as referenced
18 //!
19 //! Everything promote and rollback resolve through, because a dir that is
20 //! unreachable to them is exactly the one whose absence is discovered by an
21 //! rsync failing mid-promote:
22 //!
23 //! 1. `tier_state.current_build_id` / `previous_build_id` -> `build_runs.staged_path`.
24 //! The identity path (migration 008), and the tier's own answer to what it is
25 //! running.
26 //! 2. The newest green `build_runs` row per (version, platform) for every version
27 //! a tier names. This is what [`crate::routes::promotion`] resolves a rollback
28 //! to, and on a two-architecture product it is a different row per node.
29 //! 3. `versions.artifact_path` for those versions. The pre-identity path, still
30 //! the first thing a canary rollback reads.
31 //!
32 //! The three overlap heavily and are unioned rather than ranked: being reachable
33 //! by any of them is enough to make a directory load-bearing.
34
35 use crate::domain::AppId;
36 use anyhow::Result;
37 use sqlx::{Row, SqlitePool};
38 use std::collections::{BTreeSet, HashSet};
39 use std::path::{Path, PathBuf};
40
41 /// The release directories that must survive a gc, by the name gc matches on.
42 ///
43 /// A newtype rather than a bare `HashSet<String>` so the thing being passed
44 /// through publish and into gc says what it is at every hop, and so a caller
45 /// cannot hand it a set of paths, versions, or digests by accident.
46 #[derive(Debug, Clone, Default, PartialEq, Eq)]
47 pub struct PinnedReleases(HashSet<String>);
48
49 impl PinnedReleases {
50 /// Nothing to protect. What a caller with no deployed state passes — a unit
51 /// test, or a store that has never served anything.
52 pub fn none() -> Self {
53 Self::default()
54 }
55
56 pub fn contains(&self, dir_name: &str) -> bool {
57 self.0.contains(dir_name)
58 }
59
60 pub fn len(&self) -> usize {
61 self.0.len()
62 }
63
64 pub fn is_empty(&self) -> bool {
65 self.0.is_empty()
66 }
67
68 /// The pinned names in a stable order.
69 ///
70 /// Sorted rather than in hash order because the remote gc embeds these in a
71 /// shell script: an unordered set would rewrite the script text on every
72 /// deploy with no change of meaning, and a script that differs run to run is
73 /// one nobody can diff against the last one that worked.
74 pub fn sorted_names(&self) -> Vec<&str> {
75 let mut names: Vec<&str> = self.0.iter().map(String::as_str).collect();
76 names.sort_unstable();
77 names
78 }
79 }
80
81 impl FromIterator<String> for PinnedReleases {
82 fn from_iter<T: IntoIterator<Item = String>>(iter: T) -> Self {
83 Self(iter.into_iter().collect())
84 }
85 }
86
87 /// Which of a tier's two artifacts a reference is.
88 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
89 pub enum Role {
90 /// What the tier is running.
91 Current,
92 /// The one step of rollback history `tier_state` keeps.
93 Previous,
94 }
95
96 impl Role {
97 pub const fn as_str(self) -> &'static str {
98 match self {
99 Self::Current => "current",
100 Self::Previous => "previous",
101 }
102 }
103 }
104
105 impl std::fmt::Display for Role {
106 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
107 f.write_str(self.as_str())
108 }
109 }
110
111 /// One artifact a tier still names, and the bytes that have to be present for
112 /// the reference to be honoured.
113 #[derive(Debug, Clone, PartialEq, Eq)]
114 pub struct ArtifactRef {
115 pub tier: String,
116 pub role: Role,
117 /// The version label, when the tier records one.
118 pub version: Option<String>,
119 /// The release directory the reference resolves to.
120 pub dir: PathBuf,
121 /// The file whose absence makes the reference unusable — the primary binary
122 /// inside `dir`. A directory holding only a MANIFEST is not a rollback
123 /// target, and both stranded dirs in the 2026-08-25 measurement were exactly
124 /// that, so existence of the directory is the wrong question to ask.
125 pub binary: PathBuf,
126 }
127
128 /// The directory name gc sees under `releases/`.
129 ///
130 /// `None` for a recorded path with no final component, which is a malformed row
131 /// rather than a reference to the root.
132 fn dir_name(dir: &Path) -> Option<String> {
133 dir.file_name()
134 .and_then(|n| n.to_str())
135 .map(ToOwned::to_owned)
136 }
137
138 /// Every release directory the deployed state still points at, by the name gc
139 /// matches on.
140 ///
141 /// Scoped to one product because `release_root` is: a second app's tiers name
142 /// directories in a store this gc never walks.
143 pub async fn pinned_dirs(pool: &SqlitePool, app: &AppId) -> Result<PinnedReleases> {
144 let mut pinned = HashSet::new();
145
146 // (1) + (2): every staged_path reachable from the tier's build ids, and
147 // every newest-green build per (version, platform) for the versions the
148 // tiers name. One query: the second set is what a rollback resolves to and
149 // the first is what the tier is on, and a row can be in both.
150 let rows = sqlx::query(
151 "SELECT DISTINCT br.staged_path
152 FROM build_runs br
153 WHERE br.app = ?1
154 AND br.staged_path IS NOT NULL
155 AND (
156 br.id IN (SELECT current_build_id FROM tier_state WHERE app = ?1
157 UNION ALL
158 SELECT previous_build_id FROM tier_state WHERE app = ?1)
159 OR (br.result = 'passed'
160 AND br.version IN (SELECT current_version FROM tier_state WHERE app = ?1
161 UNION ALL
162 SELECT previous_version FROM tier_state WHERE app = ?1)
163 AND br.id = (SELECT MAX(id) FROM build_runs
164 WHERE app = ?1
165 AND version = br.version
166 AND platform IS br.platform
167 AND result = 'passed'))
168 )",
169 )
170 .bind(app)
171 .fetch_all(pool)
172 .await?;
173 for r in rows {
174 let path: String = r.get("staged_path");
175 if let Some(name) = dir_name(Path::new(&path)) {
176 pinned.insert(name);
177 }
178 }
179
180 // (3) The pre-identity path. `artifact_path` names the primary binary, so
181 // the directory is its parent.
182 let rows = sqlx::query(
183 "SELECT DISTINCT v.artifact_path
184 FROM versions v
185 WHERE v.app = ?1
186 AND v.version IN (SELECT current_version FROM tier_state WHERE app = ?1
187 UNION ALL
188 SELECT previous_version FROM tier_state WHERE app = ?1)",
189 )
190 .bind(app)
191 .fetch_all(pool)
192 .await?;
193 for r in rows {
194 let path: String = r.get("artifact_path");
195 if let Some(name) = Path::new(&path).parent().and_then(dir_name) {
196 pinned.insert(name);
197 }
198 }
199
200 Ok(PinnedReleases(pinned))
201 }
202
203 /// What each tier is running and what it would roll back to, resolved the way
204 /// promote and rollback resolve it: the build row's `staged_path` when the tier
205 /// has an identity, `versions.artifact_path` when it predates one.
206 ///
207 /// One entry per (tier, role) that resolves to a path at all. A tier with no
208 /// previous version contributes one entry, not two.
209 pub async fn tier_refs(
210 pool: &SqlitePool,
211 app: &AppId,
212 primary_bin: &str,
213 ) -> Result<Vec<ArtifactRef>> {
214 let rows = sqlx::query(
215 "SELECT ts.tier,
216 ts.current_version, cb.staged_path AS current_dir, cv.artifact_path AS current_bin,
217 ts.previous_version, pb.staged_path AS previous_dir, pv.artifact_path AS previous_bin
218 FROM tier_state ts
219 LEFT JOIN build_runs cb ON cb.id = ts.current_build_id
220 LEFT JOIN build_runs pb ON pb.id = ts.previous_build_id
221 LEFT JOIN versions cv ON cv.app = ts.app AND cv.version = ts.current_version
222 LEFT JOIN versions pv ON pv.app = ts.app AND pv.version = ts.previous_version
223 WHERE ts.app = ?
224 ORDER BY ts.tier",
225 )
226 .bind(app)
227 .fetch_all(pool)
228 .await?;
229
230 let mut refs = Vec::new();
231 for r in rows {
232 let tier: String = r.get("tier");
233 for (role, version, dir_col, bin_col) in [
234 (
235 Role::Current,
236 r.get::<Option<String>, _>("current_version"),
237 r.get::<Option<String>, _>("current_dir"),
238 r.get::<Option<String>, _>("current_bin"),
239 ),
240 (
241 Role::Previous,
242 r.get::<Option<String>, _>("previous_version"),
243 r.get::<Option<String>, _>("previous_dir"),
244 r.get::<Option<String>, _>("previous_bin"),
245 ),
246 ] {
247 // The build row wins when there is one: it is the identity, and the
248 // version label can be shared by several builds. `artifact_path` is
249 // the fallback for a tier that predates migration 008 — the same
250 // fallback the gate scope and the canary rollback take.
251 let resolved = match (dir_col, bin_col) {
252 (Some(dir), _) => {
253 let dir = PathBuf::from(dir);
254 let binary = dir.join(primary_bin);
255 Some((dir, binary))
256 }
257 (None, Some(bin)) => {
258 let binary = PathBuf::from(bin);
259 binary.parent().map(|d| (d.to_path_buf(), binary.clone()))
260 }
261 (None, None) => None,
262 };
263 if let Some((dir, binary)) = resolved {
264 refs.push(ArtifactRef {
265 tier: tier.clone(),
266 role,
267 version,
268 dir,
269 binary,
270 });
271 }
272 }
273 }
274 Ok(refs)
275 }
276
277 /// The references whose bytes are gone.
278 ///
279 /// Deliberately a `stat` per reference rather than a directory walk: there are
280 /// at most two per tier, and asking about the exact file promote would rsync is
281 /// the only question whose answer means anything.
282 pub async fn missing(refs: &[ArtifactRef]) -> Vec<ArtifactRef> {
283 let mut gone = Vec::new();
284 for r in refs {
285 if !tokio::fs::try_exists(&r.binary).await.unwrap_or(false) {
286 gone.push(r.clone());
287 }
288 }
289 gone
290 }
291
292 /// One line naming what is gone, for a log or a `/state` condition.
293 ///
294 /// Sorted and deduplicated by tier so the sentence is stable across reads —
295 /// an operator surface that reworded itself every poll would read as churn.
296 pub fn describe(missing: &[ArtifactRef]) -> String {
297 let lines: BTreeSet<String> = missing
298 .iter()
299 .map(|r| {
300 let version = r.version.as_deref().unwrap_or("unknown version");
301 format!("{} {} {version} ({})", r.tier, r.role, r.binary.display())
302 })
303 .collect();
304 lines.into_iter().collect::<Vec<_>>().join("; ")
305 }
306
307 #[cfg(test)]
308 mod tests {
309 use super::*;
310 use sqlx::sqlite::SqlitePoolOptions;
311
312 async fn fresh_pool() -> SqlitePool {
313 let pool = SqlitePoolOptions::new()
314 .max_connections(1)
315 .connect("sqlite::memory:")
316 .await
317 .unwrap();
318 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
319 pool
320 }
321
322 fn app() -> AppId {
323 AppId::default()
324 }
325
326 async fn tier(pool: &SqlitePool, name: &str, ord: i64) {
327 sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES ('mnw', ?, ?, 1)")
328 .bind(name)
329 .bind(ord)
330 .execute(pool)
331 .await
332 .unwrap();
333 sqlx::query("INSERT INTO tier_state (app, tier) VALUES ('mnw', ?)")
334 .bind(name)
335 .execute(pool)
336 .await
337 .unwrap();
338 }
339
340 /// A settled build row with an identity, as `runs::set_identity` leaves it.
341 async fn build(pool: &SqlitePool, version: &str, dir: &str, platform: Option<&str>) -> i64 {
342 sqlx::query(
343 "INSERT INTO build_runs (app, sha, version, result, started_at, bundle_digest,
344 staged_path, platform)
345 VALUES ('mnw', 'abc123', ?, 'passed', '2026-08-25T00:00:00Z', ?, ?, ?)",
346 )
347 .bind(version)
348 .bind(dir)
349 .bind(format!("/srv/sando/releases/{dir}"))
350 .bind(platform)
351 .execute(pool)
352 .await
353 .unwrap()
354 .last_insert_rowid()
355 }
356
357 async fn version_row(pool: &SqlitePool, version: &str, artifact_path: &str) {
358 sqlx::query(
359 "INSERT INTO versions (app, version, git_sha, built_at, artifact_path)
360 VALUES ('mnw', ?, 'abc123', '2026-08-25T00:00:00Z', ?)",
361 )
362 .bind(version)
363 .bind(artifact_path)
364 .execute(pool)
365 .await
366 .unwrap();
367 }
368
369 /// Fixture-only: the SET clause varies per test and every value in it is a
370 /// literal written here, so `raw_sql` is the honest tool rather than a
371 /// bound query that cannot take a clause.
372 async fn set_state(pool: &SqlitePool, tier: &str, sql: &str) {
373 sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
374 "UPDATE tier_state SET {sql} WHERE app = 'mnw' AND tier = '{tier}'"
375 )))
376 .execute(pool)
377 .await
378 .unwrap();
379 }
380
381 #[tokio::test]
382 async fn pins_what_every_tier_is_running_and_would_roll_back_to() {
383 let pool = fresh_pool().await;
384 tier(&pool, "host", 0).await;
385 tier(&pool, "a", 1).await;
386
387 let cur = build(&pool, "0.16.1", "aaaaaaaaaaaaaaaa", None).await;
388 let prev = build(&pool, "0.11.20", "bbbbbbbbbbbbbbbb", None).await;
389 set_state(
390 &pool,
391 "host",
392 &format!(
393 "current_version = '0.16.1', current_build_id = {cur},
394 previous_version = '0.11.20', previous_build_id = {prev}"
395 ),
396 )
397 .await;
398
399 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
400 assert!(pinned.contains("aaaaaaaaaaaaaaaa"));
401 assert!(pinned.contains("bbbbbbbbbbbbbbbb"));
402 assert_eq!(pinned.len(), 2, "{pinned:?}");
403 }
404
405 /// The exact shape that stranded prod: three builds of one version, the
406 /// tier on the OLDEST of them. Pinning by version alone would keep the
407 /// newest rebuild and lose the bytes actually deployed.
408 #[tokio::test]
409 async fn pins_the_build_the_tier_is_on_not_the_newest_rebuild_of_its_version() {
410 let pool = fresh_pool().await;
411 tier(&pool, "host", 0).await;
412
413 let deployed = build(&pool, "0.16.1", "0000000000000001", None).await;
414 build(&pool, "0.16.1", "0000000000000002", None).await;
415 let newest = build(&pool, "0.16.1", "0000000000000003", None).await;
416 set_state(
417 &pool,
418 "host",
419 &format!("current_version = '0.16.1', current_build_id = {deployed}"),
420 )
421 .await;
422
423 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
424 assert!(
425 pinned.contains("0000000000000001"),
426 "the deployed build must be pinned: {pinned:?}"
427 );
428 // And the newest green build of that version too: that is the row a
429 // rollback to 0.16.1 resolves through, so evicting it breaks a path the
430 // tier can still take.
431 assert!(pinned.contains("0000000000000003"), "{pinned:?}");
432 assert_ne!(deployed, newest);
433 assert!(!pinned.contains("0000000000000002"), "{pinned:?}");
434 }
435
436 /// Two architectures of one version are two artifacts, and a rollback picks
437 /// per node. Both have to survive.
438 #[tokio::test]
439 async fn pins_every_platform_of_a_referenced_version() {
440 let pool = fresh_pool().await;
441 tier(&pool, "host", 0).await;
442 build(&pool, "0.4.0", "aaaa000000000000", Some("linux/x86_64")).await;
443 build(&pool, "0.4.0", "bbbb000000000000", Some("linux/aarch64")).await;
444 set_state(&pool, "host", "current_version = '0.4.0'").await;
445
446 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
447 assert!(pinned.contains("aaaa000000000000"), "{pinned:?}");
448 assert!(pinned.contains("bbbb000000000000"), "{pinned:?}");
449 }
450
451 /// A pre-identity tier has no build id at all. `versions.artifact_path` is
452 /// the only handle, and it names the binary inside the dir.
453 #[tokio::test]
454 async fn pins_a_pre_identity_tier_through_versions() {
455 let pool = fresh_pool().await;
456 tier(&pool, "host", 0).await;
457 version_row(
458 &pool,
459 "0.8.12",
460 "/srv/sando/releases/cccccccccccccccc/makenotwork",
461 )
462 .await;
463 set_state(&pool, "host", "current_version = '0.8.12'").await;
464
465 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
466 assert!(pinned.contains("cccccccccccccccc"), "{pinned:?}");
467 }
468
469 /// `release_root` is per product, so another product's references must not
470 /// leak into this store's keep-set.
471 #[tokio::test]
472 async fn another_products_references_are_not_pinned_here() {
473 let pool = fresh_pool().await;
474 tier(&pool, "host", 0).await;
475 sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES ('pom', 'host', 0, 1)")
476 .execute(&pool)
477 .await
478 .unwrap();
479 sqlx::query("INSERT INTO tier_state (app, tier) VALUES ('pom', 'host')")
480 .execute(&pool)
481 .await
482 .unwrap();
483 sqlx::query(
484 "INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
485 VALUES ('pom', 'abc', '1.0.0', 'passed', '2026-08-25T00:00:00Z',
486 '/srv/sando-pom/releases/dddddddddddddddd')",
487 )
488 .execute(&pool)
489 .await
490 .unwrap();
491 sqlx::query(
492 "UPDATE tier_state SET current_version = '1.0.0' WHERE app = 'pom' AND tier = 'host'",
493 )
494 .execute(&pool)
495 .await
496 .unwrap();
497
498 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
499 assert!(pinned.is_empty(), "{pinned:?}");
500 }
501
502 #[tokio::test]
503 async fn a_tier_that_has_never_deployed_pins_nothing() {
504 let pool = fresh_pool().await;
505 tier(&pool, "host", 0).await;
506 assert!(pinned_dirs(&pool, &app()).await.unwrap().is_empty());
507 assert!(
508 tier_refs(&pool, &app(), "makenotwork")
509 .await
510 .unwrap()
511 .is_empty()
512 );
513 }
514
515 #[tokio::test]
516 async fn a_reference_whose_binary_is_gone_is_reported_missing() {
517 let tmp = tempfile::tempdir().unwrap();
518 let releases = tmp.path().join("releases");
519 let present = releases.join("aaaaaaaaaaaaaaaa");
520 let hollow = releases.join("bbbbbbbbbbbbbbbb");
521 tokio::fs::create_dir_all(&present).await.unwrap();
522 tokio::fs::create_dir_all(&hollow).await.unwrap();
523 tokio::fs::write(present.join("makenotwork"), b"bin")
524 .await
525 .unwrap();
526 // The measured state: MANIFEST and tree present, no binary. A directory
527 // check would call this healthy.
528 tokio::fs::write(hollow.join("MANIFEST"), b"")
529 .await
530 .unwrap();
531
532 let pool = fresh_pool().await;
533 tier(&pool, "host", 0).await;
534 let cur = sqlx::query(
535 "INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
536 VALUES ('mnw', 'abc', '0.16.1', 'passed', '2026-08-25T00:00:00Z', ?)",
537 )
538 .bind(present.to_string_lossy().as_ref())
539 .execute(&pool)
540 .await
541 .unwrap()
542 .last_insert_rowid();
543 let prev = sqlx::query(
544 "INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
545 VALUES ('mnw', 'abc', '0.11.20', 'passed', '2026-08-25T00:00:00Z', ?)",
546 )
547 .bind(hollow.to_string_lossy().as_ref())
548 .execute(&pool)
549 .await
550 .unwrap()
551 .last_insert_rowid();
552 set_state(
553 &pool,
554 "host",
555 &format!(
556 "current_version = '0.16.1', current_build_id = {cur},
557 previous_version = '0.11.20', previous_build_id = {prev}"
558 ),
559 )
560 .await;
561
562 let refs = tier_refs(&pool, &app(), "makenotwork").await.unwrap();
563 assert_eq!(refs.len(), 2);
564 let gone = missing(&refs).await;
565 assert_eq!(gone.len(), 1, "{gone:?}");
566 assert_eq!(gone[0].role, Role::Previous);
567 assert_eq!(gone[0].version.as_deref(), Some("0.11.20"));
568
569 let said = describe(&gone);
570 assert!(said.contains("host"), "{said}");
571 assert!(said.contains("previous"), "{said}");
572 assert!(said.contains("0.11.20"), "{said}");
573 }
574
575 /// The rollback path reads `versions.artifact_path` directly, so the same
576 /// question has to be asked of a pre-identity tier.
577 #[tokio::test]
578 async fn a_pre_identity_reference_is_checked_at_its_artifact_path() {
579 let tmp = tempfile::tempdir().unwrap();
580 let dir = tmp.path().join("releases").join("cccccccccccccccc");
581 tokio::fs::create_dir_all(&dir).await.unwrap();
582 let bin = dir.join("makenotwork");
583
584 let pool = fresh_pool().await;
585 tier(&pool, "host", 0).await;
586 version_row(&pool, "0.8.12", bin.to_string_lossy().as_ref()).await;
587 set_state(&pool, "host", "current_version = '0.8.12'").await;
588
589 let refs = tier_refs(&pool, &app(), "makenotwork").await.unwrap();
590 assert_eq!(refs.len(), 1);
591 assert_eq!(refs[0].dir, dir);
592 assert_eq!(missing(&refs).await.len(), 1);
593
594 tokio::fs::write(&bin, b"bin").await.unwrap();
595 assert!(missing(&refs).await.is_empty());
596 }
597 }
598