Skip to main content

max / makenotwork

21.8 KB · 595 lines History Blame Raw
1 //! Which release directories the deployed state still points at.
2 //!
3 //! <!-- wiki: release-artifact-identity -->
4 //!
5 //! Release dirs are content-addressed (`releases/<digest16>`) and trimmed after
6 //! every publish. A count alone (keep the 5 newest by mtime) cannot express
7 //! "this one is still in production": three rebuilds of a single version fill
8 //! three of the five slots, and the artifacts a tier is running and would roll
9 //! back to fall off the end.
10 //!
11 //! So the set below is computed first and set aside, and the count applies only
12 //! to what is left. The count is a floor on how much history to keep, not a
13 //! ceiling on what may be retained.
14 //!
15 //! # What counts as referenced
16 //!
17 //! Everything promote and rollback resolve through, because a dir that is
18 //! unreachable to them is exactly the one whose absence is discovered by an
19 //! rsync failing mid-promote:
20 //!
21 //! 1. `tier_state.current_build_id` / `previous_build_id` -> `build_runs.staged_path`.
22 //! The identity path (migration 008), and the tier's own answer to what it is
23 //! running.
24 //! 2. The newest green `build_runs` row per (version, platform) for every version
25 //! a tier names. This is what `crate::routes::promotion` resolves a rollback
26 //! to, and on a two-architecture product it is a different row per node.
27 //! 3. `versions.artifact_path` for those versions. The pre-identity path, still
28 //! the first thing a canary rollback reads.
29 //!
30 //! The three overlap heavily and are unioned rather than ranked: being reachable
31 //! by any of them is enough to make a directory load-bearing.
32
33 use crate::domain::AppId;
34 use anyhow::Result;
35 use sqlx::{Row, SqlitePool};
36 use std::collections::{BTreeSet, HashSet};
37 use std::path::{Path, PathBuf};
38
39 /// The release directories that must survive a gc, by the name gc matches on.
40 ///
41 /// A newtype rather than a bare `HashSet<String>` so the thing being passed
42 /// through publish and into gc says what it is at every hop, and so a caller
43 /// cannot hand it a set of paths, versions, or digests by accident.
44 #[derive(Debug, Clone, Default, PartialEq, Eq)]
45 pub struct PinnedReleases(HashSet<String>);
46
47 impl PinnedReleases {
48 /// Nothing to protect. What a caller with no deployed state passes — a unit
49 /// test, or a store that has never served anything.
50 pub fn none() -> Self {
51 Self::default()
52 }
53
54 pub fn contains(&self, dir_name: &str) -> bool {
55 self.0.contains(dir_name)
56 }
57
58 pub fn len(&self) -> usize {
59 self.0.len()
60 }
61
62 pub fn is_empty(&self) -> bool {
63 self.0.is_empty()
64 }
65
66 /// The pinned names in a stable order.
67 ///
68 /// Sorted rather than in hash order because the remote gc embeds these in a
69 /// shell script: an unordered set would rewrite the script text on every
70 /// deploy with no change of meaning, and a script that differs run to run is
71 /// one nobody can diff against the last one that worked.
72 pub fn sorted_names(&self) -> Vec<&str> {
73 let mut names: Vec<&str> = self.0.iter().map(String::as_str).collect();
74 names.sort_unstable();
75 names
76 }
77 }
78
79 impl FromIterator<String> for PinnedReleases {
80 fn from_iter<T: IntoIterator<Item = String>>(iter: T) -> Self {
81 Self(iter.into_iter().collect())
82 }
83 }
84
85 /// Which of a tier's two artifacts a reference is.
86 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
87 pub enum Role {
88 /// What the tier is running.
89 Current,
90 /// The one step of rollback history `tier_state` keeps.
91 Previous,
92 }
93
94 impl Role {
95 pub const fn as_str(self) -> &'static str {
96 match self {
97 Self::Current => "current",
98 Self::Previous => "previous",
99 }
100 }
101 }
102
103 impl std::fmt::Display for Role {
104 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
105 f.write_str(self.as_str())
106 }
107 }
108
109 /// One artifact a tier still names, and the bytes that have to be present for
110 /// the reference to be honoured.
111 #[derive(Debug, Clone, PartialEq, Eq)]
112 pub struct ArtifactRef {
113 pub tier: String,
114 pub role: Role,
115 /// The version label, when the tier records one.
116 pub version: Option<String>,
117 /// The release directory the reference resolves to.
118 pub dir: PathBuf,
119 /// The file whose absence makes the reference unusable: the primary binary
120 /// inside `dir`. A directory holding only a MANIFEST is not a rollback
121 /// target, so existence of the directory is the wrong question to ask.
122 pub binary: PathBuf,
123 }
124
125 /// The directory name gc sees under `releases/`.
126 ///
127 /// `None` for a recorded path with no final component, which is a malformed row
128 /// rather than a reference to the root.
129 fn dir_name(dir: &Path) -> Option<String> {
130 dir.file_name()
131 .and_then(|n| n.to_str())
132 .map(ToOwned::to_owned)
133 }
134
135 /// Every release directory the deployed state still points at, by the name gc
136 /// matches on.
137 ///
138 /// Scoped to one product because `release_root` is: a second app's tiers name
139 /// directories in a store this gc never walks.
140 pub async fn pinned_dirs(pool: &SqlitePool, app: &AppId) -> Result<PinnedReleases> {
141 let mut pinned = HashSet::new();
142
143 // (1) + (2): every staged_path reachable from the tier's build ids, and
144 // every newest-green build per (version, platform) for the versions the
145 // tiers name. One query: the second set is what a rollback resolves to and
146 // the first is what the tier is on, and a row can be in both.
147 let rows = sqlx::query(
148 "SELECT DISTINCT br.staged_path
149 FROM build_runs br
150 WHERE br.app = ?1
151 AND br.staged_path IS NOT NULL
152 AND (
153 br.id IN (SELECT current_build_id FROM tier_state WHERE app = ?1
154 UNION ALL
155 SELECT previous_build_id FROM tier_state WHERE app = ?1)
156 OR (br.result = 'passed'
157 AND br.version IN (SELECT current_version FROM tier_state WHERE app = ?1
158 UNION ALL
159 SELECT previous_version FROM tier_state WHERE app = ?1)
160 AND br.id = (SELECT MAX(id) FROM build_runs
161 WHERE app = ?1
162 AND version = br.version
163 AND platform IS br.platform
164 AND result = 'passed'))
165 )",
166 )
167 .bind(app)
168 .fetch_all(pool)
169 .await?;
170 for r in rows {
171 let path: String = r.get("staged_path");
172 if let Some(name) = dir_name(Path::new(&path)) {
173 pinned.insert(name);
174 }
175 }
176
177 // (3) The pre-identity path. `artifact_path` names the primary binary, so
178 // the directory is its parent.
179 let rows = sqlx::query(
180 "SELECT DISTINCT v.artifact_path
181 FROM versions v
182 WHERE v.app = ?1
183 AND v.version IN (SELECT current_version FROM tier_state WHERE app = ?1
184 UNION ALL
185 SELECT previous_version FROM tier_state WHERE app = ?1)",
186 )
187 .bind(app)
188 .fetch_all(pool)
189 .await?;
190 for r in rows {
191 let path: String = r.get("artifact_path");
192 if let Some(name) = Path::new(&path).parent().and_then(dir_name) {
193 pinned.insert(name);
194 }
195 }
196
197 Ok(PinnedReleases(pinned))
198 }
199
200 /// What each tier is running and what it would roll back to, resolved the way
201 /// promote and rollback resolve it: the build row's `staged_path` when the tier
202 /// has an identity, `versions.artifact_path` when it predates one.
203 ///
204 /// One entry per (tier, role) that resolves to a path at all. A tier with no
205 /// previous version contributes one entry, not two.
206 pub async fn tier_refs(
207 pool: &SqlitePool,
208 app: &AppId,
209 primary_bin: &str,
210 ) -> Result<Vec<ArtifactRef>> {
211 let rows = sqlx::query(
212 "SELECT ts.tier,
213 ts.current_version, cb.staged_path AS current_dir, cv.artifact_path AS current_bin,
214 ts.previous_version, pb.staged_path AS previous_dir, pv.artifact_path AS previous_bin
215 FROM tier_state ts
216 LEFT JOIN build_runs cb ON cb.id = ts.current_build_id
217 LEFT JOIN build_runs pb ON pb.id = ts.previous_build_id
218 LEFT JOIN versions cv ON cv.app = ts.app AND cv.version = ts.current_version
219 LEFT JOIN versions pv ON pv.app = ts.app AND pv.version = ts.previous_version
220 WHERE ts.app = ?
221 ORDER BY ts.tier",
222 )
223 .bind(app)
224 .fetch_all(pool)
225 .await?;
226
227 let mut refs = Vec::new();
228 for r in rows {
229 let tier: String = r.get("tier");
230 for (role, version, dir_col, bin_col) in [
231 (
232 Role::Current,
233 r.get::<Option<String>, _>("current_version"),
234 r.get::<Option<String>, _>("current_dir"),
235 r.get::<Option<String>, _>("current_bin"),
236 ),
237 (
238 Role::Previous,
239 r.get::<Option<String>, _>("previous_version"),
240 r.get::<Option<String>, _>("previous_dir"),
241 r.get::<Option<String>, _>("previous_bin"),
242 ),
243 ] {
244 // The build row wins when there is one: it is the identity, and the
245 // version label can be shared by several builds. `artifact_path` is
246 // the fallback for a tier that predates migration 008 — the same
247 // fallback the gate scope and the canary rollback take.
248 let resolved = match (dir_col, bin_col) {
249 (Some(dir), _) => {
250 let dir = PathBuf::from(dir);
251 let binary = dir.join(primary_bin);
252 Some((dir, binary))
253 }
254 (None, Some(bin)) => {
255 let binary = PathBuf::from(bin);
256 binary.parent().map(|d| (d.to_path_buf(), binary.clone()))
257 }
258 (None, None) => None,
259 };
260 if let Some((dir, binary)) = resolved {
261 refs.push(ArtifactRef {
262 tier: tier.clone(),
263 role,
264 version,
265 dir,
266 binary,
267 });
268 }
269 }
270 }
271 Ok(refs)
272 }
273
274 /// The references whose bytes are gone.
275 ///
276 /// Deliberately a `stat` per reference rather than a directory walk: there are
277 /// at most two per tier, and asking about the exact file promote would rsync is
278 /// the only question whose answer means anything.
279 pub async fn missing(refs: &[ArtifactRef]) -> Vec<ArtifactRef> {
280 let mut gone = Vec::new();
281 for r in refs {
282 if !tokio::fs::try_exists(&r.binary).await.unwrap_or(false) {
283 gone.push(r.clone());
284 }
285 }
286 gone
287 }
288
289 /// One line naming what is gone, for a log or a `/state` condition.
290 ///
291 /// Sorted and deduplicated by tier so the sentence is stable across reads —
292 /// an operator surface that reworded itself every poll would read as churn.
293 pub fn describe(missing: &[ArtifactRef]) -> String {
294 let lines: BTreeSet<String> = missing
295 .iter()
296 .map(|r| {
297 let version = r.version.as_deref().unwrap_or("unknown version");
298 format!("{} {} {version} ({})", r.tier, r.role, r.binary.display())
299 })
300 .collect();
301 lines.into_iter().collect::<Vec<_>>().join("; ")
302 }
303
304 #[cfg(test)]
305 mod tests {
306 use super::*;
307 use sqlx::sqlite::SqlitePoolOptions;
308
309 async fn fresh_pool() -> SqlitePool {
310 let pool = SqlitePoolOptions::new()
311 .max_connections(1)
312 .connect("sqlite::memory:")
313 .await
314 .unwrap();
315 sqlx::migrate!("./migrations").run(&pool).await.unwrap();
316 pool
317 }
318
319 fn app() -> AppId {
320 AppId::default()
321 }
322
323 async fn tier(pool: &SqlitePool, name: &str, ord: i64) {
324 sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES ('mnw', ?, ?, 1)")
325 .bind(name)
326 .bind(ord)
327 .execute(pool)
328 .await
329 .unwrap();
330 sqlx::query("INSERT INTO tier_state (app, tier) VALUES ('mnw', ?)")
331 .bind(name)
332 .execute(pool)
333 .await
334 .unwrap();
335 }
336
337 /// A settled build row with an identity, as `runs::set_identity` leaves it.
338 async fn build(pool: &SqlitePool, version: &str, dir: &str, platform: Option<&str>) -> i64 {
339 sqlx::query(
340 "INSERT INTO build_runs (app, sha, version, result, started_at, bundle_digest,
341 staged_path, platform)
342 VALUES ('mnw', 'abc123', ?, 'passed', '2026-08-25T00:00:00Z', ?, ?, ?)",
343 )
344 .bind(version)
345 .bind(dir)
346 .bind(format!("/srv/sando/releases/{dir}"))
347 .bind(platform)
348 .execute(pool)
349 .await
350 .unwrap()
351 .last_insert_rowid()
352 }
353
354 async fn version_row(pool: &SqlitePool, version: &str, artifact_path: &str) {
355 sqlx::query(
356 "INSERT INTO versions (app, version, git_sha, built_at, artifact_path)
357 VALUES ('mnw', ?, 'abc123', '2026-08-25T00:00:00Z', ?)",
358 )
359 .bind(version)
360 .bind(artifact_path)
361 .execute(pool)
362 .await
363 .unwrap();
364 }
365
366 /// Fixture-only: the SET clause varies per test and every value in it is a
367 /// literal written here, so `raw_sql` is the honest tool rather than a
368 /// bound query that cannot take a clause.
369 async fn set_state(pool: &SqlitePool, tier: &str, sql: &str) {
370 sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
371 "UPDATE tier_state SET {sql} WHERE app = 'mnw' AND tier = '{tier}'"
372 )))
373 .execute(pool)
374 .await
375 .unwrap();
376 }
377
378 #[tokio::test]
379 async fn pins_what_every_tier_is_running_and_would_roll_back_to() {
380 let pool = fresh_pool().await;
381 tier(&pool, "host", 0).await;
382 tier(&pool, "a", 1).await;
383
384 let cur = build(&pool, "0.16.1", "aaaaaaaaaaaaaaaa", None).await;
385 let prev = build(&pool, "0.11.20", "bbbbbbbbbbbbbbbb", None).await;
386 set_state(
387 &pool,
388 "host",
389 &format!(
390 "current_version = '0.16.1', current_build_id = {cur},
391 previous_version = '0.11.20', previous_build_id = {prev}"
392 ),
393 )
394 .await;
395
396 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
397 assert!(pinned.contains("aaaaaaaaaaaaaaaa"));
398 assert!(pinned.contains("bbbbbbbbbbbbbbbb"));
399 assert_eq!(pinned.len(), 2, "{pinned:?}");
400 }
401
402 /// The exact shape that stranded prod: three builds of one version, the
403 /// tier on the OLDEST of them. Pinning by version alone would keep the
404 /// newest rebuild and lose the bytes actually deployed.
405 #[tokio::test]
406 async fn pins_the_build_the_tier_is_on_not_the_newest_rebuild_of_its_version() {
407 let pool = fresh_pool().await;
408 tier(&pool, "host", 0).await;
409
410 let deployed = build(&pool, "0.16.1", "0000000000000001", None).await;
411 build(&pool, "0.16.1", "0000000000000002", None).await;
412 let newest = build(&pool, "0.16.1", "0000000000000003", None).await;
413 set_state(
414 &pool,
415 "host",
416 &format!("current_version = '0.16.1', current_build_id = {deployed}"),
417 )
418 .await;
419
420 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
421 assert!(
422 pinned.contains("0000000000000001"),
423 "the deployed build must be pinned: {pinned:?}"
424 );
425 // And the newest green build of that version too: that is the row a
426 // rollback to 0.16.1 resolves through, so evicting it breaks a path the
427 // tier can still take.
428 assert!(pinned.contains("0000000000000003"), "{pinned:?}");
429 assert_ne!(deployed, newest);
430 assert!(!pinned.contains("0000000000000002"), "{pinned:?}");
431 }
432
433 /// Two architectures of one version are two artifacts, and a rollback picks
434 /// per node. Both have to survive.
435 #[tokio::test]
436 async fn pins_every_platform_of_a_referenced_version() {
437 let pool = fresh_pool().await;
438 tier(&pool, "host", 0).await;
439 build(&pool, "0.4.0", "aaaa000000000000", Some("linux/x86_64")).await;
440 build(&pool, "0.4.0", "bbbb000000000000", Some("linux/aarch64")).await;
441 set_state(&pool, "host", "current_version = '0.4.0'").await;
442
443 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
444 assert!(pinned.contains("aaaa000000000000"), "{pinned:?}");
445 assert!(pinned.contains("bbbb000000000000"), "{pinned:?}");
446 }
447
448 /// A pre-identity tier has no build id at all. `versions.artifact_path` is
449 /// the only handle, and it names the binary inside the dir.
450 #[tokio::test]
451 async fn pins_a_pre_identity_tier_through_versions() {
452 let pool = fresh_pool().await;
453 tier(&pool, "host", 0).await;
454 version_row(
455 &pool,
456 "0.8.12",
457 "/srv/sando/releases/cccccccccccccccc/makenotwork",
458 )
459 .await;
460 set_state(&pool, "host", "current_version = '0.8.12'").await;
461
462 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
463 assert!(pinned.contains("cccccccccccccccc"), "{pinned:?}");
464 }
465
466 /// `release_root` is per product, so another product's references must not
467 /// leak into this store's keep-set.
468 #[tokio::test]
469 async fn another_products_references_are_not_pinned_here() {
470 let pool = fresh_pool().await;
471 tier(&pool, "host", 0).await;
472 sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES ('pom', 'host', 0, 1)")
473 .execute(&pool)
474 .await
475 .unwrap();
476 sqlx::query("INSERT INTO tier_state (app, tier) VALUES ('pom', 'host')")
477 .execute(&pool)
478 .await
479 .unwrap();
480 sqlx::query(
481 "INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
482 VALUES ('pom', 'abc', '1.0.0', 'passed', '2026-08-25T00:00:00Z',
483 '/srv/sando-pom/releases/dddddddddddddddd')",
484 )
485 .execute(&pool)
486 .await
487 .unwrap();
488 sqlx::query(
489 "UPDATE tier_state SET current_version = '1.0.0' WHERE app = 'pom' AND tier = 'host'",
490 )
491 .execute(&pool)
492 .await
493 .unwrap();
494
495 let pinned = pinned_dirs(&pool, &app()).await.unwrap();
496 assert!(pinned.is_empty(), "{pinned:?}");
497 }
498
499 #[tokio::test]
500 async fn a_tier_that_has_never_deployed_pins_nothing() {
501 let pool = fresh_pool().await;
502 tier(&pool, "host", 0).await;
503 assert!(pinned_dirs(&pool, &app()).await.unwrap().is_empty());
504 assert!(
505 tier_refs(&pool, &app(), "makenotwork")
506 .await
507 .unwrap()
508 .is_empty()
509 );
510 }
511
512 #[tokio::test]
513 async fn a_reference_whose_binary_is_gone_is_reported_missing() {
514 let tmp = tempfile::tempdir().unwrap();
515 let releases = tmp.path().join("releases");
516 let present = releases.join("aaaaaaaaaaaaaaaa");
517 let hollow = releases.join("bbbbbbbbbbbbbbbb");
518 tokio::fs::create_dir_all(&present).await.unwrap();
519 tokio::fs::create_dir_all(&hollow).await.unwrap();
520 tokio::fs::write(present.join("makenotwork"), b"bin")
521 .await
522 .unwrap();
523 // The measured state: MANIFEST and tree present, no binary. A directory
524 // check would call this healthy.
525 tokio::fs::write(hollow.join("MANIFEST"), b"")
526 .await
527 .unwrap();
528
529 let pool = fresh_pool().await;
530 tier(&pool, "host", 0).await;
531 let cur = sqlx::query(
532 "INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
533 VALUES ('mnw', 'abc', '0.16.1', 'passed', '2026-08-25T00:00:00Z', ?)",
534 )
535 .bind(present.to_string_lossy().as_ref())
536 .execute(&pool)
537 .await
538 .unwrap()
539 .last_insert_rowid();
540 let prev = sqlx::query(
541 "INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
542 VALUES ('mnw', 'abc', '0.11.20', 'passed', '2026-08-25T00:00:00Z', ?)",
543 )
544 .bind(hollow.to_string_lossy().as_ref())
545 .execute(&pool)
546 .await
547 .unwrap()
548 .last_insert_rowid();
549 set_state(
550 &pool,
551 "host",
552 &format!(
553 "current_version = '0.16.1', current_build_id = {cur},
554 previous_version = '0.11.20', previous_build_id = {prev}"
555 ),
556 )
557 .await;
558
559 let refs = tier_refs(&pool, &app(), "makenotwork").await.unwrap();
560 assert_eq!(refs.len(), 2);
561 let gone = missing(&refs).await;
562 assert_eq!(gone.len(), 1, "{gone:?}");
563 assert_eq!(gone[0].role, Role::Previous);
564 assert_eq!(gone[0].version.as_deref(), Some("0.11.20"));
565
566 let said = describe(&gone);
567 assert!(said.contains("host"), "{said}");
568 assert!(said.contains("previous"), "{said}");
569 assert!(said.contains("0.11.20"), "{said}");
570 }
571
572 /// The rollback path reads `versions.artifact_path` directly, so the same
573 /// question has to be asked of a pre-identity tier.
574 #[tokio::test]
575 async fn a_pre_identity_reference_is_checked_at_its_artifact_path() {
576 let tmp = tempfile::tempdir().unwrap();
577 let dir = tmp.path().join("releases").join("cccccccccccccccc");
578 tokio::fs::create_dir_all(&dir).await.unwrap();
579 let bin = dir.join("makenotwork");
580
581 let pool = fresh_pool().await;
582 tier(&pool, "host", 0).await;
583 version_row(&pool, "0.8.12", bin.to_string_lossy().as_ref()).await;
584 set_state(&pool, "host", "current_version = '0.8.12'").await;
585
586 let refs = tier_refs(&pool, &app(), "makenotwork").await.unwrap();
587 assert_eq!(refs.len(), 1);
588 assert_eq!(refs[0].dir, dir);
589 assert_eq!(missing(&refs).await.len(), 1);
590
591 tokio::fs::write(&bin, b"bin").await.unwrap();
592 assert!(missing(&refs).await.is_empty());
593 }
594 }
595