max / makenotwork

sando: never gc an artifact a tier still references gc_local_releases kept the five newest release dirs by mtime and nothing else, which cannot express "production is running this one". Content addressing means three rebuilds of one version are three directories, so a single version can eat three of five slots. It evicted production twice: 2026-08-19 took prod's running 0.11.5 and tier a's burned-in 0.11.18; 2026-08-25 left three 0.16.1 rebuilds plus two directories holding a MANIFEST and no binary, so prod had no rollback target at all. Per the ruling on d085ffe0: pin what tiers reference, keep the count as a floor. retention.rs computes the referenced set and gc subtracts it before ordering anything, so a pinned dir never occupies one of the five slots. Referenced means everything promote and rollback resolve through: the tier's own build ids, the newest green build per (version, platform) for every version a tier names, and versions.artifact_path for the pre-identity path. Second half: a row kept naming a path long after the bytes were gone and nothing reconciled the two, so the failure surfaced as an rsync failing mid-promote while /state showed the tier green. /state now carries missing_artifact per tier and /status.json a degraded artifact condition, computed live from a stat of the primary binary inside the release dir -- not the directory, which both stranded dirs still had. Also logged once at startup.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-08-25 19:04 UTC

Signed with PGP, not checked

Commit: 6849120942dfe61fa650f4776811900d2d5839a4

Parent: f6be917

10 files changed, +846 insertions, -43 deletions

M sando/Cargo.lock +17 -17

			@@ -1228,7 +1228,7 @@
1228	1228
1229	1229		[[package]]
1230	1230		name = "sando-daemon"
1231		-	version = "0.2.13"
	1231	+	version = "0.2.14"
1232	1232		dependencies = [
1233	1233		"anyhow",
1234	1234		"async-trait",
			@@ -2270,29 +2270,37 @@
2270	2270		name = "docengine"
2271	2271		version = "0.7.0"
2272	2272
	2273	+	[[patch.unused]]
	2274	+	name = "synckit-client"
	2275	+	version = "0.9.1"
	2276	+
	2277	+	[[patch.unused]]
	2278	+	name = "synckit-config"
	2279	+	version = "0.2.0"
	2280	+
2273	2281		[[patch.unused]]
2274	2282		name = "quasi-axum"
2275		-	version = "0.56.0"
	2283	+	version = "0.59.0"
2276	2284
2277	2285		[[patch.unused]]
2278	2286		name = "quasi-basics"
2279		-	version = "0.56.0"
	2287	+	version = "0.59.0"
2280	2288
2281	2289		[[patch.unused]]
2282	2290		name = "quasi-http"
2283		-	version = "0.56.0"
	2291	+	version = "0.59.0"
2284	2292
2285	2293		[[patch.unused]]
2286	2294		name = "quasi-immediate"
2287		-	version = "0.56.0"
	2295	+	version = "0.59.0"
2288	2296
2289	2297		[[patch.unused]]
2290	2298		name = "quasi-notifs"
2291		-	version = "0.56.0"
	2299	+	version = "0.59.0"
2292	2300
2293	2301		[[patch.unused]]
2294	2302		name = "quasi-router"
2295		-	version = "0.56.0"
	2303	+	version = "0.59.0"
2296	2304
2297	2305		[[patch.unused]]
2298	2306		name = "quasi-store"
			@@ -2300,24 +2308,16 @@
2300	2308
2301	2309		[[patch.unused]]
2302	2310		name = "quasi-tauri"
2303		-	version = "0.56.0"
	2311	+	version = "0.59.0"
2304	2312
2305	2313		[[patch.unused]]
2306	2314		name = "quasi-webview"
2307		-	version = "0.56.0"
	2315	+	version = "0.59.0"
2308	2316
2309	2317		[[patch.unused]]
2310	2318		name = "quasi-type"
2311	2319		version = "0.1.0"
2312	2320
2313		-	[[patch.unused]]
2314		-	name = "synckit-client"
2315		-	version = "0.9.1"
2316		-
2317		-	[[patch.unused]]
2318		-	name = "synckit-config"
2319		-	version = "0.2.0"
2320		-
2321	2321		[[patch.unused]]
2322	2322		name = "kberg"
2323	2323		version = "0.1.0"

M sando/daemon/Cargo.toml +1 -1

			@@ -1,6 +1,6 @@
1	1		[package]
2	2		name = "sando-daemon"
3		-	version = "0.2.13"
	3	+	version = "0.2.14"
4	4		edition = "2024"
5	5		license = "MIT"
6	6

M sando/daemon/src/build.rs +5 -2

			@@ -465,7 +465,8 @@
465	465		.await
466	466		.ok();
467	467
468		-	let accepted = crate::intake::accept(&cfg.release_root, staged, record_json)
	468	+	let pinned = crate::retention::pinned_dirs(pool, &cfg.id).await?;
	469	+	let accepted = crate::intake::accept(&cfg.release_root, staged, record_json, &pinned)
469	470		.await
470	471		.map_err(\|e\| anyhow::anyhow!("{e}"))?;
471	472
			@@ -656,8 +657,10 @@
656	657		)
657	658		.await
658	659		.context("writing bundle MANIFEST")?;
	660	+	let pinned = crate::retention::pinned_dirs(pool, &cfg.id).await?;
659	661		let released =
660		-	deploy::finalize_local_release(&cfg.release_root, &staged.staging, digest.short()).await?;
	662	+	deploy::finalize_local_release(&cfg.release_root, &staged.staging, digest.short(), &pinned)
	663	+	.await?;
661	664
662	665		let staged_bin = released.join(cfg.primary_bin());
663	666		sqlx::query("UPDATE versions SET artifact_path = ? WHERE app = ? AND version = ?")

M sando/daemon/src/deploy.rs +118 -13

			@@ -27,6 +27,7 @@
27	27		//! not a model change.
28	28
29	29		use crate::domain::Platform;
	30	+	use crate::retention::PinnedReleases;
30	31		use crate::topology::Node;
31	32		use anyhow::{Context, Result};
32	33		use async_trait::async_trait;
			@@ -133,9 +134,13 @@
133	134		}
134	135		}
135	136
136		-	/// Keep this many release dirs per node; older ones get gc'd after a
	137	+	/// Keep at least this many release dirs per node; older ones get gc'd after a
137	138		/// successful deploy. Fixed for now; promote to config if the constant ever
138	139		/// needs to vary by tier.
	140	+	///
	141	+	/// A floor, not a ceiling. Whatever the deployed state still references is set
	142	+	/// aside first and this count applies to the remainder — see
	143	+	/// [`crate::retention`] for why a count alone could not express that.
139	144		const RELEASES_TO_KEEP: usize = 5;
140	145
141	146		/// A sink that drops streamed bytes. Deploy steps don't have a live-log handle
			@@ -211,10 +216,16 @@
211	216		/// rows and burn-in clock stops being something to guard against and becomes
212	217		/// something that cannot be expressed. If a release with this digest already
213	218		/// exists (identical bytes rebuilt), the staging copy is redundant and dropped.
	219	+	///
	220	+	/// `pinned` names the release dirs the deployed state still points at
	221	+	/// ([`crate::retention::pinned_dirs`]); they are never gc'd here, however old
	222	+	/// they are. It arrives as data rather than as a pool handle so this stays a
	223	+	/// filesystem operation and the intake seam keeps working without a database.
214	224		pub async fn finalize_local_release(
215	225		release_root: &Path,
216	226		staging: &Path,
217	227		digest16: &str,
	228	+	pinned: &PinnedReleases,
218	229		) -> Result<PathBuf> {
219	230		let releases = release_root.join("releases");
220	231		tokio::fs::create_dir_all(&releases).await?;
			@@ -242,7 +253,7 @@
242	253		String::from_utf8_lossy(&out.stderr),
243	254		);
244	255
245		-	if let Err(e) = gc_local_releases(release_root).await {
	256	+	if let Err(e) = gc_local_releases(release_root, pinned).await {
246	257		tracing::warn!(error = %e, "local release GC failed (non-fatal)");
247	258		}
248	259		Ok(released)
			@@ -932,7 +943,13 @@
932	943		)
933	944		}
934	945
935		-	async fn gc_local_releases(release_root: &Path) -> Result<()> {
	946	+	/// Trim `releases/` to the pinned set plus the [`RELEASES_TO_KEEP`] newest of
	947	+	/// what is left.
	948	+	///
	949	+	/// Pinning is applied before the count, so a referenced artifact cannot be aged
	950	+	/// out by rebuilds of a newer version — the failure that stranded production
	951	+	/// twice. See [`crate::retention`].
	952	+	async fn gc_local_releases(release_root: &Path, pinned: &PinnedReleases) -> Result<()> {
936	953		let releases = release_root.join("releases");
937	954		if !releases.exists() {
938	955		return Ok(());
			@@ -943,6 +960,15 @@
943	960		if !entry.file_type().await?.is_dir() {
944	961		continue;
945	962		}
	963	+	// Set aside before anything is ordered or counted: a pinned dir is not
	964	+	// a candidate, so it can never occupy one of the count's slots either.
	965	+	if entry
	966	+	.file_name()
	967	+	.to_str()
	968	+	.is_some_and(\|n\| pinned.contains(n))
	969	+	{
	970	+	continue;
	971	+	}
946	972		let meta = entry.metadata().await?;
947	973		entries.push((entry.path(), meta.modified()?));
948	974		}
			@@ -974,6 +1000,13 @@
974	1000		#[cfg(test)]
975	1001		mod tests {
976	1002		use super::*;
	1003	+
	1004	+	/// Nothing deployed, so nothing pinned: the tests that exercise the count
	1005	+	/// alone pass this, and the ones that exercise pinning build their own set.
	1006	+	fn no_pins() -> PinnedReleases {
	1007	+	PinnedReleases::none()
	1008	+	}
	1009	+
977	1010		use crate::topology::NodeCompanion;
978	1011		use ops_exec::{CapabilitySet, LocalExec, SshExec};
979	1012		use std::os::unix::process::ExitStatusExt;
			@@ -1294,9 +1327,10 @@
1294	1327		);
1295	1328
1296	1329		// Publish content-addressed at releases/<digest16>.
1297		-	let released = finalize_local_release(&release_root, &staging, "deadbeefcafe0000")
1298		-	.await
1299		-	.expect("finalize_local_release should succeed");
	1330	+	let released =
	1331	+	finalize_local_release(&release_root, &staging, "deadbeefcafe0000", &no_pins())
	1332	+	.await
	1333	+	.expect("finalize_local_release should succeed");
1300	1334		assert_eq!(
1301	1335		released,
1302	1336		release_root.join("releases").join("deadbeefcafe0000")
			@@ -1337,14 +1371,14 @@
1337	1371		let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1338	1372		.await
1339	1373		.unwrap();
1340		-	finalize_local_release(&release_root, &s1, "1111111111111111")
	1374	+	finalize_local_release(&release_root, &s1, "1111111111111111", &no_pins())
1341	1375		.await
1342	1376		.unwrap();
1343	1377		tokio::fs::write(&bin, b"V2").await.unwrap();
1344	1378		let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1345	1379		.await
1346	1380		.unwrap();
1347		-	finalize_local_release(&release_root, &s2, "2222222222222222")
	1381	+	finalize_local_release(&release_root, &s2, "2222222222222222", &no_pins())
1348	1382		.await
1349	1383		.unwrap();
1350	1384
			@@ -1380,7 +1414,7 @@
1380	1414		let s1 = stage_local_bundle(&release_root, 1, std::slice::from_ref(&bin))
1381	1415		.await
1382	1416		.unwrap();
1383		-	finalize_local_release(&release_root, &s1, "abc123abc123abc1")
	1417	+	finalize_local_release(&release_root, &s1, "abc123abc123abc1", &no_pins())
1384	1418		.await
1385	1419		.unwrap();
1386	1420		// Same digest rebuilt (e.g. a re-run at the same content): finalize must
			@@ -1388,7 +1422,7 @@
1388	1422		let s2 = stage_local_bundle(&release_root, 2, std::slice::from_ref(&bin))
1389	1423		.await
1390	1424		.unwrap();
1391		-	let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1")
	1425	+	let released = finalize_local_release(&release_root, &s2, "abc123abc123abc1", &no_pins())
1392	1426		.await
1393	1427		.expect("finalize is idempotent on a repeated digest");
1394	1428		assert_eq!(released, release_root.join("releases/abc123abc123abc1"));
			@@ -1474,7 +1508,7 @@
1474	1508		names.push(name);
1475	1509		}
1476	1510
1477		-	gc_local_releases(root).await.unwrap();
	1511	+	gc_local_releases(root, &no_pins()).await.unwrap();
1478	1512
1479	1513		let surviving_expected: Vec<_> = names
1480	1514		.iter()
			@@ -1492,6 +1526,77 @@
1492	1526		}
1493	1527		}
1494	1528
	1529	+	#[tokio::test]
	1530	+	async fn gc_local_releases_never_evicts_a_pinned_dir() {
	1531	+	// The 2026-08-25 shape exactly: the oldest dir is the one production is
	1532	+	// running, and enough newer rebuilds exist to push it past the count.
	1533	+	// Under the count alone it was the first thing deleted.
	1534	+	let tmp = tempfile::tempdir().unwrap();
	1535	+	let root = tmp.path();
	1536	+	let releases = root.join("releases");
	1537	+	tokio::fs::create_dir_all(&releases).await.unwrap();
	1538	+
	1539	+	let total = RELEASES_TO_KEEP + 3;
	1540	+	let mut names = Vec::new();
	1541	+	for i in 0..total {
	1542	+	let name = format!("v{i:02}");
	1543	+	let dir = releases.join(&name);
	1544	+	tokio::fs::create_dir(&dir).await.unwrap();
	1545	+	let f = std::fs::File::open(&dir).unwrap();
	1546	+	let when =
	1547	+	SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000 + i as u64);
	1548	+	f.set_times(std::fs::FileTimes::new().set_modified(when))
	1549	+	.unwrap();
	1550	+	names.push(name);
	1551	+	}
	1552	+
	1553	+	// The two oldest: what a tier is running and what it would roll back to.
	1554	+	let pinned: PinnedReleases = [names[0].clone(), names[1].clone()].into_iter().collect();
	1555	+	gc_local_releases(root, &pinned).await.unwrap();
	1556	+
	1557	+	for name in [&names[0], &names[1]] {
	1558	+	assert!(
	1559	+	releases.join(name).exists(),
	1560	+	"a referenced artifact was evicted: {name}"
	1561	+	);
	1562	+	}
	1563	+	// And the count still applies to everything else, from a floor that the
	1564	+	// pinned pair did not eat into: the newest RELEASES_TO_KEEP unpinned
	1565	+	// dirs survive, so pinning two costs two extra slots rather than two of
	1566	+	// the five.
	1567	+	let unpinned: Vec<_> = names.iter().filter(\|n\| !pinned.contains(n)).collect();
	1568	+	let cut = unpinned.len() - RELEASES_TO_KEEP;
	1569	+	for name in unpinned.iter().take(cut) {
	1570	+	assert!(
	1571	+	!releases.join(name).exists(),
	1572	+	"expected to be pruned: {name}"
	1573	+	);
	1574	+	}
	1575	+	for name in unpinned.iter().skip(cut) {
	1576	+	assert!(releases.join(name).exists(), "expected to survive: {name}");
	1577	+	}
	1578	+	}
	1579	+
	1580	+	#[tokio::test]
	1581	+	async fn gc_local_releases_keeps_a_pinned_dir_that_is_not_even_present() {
	1582	+	// A pinned name with nothing on disk must not disturb the count. This is
	1583	+	// the state the bug leaves behind, and gc runs again while it holds.
	1584	+	let tmp = tempfile::tempdir().unwrap();
	1585	+	let root = tmp.path();
	1586	+	let releases = root.join("releases");
	1587	+	tokio::fs::create_dir_all(&releases).await.unwrap();
	1588	+	for i in 0..=RELEASES_TO_KEEP {
	1589	+	tokio::fs::create_dir(releases.join(format!("v{i}")))
	1590	+	.await
	1591	+	.unwrap();
	1592	+	}
	1593	+	let pinned: PinnedReleases = ["gone-already".to_string()].into_iter().collect();
	1594	+	gc_local_releases(root, &pinned).await.unwrap();
	1595	+
	1596	+	let left = std::fs::read_dir(&releases).unwrap().count();
	1597	+	assert_eq!(left, RELEASES_TO_KEEP);
	1598	+	}
	1599	+
1495	1600		#[tokio::test]
1496	1601		async fn gc_local_releases_noop_when_below_threshold() {
1497	1602		let tmp = tempfile::tempdir().unwrap();
			@@ -1503,7 +1608,7 @@
1503	1608		.await
1504	1609		.unwrap();
1505	1610		}
1506		-	gc_local_releases(root).await.unwrap();
	1611	+	gc_local_releases(root, &no_pins()).await.unwrap();
1507	1612		for i in 0..3 {
1508	1613		assert!(releases.join(format!("v{i}")).exists());
1509	1614		}
			@@ -1512,7 +1617,7 @@
1512	1617		#[tokio::test]
1513	1618		async fn gc_local_releases_noop_when_releases_dir_missing() {
1514	1619		let tmp = tempfile::tempdir().unwrap();
1515		-	gc_local_releases(tmp.path()).await.unwrap();
	1620	+	gc_local_releases(tmp.path(), &no_pins()).await.unwrap();
1516	1621		}
1517	1622
1518	1623		#[tokio::test]

M sando/daemon/src/intake.rs +32 -9

			@@ -111,10 +111,16 @@
111	111		/// publishing is an atomic same-filesystem rename and a cross-device staging dir
112	112		/// would silently become a copy. The caller owns getting the bytes there; this
113	113		/// owns deciding whether they may stay.
	114	+	///
	115	+	/// `pinned` is passed straight through to publishing, which gc's the store: it
	116	+	/// names the release dirs the deployed state still points at
	117	+	/// ([`crate::retention::pinned_dirs`]). Passing an empty set is what a caller
	118	+	/// with no deployed state to protect does, not a shortcut.
114	119		pub async fn accept(
115	120		release_root: &Path,
116	121		staged: &Path,
117	122		record_json: &str,
	123	+	pinned: &crate::retention::PinnedReleases,
118	124		) -> Result<AcceptedArtifact, IntakeError> {
119	125		let record = ArtifactRecord::parse(record_json).map_err(IntakeError::BadRecord)?;
120	126
			@@ -147,9 +153,10 @@
147	153		.await
148	154		.map_err(\|e\| IntakeError::Publish(format!("writing MANIFEST: {e}")))?;
149	155
150		-	let released = crate::deploy::finalize_local_release(release_root, staged, computed.short())
151		-	.await
152		-	.map_err(\|e\| IntakeError::Publish(format!("{e:#}")))?;
	156	+	let released =
	157	+	crate::deploy::finalize_local_release(release_root, staged, computed.short(), pinned)
	158	+	.await
	159	+	.map_err(\|e\| IntakeError::Publish(format!("{e:#}")))?;
153	160
154	161		tracing::info!(
155	162		app = %record.provenance.app,
			@@ -189,6 +196,12 @@
189	196
190	197		#[cfg(test)]
191	198		mod tests {
	199	+	/// No deployed state in a unit test, so nothing is pinned and gc is free to
	200	+	/// apply the count alone. Retention is exercised in `deploy`'s own tests.
	201	+	fn no_pins() -> crate::retention::PinnedReleases {
	202	+	crate::retention::PinnedReleases::none()
	203	+	}
	204	+
192	205		use super::*;
193	206		use chrono::{DateTime, Utc};
194	207		use ops_artifact::{GateRecord, Manifest, Provenance, Scope, Verdict};
			@@ -265,7 +278,9 @@
265	278		let staged = staged_bundle(dir.path()).await;
266	279		let json = record_for(&staged, vec![passing_gate()]).await;
267	280
268		-	let accepted = accept(dir.path(), &staged, &json).await.unwrap();
	281	+	let accepted = accept(dir.path(), &staged, &json, &no_pins())
	282	+	.await
	283	+	.unwrap();
269	284		assert!(accepted.builder_gates_passed());
270	285		assert!(
271	286		accepted.released.ends_with(accepted.record.digest.short()),
			@@ -287,7 +302,9 @@
287	302		let json = record_for(&staged, vec![passing_gate()]).await;
288	303		write(&staged, "static/app.css", b"tampered").await;
289	304
290		-	let err = accept(dir.path(), &staged, &json).await.unwrap_err();
	305	+	let err = accept(dir.path(), &staged, &json, &no_pins())
	306	+	.await
	307	+	.unwrap_err();
291	308		match err {
292	309		IntakeError::ManifestMismatch {
293	310		first_difference: Some(p),
			@@ -308,7 +325,9 @@
308	325		let json = record_for(&staged, vec![passing_gate()]).await;
309	326		write(&staged, "companions/unexpected", b"who put this here").await;
310	327
311		-	let err = accept(dir.path(), &staged, &json).await.unwrap_err();
	328	+	let err = accept(dir.path(), &staged, &json, &no_pins())
	329	+	.await
	330	+	.unwrap_err();
312	331		match err {
313	332		IntakeError::ManifestMismatch {
314	333		first_difference: Some(p),
			@@ -322,7 +341,7 @@
322	341		async fn a_record_that_does_not_parse_is_refused_before_anything_is_hashed() {
323	342		let dir = tempfile::tempdir().unwrap();
324	343		let staged = staged_bundle(dir.path()).await;
325		-	let err = accept(dir.path(), &staged, "{\"not\": \"a record\"}")
	344	+	let err = accept(dir.path(), &staged, "{\"not\": \"a record\"}", &no_pins())
326	345		.await
327	346		.unwrap_err();
328	347		assert!(matches!(err, IntakeError::BadRecord(_)), "{err}");
			@@ -337,7 +356,9 @@
337	356		let staged = staged_bundle(dir.path()).await;
338	357		let json = record_for(&staged, vec![passing_gate()]).await;
339	358		let forged = json.replace(&record_digest(&json), &"f".repeat(64));
340		-	let err = accept(dir.path(), &staged, &forged).await.unwrap_err();
	359	+	let err = accept(dir.path(), &staged, &forged, &no_pins())
	360	+	.await
	361	+	.unwrap_err();
341	362		assert!(matches!(err, IntakeError::BadRecord(_)), "{err}");
342	363		}
343	364
			@@ -355,7 +376,9 @@
355	376		at(),
356	377		);
357	378		let json = record_for(&staged, vec![failed]).await;
358		-	let accepted = accept(dir.path(), &staged, &json).await.unwrap();
	379	+	let accepted = accept(dir.path(), &staged, &json, &no_pins())
	380	+	.await
	381	+	.unwrap();
359	382		assert!(!accepted.builder_gates_passed());
360	383		}
361	384

M sando/daemon/src/lib.rs +1

			@@ -29,6 +29,7 @@
29	29		pub mod intake;
30	30		pub mod outcome;
31	31		pub mod reconcile;
	32	+	pub mod retention;
32	33		pub mod routes;
33	34		pub mod runs;
34	35		pub mod state;

M sando/daemon/src/main.rs +32 -1

			@@ -1,5 +1,7 @@
1	1		use anyhow::Result;
2		-	use sando_daemon::{config, db, events, git, reconcile, routes, runs, state, sync, topology};
	2	+	use sando_daemon::{
	3	+	config, db, events, git, reconcile, retention, routes, runs, state, sync, topology,
	4	+	};
3	5		use std::net::SocketAddr;
4	6		use std::path::Path;
5	7		use std::sync::Arc;
			@@ -160,6 +162,35 @@
160	162		}
161	163		}
162	164
	165	+	// Say at startup whether every artifact the deployed state names is still on
	166	+	// disk. `/state` answers the same question on every poll; this is so the
	167	+	// journal carries it too, because the two occurrences of the retention bug
	168	+	// were both discovered by an rsync failing mid-promote rather than by
	169	+	// anything that had looked. Never fatal: a stranded artifact is a lost
	170	+	// rollback, not a reason to refuse to start — refusing would take the
	171	+	// controller down as well.
	172	+	for (id, app) in apps.iter() {
	173	+	match retention::tier_refs(&pool, id, app.cfg.primary_bin()).await {
	174	+	Ok(refs) => {
	175	+	let gone = retention::missing(&refs).await;
	176	+	if gone.is_empty() {
	177	+	tracing::info!(app = %id, referenced = refs.len(), "every referenced release artifact is present");
	178	+	} else {
	179	+	tracing::error!(
	180	+	app = %id,
	181	+	missing = gone.len(),
	182	+	detail = %retention::describe(&gone),
	183	+	"referenced release artifact(s) missing from the store; promote and rollback \
	184	+	through them will fail at the rsync. See /state missing_artifact"
	185	+	);
	186	+	}
	187	+	}
	188	+	Err(e) => {
	189	+	tracing::error!(app = %id, error = %e, "could not check referenced release artifacts");
	190	+	}
	191	+	}
	192	+	}
	193	+
163	194		// Fail closed on the scratch cluster's privileges before any gate can hit
164	195		// them, rather than letting the first migration_dry_run discover it as an
165	196		// opaque "permission denied". Unset scratch_db_url is already a per-gate

M sando/daemon/src/status.rs +62

			@@ -195,6 +195,18 @@
195	195		},
196	196		);
197	197		}
	198	+	// An artifact the tier still names is gone from the store. The tier is
	199	+	// serving, so this is not a `consistent` failure — what is lost is the
	200	+	// ability to redeploy or roll back, which nothing else on this node would
	201	+	// say. Reported after `consistent` and before the routine conditions.
	202	+	if let Some(detail) = &tier.missing_artifact {
	203	+	conditions.push(Condition {
	204	+	condition_type: "artifact".into(),
	205	+	status: Status::Degraded,
	206	+	since: None,
	207	+	detail: Some(format!("referenced artifact missing: {detail}")),
	208	+	});
	209	+	}
198	210		if !tier.provisioned {
199	211		conditions.push(Condition {
200	212		condition_type: "provisioned".into(),
			@@ -266,6 +278,13 @@
266	278		{
267	279		return Status::Failed;
268	280		}
	281	+	// Degraded rather than failed: the tier is serving the version it says it
	282	+	// is. What is broken is the next promote and every rollback, which is worse
	283	+	// than routine and less than an outage. Reporting it green is what let two
	284	+	// occurrences go unnoticed until an operator reached for a rollback.
	285	+	if tier.missing_artifact.is_some() {
	286	+	return Status::Degraded;
	287	+	}
269	288		// An unrecognized gate status is contract drift between daemon and viewer,
270	289		// which is worth surfacing rather than smoothing over. The TUI renders it
271	290		// magenta for the same reason.
			@@ -537,6 +556,7 @@
537	556		previous_version: Some("0.10.13".into()),
538	557		burn_in_started_at: Some("2026-07-21T14:02:00Z".into()),
539	558		partial_reason: None,
	559	+	missing_artifact: None,
540	560		nodes: vec![format!("{name}-1")],
541	561		gates: Vec::new(),
542	562		}
			@@ -844,4 +864,46 @@
844	864		assert!(node(&p, "tier:b").conditions[0].since.is_none());
845	865		assert_eq!(node(&p, "tier:b").status, Status::Ok);
846	866		}
	867	+
	868	+	#[test]
	869	+	fn a_tier_missing_a_referenced_artifact_is_degraded_and_says_which() {
	870	+	// The state that reported green through two occurrences: the tier is
	871	+	// serving, and the artifact behind its rollback is gone.
	872	+	let mut t = tier("b");
	873	+	t.missing_artifact = Some("b previous 0.11.20 (/srv/sando/releases/ed29/mnw)".into());
	874	+	let p = payload(&view(vec![t]), now());
	875	+
	876	+	let n = node(&p, "tier:b");
	877	+	assert_eq!(n.status, Status::Degraded);
	878	+	let c = n
	879	+	.conditions
	880	+	.iter()
	881	+	.find(\|c\| c.condition_type == "artifact")
	882	+	.expect("an artifact condition");
	883	+	assert_eq!(c.status, Status::Degraded);
	884	+	assert!(
	885	+	c.detail.as_deref().unwrap_or_default().contains("0.11.20"),
	886	+	"the condition has to name what is gone: {:?}",
	887	+	c.detail
	888	+	);
	889	+	assert_eq!(p.validate(), Ok(()));
	890	+	}
	891	+
	892	+	#[test]
	893	+	fn a_partial_tier_stays_failed_even_with_a_missing_artifact() {
	894	+	// Degraded must not mask the loudest thing Sando can report.
	895	+	let mut t = tier("b");
	896	+	t.partial_reason = Some("mixed versions after a failed promote".into());
	897	+	t.missing_artifact = Some("b previous 0.11.20 (/srv/sando/releases/ed29/mnw)".into());
	898	+	let p = payload(&view(vec![t]), now());
	899	+
	900	+	let n = node(&p, "tier:b");
	901	+	assert_eq!(n.status, Status::Failed);
	902	+	assert!(n.conditions.iter().any(\|c\| c.condition_type == "artifact"));
	903	+	assert!(
	904	+	n.conditions
	905	+	.iter()
	906	+	.any(\|c\| c.condition_type == "consistent")
	907	+	);
	908	+	}
847	909		}

M sando/daemon/src/routes/mod.rs +78

			@@ -174,6 +174,15 @@
174	174		/// failed promote or rollback whose compensation could not fully restore
175	175		/// consistency. NULL when the tier is consistent. The TUI flags it red.
176	176		pub(crate) partial_reason: Option<String>,
	177	+	/// Non-null when an artifact this tier still names is no longer on disk:
	178	+	/// the row points at a release dir whose primary binary is gone, so a
	179	+	/// promote or rollback resolving through it would fail at the rsync.
	180	+	///
	181	+	/// The tier keeps serving — the running process holds its own inode — which
	182	+	/// is why this was invisible until an operator reached for a rollback that
	183	+	/// was not there. Read live on every `/state`, so a store repaired by hand
	184	+	/// clears it on the next poll with nothing to reset.
	185	+	pub(crate) missing_artifact: Option<String>,
177	186		pub(crate) nodes: Vec<String>,
178	187		pub(crate) gates: Vec<GateView>,
179	188		}
			@@ -218,6 +227,12 @@
218	227		.fetch_all(&s.pool)
219	228		.await?;
220	229
	230	+	// What the deployed state points at, and which of it is actually there.
	231	+	// One pass for every tier: the read is a handful of stats, and doing it here
	232	+	// rather than on a timer means `/state` cannot report a stale verdict.
	233	+	let refs = crate::retention::tier_refs(&s.pool, &s.cfg.id, s.cfg.primary_bin()).await?;
	234	+	let gone = crate::retention::missing(&refs).await;
	235	+
221	236		let mut tiers: Vec<TierView> = Vec::with_capacity(rows.len());
222	237		for r in rows {
223	238		let name: String = r.get("name");
			@@ -317,6 +332,11 @@
317	332		})
318	333		.collect();
319	334
	335	+	let missing_artifact = {
	336	+	let mine: Vec<_> = gone.iter().filter(\|g\| g.tier == name).cloned().collect();
	337	+	(!mine.is_empty()).then(\|\| crate::retention::describe(&mine))
	338	+	};
	339	+
320	340		tiers.push(TierView {
321	341		name,
322	342		ord: r.get("ord"),
			@@ -326,6 +346,7 @@
326	346		previous_version: r.get("previous_version"),
327	347		burn_in_started_at: r.get("burn_in_started_at"),
328	348		partial_reason: r.get("partial_reason"),
	349	+	missing_artifact,
329	350		nodes,
330	351		gates,
331	352		});
			@@ -1508,6 +1529,63 @@
1508	1529		.unwrap();
1509	1530		}
1510	1531
	1532	+	/// `/state` says when a tier's artifact is gone, and stays quiet when it is
	1533	+	/// not.
	1534	+	///
	1535	+	/// The gap this closes: `versions`/`build_runs` keep naming a path long
	1536	+	/// after gc removed the bytes, and nothing reconciled the two. Both times it
	1537	+	/// happened, the tier read green until an rsync failed mid-promote.
	1538	+	#[tokio::test]
	1539	+	async fn state_reports_a_referenced_artifact_whose_bytes_are_gone() {
	1540	+	let tmp = tempfile::tempdir().unwrap();
	1541	+	let present = tmp.path().join("releases").join("1111111111111111");
	1542	+	tokio::fs::create_dir_all(&present).await.unwrap();
	1543	+
	1544	+	let state = test_state().await;
	1545	+	let bin = state.cfg.primary_bin().to_string();
	1546	+	tokio::fs::write(present.join(&bin), b"bin").await.unwrap();
	1547	+
	1548	+	seed(&state.pool, "a", "0.11.20").await;
	1549	+	let build = seed_build(
	1550	+	&state.pool,
	1551	+	"2a53c900",
	1552	+	"0.11.20",
	1553	+	present.to_string_lossy().as_ref(),
	1554	+	)
	1555	+	.await;
	1556	+	sqlx::query(
	1557	+	"UPDATE tier_state SET current_version = ?, current_build_id = ? WHERE tier = 'a'",
	1558	+	)
	1559	+	.bind("0.11.20")
	1560	+	.bind(build)
	1561	+	.execute(&state.pool)
	1562	+	.await
	1563	+	.unwrap();
	1564	+
	1565	+	let tier_a = \|v: &StateView\| {
	1566	+	v.tiers
	1567	+	.iter()
	1568	+	.find(\|t\| t.name == "a")
	1569	+	.expect("tier a")
	1570	+	.missing_artifact
	1571	+	.clone()
	1572	+	};
	1573	+
	1574	+	assert_eq!(
	1575	+	tier_a(&state_view(&state).await.unwrap()),
	1576	+	None,
	1577	+	"the artifact is on disk; nothing to report"
	1578	+	);
	1579	+
	1580	+	// gc takes it, as it did twice in production. Nothing in the database
	1581	+	// changes, which is the whole defect.
	1582	+	tokio::fs::remove_dir_all(&present).await.unwrap();
	1583	+
	1584	+	let said = tier_a(&state_view(&state).await.unwrap()).expect("a missing-artifact report");
	1585	+	assert!(said.contains("0.11.20"), "{said}");
	1586	+	assert!(said.contains("current"), "{said}");
	1587	+	}
	1588	+
1511	1589		/// A rebuild at an unchanged version must not move what a tier reports.
1512	1590		///
1513	1591		/// `/state` keyed its gate rows on (tier, version), so two runs of one

A sando/daemon/src/retention.rs +500

		@@ -1,0 +1,585 @@
1	+	//! Which release directories the deployed state still points at.
2	+	//!
3	+	//! <!-- wiki: release-artifact-identity -->
4	+	//!
5	+	//! Release dirs are content-addressed (`releases/<digest16>`) and trimmed after
6	+	//! every publish. Trimming used to be a count alone — keep the 5 newest by
7	+	//! mtime — which cannot express "this one is still in production". Three
8	+	//! rebuilds of a single version fill three of the five slots, and the artifacts
9	+	//! a tier is running and would roll back to fall off the end. That happened
10	+	//! twice (2026-08-19, 2026-08-25); the second time neither surviving directory
11	+	//! held a runnable binary and prod had nothing to roll back to.
12	+	//!
13	+	//! So the set below is computed first and set aside, and the count applies only
14	+	//! to what is left. The count is a floor on how much history to keep, not a
15	+	//! ceiling on what may be retained.
16	+	//!
17	+	//! # What counts as referenced
18	+	//!
19	+	//! Everything promote and rollback resolve through, because a dir that is
20	+	//! unreachable to them is exactly the one whose absence is discovered by an
21	+	//! rsync failing mid-promote:
22	+	//!
23	+	//! 1. `tier_state.current_build_id` / `previous_build_id` -> `build_runs.staged_path`.
24	+	//! The identity path (migration 008), and the tier's own answer to what it is
25	+	//! running.
26	+	//! 2. The newest green `build_runs` row per (version, platform) for every version
27	+	//! a tier names. This is what [`crate::routes::promotion`] resolves a rollback
28	+	//! to, and on a two-architecture product it is a different row per node.
29	+	//! 3. `versions.artifact_path` for those versions. The pre-identity path, still
30	+	//! the first thing a canary rollback reads.
31	+	//!
32	+	//! The three overlap heavily and are unioned rather than ranked: being reachable
33	+	//! by any of them is enough to make a directory load-bearing.
34	+
35	+	use crate::domain::AppId;
36	+	use anyhow::Result;
37	+	use sqlx::{Row, SqlitePool};
38	+	use std::collections::{BTreeSet, HashSet};
39	+	use std::path::{Path, PathBuf};
40	+
41	+	/// The release directories that must survive a gc, by the name gc matches on.
42	+	///
43	+	/// A newtype rather than a bare `HashSet<String>` so the thing being passed
44	+	/// through publish and into gc says what it is at every hop, and so a caller
45	+	/// cannot hand it a set of paths, versions, or digests by accident.
46	+	#[derive(Debug, Clone, Default, PartialEq, Eq)]
47	+	pub struct PinnedReleases(HashSet<String>);
48	+
49	+	impl PinnedReleases {
50	+	/// Nothing to protect. What a caller with no deployed state passes — a unit
51	+	/// test, or a store that has never served anything.
52	+	pub fn none() -> Self {
53	+	Self::default()
54	+	}
55	+
56	+	pub fn contains(&self, dir_name: &str) -> bool {
57	+	self.0.contains(dir_name)
58	+	}
59	+
60	+	pub fn len(&self) -> usize {
61	+	self.0.len()
62	+	}
63	+
64	+	pub fn is_empty(&self) -> bool {
65	+	self.0.is_empty()
66	+	}
67	+	}
68	+
69	+	impl FromIterator<String> for PinnedReleases {
70	+	fn from_iter<T: IntoIterator<Item = String>>(iter: T) -> Self {
71	+	Self(iter.into_iter().collect())
72	+	}
73	+	}
74	+
75	+	/// Which of a tier's two artifacts a reference is.
76	+	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77	+	pub enum Role {
78	+	/// What the tier is running.
79	+	Current,
80	+	/// The one step of rollback history `tier_state` keeps.
81	+	Previous,
82	+	}
83	+
84	+	impl Role {
85	+	pub const fn as_str(self) -> &'static str {
86	+	match self {
87	+	Self::Current => "current",
88	+	Self::Previous => "previous",
89	+	}
90	+	}
91	+	}
92	+
93	+	impl std::fmt::Display for Role {
94	+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
95	+	f.write_str(self.as_str())
96	+	}
97	+	}
98	+
99	+	/// One artifact a tier still names, and the bytes that have to be present for
100	+	/// the reference to be honoured.
101	+	#[derive(Debug, Clone, PartialEq, Eq)]
102	+	pub struct ArtifactRef {
103	+	pub tier: String,
104	+	pub role: Role,
105	+	/// The version label, when the tier records one.
106	+	pub version: Option<String>,
107	+	/// The release directory the reference resolves to.
108	+	pub dir: PathBuf,
109	+	/// The file whose absence makes the reference unusable — the primary binary
110	+	/// inside `dir`. A directory holding only a MANIFEST is not a rollback
111	+	/// target, and both stranded dirs in the 2026-08-25 measurement were exactly
112	+	/// that, so existence of the directory is the wrong question to ask.
113	+	pub binary: PathBuf,
114	+	}
115	+
116	+	/// The directory name gc sees under `releases/`.
117	+	///
118	+	/// `None` for a recorded path with no final component, which is a malformed row
119	+	/// rather than a reference to the root.
120	+	fn dir_name(dir: &Path) -> Option<String> {
121	+	dir.file_name()
122	+	.and_then(\|n\| n.to_str())
123	+	.map(ToOwned::to_owned)
124	+	}
125	+
126	+	/// Every release directory the deployed state still points at, by the name gc
127	+	/// matches on.
128	+	///
129	+	/// Scoped to one product because `release_root` is: a second app's tiers name
130	+	/// directories in a store this gc never walks.
131	+	pub async fn pinned_dirs(pool: &SqlitePool, app: &AppId) -> Result<PinnedReleases> {
132	+	let mut pinned = HashSet::new();
133	+
134	+	// (1) + (2): every staged_path reachable from the tier's build ids, and
135	+	// every newest-green build per (version, platform) for the versions the
136	+	// tiers name. One query: the second set is what a rollback resolves to and
137	+	// the first is what the tier is on, and a row can be in both.
138	+	let rows = sqlx::query(
139	+	"SELECT DISTINCT br.staged_path
140	+	FROM build_runs br
141	+	WHERE br.app = ?1
142	+	AND br.staged_path IS NOT NULL
143	+	AND (
144	+	br.id IN (SELECT current_build_id FROM tier_state WHERE app = ?1
145	+	UNION ALL
146	+	SELECT previous_build_id FROM tier_state WHERE app = ?1)
147	+	OR (br.result = 'passed'
148	+	AND br.version IN (SELECT current_version FROM tier_state WHERE app = ?1
149	+	UNION ALL
150	+	SELECT previous_version FROM tier_state WHERE app = ?1)
151	+	AND br.id = (SELECT MAX(id) FROM build_runs
152	+	WHERE app = ?1
153	+	AND version = br.version
154	+	AND platform IS br.platform
155	+	AND result = 'passed'))
156	+	)",
157	+	)
158	+	.bind(app)
159	+	.fetch_all(pool)
160	+	.await?;
161	+	for r in rows {
162	+	let path: String = r.get("staged_path");
163	+	if let Some(name) = dir_name(Path::new(&path)) {
164	+	pinned.insert(name);
165	+	}
166	+	}
167	+
168	+	// (3) The pre-identity path. `artifact_path` names the primary binary, so
169	+	// the directory is its parent.
170	+	let rows = sqlx::query(
171	+	"SELECT DISTINCT v.artifact_path
172	+	FROM versions v
173	+	WHERE v.app = ?1
174	+	AND v.version IN (SELECT current_version FROM tier_state WHERE app = ?1
175	+	UNION ALL
176	+	SELECT previous_version FROM tier_state WHERE app = ?1)",
177	+	)
178	+	.bind(app)
179	+	.fetch_all(pool)
180	+	.await?;
181	+	for r in rows {
182	+	let path: String = r.get("artifact_path");
183	+	if let Some(name) = Path::new(&path).parent().and_then(dir_name) {
184	+	pinned.insert(name);
185	+	}
186	+	}
187	+
188	+	Ok(PinnedReleases(pinned))
189	+	}
190	+
191	+	/// What each tier is running and what it would roll back to, resolved the way
192	+	/// promote and rollback resolve it: the build row's `staged_path` when the tier
193	+	/// has an identity, `versions.artifact_path` when it predates one.
194	+	///
195	+	/// One entry per (tier, role) that resolves to a path at all. A tier with no
196	+	/// previous version contributes one entry, not two.
197	+	pub async fn tier_refs(
198	+	pool: &SqlitePool,
199	+	app: &AppId,
200	+	primary_bin: &str,
201	+	) -> Result<Vec<ArtifactRef>> {
202	+	let rows = sqlx::query(
203	+	"SELECT ts.tier,
204	+	ts.current_version, cb.staged_path AS current_dir, cv.artifact_path AS current_bin,
205	+	ts.previous_version, pb.staged_path AS previous_dir, pv.artifact_path AS previous_bin
206	+	FROM tier_state ts
207	+	LEFT JOIN build_runs cb ON cb.id = ts.current_build_id
208	+	LEFT JOIN build_runs pb ON pb.id = ts.previous_build_id
209	+	LEFT JOIN versions cv ON cv.app = ts.app AND cv.version = ts.current_version
210	+	LEFT JOIN versions pv ON pv.app = ts.app AND pv.version = ts.previous_version
211	+	WHERE ts.app = ?
212	+	ORDER BY ts.tier",
213	+	)
214	+	.bind(app)
215	+	.fetch_all(pool)
216	+	.await?;
217	+
218	+	let mut refs = Vec::new();
219	+	for r in rows {
220	+	let tier: String = r.get("tier");
221	+	for (role, version, dir_col, bin_col) in [
222	+	(
223	+	Role::Current,
224	+	r.get::<Option<String>, _>("current_version"),
225	+	r.get::<Option<String>, _>("current_dir"),
226	+	r.get::<Option<String>, _>("current_bin"),
227	+	),
228	+	(
229	+	Role::Previous,
230	+	r.get::<Option<String>, _>("previous_version"),
231	+	r.get::<Option<String>, _>("previous_dir"),
232	+	r.get::<Option<String>, _>("previous_bin"),
233	+	),
234	+	] {
235	+	// The build row wins when there is one: it is the identity, and the
236	+	// version label can be shared by several builds. `artifact_path` is
237	+	// the fallback for a tier that predates migration 008 — the same
238	+	// fallback the gate scope and the canary rollback take.
239	+	let resolved = match (dir_col, bin_col) {
240	+	(Some(dir), _) => {
241	+	let dir = PathBuf::from(dir);
242	+	let binary = dir.join(primary_bin);
243	+	Some((dir, binary))
244	+	}
245	+	(None, Some(bin)) => {
246	+	let binary = PathBuf::from(bin);
247	+	binary.parent().map(\|d\| (d.to_path_buf(), binary.clone()))
248	+	}
249	+	(None, None) => None,
250	+	};
251	+	if let Some((dir, binary)) = resolved {
252	+	refs.push(ArtifactRef {
253	+	tier: tier.clone(),
254	+	role,
255	+	version,
256	+	dir,
257	+	binary,
258	+	});
259	+	}
260	+	}
261	+	}
262	+	Ok(refs)
263	+	}
264	+
265	+	/// The references whose bytes are gone.
266	+	///
267	+	/// Deliberately a `stat` per reference rather than a directory walk: there are
268	+	/// at most two per tier, and asking about the exact file promote would rsync is
269	+	/// the only question whose answer means anything.
270	+	pub async fn missing(refs: &[ArtifactRef]) -> Vec<ArtifactRef> {
271	+	let mut gone = Vec::new();
272	+	for r in refs {
273	+	if !tokio::fs::try_exists(&r.binary).await.unwrap_or(false) {
274	+	gone.push(r.clone());
275	+	}
276	+	}
277	+	gone
278	+	}
279	+
280	+	/// One line naming what is gone, for a log or a `/state` condition.
281	+	///
282	+	/// Sorted and deduplicated by tier so the sentence is stable across reads —
283	+	/// an operator surface that reworded itself every poll would read as churn.
284	+	pub fn describe(missing: &[ArtifactRef]) -> String {
285	+	let lines: BTreeSet<String> = missing
286	+	.iter()
287	+	.map(\|r\| {
288	+	let version = r.version.as_deref().unwrap_or("unknown version");
289	+	format!("{} {} {version} ({})", r.tier, r.role, r.binary.display())
290	+	})
291	+	.collect();
292	+	lines.into_iter().collect::<Vec<_>>().join("; ")
293	+	}
294	+
295	+	#[cfg(test)]
296	+	mod tests {
297	+	use super::*;
298	+	use sqlx::sqlite::SqlitePoolOptions;
299	+
300	+	async fn fresh_pool() -> SqlitePool {
301	+	let pool = SqlitePoolOptions::new()
302	+	.max_connections(1)
303	+	.connect("sqlite::memory:")
304	+	.await
305	+	.unwrap();
306	+	sqlx::migrate!("./migrations").run(&pool).await.unwrap();
307	+	pool
308	+	}
309	+
310	+	fn app() -> AppId {
311	+	AppId::default()
312	+	}
313	+
314	+	async fn tier(pool: &SqlitePool, name: &str, ord: i64) {
315	+	sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES ('mnw', ?, ?, 1)")
316	+	.bind(name)
317	+	.bind(ord)
318	+	.execute(pool)
319	+	.await
320	+	.unwrap();
321	+	sqlx::query("INSERT INTO tier_state (app, tier) VALUES ('mnw', ?)")
322	+	.bind(name)
323	+	.execute(pool)
324	+	.await
325	+	.unwrap();
326	+	}
327	+
328	+	/// A settled build row with an identity, as `runs::set_identity` leaves it.
329	+	async fn build(pool: &SqlitePool, version: &str, dir: &str, platform: Option<&str>) -> i64 {
330	+	sqlx::query(
331	+	"INSERT INTO build_runs (app, sha, version, result, started_at, bundle_digest,
332	+	staged_path, platform)
333	+	VALUES ('mnw', 'abc123', ?, 'passed', '2026-08-25T00:00:00Z', ?, ?, ?)",
334	+	)
335	+	.bind(version)
336	+	.bind(dir)
337	+	.bind(format!("/srv/sando/releases/{dir}"))
338	+	.bind(platform)
339	+	.execute(pool)
340	+	.await
341	+	.unwrap()
342	+	.last_insert_rowid()
343	+	}
344	+
345	+	async fn version_row(pool: &SqlitePool, version: &str, artifact_path: &str) {
346	+	sqlx::query(
347	+	"INSERT INTO versions (app, version, git_sha, built_at, artifact_path)
348	+	VALUES ('mnw', ?, 'abc123', '2026-08-25T00:00:00Z', ?)",
349	+	)
350	+	.bind(version)
351	+	.bind(artifact_path)
352	+	.execute(pool)
353	+	.await
354	+	.unwrap();
355	+	}
356	+
357	+	/// Fixture-only: the SET clause varies per test and every value in it is a
358	+	/// literal written here, so `raw_sql` is the honest tool rather than a
359	+	/// bound query that cannot take a clause.
360	+	async fn set_state(pool: &SqlitePool, tier: &str, sql: &str) {
361	+	sqlx::raw_sql(sqlx::AssertSqlSafe(format!(
362	+	"UPDATE tier_state SET {sql} WHERE app = 'mnw' AND tier = '{tier}'"
363	+	)))
364	+	.execute(pool)
365	+	.await
366	+	.unwrap();
367	+	}
368	+
369	+	#[tokio::test]
370	+	async fn pins_what_every_tier_is_running_and_would_roll_back_to() {
371	+	let pool = fresh_pool().await;
372	+	tier(&pool, "host", 0).await;
373	+	tier(&pool, "a", 1).await;
374	+
375	+	let cur = build(&pool, "0.16.1", "aaaaaaaaaaaaaaaa", None).await;
376	+	let prev = build(&pool, "0.11.20", "bbbbbbbbbbbbbbbb", None).await;
377	+	set_state(
378	+	&pool,
379	+	"host",
380	+	&format!(
381	+	"current_version = '0.16.1', current_build_id = {cur},
382	+	previous_version = '0.11.20', previous_build_id = {prev}"
383	+	),
384	+	)
385	+	.await;
386	+
387	+	let pinned = pinned_dirs(&pool, &app()).await.unwrap();
388	+	assert!(pinned.contains("aaaaaaaaaaaaaaaa"));
389	+	assert!(pinned.contains("bbbbbbbbbbbbbbbb"));
390	+	assert_eq!(pinned.len(), 2, "{pinned:?}");
391	+	}
392	+
393	+	/// The exact shape that stranded prod: three builds of one version, the
394	+	/// tier on the OLDEST of them. Pinning by version alone would keep the
395	+	/// newest rebuild and lose the bytes actually deployed.
396	+	#[tokio::test]
397	+	async fn pins_the_build_the_tier_is_on_not_the_newest_rebuild_of_its_version() {
398	+	let pool = fresh_pool().await;
399	+	tier(&pool, "host", 0).await;
400	+
401	+	let deployed = build(&pool, "0.16.1", "0000000000000001", None).await;
402	+	build(&pool, "0.16.1", "0000000000000002", None).await;
403	+	let newest = build(&pool, "0.16.1", "0000000000000003", None).await;
404	+	set_state(
405	+	&pool,
406	+	"host",
407	+	&format!("current_version = '0.16.1', current_build_id = {deployed}"),
408	+	)
409	+	.await;
410	+
411	+	let pinned = pinned_dirs(&pool, &app()).await.unwrap();
412	+	assert!(
413	+	pinned.contains("0000000000000001"),
414	+	"the deployed build must be pinned: {pinned:?}"
415	+	);
416	+	// And the newest green build of that version too: that is the row a
417	+	// rollback to 0.16.1 resolves through, so evicting it breaks a path the
418	+	// tier can still take.
419	+	assert!(pinned.contains("0000000000000003"), "{pinned:?}");
420	+	assert_ne!(deployed, newest);
421	+	assert!(!pinned.contains("0000000000000002"), "{pinned:?}");
422	+	}
423	+
424	+	/// Two architectures of one version are two artifacts, and a rollback picks
425	+	/// per node. Both have to survive.
426	+	#[tokio::test]
427	+	async fn pins_every_platform_of_a_referenced_version() {
428	+	let pool = fresh_pool().await;
429	+	tier(&pool, "host", 0).await;
430	+	build(&pool, "0.4.0", "aaaa000000000000", Some("linux/x86_64")).await;
431	+	build(&pool, "0.4.0", "bbbb000000000000", Some("linux/aarch64")).await;
432	+	set_state(&pool, "host", "current_version = '0.4.0'").await;
433	+
434	+	let pinned = pinned_dirs(&pool, &app()).await.unwrap();
435	+	assert!(pinned.contains("aaaa000000000000"), "{pinned:?}");
436	+	assert!(pinned.contains("bbbb000000000000"), "{pinned:?}");
437	+	}
438	+
439	+	/// A pre-identity tier has no build id at all. `versions.artifact_path` is
440	+	/// the only handle, and it names the binary inside the dir.
441	+	#[tokio::test]
442	+	async fn pins_a_pre_identity_tier_through_versions() {
443	+	let pool = fresh_pool().await;
444	+	tier(&pool, "host", 0).await;
445	+	version_row(
446	+	&pool,
447	+	"0.8.12",
448	+	"/srv/sando/releases/cccccccccccccccc/makenotwork",
449	+	)
450	+	.await;
451	+	set_state(&pool, "host", "current_version = '0.8.12'").await;
452	+
453	+	let pinned = pinned_dirs(&pool, &app()).await.unwrap();
454	+	assert!(pinned.contains("cccccccccccccccc"), "{pinned:?}");
455	+	}
456	+
457	+	/// `release_root` is per product, so another product's references must not
458	+	/// leak into this store's keep-set.
459	+	#[tokio::test]
460	+	async fn another_products_references_are_not_pinned_here() {
461	+	let pool = fresh_pool().await;
462	+	tier(&pool, "host", 0).await;
463	+	sqlx::query("INSERT INTO tiers (app, name, ord, provisioned) VALUES ('pom', 'host', 0, 1)")
464	+	.execute(&pool)
465	+	.await
466	+	.unwrap();
467	+	sqlx::query("INSERT INTO tier_state (app, tier) VALUES ('pom', 'host')")
468	+	.execute(&pool)
469	+	.await
470	+	.unwrap();
471	+	sqlx::query(
472	+	"INSERT INTO build_runs (app, sha, version, result, started_at, staged_path)
473	+	VALUES ('pom', 'abc', '1.0.0', 'passed', '2026-08-25T00:00:00Z',
474	+	'/srv/sando-pom/releases/dddddddddddddddd')",
475	+	)
476	+	.execute(&pool)
477	+	.await
478	+	.unwrap();
479	+	sqlx::query(
480	+	"UPDATE tier_state SET current_version = '1.0.0' WHERE app = 'pom' AND tier = 'host'",
481	+	)
482	+	.execute(&pool)
483	+	.await
484	+	.unwrap();
485	+
486	+	let pinned = pinned_dirs(&pool, &app()).await.unwrap();
487	+	assert!(pinned.is_empty(), "{pinned:?}");
488	+	}
489	+
490	+	#[tokio::test]
491	+	async fn a_tier_that_has_never_deployed_pins_nothing() {
492	+	let pool = fresh_pool().await;
493	+	tier(&pool, "host", 0).await;
494	+	assert!(pinned_dirs(&pool, &app()).await.unwrap().is_empty());
495	+	assert!(
496	+	tier_refs(&pool, &app(), "makenotwork")
497	+	.await
498	+	.unwrap()
499	+	.is_empty()
500	+	);

Lines truncated