max / makenotwork

7.8 KB · 196 lines History Blame Raw

1	//! Disk retention.
2	//!
3	//! Build logs (`logs_root`) and collected artifacts (`dist_root`) are written
4	//! per `(app, version)` and never removed, so both roots grow without bound —
5	//! a build host accumulates every historical version forever until the disk
6	//! fills (which then surfaces as silent SQLite write failures). This module
7	//! keeps only the most recent [`KEEP_VERSIONS_PER_APP`] versions per app under
8	//! each root; `main` runs it at startup and on a periodic timer.
9
10	use crate::config::Config;
11	use crate::domain::Version;
12	use std::collections::HashSet;
13	use std::path::{Path, PathBuf};
14
15	/// How many of the newest versions to keep per app, per root. Chosen for
16	/// rollback/debug depth without unbounded growth (operator decision, Run 2).
17	pub const KEEP_VERSIONS_PER_APP: usize = 5;
18
19	/// A set of `(app, version)` pairs retention must never delete, even when they
20	/// rank outside the newest N. Two classes belong here:
21	///
22	/// - In-flight — a build is currently writing into the version's dir, so
23	/// pruning it would pull a directory out from under a live `collect`/log
24	/// stream.
25	/// - Released — a published release (an OTA manifest / the `releases` table)
26	/// still points at the version's artifacts. A slow-adopting `0.5.0` followed
27	/// by five quick patch releases would otherwise fall outside the window and be
28	/// deleted while clients are still fetching it.
29	///
30	/// The caller assembles the set; retention only honours it.
31	pub type ProtectedVersions = HashSet<(String, String)>;
32
33	/// Prune both roots to the newest [`KEEP_VERSIONS_PER_APP`] versions per app,
34	/// skipping any `(app, version)` in `protected`. Best-effort and blocking: IO
35	/// errors are logged and skipped, never fatal. Intended to run inside
36	/// `spawn_blocking`.
37	pub fn prune_once(cfg: &Config, protected: &ProtectedVersions) {
38	// The daemon's own copies only. The archive host is deliberately left alone:
39	// it is the copy meant to outlive the build box's disk pressure.
40	prune_root(&cfg.dist_root, "dist", protected);
41	prune_root(&cfg.logs_root, "logs", protected);
42	}
43
44	fn prune_root(root: &Path, label: &str, protected: &ProtectedVersions) {
45	let Ok(apps) = std::fs::read_dir(root) else {
46	return;
47	};
48	for app in apps.filter_map(std::result::Result::ok) {
49	let app_dir = app.path();
50	if app_dir.is_dir() {
51	let app_name = app.file_name().to_string_lossy().into_owned();
52	prune_app_dir(&app_dir, &app_name, label, protected);
53	}
54	}
55	}
56
57	fn prune_app_dir(app_dir: &Path, app_name: &str, label: &str, protected: &ProtectedVersions) {
58	let Ok(entries) = std::fs::read_dir(app_dir) else {
59	return;
60	};
61	// Only consider subdirectories whose name parses as a semver — anything
62	// else (a stray file, a non-version dir) is left untouched.
63	let mut versions: Vec<(Version, String, PathBuf)> = entries
64	.filter_map(std::result::Result::ok)
65	.filter(\|e\| e.path().is_dir())
66	.filter_map(\|e\| {
67	let name = e.file_name().to_string_lossy().into_owned();
68	let v = Version::parse(&name).ok()?;
69	Some((v, name, e.path()))
70	})
71	.collect();
72	if versions.len() <= KEEP_VERSIONS_PER_APP {
73	return;
74	}
75	versions.sort_by(\|a, b\| b.0.cmp(&a.0)); // newest first
76	for (ver, name, path) in versions.into_iter().skip(KEEP_VERSIONS_PER_APP) {
77	// Never delete a version an in-flight build is writing to, or one a
78	// published release still points at.
79	if protected.contains(&(app_name.to_string(), name)) {
80	tracing::debug!(%ver, app = app_name, "retention: skipping protected {label} version");
81	continue;
82	}
83	match std::fs::remove_dir_all(&path) {
84	Ok(()) => {
85	tracing::info!(%ver, dir = %path.display(), "retention: pruned old {label} version");
86	}
87	Err(e) => {
88	tracing::warn!(error = %e, dir = %path.display(), "retention: could not prune {label} dir");
89	}
90	}
91	}
92	}
93
94	#[cfg(test)]
95	mod tests {
96	use super::*;
97
98	fn touch_version(root: &Path, app: &str, ver: &str) {
99	let d = root.join(app).join(ver);
100	std::fs::create_dir_all(&d).unwrap();
101	std::fs::write(d.join("artifact.bin"), b"x").unwrap();
102	}
103
104	#[test]
105	fn keeps_newest_n_and_prunes_older_by_semver() {
106	let tmp = tempfile::tempdir().unwrap();
107	let root = tmp.path().join("dist");
108	// Seven versions; lexical order would mis-rank 0.4.10 vs 0.4.9.
109	for v in [
110	"0.4.0", "0.4.1", "0.4.2", "0.4.9", "0.4.10", "0.5.0", "0.5.1",
111	] {
112	touch_version(&root, "demo", v);
113	}
114	// A non-version dir must be left alone.
115	std::fs::create_dir_all(root.join("demo").join("scratch")).unwrap();
116
117	prune_root(&root, "dist", &ProtectedVersions::new());
118
119	let mut kept: Vec<String> = std::fs::read_dir(root.join("demo"))
120	.unwrap()
121	.filter_map(std::result::Result::ok)
122	.map(\|e\| e.file_name().to_string_lossy().into_owned())
123	.collect();
124	kept.sort();
125	// Newest 5 by semver: 0.5.1, 0.5.0, 0.4.10, 0.4.9, 0.4.2 (+ the non-version dir).
126	assert_eq!(
127	kept,
128	vec!["0.4.10", "0.4.2", "0.4.9", "0.5.0", "0.5.1", "scratch"]
129	);
130	}
131
132	#[test]
133	fn no_op_when_under_limit() {
134	let tmp = tempfile::tempdir().unwrap();
135	let root = tmp.path().join("logs");
136	for v in ["0.1.0", "0.2.0"] {
137	touch_version(&root, "demo", v);
138	}
139	prune_root(&root, "logs", &ProtectedVersions::new());
140	assert_eq!(std::fs::read_dir(root.join("demo")).unwrap().count(), 2);
141	}
142
143	#[test]
144	fn never_prunes_a_protected_version() {
145	let tmp = tempfile::tempdir().unwrap();
146	let root = tmp.path().join("dist");
147	// Seven versions, keep 5 -> 0.1.0 and 0.2.0 are beyond the window.
148	for v in [
149	"0.1.0", "0.2.0", "0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.7.0",
150	] {
151	touch_version(&root, "demo", v);
152	}
153	// 0.1.0 is beyond the window but is protected (in-flight build, or a
154	// still-referenced release).
155	let protected: ProtectedVersions = [("demo".to_string(), "0.1.0".to_string())]
156	.into_iter()
157	.collect();
158	prune_root(&root, "dist", &protected);
159	assert!(
160	root.join("demo").join("0.1.0").exists(),
161	"protected version must survive"
162	);
163	// 0.2.0 (also beyond the window, not protected) is pruned instead.
164	assert!(!root.join("demo").join("0.2.0").exists());
165	assert!(
166	root.join("demo").join("0.3.0").exists(),
167	"newest 5 are kept"
168	);
169	}
170
171	#[test]
172	fn protects_a_released_version_far_outside_the_window() {
173	let tmp = tempfile::tempdir().unwrap();
174	let root = tmp.path().join("dist");
175	// A slow-adopting 0.5.0 followed by six newer patch releases: 0.5.0 is
176	// seventh-newest, two slots beyond the keep window.
177	for v in [
178	"0.5.0", "0.5.1", "0.5.2", "0.5.3", "0.5.4", "0.5.5", "0.5.6",
179	] {
180	touch_version(&root, "demo", v);
181	}
182	// A live release still points at 0.5.0 (the caller unions the releases
183	// table into the protected set).
184	let protected: ProtectedVersions = [("demo".to_string(), "0.5.0".to_string())]
185	.into_iter()
186	.collect();
187	prune_root(&root, "dist", &protected);
188	assert!(
189	root.join("demo").join("0.5.0").exists(),
190	"a released version must survive even far outside the keep window"
191	);
192	// The next-oldest non-released version (0.5.1) is pruned in its place.
193	assert!(!root.join("demo").join("0.5.1").exists());
194	}
195	}
196