Skip to main content

max / makenotwork

bento: retention skips in-flight versions (Run 3 storage) The 6h/startup pruner could remove_dir_all a version dir out from under a live collect/log stream if that version ranked beyond the newest 5. prune_once now takes the set of (app, version) pairs an in-flight build is writing — queried from target_runs WHERE status='running' each tick — and never prunes one, even if it's beyond the keep window. The next-oldest non-active version is pruned instead. bento-daemon 48 tests; clippy -D clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-23 02:47 UTC
Commit: 5e7165c9994e602a03fd48843b137efb8b60f409
Parent: 142252f
2 files changed, +61 insertions, -18 deletions
M bento/daemon/src/main.rs +15 -2
@@ -68,15 +68,28 @@ async fn main() -> Result<()> {
68 68 api_token,
69 69 };
70 70 // Disk retention: prune logs_root + dist_root to the newest N versions per
71 - // app, at startup and every 6h, so neither root grows without bound.
71 + // app, at startup and every 6h, so neither root grows without bound. Skips
72 + // any (app, version) an in-flight build is writing to.
72 73 {
73 74 let cfg = app_state.cfg.clone();
75 + let pool = app_state.pool.clone();
74 76 tokio::spawn(async move {
75 77 let mut tick = tokio::time::interval(std::time::Duration::from_secs(6 * 60 * 60));
76 78 loop {
77 79 tick.tick().await; // fires immediately, then every 6h
80 + let active: bento_daemon::retention::ActiveVersions = sqlx::query_as::<_, (String, String)>(
81 + "SELECT DISTINCT app, version FROM target_runs WHERE status = 'running'",
82 + )
83 + .fetch_all(&pool)
84 + .await
85 + .unwrap_or_default()
86 + .into_iter()
87 + .collect();
78 88 let cfg = cfg.clone();
79 - let _ = tokio::task::spawn_blocking(move || bento_daemon::retention::prune_once(&cfg)).await;
89 + let _ = tokio::task::spawn_blocking(move || {
90 + bento_daemon::retention::prune_once(&cfg, &active)
91 + })
92 + .await;
80 93 }
81 94 });
82 95 }
M bento/daemon/src/retention.rs +46 -16
@@ -9,48 +9,61 @@
9 9
10 10 use crate::config::Config;
11 11 use crate::domain::Version;
12 + use std::collections::HashSet;
12 13 use std::path::{Path, PathBuf};
13 14
14 15 /// How many of the newest versions to keep per app, per root. Chosen for
15 16 /// rollback/debug depth without unbounded growth (operator decision, Run 2).
16 17 pub const KEEP_VERSIONS_PER_APP: usize = 5;
17 18
18 - /// Prune both roots to the newest [`KEEP_VERSIONS_PER_APP`] versions per app.
19 - /// Best-effort and blocking: IO errors are logged and skipped, never fatal.
20 - /// Intended to run inside `spawn_blocking`.
21 - pub fn prune_once(cfg: &Config) {
22 - prune_root(&cfg.dist_root, "dist");
23 - prune_root(&cfg.logs_root, "logs");
19 + /// A `(app, version)` pair an in-flight build is currently writing into. Such a
20 + /// version is never pruned even if it ranks outside the newest N, so retention
21 + /// can't delete a directory out from under a live `collect`/log stream.
22 + pub type ActiveVersions = HashSet<(String, String)>;
23 +
24 + /// Prune both roots to the newest [`KEEP_VERSIONS_PER_APP`] versions per app,
25 + /// skipping any `(app, version)` in `active`. Best-effort and blocking: IO
26 + /// errors are logged and skipped, never fatal. Intended to run inside
27 + /// `spawn_blocking`.
28 + pub fn prune_once(cfg: &Config, active: &ActiveVersions) {
29 + prune_root(&cfg.dist_root, "dist", active);
30 + prune_root(&cfg.logs_root, "logs", active);
24 31 }
25 32
26 - fn prune_root(root: &Path, label: &str) {
33 + fn prune_root(root: &Path, label: &str, active: &ActiveVersions) {
27 34 let Ok(apps) = std::fs::read_dir(root) else { return };
28 35 for app in apps.filter_map(|e| e.ok()) {
29 36 let app_dir = app.path();
30 37 if app_dir.is_dir() {
31 - prune_app_dir(&app_dir, label);
38 + let app_name = app.file_name().to_string_lossy().into_owned();
39 + prune_app_dir(&app_dir, &app_name, label, active);
32 40 }
33 41 }
34 42 }
35 43
36 - fn prune_app_dir(app_dir: &Path, label: &str) {
44 + fn prune_app_dir(app_dir: &Path, app_name: &str, label: &str, active: &ActiveVersions) {
37 45 let Ok(entries) = std::fs::read_dir(app_dir) else { return };
38 46 // Only consider subdirectories whose name parses as a semver — anything
39 47 // else (a stray file, a non-version dir) is left untouched.
40 - let mut versions: Vec<(Version, PathBuf)> = entries
48 + let mut versions: Vec<(Version, String, PathBuf)> = entries
41 49 .filter_map(|e| e.ok())
42 50 .filter(|e| e.path().is_dir())
43 51 .filter_map(|e| {
44 - let name = e.file_name();
45 - let v = Version::parse(name.to_string_lossy().as_ref()).ok()?;
46 - Some((v, e.path()))
52 + let name = e.file_name().to_string_lossy().into_owned();
53 + let v = Version::parse(&name).ok()?;
54 + Some((v, name, e.path()))
47 55 })
48 56 .collect();
49 57 if versions.len() <= KEEP_VERSIONS_PER_APP {
50 58 return;
51 59 }
52 60 versions.sort_by(|a, b| b.0.cmp(&a.0)); // newest first
53 - for (ver, path) in versions.into_iter().skip(KEEP_VERSIONS_PER_APP) {
61 + for (ver, name, path) in versions.into_iter().skip(KEEP_VERSIONS_PER_APP) {
62 + // Never delete a version an in-flight build is writing to.
63 + if active.contains(&(app_name.to_string(), name)) {
64 + tracing::debug!(%ver, app = app_name, "retention: skipping in-flight {label} version");
65 + continue;
66 + }
54 67 match std::fs::remove_dir_all(&path) {
55 68 Ok(()) => tracing::info!(%ver, dir = %path.display(), "retention: pruned old {label} version"),
56 69 Err(e) => tracing::warn!(error = %e, dir = %path.display(), "retention: could not prune {label} dir"),
@@ -79,7 +92,7 @@ mod tests {
79 92 // A non-version dir must be left alone.
80 93 std::fs::create_dir_all(root.join("demo").join("scratch")).unwrap();
81 94
82 - prune_root(&root, "dist");
95 + prune_root(&root, "dist", &ActiveVersions::new());
83 96
84 97 let mut kept: Vec<String> = std::fs::read_dir(root.join("demo"))
85 98 .unwrap()
@@ -98,7 +111,24 @@ mod tests {
98 111 for v in ["0.1.0", "0.2.0"] {
99 112 touch_version(&root, "demo", v);
100 113 }
101 - prune_root(&root, "logs");
114 + prune_root(&root, "logs", &ActiveVersions::new());
102 115 assert_eq!(std::fs::read_dir(root.join("demo")).unwrap().count(), 2);
103 116 }
117 +
118 + #[test]
119 + fn never_prunes_an_in_flight_version() {
120 + let tmp = tempfile::tempdir().unwrap();
121 + let root = tmp.path().join("dist");
122 + // Seven versions, keep 5 -> 0.1.0 and 0.2.0 are beyond the window.
123 + for v in ["0.1.0", "0.2.0", "0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.7.0"] {
124 + touch_version(&root, "demo", v);
125 + }
126 + // 0.1.0 is beyond the window but is being built right now.
127 + let active: ActiveVersions = [("demo".to_string(), "0.1.0".to_string())].into_iter().collect();
128 + prune_root(&root, "dist", &active);
129 + assert!(root.join("demo").join("0.1.0").exists(), "in-flight version must survive");
130 + // 0.2.0 (also beyond the window, not active) is pruned instead.
131 + assert!(!root.join("demo").join("0.2.0").exists());
132 + assert!(root.join("demo").join("0.3.0").exists(), "newest 5 are kept");
133 + }
104 134 }