Skip to main content

max / makenotwork

sando: backup plausibility floor + checked terminal writes + tx prune (Storage A+, ultra-fuzz Run 2 deep Phase 3) Drive the Build/Sync/Backup/DB axis from A to A+: - backup plausibility floor (Run-2 mandatory surprise): verify_backup now rejects a dump below half the last verified backup's size, not just the 64-byte absolute floor. A source-side truncation that still closed a valid gzip (passing gzip -t) is caught before it becomes the live dump the migration_dry_run gate restores. First-ever fetch falls back to the absolute floor. - checked terminal writes: the host pipeline's mark_passed/mark_failed (incl. the compile-fail path) no longer swallow their error with .ok() — a dropped terminal verdict leaves a run wedged at 'building', so it now logs loudly (the Phase 4 startup reconcile is the backstop). Phase pings stay best-effort. - backup record + 30-day prune now run in one transaction, so a crash between them can't half-apply. Tests: a valid gzip far below the last backup size is rejected and records no row. clippy -D warnings clean; native fw13. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-06-23 03:34 UTC
Commit: 215112bbb6ef9b19401ce5ac70e2e8195d44f669
Parent: 675dec1
2 files changed, +84 insertions, -23 deletions
M sando/daemon/src/backup.rs +71 -20
@@ -81,25 +81,31 @@ pub(crate) fn parse_source(s: &str) -> Result<BackupSource> {
81 81 bail!("unsupported backup source scheme: {s}");
82 82 }
83 83
84 - /// Backups smaller than this are treated as a failed/truncated transfer rather
85 - /// than a real dump. The prod backup is ~885M; even a dev fixture is well above
86 - /// this. The floor exists to catch an empty or header-only file (an rsync that
87 - /// wrote zero bytes, an empty source); `gzip -t` catches mid-stream truncation
88 - /// of a `.gz`.
84 + /// Absolute floor: backups smaller than this are an empty or header-only file
85 + /// (an rsync that wrote zero bytes, an empty source). This is the fallback when
86 + /// there is no prior backup to compare against (the first-ever fetch).
89 87 const MIN_BACKUP_BYTES: u64 = 64;
90 88
89 + /// Plausibility floor as a fraction of the last verified backup's size. A real
90 + /// dump never abruptly halves; a source-side truncation that still closed a
91 + /// valid gzip would pass `gzip -t` and the absolute floor, but not this. Every
92 + /// row in `backups` is a previously-verified dump, so the last one is a sound
93 + /// reference. Integer-halved at the call site.
94 + const MIN_BACKUP_FRACTION_DENOM: i64 = 2;
95 +
91 96 /// Verify a freshly-downloaded backup before it is allowed to become the live
92 - /// dump: reject anything implausibly small, and for a gzip require a complete,
93 - /// valid stream (`gzip -t` fails on truncation/corruption). `is_gz` is taken
94 - /// from the *destination* name, not the temp path (which carries a `.partial`
95 - /// suffix).
96 - async fn verify_backup(tmp_path: &str, is_gz: bool) -> Result<()> {
97 + /// dump: reject anything below `min_bytes` (the plausibility floor derived from
98 + /// history), and for a gzip require a complete, valid stream (`gzip -t` fails on
99 + /// truncation/corruption). `is_gz` is taken from the *destination* name, not the
100 + /// temp path (which carries a `.partial` suffix).
101 + async fn verify_backup(tmp_path: &str, is_gz: bool, min_bytes: u64) -> Result<()> {
97 102 let meta = tokio::fs::metadata(tmp_path)
98 103 .await
99 104 .with_context(|| format!("stat fetched backup {tmp_path}"))?;
100 105 anyhow::ensure!(
101 - meta.len() >= MIN_BACKUP_BYTES,
102 - "fetched backup {tmp_path} is implausibly small ({} bytes); treating as a failed/truncated transfer",
106 + meta.len() >= min_bytes,
107 + "fetched backup {tmp_path} is implausibly small ({} bytes, floor {min_bytes}); \
108 + treating as a failed/truncated transfer",
103 109 meta.len(),
104 110 );
105 111 if is_gz {
@@ -138,6 +144,18 @@ pub async fn fetch(
138 144 let tmp_path = format!("{local_path}.partial");
139 145 let is_gz = local_path.ends_with(".gz");
140 146
147 + // Plausibility floor: half the last verified backup's size, never below the
148 + // absolute floor. The first-ever fetch (no prior row) falls back to the
149 + // absolute floor.
150 + let last_size: Option<i64> = sqlx::query_scalar(
151 + "SELECT byte_size FROM backups ORDER BY fetched_at DESC LIMIT 1",
152 + )
153 + .fetch_optional(pool)
154 + .await?;
155 + let min_bytes = last_size
156 + .map(|s| ((s / MIN_BACKUP_FRACTION_DENOM) as u64).max(MIN_BACKUP_BYTES))
157 + .unwrap_or(MIN_BACKUP_BYTES);
158 +
141 159 let parsed = parse_source(&source)?;
142 160 let downloaded: Result<()> = async {
143 161 match parsed {
@@ -179,7 +197,7 @@ pub async fn fetch(
179 197 );
180 198 }
181 199 }
182 - verify_backup(&tmp_path, is_gz).await
200 + verify_backup(&tmp_path, is_gz, min_bytes).await
183 201 }
184 202 .await;
185 203
@@ -197,6 +215,12 @@ pub async fn fetch(
197 215 let meta = tokio::fs::metadata(&local_path).await?;
198 216 let size = meta.len() as i64;
199 217
218 + // Record the new backup and prune stale rows in one transaction, so a crash
219 + // between the two can't leave the insert without the prune (or, worse, lose
220 + // the insert while keeping a half-applied delete). The on-disk file is
221 + // overwritten each fetch (single `local_path`), so rows older than 30 days
222 + // reference a path that no longer exists — keep the table from growing.
223 + let mut tx = pool.begin().await?;
200 224 sqlx::query(
201 225 "INSERT INTO backups (fetched_at, source, local_path, byte_size) VALUES (?, ?, ?, ?)",
202 226 )
@@ -204,18 +228,14 @@ pub async fn fetch(
204 228 .bind(&source)
205 229 .bind(&local_path)
206 230 .bind(size)
207 - .execute(pool)
231 + .execute(&mut *tx)
208 232 .await?;
209 -
210 - // Retention: prune rows fetched more than 30 days ago. The on-disk file
211 - // is overwritten each fetch (single `local_path`), so old rows reference
212 - // a path that no longer exists — keep the table from growing for no
213 - // good reason.
214 233 sqlx::query(
215 234 "DELETE FROM backups WHERE fetched_at < datetime('now', '-30 days')",
216 235 )
217 - .execute(pool)
236 + .execute(&mut *tx)
218 237 .await?;
238 + tx.commit().await?;
219 239
220 240 Ok(FetchedBackup { source, local_path, byte_size: Some(size) })
221 241 }
@@ -289,6 +309,37 @@ mod tests {
289 309 }
290 310
291 311 #[tokio::test]
312 + async fn fetch_rejects_a_dump_far_below_the_last_backup_size() {
313 + // Plausibility floor: a complete, valid gzip that is far smaller than the
314 + // last verified backup is a likely source-side truncation and is rejected
315 + // even though `gzip -t` passes — the gap the 64-byte absolute floor missed.
316 + let tmp = tempfile::tempdir().unwrap();
317 + let src = tmp.path().join("src.sql.gz");
318 + write_valid_gz(&src).await; // ~4 KB valid gzip
319 + let dest = tmp.path().join("backups/latest.sql.gz");
320 + let topo = Arc::new(topo_with_backup(
321 + format!("file://{}", src.display()),
322 + dest.to_string_lossy().into_owned(),
323 + ));
324 + let pool = mem_pool().await;
325 + let cfg = Arc::new(Config::for_tests());
326 +
327 + // Seed a prior verified backup far larger than the incoming one; the floor
328 + // becomes 500_000, well above the ~4 KB dump.
329 + sqlx::query("INSERT INTO backups (fetched_at, source, local_path, byte_size) VALUES (?, 'x', ?, 1000000)")
330 + .bind(Utc::now().to_rfc3339())
331 + .bind(dest.to_string_lossy().into_owned())
332 + .execute(&pool).await.unwrap();
333 +
334 + let err = fetch(&pool, &cfg, &topo).await.unwrap_err().to_string();
335 + assert!(err.contains("implausibly small"), "{err}");
336 + assert!(!dest.exists(), "a rejected dump never becomes the live backup");
337 + let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM backups")
338 + .fetch_one(&pool).await.unwrap();
339 + assert_eq!(count.0, 1, "the rejected fetch records no new row");
340 + }
341 +
342 + #[tokio::test]
292 343 async fn fetch_rejects_truncated_gz_and_leaves_no_live_file() {
293 344 let tmp = tempfile::tempdir().unwrap();
294 345 let src = tmp.path().join("src.sql.gz");
M sando/daemon/src/build.rs +13 -3
@@ -142,7 +142,9 @@ pub async fn run(
142 142 // Settle the run with the headline compiler diagnostic (not the raw
143 143 // 4 KB tail) so `GET /runs/{id}` answers "why" without a journald dive.
144 144 let summary = crate::classify::classify_compile_error(&out.stdout, &out.stderr).summary();
145 - crate::runs::mark_failed(&pool, run_id, &summary).await.ok();
145 + if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
146 + tracing::error!(run_id = %run_id, error = %e, "persisting compile-fail verdict failed; run may show stale 'building' until restart-reconcile");
147 + }
146 148 anyhow::bail!("cargo build --release failed:\n{}", tail(&out.stderr, 4_000));
147 149 }
148 150 tracing::info!(sha = %sha, version = %version, elapsed_s, "cargo build --release ok");
@@ -246,7 +248,13 @@ pub async fn build_and_run_host(
246 248 let _deploy_guard = deploy_lock.lock().await;
247 249 crate::runs::advance_tier(&pool, "host", &art.version).await?;
248 250 }
249 - crate::runs::mark_passed(&pool, run_id).await.ok();
251 + // Terminal verdict: unlike the phase pings above (best-effort), a dropped
252 + // pass/fail write leaves the run wedged at `building`. Log it loudly if it
253 + // fails — the startup reconcile (main) is the backstop that settles such a
254 + // row on the next restart.
255 + if let Err(e) = crate::runs::mark_passed(&pool, run_id).await {
256 + tracing::error!(run_id = %run_id, error = %e, "persisting host-green verdict failed; run may show stale 'building' until restart-reconcile");
257 + }
250 258 tracing::info!(version = %art.version, "host pipeline green; ready to promote to next tier");
251 259 } else {
252 260 // Pull the first red gate's typed summary into the run so the API
@@ -254,7 +262,9 @@ pub async fn build_and_run_host(
254 262 let summary = crate::runs::first_failed_gate_summary(&pool, &art.version)
255 263 .await
256 264 .unwrap_or_else(|| "host pipeline red".to_string());
257 - crate::runs::mark_failed(&pool, run_id, &summary).await.ok();
265 + if let Err(e) = crate::runs::mark_failed(&pool, run_id, &summary).await {
266 + tracing::error!(run_id = %run_id, error = %e, "persisting host-red verdict failed; run may show stale 'building' until restart-reconcile");
267 + }
258 268 tracing::warn!(version = %art.version, "host pipeline red; not advancing tier_state");
259 269 }
260 270 Ok(())