| 1 |
|
| 2 |
|
| 3 |
|
| 4 |
|
| 5 |
|
| 6 |
|
| 7 |
|
| 8 |
|
| 9 |
|
| 10 |
|
| 11 |
|
| 12 |
|
| 13 |
|
| 14 |
|
| 15 |
|
| 16 |
|
| 17 |
|
| 18 |
|
| 19 |
|
| 20 |
|
| 21 |
|
| 22 |
|
| 23 |
|
| 24 |
|
| 25 |
|
| 26 |
|
| 27 |
|
| 28 |
|
| 29 |
|
| 30 |
|
| 31 |
|
| 32 |
|
| 33 |
|
| 34 |
|
| 35 |
|
| 36 |
|
| 37 |
|
| 38 |
use std::path::{Path, PathBuf}; |
| 39 |
use std::sync::atomic::{AtomicBool, Ordering}; |
| 40 |
use std::time::Instant; |
| 41 |
|
| 42 |
use audiofiles_core::config_key::ConfigKey; |
| 43 |
use audiofiles_core::db::Database; |
| 44 |
use audiofiles_core::id_types::SampleHash; |
| 45 |
use audiofiles_core::store::layout::{ |
| 46 |
BlobLayout, LayoutMigration, count_flat_blobs, migrate_to_sharded, recorded_layout, |
| 47 |
}; |
| 48 |
use audiofiles_core::store::{ |
| 49 |
SampleStore, existing_blob_path, hash_file, legacy_flat_blob_path, store_blob_path, |
| 50 |
}; |
| 51 |
use audiofiles_core::vfs; |
| 52 |
use audiofiles_core::vfs_mirror::{MirrorConfig, sync_mirror}; |
| 53 |
|
| 54 |
use crate::report::Report; |
| 55 |
use crate::storage; |
| 56 |
|
| 57 |
|
| 58 |
|
| 59 |
|
| 60 |
|
| 61 |
|
| 62 |
const SCENARIO_BLOBS: usize = 2_000; |
| 63 |
|
| 64 |
|
| 65 |
|
| 66 |
|
| 67 |
|
| 68 |
|
| 69 |
pub(crate) const DEFAULT_TIMED_BLOBS: usize = 50_000; |
| 70 |
|
| 71 |
|
| 72 |
type Blob = (String, String); |
| 73 |
|
| 74 |
|
| 75 |
|
| 76 |
|
| 77 |
|
| 78 |
|
| 79 |
fn payload(i: usize) -> Vec<u8> { |
| 80 |
let mut bytes = format!("af-bench-layout blob {i}\n").into_bytes(); |
| 81 |
bytes.resize(64 + (i % 97) * 8, b'\0'); |
| 82 |
bytes |
| 83 |
} |
| 84 |
|
| 85 |
|
| 86 |
|
| 87 |
|
| 88 |
|
| 89 |
|
| 90 |
fn set_readonly(path: &Path) { |
| 91 |
if let Ok(meta) = std::fs::metadata(path) { |
| 92 |
let mut perms = meta.permissions(); |
| 93 |
#[cfg(unix)] |
| 94 |
{ |
| 95 |
use std::os::unix::fs::PermissionsExt; |
| 96 |
perms.set_mode(0o444); |
| 97 |
} |
| 98 |
#[cfg(not(unix))] |
| 99 |
perms.set_readonly(true); |
| 100 |
let _ = std::fs::set_permissions(path, perms); |
| 101 |
} |
| 102 |
} |
| 103 |
|
| 104 |
|
| 105 |
struct FlatVault { |
| 106 |
db: Database, |
| 107 |
store: SampleStore, |
| 108 |
blobs: Vec<Blob>, |
| 109 |
root: PathBuf, |
| 110 |
} |
| 111 |
|
| 112 |
|
| 113 |
|
| 114 |
|
| 115 |
|
| 116 |
|
| 117 |
fn fabricate(vault: &Path, n: usize, with_vfs: bool) -> Option<FlatVault> { |
| 118 |
if vault.exists() |
| 119 |
&& let Err(e) = std::fs::remove_dir_all(vault) |
| 120 |
{ |
| 121 |
eprintln!("could not clear scratch vault: {e}"); |
| 122 |
return None; |
| 123 |
} |
| 124 |
let root = vault.join("samples"); |
| 125 |
let stage = vault.join("stage"); |
| 126 |
for dir in [&root, &stage] { |
| 127 |
if let Err(e) = std::fs::create_dir_all(dir) { |
| 128 |
eprintln!("could not create {}: {e}", dir.display()); |
| 129 |
return None; |
| 130 |
} |
| 131 |
} |
| 132 |
|
| 133 |
let db = match Database::open(vault.join("audiofiles.db")) { |
| 134 |
Ok(db) => db, |
| 135 |
Err(e) => { |
| 136 |
eprintln!("Database::open failed (WAL unsupported on this fs?): {e}"); |
| 137 |
return None; |
| 138 |
} |
| 139 |
}; |
| 140 |
let store = match SampleStore::new(&root) { |
| 141 |
Ok(s) => s, |
| 142 |
Err(e) => { |
| 143 |
eprintln!("SampleStore::new failed: {e}"); |
| 144 |
return None; |
| 145 |
} |
| 146 |
}; |
| 147 |
|
| 148 |
let vfs_id = if with_vfs { |
| 149 |
match vfs::create_vfs(&db, "bench") { |
| 150 |
Ok(id) => Some(id), |
| 151 |
Err(e) => { |
| 152 |
eprintln!("could not create bench vfs: {e}"); |
| 153 |
return None; |
| 154 |
} |
| 155 |
} |
| 156 |
} else { |
| 157 |
None |
| 158 |
}; |
| 159 |
|
| 160 |
let mut blobs = Vec::with_capacity(n); |
| 161 |
|
| 162 |
|
| 163 |
|
| 164 |
let staged = stage.join("blob.wav"); |
| 165 |
|
| 166 |
|
| 167 |
if db.conn().execute_batch("BEGIN").is_err() { |
| 168 |
eprintln!("could not open the fabrication transaction"); |
| 169 |
return None; |
| 170 |
} |
| 171 |
for i in 0..n { |
| 172 |
let content = payload(i); |
| 173 |
if std::fs::write(&staged, &content).is_err() { |
| 174 |
eprintln!("could not stage blob {i}"); |
| 175 |
return None; |
| 176 |
} |
| 177 |
let (hash, size) = match hash_file(&staged) { |
| 178 |
Ok(pair) => pair, |
| 179 |
Err(e) => { |
| 180 |
eprintln!("could not hash blob {i}: {e}"); |
| 181 |
return None; |
| 182 |
} |
| 183 |
}; |
| 184 |
let dest = legacy_flat_blob_path(&root, &hash, "wav"); |
| 185 |
if std::fs::rename(&staged, &dest).is_err() { |
| 186 |
eprintln!("could not place flat blob {i}"); |
| 187 |
return None; |
| 188 |
} |
| 189 |
set_readonly(&dest); |
| 190 |
let now = i as i64; |
| 191 |
if db |
| 192 |
.conn() |
| 193 |
.execute( |
| 194 |
"INSERT OR IGNORE INTO samples |
| 195 |
(hash, original_name, file_extension, file_size, import_date, last_modified) |
| 196 |
VALUES (?1, ?2, 'wav', ?3, ?4, ?4)", |
| 197 |
rusqlite::params![hash, format!("blob-{i:06}.wav"), size, now], |
| 198 |
) |
| 199 |
.is_err() |
| 200 |
{ |
| 201 |
eprintln!("could not insert sample row {i}"); |
| 202 |
return None; |
| 203 |
} |
| 204 |
if let Some(vfs_id) = vfs_id |
| 205 |
&& vfs::create_sample_link( |
| 206 |
&db, |
| 207 |
vfs_id, |
| 208 |
None, |
| 209 |
&format!("blob-{i:06}.wav"), |
| 210 |
&SampleHash::from_trusted(hash.clone()), |
| 211 |
) |
| 212 |
.is_err() |
| 213 |
{ |
| 214 |
eprintln!("could not link blob {i} into the vfs"); |
| 215 |
return None; |
| 216 |
} |
| 217 |
blobs.push((hash, "wav".to_string())); |
| 218 |
} |
| 219 |
if db.conn().execute_batch("COMMIT").is_err() { |
| 220 |
eprintln!("could not commit the fabrication transaction"); |
| 221 |
return None; |
| 222 |
} |
| 223 |
let _ = std::fs::remove_dir_all(&stage); |
| 224 |
|
| 225 |
Some(FlatVault { |
| 226 |
db, |
| 227 |
store, |
| 228 |
blobs, |
| 229 |
root, |
| 230 |
}) |
| 231 |
} |
| 232 |
|
| 233 |
|
| 234 |
fn sweep(v: &FlatVault) -> Option<LayoutMigration> { |
| 235 |
let cancel = AtomicBool::new(false); |
| 236 |
migrate_to_sharded(&v.store, &v.db, &cancel, &mut |_, _| {}) |
| 237 |
.inspect_err(|e| eprintln!("sweep failed: {e}")) |
| 238 |
.ok() |
| 239 |
} |
| 240 |
|
| 241 |
|
| 242 |
|
| 243 |
|
| 244 |
|
| 245 |
|
| 246 |
fn all_resolve(v: &FlatVault) -> bool { |
| 247 |
v.blobs |
| 248 |
.iter() |
| 249 |
.all(|(hash, ext)| existing_blob_path(&v.root, hash, ext).is_some_and(|p| p.is_file())) |
| 250 |
} |
| 251 |
|
| 252 |
|
| 253 |
fn all_sharded(v: &FlatVault) -> bool { |
| 254 |
v.blobs.iter().all(|(hash, ext)| { |
| 255 |
store_blob_path(&v.root, hash, ext).is_file() |
| 256 |
&& !legacy_flat_blob_path(&v.root, hash, ext).exists() |
| 257 |
}) |
| 258 |
} |
| 259 |
|
| 260 |
|
| 261 |
fn root_strays(root: &Path) -> Vec<String> { |
| 262 |
let Ok(entries) = std::fs::read_dir(root) else { |
| 263 |
return Vec::new(); |
| 264 |
}; |
| 265 |
entries |
| 266 |
.flatten() |
| 267 |
.filter(|e| !e.file_type().is_ok_and(|t| t.is_dir())) |
| 268 |
.map(|e| e.file_name().to_string_lossy().into_owned()) |
| 269 |
.collect() |
| 270 |
} |
| 271 |
|
| 272 |
|
| 273 |
struct Checks { |
| 274 |
rows: Vec<(String, bool, String)>, |
| 275 |
} |
| 276 |
|
| 277 |
impl Checks { |
| 278 |
fn new() -> Self { |
| 279 |
Self { rows: Vec::new() } |
| 280 |
} |
| 281 |
|
| 282 |
fn add(&mut self, name: &str, ok: bool, detail: impl Into<String>) { |
| 283 |
let detail = detail.into(); |
| 284 |
println!( |
| 285 |
" {:<44} {} {detail}", |
| 286 |
name, |
| 287 |
if ok { "PASS" } else { "FAIL" } |
| 288 |
); |
| 289 |
self.rows.push((name.to_string(), ok, detail)); |
| 290 |
} |
| 291 |
|
| 292 |
fn failed(&self) -> usize { |
| 293 |
self.rows.iter().filter(|(_, ok, _)| !ok).count() |
| 294 |
} |
| 295 |
} |
| 296 |
|
| 297 |
|
| 298 |
fn scenario_full(vault: &Path, checks: &mut Checks) { |
| 299 |
let Some(v) = fabricate(vault, SCENARIO_BLOBS, false) else { |
| 300 |
checks.add("full sweep", false, "could not fabricate the vault"); |
| 301 |
return; |
| 302 |
}; |
| 303 |
checks.add( |
| 304 |
"pending before the sweep", |
| 305 |
!matches!(recorded_layout(&v.db), Ok(BlobLayout::Sharded)) |
| 306 |
&& count_flat_blobs(&v.root).unwrap_or(0) == SCENARIO_BLOBS, |
| 307 |
format!("{SCENARIO_BLOBS} flat blobs, layout unrecorded"), |
| 308 |
); |
| 309 |
|
| 310 |
|
| 311 |
|
| 312 |
|
| 313 |
let mut seen: Vec<(usize, usize)> = Vec::new(); |
| 314 |
let cancel = AtomicBool::new(false); |
| 315 |
let Ok(report) = migrate_to_sharded(&v.store, &v.db, &cancel, &mut |done, total| { |
| 316 |
seen.push((done, total)); |
| 317 |
}) else { |
| 318 |
checks.add("full sweep", false, "sweep returned an error"); |
| 319 |
return; |
| 320 |
}; |
| 321 |
|
| 322 |
checks.add( |
| 323 |
"full sweep relocates every blob", |
| 324 |
report.moved == SCENARIO_BLOBS |
| 325 |
&& report.deduped == 0 |
| 326 |
&& report.errors == 0 |
| 327 |
&& report.completed |
| 328 |
&& !report.cancelled, |
| 329 |
format!( |
| 330 |
"moved {} deduped {} errors {}", |
| 331 |
report.moved, report.deduped, report.errors |
| 332 |
), |
| 333 |
); |
| 334 |
checks.add( |
| 335 |
"progress is monotonic and ends full", |
| 336 |
seen.windows(2).all(|w| w[1].0 == w[0].0 + 1) |
| 337 |
&& seen.last() == Some(&(SCENARIO_BLOBS, SCENARIO_BLOBS)), |
| 338 |
format!("{} callbacks", seen.len()), |
| 339 |
); |
| 340 |
checks.add( |
| 341 |
"blobs land under their hash prefix", |
| 342 |
all_sharded(&v), |
| 343 |
"{root}/{ab}/{hash}.wav".to_string(), |
| 344 |
); |
| 345 |
let strays = root_strays(&v.root); |
| 346 |
checks.add( |
| 347 |
"root holds only shard directories", |
| 348 |
strays.is_empty(), |
| 349 |
if strays.is_empty() { |
| 350 |
"clean".to_string() |
| 351 |
} else { |
| 352 |
format!("{} left: {}", strays.len(), strays.join(", ")) |
| 353 |
}, |
| 354 |
); |
| 355 |
checks.add( |
| 356 |
"layout recorded as sharded", |
| 357 |
matches!(recorded_layout(&v.db), Ok(BlobLayout::Sharded)), |
| 358 |
"blob_layout=sharded".to_string(), |
| 359 |
); |
| 360 |
|
| 361 |
|
| 362 |
let Some(again) = sweep(&v) else { |
| 363 |
checks.add( |
| 364 |
"re-open does not re-sweep", |
| 365 |
false, |
| 366 |
"sweep returned an error", |
| 367 |
); |
| 368 |
return; |
| 369 |
}; |
| 370 |
checks.add( |
| 371 |
"re-open does not re-sweep", |
| 372 |
again.moved == 0 && again.deduped == 0 && again.errors == 0 && again.completed, |
| 373 |
format!("moved {} on the second pass", again.moved), |
| 374 |
); |
| 375 |
} |
| 376 |
|
| 377 |
|
| 378 |
fn scenario_resume(vault: &Path, checks: &mut Checks) { |
| 379 |
let Some(v) = fabricate(vault, SCENARIO_BLOBS, false) else { |
| 380 |
checks.add("resume", false, "could not fabricate the vault"); |
| 381 |
return; |
| 382 |
}; |
| 383 |
|
| 384 |
|
| 385 |
|
| 386 |
let cancel = AtomicBool::new(false); |
| 387 |
let stop_at = SCENARIO_BLOBS / 3; |
| 388 |
let Ok(first) = migrate_to_sharded(&v.store, &v.db, &cancel, &mut |done, _| { |
| 389 |
if done >= stop_at { |
| 390 |
cancel.store(true, Ordering::Release); |
| 391 |
} |
| 392 |
}) else { |
| 393 |
checks.add("resume", false, "first pass returned an error"); |
| 394 |
return; |
| 395 |
}; |
| 396 |
|
| 397 |
checks.add( |
| 398 |
"cancel stops the pass early", |
| 399 |
first.cancelled |
| 400 |
&& !first.completed |
| 401 |
&& first.moved >= stop_at |
| 402 |
&& first.moved < SCENARIO_BLOBS, |
| 403 |
format!("moved {} of {SCENARIO_BLOBS} then stopped", first.moved), |
| 404 |
); |
| 405 |
checks.add( |
| 406 |
"cancelled vault stays recorded flat", |
| 407 |
matches!(recorded_layout(&v.db), Ok(BlobLayout::Flat)), |
| 408 |
"so the next open resumes".to_string(), |
| 409 |
); |
| 410 |
checks.add( |
| 411 |
"every blob resolves mid-migration", |
| 412 |
all_resolve(&v), |
| 413 |
"reads span both layouts".to_string(), |
| 414 |
); |
| 415 |
|
| 416 |
let remaining = count_flat_blobs(&v.root).unwrap_or(0); |
| 417 |
let Some(second) = sweep(&v) else { |
| 418 |
checks.add("resume", false, "second pass returned an error"); |
| 419 |
return; |
| 420 |
}; |
| 421 |
checks.add( |
| 422 |
"the next pass resumes rather than restarts", |
| 423 |
second.moved == remaining && first.moved + second.moved == SCENARIO_BLOBS, |
| 424 |
format!("{} + {} = {SCENARIO_BLOBS}", first.moved, second.moved), |
| 425 |
); |
| 426 |
checks.add( |
| 427 |
"resumed vault ends fully sharded", |
| 428 |
second.completed && all_sharded(&v) && root_strays(&v.root).is_empty(), |
| 429 |
"blob_layout=sharded".to_string(), |
| 430 |
); |
| 431 |
} |
| 432 |
|
| 433 |
|
| 434 |
fn scenario_mismatch(vault: &Path, checks: &mut Checks) { |
| 435 |
let Some(v) = fabricate(vault, 32, false) else { |
| 436 |
checks.add("size mismatch", false, "could not fabricate the vault"); |
| 437 |
return; |
| 438 |
}; |
| 439 |
|
| 440 |
|
| 441 |
|
| 442 |
|
| 443 |
let (mismatch, ext) = v.blobs[0].clone(); |
| 444 |
let (dupe, _) = v.blobs[1].clone(); |
| 445 |
for (hash, content) in [(&mismatch, b"short".to_vec()), (&dupe, payload(1))] { |
| 446 |
let dest = store_blob_path(&v.root, hash, &ext); |
| 447 |
let Some(shard) = dest.parent() else { continue }; |
| 448 |
if std::fs::create_dir_all(shard).is_err() || std::fs::write(&dest, &content).is_err() { |
| 449 |
checks.add("size mismatch", false, "could not plant the shard copy"); |
| 450 |
return; |
| 451 |
} |
| 452 |
} |
| 453 |
|
| 454 |
|
| 455 |
let tmp = v.root.join(format!("{mismatch}.wav.12345.tmp")); |
| 456 |
let note = v.root.join("notes.txt"); |
| 457 |
let _ = std::fs::write(&tmp, b"partial"); |
| 458 |
let _ = std::fs::write(¬e, b"mine"); |
| 459 |
|
| 460 |
let Some(report) = sweep(&v) else { |
| 461 |
checks.add("size mismatch", false, "sweep returned an error"); |
| 462 |
return; |
| 463 |
}; |
| 464 |
|
| 465 |
checks.add( |
| 466 |
"size mismatch is counted, not resolved", |
| 467 |
report.errors == 1 && !report.completed, |
| 468 |
format!("errors {} completed {}", report.errors, report.completed), |
| 469 |
); |
| 470 |
checks.add( |
| 471 |
"both copies of a mismatch survive", |
| 472 |
legacy_flat_blob_path(&v.root, &mismatch, &ext).is_file() |
| 473 |
&& store_blob_path(&v.root, &mismatch, &ext).is_file(), |
| 474 |
"left for inspection".to_string(), |
| 475 |
); |
| 476 |
checks.add( |
| 477 |
"redundant flat copy is discarded", |
| 478 |
report.deduped == 1 && !legacy_flat_blob_path(&v.root, &dupe, &ext).exists(), |
| 479 |
format!("deduped {}", report.deduped), |
| 480 |
); |
| 481 |
checks.add( |
| 482 |
"an errored pass stays recorded flat", |
| 483 |
matches!(recorded_layout(&v.db), Ok(BlobLayout::Flat)), |
| 484 |
"so the mismatch is swept again after repair".to_string(), |
| 485 |
); |
| 486 |
checks.add( |
| 487 |
"temp leftovers and stray files are untouched", |
| 488 |
tmp.is_file() && note.is_file(), |
| 489 |
"not blobs, never renamed".to_string(), |
| 490 |
); |
| 491 |
} |
| 492 |
|
| 493 |
|
| 494 |
fn scenario_mirror(vault: &Path, checks: &mut Checks) { |
| 495 |
let Some(v) = fabricate(vault, 256, true) else { |
| 496 |
checks.add("mirror", false, "could not fabricate the vault"); |
| 497 |
return; |
| 498 |
}; |
| 499 |
let mirror_root = vault.join("mirror"); |
| 500 |
let config = MirrorConfig { |
| 501 |
mirror_root: mirror_root.clone(), |
| 502 |
store_root: v.root.clone(), |
| 503 |
}; |
| 504 |
if v.db.set_config(ConfigKey::MirrorEnabled, "true").is_err() |
| 505 |
|| v.db |
| 506 |
.set_config(ConfigKey::MirrorPath, &mirror_root.to_string_lossy()) |
| 507 |
.is_err() |
| 508 |
{ |
| 509 |
checks.add("mirror", false, "could not record the mirror config"); |
| 510 |
return; |
| 511 |
} |
| 512 |
|
| 513 |
|
| 514 |
|
| 515 |
let Ok(before) = sync_mirror(&v.db, &config) else { |
| 516 |
checks.add("mirror", false, "the first sync failed"); |
| 517 |
return; |
| 518 |
}; |
| 519 |
checks.add( |
| 520 |
"mirror links a flat vault", |
| 521 |
before.links_created == 256 && dangling(&mirror_root) == 0, |
| 522 |
format!("{} links, none dangling", before.links_created), |
| 523 |
); |
| 524 |
|
| 525 |
let Some(report) = sweep(&v) else { |
| 526 |
checks.add("mirror", false, "sweep returned an error"); |
| 527 |
return; |
| 528 |
}; |
| 529 |
let stale = dangling(&mirror_root); |
| 530 |
checks.add( |
| 531 |
"sweep leaves the old links dangling", |
| 532 |
report.completed && stale == 256, |
| 533 |
format!("{stale} of 256 dangle, which is why a rebuild fires"), |
| 534 |
); |
| 535 |
|
| 536 |
let Ok(after) = sync_mirror(&v.db, &config) else { |
| 537 |
checks.add("mirror", false, "the rebuild failed"); |
| 538 |
return; |
| 539 |
}; |
| 540 |
checks.add( |
| 541 |
"rebuild repoints every link", |
| 542 |
dangling(&mirror_root) == 0, |
| 543 |
format!("{} relinked", after.links_created), |
| 544 |
); |
| 545 |
} |
| 546 |
|
| 547 |
|
| 548 |
fn dangling(root: &Path) -> usize { |
| 549 |
let Ok(entries) = std::fs::read_dir(root) else { |
| 550 |
return 0; |
| 551 |
}; |
| 552 |
entries |
| 553 |
.flatten() |
| 554 |
.map(|e| { |
| 555 |
let path = e.path(); |
| 556 |
if e.file_type().is_ok_and(|t| t.is_dir()) { |
| 557 |
dangling(&path) |
| 558 |
} else { |
| 559 |
|
| 560 |
|
| 561 |
usize::from(path.symlink_metadata().is_ok() && !path.exists()) |
| 562 |
} |
| 563 |
}) |
| 564 |
.sum() |
| 565 |
} |
| 566 |
|
| 567 |
|
| 568 |
fn timed_sweep(vault: &Path, n: usize, report: &mut Report, checks: &mut Checks) { |
| 569 |
println!(); |
| 570 |
println!("━━━ TIMED SWEEP ({n} blobs) ━━━"); |
| 571 |
println!(); |
| 572 |
|
| 573 |
let build = Instant::now(); |
| 574 |
let Some(v) = fabricate(vault, n, false) else { |
| 575 |
checks.add("timed sweep", false, "could not fabricate the vault"); |
| 576 |
return; |
| 577 |
}; |
| 578 |
println!(" fabricated in {:.1}s", build.elapsed().as_secs_f64()); |
| 579 |
|
| 580 |
let mut callbacks = 0usize; |
| 581 |
let cancel = AtomicBool::new(false); |
| 582 |
let start = Instant::now(); |
| 583 |
let Ok(pass) = migrate_to_sharded(&v.store, &v.db, &cancel, &mut |_, _| callbacks += 1) else { |
| 584 |
checks.add("timed sweep", false, "sweep returned an error"); |
| 585 |
return; |
| 586 |
}; |
| 587 |
let elapsed = start.elapsed().as_secs_f64(); |
| 588 |
let per_sec = if elapsed > 0.0 { |
| 589 |
pass.moved as f64 / elapsed |
| 590 |
} else { |
| 591 |
0.0 |
| 592 |
}; |
| 593 |
|
| 594 |
println!(); |
| 595 |
println!(" moved {} blobs in {elapsed:.2}s", pass.moved); |
| 596 |
println!(" {per_sec:.0} blobs/s {:.3} ms/blob", 1000.0 / per_sec); |
| 597 |
println!(); |
| 598 |
report.set("layout_blobs", n); |
| 599 |
report.set("layout_sweep_s", (elapsed * 100.0).round() / 100.0); |
| 600 |
report.set("layout_blobs_per_sec", per_sec.round()); |
| 601 |
|
| 602 |
checks.add( |
| 603 |
"timed sweep completes cleanly", |
| 604 |
pass.moved == n && pass.errors == 0 && pass.completed && callbacks == n, |
| 605 |
format!("{n} moved, {callbacks} progress callbacks"), |
| 606 |
); |
| 607 |
checks.add( |
| 608 |
"timed sweep leaves a clean root", |
| 609 |
root_strays(&v.root).is_empty() && count_flat_blobs(&v.root).unwrap_or(1) == 0, |
| 610 |
"only shard directories".to_string(), |
| 611 |
); |
| 612 |
} |
| 613 |
|
| 614 |
|
| 615 |
pub(crate) fn run(vault: &Path, timed_blobs: usize) { |
| 616 |
println!("━━━ BLOB LAYOUT MIGRATION ━━━"); |
| 617 |
println!(); |
| 618 |
println!(" vault: {}", vault.display()); |
| 619 |
println!(); |
| 620 |
|
| 621 |
let mut report = Report::new("layout"); |
| 622 |
let vault_storage = storage::describe( |
| 623 |
vault |
| 624 |
.parent() |
| 625 |
.filter(|p| p.exists()) |
| 626 |
.unwrap_or_else(|| Path::new(".")), |
| 627 |
); |
| 628 |
report.set_storage("vault", &vault_storage); |
| 629 |
storage::print_conditions(&[("vault", &vault_storage)], None); |
| 630 |
|
| 631 |
let mut checks = Checks::new(); |
| 632 |
println!("━━━ SCENARIOS ({SCENARIO_BLOBS} blobs unless stated) ━━━"); |
| 633 |
println!(); |
| 634 |
scenario_full(vault, &mut checks); |
| 635 |
println!(); |
| 636 |
scenario_resume(vault, &mut checks); |
| 637 |
println!(); |
| 638 |
scenario_mismatch(vault, &mut checks); |
| 639 |
println!(); |
| 640 |
scenario_mirror(vault, &mut checks); |
| 641 |
|
| 642 |
timed_sweep(vault, timed_blobs, &mut report, &mut checks); |
| 643 |
|
| 644 |
let failed = checks.failed(); |
| 645 |
report.set("layout_checks", checks.rows.len()); |
| 646 |
report.set("layout_checks_failed", failed); |
| 647 |
report.write(); |
| 648 |
|
| 649 |
println!(); |
| 650 |
if failed == 0 { |
| 651 |
println!(" {} checks, all passed", checks.rows.len()); |
| 652 |
} else { |
| 653 |
println!(" {failed} of {} checks FAILED", checks.rows.len()); |
| 654 |
} |
| 655 |
|
| 656 |
|
| 657 |
if failed == 0 { |
| 658 |
let _ = std::fs::remove_dir_all(vault); |
| 659 |
} else { |
| 660 |
println!(" vault left at {} for inspection", vault.display()); |
| 661 |
std::process::exit(1); |
| 662 |
} |
| 663 |
} |
| 664 |
|
| 665 |
|
| 666 |
|
| 667 |
|
| 668 |
#[cfg(test)] |
| 669 |
mod tests { |
| 670 |
use super::*; |
| 671 |
|
| 672 |
#[test] |
| 673 |
fn payloads_differ_in_length_across_the_cycle() { |
| 674 |
assert_ne!(payload(0).len(), payload(1).len()); |
| 675 |
assert_eq!(payload(0).len(), payload(97).len()); |
| 676 |
} |
| 677 |
|
| 678 |
#[test] |
| 679 |
fn fabricated_vault_is_flat_and_countable() { |
| 680 |
let dir = tempfile::TempDir::new().unwrap(); |
| 681 |
let vault = dir.path().join("vault"); |
| 682 |
let v = fabricate(&vault, 8, true).expect("fabrication failed"); |
| 683 |
assert_eq!(v.blobs.len(), 8); |
| 684 |
assert_eq!(count_flat_blobs(&v.root).unwrap(), 8); |
| 685 |
assert!(v.blobs.iter().all(|(h, e)| { |
| 686 |
legacy_flat_blob_path(&v.root, h, e).is_file() |
| 687 |
&& !store_blob_path(&v.root, h, e).exists() |
| 688 |
})); |
| 689 |
assert!(all_resolve(&v)); |
| 690 |
|
| 691 |
let unique: std::collections::HashSet<_> = v.blobs.iter().map(|(h, _)| h).collect(); |
| 692 |
assert_eq!(unique.len(), 8); |
| 693 |
} |
| 694 |
|
| 695 |
#[test] |
| 696 |
fn dangling_counts_only_broken_links() { |
| 697 |
let dir = tempfile::TempDir::new().unwrap(); |
| 698 |
let root = dir.path(); |
| 699 |
let real = root.join("real"); |
| 700 |
std::fs::write(&real, b"x").unwrap(); |
| 701 |
#[cfg(unix)] |
| 702 |
{ |
| 703 |
std::os::unix::fs::symlink(&real, root.join("good")).unwrap(); |
| 704 |
std::os::unix::fs::symlink(root.join("gone"), root.join("bad")).unwrap(); |
| 705 |
assert_eq!(dangling(root), 1); |
| 706 |
} |
| 707 |
} |
| 708 |
} |
| 709 |
|