max / audiofiles

bench: checkpoint AF_BENCH_JSON per batch, dedupe the pipeline config Report::write ran once at the end, so the runs most worth recording were the ones that produced nothing: the NSynth control was killed after 50,000 files and left only a stdout table behind. The import loop now serialises the per-batch curve and rewrites the file after every batch, so the record is never more than one batch behind the terminal. Each write replaces the whole document, so a run killed mid-loop still leaves valid JSON. import_complete distinguishes a finished run from a checkpoint of an abandoned one. Also folds the two verbatim AnalysisConfig literals into full_pipeline_config, so the throughput and edge-case sections cannot drift into measuring different work. Closes GoingsOn problems 5f5002c9, c0b4b283.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-08-01 20:23 UTC

Signed with PGP, not checked

Commit: 926c309e0374fd50c81f7f78fa98ee75e1c2c3d8

Parent: ffb6215

3 files changed, +97 insertions, -29 deletions

M crates/audiofiles-bench/src/ingest.rs +39

			@@ -92,6 +92,32 @@
92	92		}
93	93		}
94	94
	95	+	/// The per-batch curve as JSON, one object per batch in order.
	96	+	///
	97	+	/// The shape of the curve is what the import benchmark is actually asking
	98	+	/// about (does throughput degrade as the vault grows), and it used to live only
	99	+	/// in the stdout table. Serialising it means a killed run still leaves the
	100	+	/// measurement behind.
	101	+	fn batch_series(stats: &[BatchStat]) -> serde_json::Value {
	102	+	serde_json::Value::Array(
	103	+	stats
	104	+	.iter()
	105	+	.enumerate()
	106	+	.map(\|(i, s)\| {
	107	+	serde_json::json!({
	108	+	"batch": i + 1,
	109	+	"cumulative": s.cumulative,
	110	+	"files": s.files,
	111	+	"bytes": s.bytes,
	112	+	"elapsed_s": (s.elapsed_s * 100.0).round() / 100.0,
	113	+	"files_per_sec": (s.files_per_sec() * 10.0).round() / 10.0,
	114	+	"mb_per_sec": (s.mb_per_sec() * 10.0).round() / 10.0,
	115	+	})
	116	+	})
	117	+	.collect(),
	118	+	)
	119	+	}
	120	+
95	121		/// Collect audio files under `dir`, recursively.
96	122		fn collect(dir: &Path, out: &mut Vec<PathBuf>) {
97	123		let Ok(entries) = std::fs::read_dir(dir) else {
			@@ -360,6 +386,15 @@
360	386		stat.elapsed_s,
361	387		);
362	388		stats.push(stat);
	389	+
	390	+	// Checkpoint the curve after every batch. A long import is the run most
	391	+	// worth recording and the one most likely to be killed part way, and
	392	+	// the summary metrics below only exist once the loop finishes.
	393	+	report.set("import_batches", batch_series(&stats));
	394	+	report.set("import_files", stats.iter().map(\|s\| s.files).sum::<usize>());
	395	+	report.set("import_bytes", stats.iter().map(\|s\| s.bytes).sum::<u64>());
	396	+	report.set("import_complete", false);
	397	+	report.checkpoint();
363	398		}
364	399
365	400		println!();
			@@ -398,6 +433,10 @@
398	433		total_files as f64 / total_s,
399	434		(total_bytes as f64 / 1e6) / total_s,
400	435		);
	436	+	// `import_complete` separates a finished import from a checkpoint of one
	437	+	// that was killed, so a comparison tool does not read a partial curve as a
	438	+	// regression.
	439	+	report.set("import_complete", true);
401	440		report.set("import_files", total_files);
402	441		report.set("import_bytes", total_bytes);
403	442		report.set(

M crates/audiofiles-bench/src/main.rs +18 -23

			@@ -225,6 +225,23 @@
225	225		/// The corpus is gitignored and rebuilt per machine (see `scripts/corpus.py`),
226	226		/// and on a dev box it is usually on an external drive rather than in the
227	227		/// checkout, so the in-repo path is only the fallback.
	228	+	/// Every stage on, 30-second cap, no tag suggestion and no smart skip. The
	229	+	/// throughput and edge-case sections both want the full pipeline, and a bench
	230	+	/// only measures what it says it measures if they ask for exactly the same work.
	231	+	fn full_pipeline_config() -> AnalysisConfig {
	232	+	AnalysisConfig {
	233	+	loudness: true,
	234	+	spectral: true,
	235	+	bpm: true,
	236	+	key: true,
	237	+	loop_detect: true,
	238	+	fingerprint: true,
	239	+	auto_suggest_tags: false,
	240	+	max_analysis_seconds: Some(30.0),
	241	+	smart_skip: false,
	242	+	}
	243	+	}
	244	+
228	245		fn corpus_dir() -> PathBuf {
229	246		if let Ok(dir) = std::env::var("AF_BENCH_CORPUS") {
230	247		return PathBuf::from(dir);
			@@ -555,17 +572,7 @@
555	572		let mut throughput_files_ext = throughput_files;
556	573		throughput_files_ext.extend(collect_audio_files(&training_dir.join("snare"), Some(250)));
557	574
558		-	let config = AnalysisConfig {
559		-	loudness: true,
560		-	spectral: true,
561		-	bpm: true,
562		-	key: true,
563		-	loop_detect: true,
564		-	fingerprint: true,
565		-	auto_suggest_tags: false,
566		-	max_analysis_seconds: Some(30.0),
567		-	smart_skip: false,
568		-	};
	575	+	let config = full_pipeline_config();
569	576
570	577		let tp_count = throughput_files_ext.len();
571	578		let tp_start = Instant::now();
			@@ -670,18 +677,6 @@
670	677		];
671	678
672	679		for (name, path) in &edge_cases {
673		-	let config = AnalysisConfig {
674		-	loudness: true,
675		-	spectral: true,
676		-	bpm: true,
677		-	key: true,
678		-	loop_detect: true,
679		-	fingerprint: true,
680		-	auto_suggest_tags: false,
681		-	max_analysis_seconds: Some(30.0),
682		-	smart_skip: false,
683		-	};
684		-
685	680		match analysis::analyze_sample("edge", path, &config) {
686	681		Ok(r) => {
687	682		println!(

M crates/audiofiles-bench/src/report.rs +40 -6

			@@ -54,11 +54,48 @@
54	54		self.metrics.insert(key.to_string(), value.into());
55	55		}
56	56
	57	+	/// Write what has been recorded so far, without announcing it.
	58	+	///
	59	+	/// The runs worth recording are the long ones, and those are exactly the
	60	+	/// ones likely to be cut short: a 50,000-file import killed part way used
	61	+	/// to produce no JSON at all, because [`Report::write`] runs once at the
	62	+	/// end. Callers with a batch loop call this after each batch so the file on
	63	+	/// disk is never more than one batch behind the terminal.
	64	+	///
	65	+	/// Silent on success (a checkpoint per batch would otherwise bury the batch
	66	+	/// table it sits under) and silent on repeat failures, but the first
	67	+	/// failure is reported: a checkpoint that cannot write is how a run ends up
	68	+	/// with nothing again, so it must not fail invisibly.
	69	+	pub(crate) fn checkpoint(&self) {
	70	+	static REPORTED: std::sync::Once = std::sync::Once::new();
	71	+	if let Err(e) = self.write_json() {
	72	+	REPORTED.call_once(\|\| eprintln!("\n could not checkpoint AF_BENCH_JSON: {e}"));
	73	+	}
	74	+	}
	75	+
57	76		/// Write to `AF_BENCH_JSON` if set. A run with the variable unset is the
58	77		/// normal case and must stay silent.
59	78		pub(crate) fn write(&self) {
60		-	let Ok(path) = std::env::var("AF_BENCH_JSON") else {
	79	+	if std::env::var("AF_BENCH_JSON").is_err() {
61	80		return;
	81	+	}
	82	+	match self.write_json() {
	83	+	Ok(Some(path)) => println!("\n wrote {path}"),
	84	+	Ok(None) => {}
	85	+	// A failed write must not look like a failed benchmark.
	86	+	Err(e) => eprintln!("\n could not write AF_BENCH_JSON: {e}"),
	87	+	}
	88	+	}
	89	+
	90	+	/// Render the current state and replace the file at `AF_BENCH_JSON`.
	91	+	///
	92	+	/// Returns the path written, or `None` when the variable is unset. Every
	93	+	/// call rewrites the whole object rather than appending, so a run killed
	94	+	/// between two calls leaves a complete, parseable document rather than a
	95	+	/// truncated one.
	96	+	fn write_json(&self) -> std::io::Result<Option<String>> {
	97	+	let Ok(path) = std::env::var("AF_BENCH_JSON") else {
	98	+	return Ok(None);
62	99		};
63	100		let mut root = Map::new();
64	101		root.insert("schema".into(), SCHEMA.into());
			@@ -83,11 +120,8 @@
83	120		root.insert("metrics".into(), Value::Object(self.metrics.clone()));
84	121
85	122		let text = serde_json::to_string_pretty(&Value::Object(root)).unwrap_or_default();
86		-	match std::fs::File::create(&path).and_then(\|mut f\| f.write_all(text.as_bytes())) {
87		-	Ok(()) => println!("\n wrote {path}"),
88		-	// A failed write must not look like a failed benchmark.
89		-	Err(e) => eprintln!("\n could not write {path}: {e}"),
90		-	}
	123	+	std::fs::File::create(&path).and_then(\|mut f\| f.write_all(text.as_bytes()))?;
	124	+	Ok(Some(path))
91	125		}
92	126		}
93	127