//! Does an export actually satisfy the device it was exported for? //! //! A checker rather than a measurement, like [`layout`](crate::layout): it //! exports to each of the fourteen bundled targets, reads every file it wrote //! back off disk, and scores it against that device's manifest. Per-target //! pass/fail, and a non-zero exit if any target fails. //! //! **Why not unit tests.** The export module has plenty, and they assert what a //! config the test wrote produces. That cannot catch a profile whose constraints //! never reach the config, or a config the pipeline quietly ignores, because //! both ends of the comparison were written by the same hand. This runs the real //! resolution ([`DeviceProfile::apply_to`]) over the real manifest, exports //! through the real pipeline, and compares against the manifest again. The //! manifest is the ground truth, and this is the rare corner where the truth is //! written down rather than annotated by somebody. //! //! **The header is the oracle, not a decoder.** Sample rate, bit depth and //! channel count are read out of the WAV `fmt ` and AIFF `COMM` chunks by the //! parser below. A device reads the file's header, so the header is the thing //! that has to be right, and going through symphonia would let a decoder's //! tolerance for an odd header hide exactly the defect this is looking for. //! //! **Sources are fabricated, not drawn from the corpus.** The corpus is real //! audio with ordinary names, and what this checks hardest is what happens to a //! name a device cannot take: spaces, punctuation, case, accents, and a stem //! four times the longest limit any of the fourteen declare. Those have to be //! constructed. It also means the mode runs anywhere, with no `AF_BENCH_CORPUS`. //! //! **What it does not check, stated so a green run is not read as more than it //! is.** Two manifest fields go unexercised because only one device declares //! each and neither can be reached with sources of a sane size: the Volca's //! `max_file_size_bytes` (128 MiB, against sources of a tenth of a second) and //! the SP-404's `max_sample_count` (100, against seven). `check_size` is written //! and will fire if a device ever declares a limit a real sample can cross; //! `max_sample_count` has no check at all, because nothing in the pipeline //! enforces it yet. Audio content is not compared either: this asks whether the //! file is one the device can read, not whether it still sounds like the source. //! //! Usage: `cargo run --release -p audiofiles-bench -- device-export` use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicBool; use audiofiles_core::export::profile::{ChannelConstraint, DeviceProfile, NamingCase}; use audiofiles_core::export::{ExportChannels, ExportConfig, ExportFormat, ExportItem, run_export}; use audiofiles_core::store::SampleStore; /// The names every device is asked to cope with. /// /// Each is a stem, and each is here for a rule in the manifests. Nothing in the /// list is exotic for a sample library: these are the names a pack ships with. const AWKWARD_STEMS: &[&str] = &[ // Ordinary, so a device that mangles the easy case shows up too. "kick", // Spaces and mixed case, the two most common things in a sample pack. "Deep House Kick 01", // Punctuation a filesystem allows and a sampler often does not. "snare (bright) [wet] #2!", // Non-ASCII, which `strip_special` has to answer for one way or the other. "cafe\u{301} cra\u{300}sh", // Longer than the longest `max_length` in the bundled set (128) by enough // that a truncation off by a character is visible. "a_very_long_stem_that_keeps_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going_and_going", // Two stems that collide once case and separators are normalised, so the // deduplication has to hold the names apart. "Clap A", "clap_a", ]; /// What one device's run came to. struct DeviceReport { device: String, slug: String, exported: usize, violations: Vec, } impl DeviceReport { fn passed(&self) -> bool { self.violations.is_empty() } } /// What a container's header says it holds. #[derive(Debug, Clone, Copy)] struct Header { sample_rate: u32, bit_depth: u16, channels: u16, } /// Run the whole matrix. Returns false if any target failed. pub(crate) fn run() -> bool { let mut registry = audiofiles_rhai::registry::PluginRegistry::new(); if let Err(e) = audiofiles_rhai::bundled::load_bundled(&mut registry) { eprintln!("could not load the bundled plugins: {e}"); return false; } let scratch = match tempdir() { Ok(dir) => dir, Err(e) => { eprintln!("could not make a scratch directory: {e}"); return false; } }; let sources = match fabricate_sources(&scratch.join("sources")) { Ok(sources) => sources, Err(e) => { eprintln!("could not write the source files: {e}"); return false; } }; println!("Device export conformance"); println!(" {} sources, {} targets", sources.len(), registry.len()); println!(); let mut reports: Vec = registry .list() .iter() .filter_map(|summary| registry.get(&summary.name)) .map(|plugin| score(&plugin.profile, &sources, &scratch)) .collect(); reports.sort_by(|a, b| a.slug.cmp(&b.slug)); for report in &reports { let mark = if report.passed() { "pass" } else { "FAIL" }; println!( " {mark} {:<20} {} exported", report.device, report.exported ); for violation in &report.violations { println!(" {violation}"); } } let failed = reports.iter().filter(|r| !r.passed()).count(); println!(); println!( " {} of {} targets conform", reports.len() - failed, reports.len() ); let _ = std::fs::remove_dir_all(&scratch); failed == 0 } /// Export every source for one device and check what came out. fn score(profile: &DeviceProfile, sources: &[PathBuf], scratch: &Path) -> DeviceReport { let slug = slug(&profile.name); let destination = scratch.join("out").join(&slug); let mut violations = Vec::new(); let items: Vec = sources .iter() .map(|path| ExportItem { // Never read: `source_path` is set, so the runner takes the file // from disk rather than from the store. hash: audiofiles_core::SampleHash::from_trusted("0".repeat(64)), ext: "wav".to_string(), relative_path: PathBuf::from(file_stem(path)), name: file_stem(path), bpm: None, musical_key: None, duration: None, tags: Vec::new(), source_path: Some(path.clone()), }) .collect(); // Everything left open, so the profile is what decides. This is the state // the picker hands the backend when a user chooses a device and touches // nothing else, which is the case worth checking. let mut config = ExportConfig { format: ExportFormat::Original, sample_rate: None, bit_depth: None, channels: ExportChannels::Original, naming_pattern: None, flatten: true, metadata_sidecar: false, destination: destination.clone(), device_profile: Some(profile.name.clone()), naming_rules: None, max_file_size_bytes: None, name_overrides: None, }; profile.apply_to(&mut config); let store = match SampleStore::new(scratch.join("store")) { Ok(store) => store, Err(e) => { violations.push(format!("could not open a store: {e}")); return DeviceReport { device: profile.name.clone(), slug, exported: 0, violations, }; } }; let summary = match run_export( &items, &config, &store, &AtomicBool::new(false), |_, _, _| true, ) { Ok(summary) => summary, Err(e) => { violations.push(format!("the export failed outright: {e}")); return DeviceReport { device: profile.name.clone(), slug, exported: 0, violations, }; } }; for (name, error) in &summary.errors { violations.push(format!("{name}: the pipeline reported {error}")); } let written = match list_files(&destination) { Ok(written) => written, Err(e) => { violations.push(format!("nothing to read back: {e}")); Vec::new() } }; // A name that collided and was silently overwritten is a lost sample, and // the count is the only place it shows. let expected = items.len() - summary.errors.len(); if written.len() != expected { violations.push(format!( "{} files written for {expected} samples that did not error: names collided, \ or something was dropped without saying so", written.len() )); } let mut stems = BTreeSet::new(); for path in &written { check_format(profile, path, &mut violations); check_header(profile, path, &mut violations); check_name(profile, path, &mut violations); check_size(profile, path, &mut violations); stems.insert(file_stem(path)); } if stems.len() != written.len() { violations.push("two files share a stem after normalisation".to_string()); } DeviceReport { device: profile.name.clone(), slug, exported: written.len(), violations, } } /// The extension has to be one the device reads. fn check_format(profile: &DeviceProfile, path: &Path, violations: &mut Vec) { let ext = path .extension() .and_then(|e| e.to_str()) .unwrap_or_default() .to_ascii_lowercase(); let allowed: Vec<&str> = profile .audio .formats .iter() .filter_map(|f| match f { ExportFormat::Wav => Some("wav"), ExportFormat::Aiff => Some("aiff"), // "whatever came in" is not a format a device declares support for. ExportFormat::Original => None, }) .collect(); if !allowed.is_empty() && !allowed.contains(&ext.as_str()) { violations.push(format!( "{}: extension .{ext}, device reads {}", name(path), allowed.join("/") )); } } /// Rate, depth and channel count, read off the container. fn check_header(profile: &DeviceProfile, path: &Path, violations: &mut Vec) { let header = match read_header(path) { Ok(header) => header, Err(e) => { violations.push(format!("{}: unreadable header ({e})", name(path))); return; } }; if !profile.audio.sample_rates.contains(&header.sample_rate) { violations.push(format!( "{}: {} Hz, device takes {:?}", name(path), header.sample_rate, profile.audio.sample_rates )); } if !profile.audio.bit_depths.contains(&header.bit_depth) { violations.push(format!( "{}: {}-bit, device takes {:?}", name(path), header.bit_depth, profile.audio.bit_depths )); } let channels_ok = match profile.audio.channels { ChannelConstraint::Mono => header.channels == 1, ChannelConstraint::Stereo => header.channels == 2, ChannelConstraint::Both => header.channels == 1 || header.channels == 2, }; if !channels_ok { violations.push(format!( "{}: {} channels, device is {:?}", name(path), header.channels, profile.audio.channels )); } } /// Case, separator, length and the character set. fn check_name(profile: &DeviceProfile, path: &Path, violations: &mut Vec) { let Some(rules) = &profile.naming else { return; }; let stem = file_stem(path); match rules.case { NamingCase::Lower if stem != stem.to_lowercase() => { violations.push(format!("{stem}: not lowercased")); } NamingCase::Upper if stem != stem.to_uppercase() => { violations.push(format!("{stem}: not uppercased")); } _ => {} } if stem.chars().count() > rules.max_length { violations.push(format!( "{stem}: {} characters, device takes {}", stem.chars().count(), rules.max_length )); } if rules.strip_special { let bad: String = stem .chars() .filter(|c| !c.is_ascii_alphanumeric() && *c != rules.separator) .collect(); if !bad.is_empty() { violations.push(format!( "{stem}: keeps {bad:?}, which strip_special forbids" )); } } } /// The written file has to fit. fn check_size(profile: &DeviceProfile, path: &Path, violations: &mut Vec) { let Some(limit) = profile.limits.as_ref().and_then(|l| l.max_file_size_bytes) else { return; }; let Ok(meta) = std::fs::metadata(path) else { return; }; if meta.len() > limit { violations.push(format!( "{}: {} bytes, device takes {limit}", name(path), meta.len() )); } } // ── Reading a container's header ── /// Sample rate, bit depth and channel count, from the file itself. fn read_header(path: &Path) -> Result { let bytes = std::fs::read(path).map_err(|e| e.to_string())?; if bytes.len() < 12 { return Err("shorter than a container header".to_string()); } match &bytes[0..4] { b"RIFF" => wav_header(&bytes), b"FORM" => aiff_header(&bytes), other => Err(format!( "unknown container {:?}", String::from_utf8_lossy(other) )), } } /// The `fmt ` chunk of a RIFF/WAVE file. fn wav_header(bytes: &[u8]) -> Result { if &bytes[8..12] != b"WAVE" { return Err("RIFF but not WAVE".to_string()); } let mut at = 12; while at + 8 <= bytes.len() { let id = &bytes[at..at + 4]; let size = u32::from_le_bytes(take4(bytes, at + 4)?) as usize; let body = at + 8; if id == b"fmt " { if body + 16 > bytes.len() { return Err("truncated fmt chunk".to_string()); } return Ok(Header { channels: u16::from_le_bytes([bytes[body + 2], bytes[body + 3]]), sample_rate: u32::from_le_bytes(take4(bytes, body + 4)?), bit_depth: u16::from_le_bytes([bytes[body + 14], bytes[body + 15]]), }); } // Chunks are word-aligned: an odd size carries a pad byte. at = body + size + (size % 2); } Err("no fmt chunk".to_string()) } /// The `COMM` chunk of an AIFF file. /// /// The sample rate is an 80-bit IEEE 754 extended float, which is why this /// reads longer than the WAV case. Only the integral part is wanted and every /// rate in the manifests is a small positive integer, so the mantissa's /// fractional bits are dropped rather than accumulated. fn aiff_header(bytes: &[u8]) -> Result { if &bytes[8..12] != b"AIFF" && &bytes[8..12] != b"AIFC" { return Err("FORM but not AIFF".to_string()); } let mut at = 12; while at + 8 <= bytes.len() { let id = &bytes[at..at + 4]; let size = u32::from_be_bytes(take4(bytes, at + 4)?) as usize; let body = at + 8; if id == b"COMM" { if body + 18 > bytes.len() { return Err("truncated COMM chunk".to_string()); } let exponent = u16::from_be_bytes([bytes[body + 8], bytes[body + 9]]); let mantissa = u64::from_be_bytes([ bytes[body + 10], bytes[body + 11], bytes[body + 12], bytes[body + 13], bytes[body + 14], bytes[body + 15], bytes[body + 16], bytes[body + 17], ]); let shift = i32::from(exponent & 0x7fff) - 16383 - 63; let rate = if shift >= 0 { mantissa << shift.min(63) } else { mantissa >> (-shift).min(63) }; return Ok(Header { channels: u16::from_be_bytes([bytes[body], bytes[body + 1]]), sample_rate: u32::try_from(rate).map_err(|_| "absurd sample rate".to_string())?, bit_depth: u16::from_be_bytes([bytes[body + 6], bytes[body + 7]]), }); } at = body + size + (size % 2); } Err("no COMM chunk".to_string()) } fn take4(bytes: &[u8], at: usize) -> Result<[u8; 4], String> { bytes .get(at..at + 4) .and_then(|s| s.try_into().ok()) .ok_or_else(|| "truncated chunk header".to_string()) } // ── Fabricating the sources ── /// One source per awkward stem, cycling through four shapes. /// /// **Every shape is one at least one device has to convert.** A source that /// already conformed would let a pipeline that did nothing at all pass. The /// four together cover conversion in both directions on each axis: 44.1k up to /// the 48k-only devices and 48k down to the 44.1k-only ones, 24-bit down to the /// seven that take only 16, and stereo down to the five that are mono. /// /// 96k is here for the widest device rather than the narrowest. MPC One/Live /// takes 44.1k and 48k, 16 and 24 bit, mono and stereo, so without a rate none /// of the fourteen accept it would be handed four sources it could pass through /// untouched, and its green cell would say nothing at all. /// `no_device_can_pass_without_converting_something` is what keeps that true. /// /// 31250 Hz is deliberately not among them. It is the Volca's only rate, so /// resampling to it is exercised from every source rather than needing one of /// its own. const SOURCE_SHAPES: &[(u32, u16, u16)] = &[ (44100, 16, 2), (48000, 24, 2), (44100, 24, 1), (48000, 16, 1), (96000, 24, 2), ]; /// Write one WAV per awkward stem, rotating through [`SOURCE_SHAPES`]. fn fabricate_sources(dir: &Path) -> std::io::Result> { std::fs::create_dir_all(dir)?; let mut written = Vec::new(); for (i, stem) in AWKWARD_STEMS.iter().enumerate() { let (rate, bits, channels) = SOURCE_SHAPES[i % SOURCE_SHAPES.len()]; let path = dir.join(format!("{stem}.wav")); std::fs::write( &path, wav(&tone(i, rate, bits, channels), rate, bits, channels), )?; written.push(path); } Ok(written) } /// A tenth of a second of a sine, at the given shape. /// /// The frequency varies with `i` so two sources are never byte-identical; a /// pipeline that wrote the same file twice would otherwise be invisible to the /// name checks, which are all that read the output today. fn tone(i: usize, rate: u32, bits: u16, channels: u16) -> Vec { let frames = rate as usize / 10; let mut pcm = Vec::with_capacity(frames * usize::from(channels) * usize::from(bits / 8)); for frame in 0..frames { #[allow(clippy::cast_precision_loss)] let t = frame as f64 / f64::from(rate); #[allow(clippy::cast_precision_loss)] let hz = 220.0 * (i + 1) as f64; let value = (t * hz * std::f64::consts::TAU).sin() * 0.25; for _ in 0..channels { match bits { 24 => { #[allow(clippy::cast_possible_truncation)] let scaled = (value * f64::from(1 << 23)) as i32; pcm.extend_from_slice(&scaled.to_le_bytes()[0..3]); } _ => { #[allow(clippy::cast_possible_truncation)] let scaled = (value * f64::from(i16::MAX)) as i16; pcm.extend_from_slice(&scaled.to_le_bytes()); } } } } pcm } /// A canonical 44-byte-header RIFF/WAVE file around `pcm`. fn wav(pcm: &[u8], rate: u32, bits: u16, channels: u16) -> Vec { let block_align = channels * bits / 8; let byte_rate = rate * u32::from(block_align); let mut out = Vec::with_capacity(44 + pcm.len()); out.extend_from_slice(b"RIFF"); out.extend_from_slice( &u32::try_from(36 + pcm.len()) .unwrap_or(u32::MAX) .to_le_bytes(), ); out.extend_from_slice(b"WAVEfmt "); out.extend_from_slice(&16u32.to_le_bytes()); out.extend_from_slice(&1u16.to_le_bytes()); // PCM out.extend_from_slice(&channels.to_le_bytes()); out.extend_from_slice(&rate.to_le_bytes()); out.extend_from_slice(&byte_rate.to_le_bytes()); out.extend_from_slice(&block_align.to_le_bytes()); out.extend_from_slice(&bits.to_le_bytes()); out.extend_from_slice(b"data"); out.extend_from_slice(&u32::try_from(pcm.len()).unwrap_or(u32::MAX).to_le_bytes()); out.extend_from_slice(pcm); out } // ── Small helpers ── fn tempdir() -> std::io::Result { let dir = std::env::temp_dir().join(format!("af-device-export-{}", std::process::id())); std::fs::create_dir_all(&dir)?; Ok(dir) } fn list_files(dir: &Path) -> std::io::Result> { let mut out = Vec::new(); let mut stack = vec![dir.to_path_buf()]; while let Some(at) = stack.pop() { for entry in std::fs::read_dir(&at)? { let path = entry?.path(); if path.is_dir() { stack.push(path); } else { out.push(path); } } } out.sort(); Ok(out) } fn name(path: &Path) -> String { path.file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default() } fn file_stem(path: &Path) -> String { path.file_stem() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default() } fn slug(device: &str) -> String { device .chars() .map(|c| { if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '_' } }) .collect() } #[cfg(test)] mod tests { use super::*; /// The header parser is the oracle here, so it is the one thing that cannot /// be checked by the harness it serves. Round-trip it against the writer. #[test] fn the_wav_parser_reads_back_what_the_writer_wrote() { for (rate, bits, channels) in [ (44100, 16, 1), (48000, 16, 2), (44100, 24, 2), (48000, 24, 1), ] { let bytes = wav(&[0u8; 64], rate, bits, channels); let dir = tempdir().unwrap(); let path = dir.join(format!("probe-{rate}-{bits}-{channels}.wav")); std::fs::write(&path, &bytes).unwrap(); let header = read_header(&path).unwrap(); assert_eq!(header.sample_rate, rate); assert_eq!(header.bit_depth, bits); assert_eq!(header.channels, channels); std::fs::remove_file(&path).unwrap(); } } /// A chunk before `fmt ` must not throw the walk off, and an odd-sized one /// carries a pad byte the walk has to skip or every later offset is wrong. #[test] fn the_wav_parser_walks_past_an_odd_sized_chunk() { let mut bytes = Vec::new(); bytes.extend_from_slice(b"RIFF"); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(b"WAVE"); // A 3-byte LIST chunk, so the pad byte matters. bytes.extend_from_slice(b"LIST"); bytes.extend_from_slice(&3u32.to_le_bytes()); bytes.extend_from_slice(&[1, 2, 3, 0]); let canonical = wav(&[0u8; 8], 48000, 24, 2); bytes.extend_from_slice(&canonical[12..]); let dir = tempdir().unwrap(); let path = dir.join("odd-chunk.wav"); std::fs::write(&path, &bytes).unwrap(); let header = read_header(&path).unwrap(); assert_eq!(header.sample_rate, 48000); assert_eq!(header.bit_depth, 24); assert_eq!(header.channels, 2); std::fs::remove_file(&path).unwrap(); } /// Every stem in the list has to reach the harness as a real file, or a /// case silently stops being covered. #[test] fn every_awkward_stem_becomes_a_source_of_the_shape_it_was_meant_to_have() { let dir = tempdir().unwrap().join("fabricate-test"); let sources = fabricate_sources(&dir).unwrap(); assert_eq!(sources.len(), AWKWARD_STEMS.len()); for (i, path) in sources.iter().enumerate() { let (rate, bits, channels) = SOURCE_SHAPES[i % SOURCE_SHAPES.len()]; let header = read_header(path).unwrap(); assert_eq!(header.sample_rate, rate); assert_eq!(header.bit_depth, bits); assert_eq!(header.channels, channels); } std::fs::remove_dir_all(&dir).unwrap(); } /// The point of the four shapes is that no device is handed a source it can /// take as-is on every axis. If that ever stops being true the harness goes /// quietly weaker, so it is asserted rather than left to the comment. #[test] fn no_device_can_pass_without_converting_something() { let mut registry = audiofiles_rhai::registry::PluginRegistry::new(); audiofiles_rhai::bundled::load_bundled(&mut registry).unwrap(); for summary in registry.list() { let profile = ®istry.get(&summary.name).unwrap().profile; let conforming = SOURCE_SHAPES.iter().filter(|(rate, bits, channels)| { profile.audio.sample_rates.contains(rate) && profile.audio.bit_depths.contains(bits) && match profile.audio.channels { ChannelConstraint::Mono => *channels == 1, ChannelConstraint::Stereo => *channels == 2, ChannelConstraint::Both => true, } }); assert!( conforming.count() < SOURCE_SHAPES.len(), "{} takes every source shape unchanged, so its pass says nothing", profile.name ); } } }