use std::collections::BTreeMap; use std::fs; use std::path::Path; use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use crate::workloads::WorkloadResult; /// A frozen set of per-workload metrics captured on one machine. Not committed /// to git — regenerable via `shop-bench save`. #[derive(Debug, Serialize, Deserialize)] pub(crate) struct Baseline { pub machine: String, pub captured_at: String, pub shop_commit: Option, pub workloads: BTreeMap>, } impl Baseline { pub(crate) fn load(path: &str) -> Result { let s = fs::read_to_string(path).with_context(|| format!("read baseline {path}"))?; let b: Self = serde_json::from_str(&s).with_context(|| format!("parse baseline {path}"))?; Ok(b) } pub(crate) fn save(&self, path: &str) -> Result<()> { if let Some(parent) = Path::new(path).parent() { fs::create_dir_all(parent).ok(); } let s = serde_json::to_string_pretty(self)?; fs::write(path, s).with_context(|| format!("write baseline {path}"))?; Ok(()) } pub(crate) fn from_results(results: &[WorkloadResult]) -> Self { let workloads = results .iter() .map(|r| (r.name.clone(), r.metrics.iter().cloned().collect())) .collect(); Self { machine: hostname(), captured_at: now_rfc3339(), shop_commit: git_head(), workloads, } } pub(crate) fn metric(&self, workload: &str, metric: &str) -> Option<&f64> { self.workloads.get(workload).and_then(|m| m.get(metric)) } /// Enforce the Phase-0 gates from `shop-typed-protocol.md`. Additive: /// unknown metrics or unknown workloads are not gated. pub(crate) fn check_gate(&self, r: &WorkloadResult) -> Result<(), String> { for gate in GATES { if gate.workload != r.name { continue; } let Some(&prev) = self.metric(&r.name, gate.metric) else { continue; }; let Some(&new) = r .metrics .iter() .find(|(k, _)| k == gate.metric) .map(|(_, v)| v) else { continue; }; gate.check(prev, new)?; } Ok(()) } } struct Gate { workload: &'static str, metric: &'static str, kind: GateKind, } enum GateKind { /// New value must be at least `prev * (1 - tol)`. For throughput. ThroughputRegressionPct(f64), /// New value must be at most `prev * (1 + tol)`. For latency / frame time. LatencyRegressionPct(f64), /// Absolute cap on regression, in the metric's units. For raw latency. LatencyRegressionAbs(f64), } impl Gate { fn check(&self, prev: f64, new: f64) -> Result<(), String> { match self.kind { GateKind::ThroughputRegressionPct(tol) => { let floor = prev * (1.0 - tol); if new < floor { return Err(format!( "{} throughput {new:.4} below floor {floor:.4} (prev {prev:.4}, tol {:.2}%)", self.metric, tol * 100.0 )); } } GateKind::LatencyRegressionPct(tol) => { let ceil = prev * (1.0 + tol); if new > ceil { return Err(format!( "{} {new:.4} above ceil {ceil:.4} (prev {prev:.4}, tol {:.2}%)", self.metric, tol * 100.0 )); } } GateKind::LatencyRegressionAbs(bound) => { if new - prev > bound { return Err(format!( "{} {new:.4} exceeds prev {prev:.4} by more than {bound}", self.metric )); } } } Ok(()) } } // Gate table sourced from shop-typed-protocol.md Phase 0. Table-specific // metrics (typed-*) are recorded, not gated, until their respective phase. const GATES: &[Gate] = &[ Gate { workload: "vtebench-cat", metric: "cell_throughput_mb_s", kind: GateKind::ThroughputRegressionPct(0.01), }, Gate { workload: "vtebench-scrolling", metric: "frame_time_p99_ms", kind: GateKind::LatencyRegressionPct(0.02), }, Gate { workload: "vtebench-unicode", metric: "cell_throughput_mb_s", kind: GateKind::ThroughputRegressionPct(0.01), }, Gate { workload: "kitty-image-flood", metric: "throughput_mb_s", kind: GateKind::ThroughputRegressionPct(0.0), }, Gate { workload: "pty-to-pixel", metric: "latency_p50_us", kind: GateKind::LatencyRegressionAbs(200.0), }, ]; fn hostname() -> String { std::env::var("HOSTNAME") .or_else(|_| std::env::var("HOST")) .unwrap_or_else(|_| "unknown".into()) } fn now_rfc3339() -> String { // Minimal RFC3339 formatter to avoid a chrono dep for a stamp field. let secs = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); format!("@unix:{secs}") } fn git_head() -> Option { std::process::Command::new("git") .args(["rev-parse", "--short", "HEAD"]) .output() .ok() .and_then(|o| String::from_utf8(o.stdout).ok()) .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) }