Skip to main content

max / shop

5.5 KB · 181 lines History Blame Raw
1 use std::collections::BTreeMap;
2 use std::fs;
3 use std::path::Path;
4
5 use anyhow::{Context, Result};
6 use serde::{Deserialize, Serialize};
7
8 use crate::workloads::WorkloadResult;
9
10 /// A frozen set of per-workload metrics captured on one machine. Not committed
11 /// to git — regenerable via `shop-bench save`.
12 #[derive(Debug, Serialize, Deserialize)]
13 pub(crate) struct Baseline {
14 pub machine: String,
15 pub captured_at: String,
16 pub shop_commit: Option<String>,
17 pub workloads: BTreeMap<String, BTreeMap<String, f64>>,
18 }
19
20 impl Baseline {
21 pub(crate) fn load(path: &str) -> Result<Self> {
22 let s = fs::read_to_string(path).with_context(|| format!("read baseline {path}"))?;
23 let b: Self = serde_json::from_str(&s).with_context(|| format!("parse baseline {path}"))?;
24 Ok(b)
25 }
26
27 pub(crate) fn save(&self, path: &str) -> Result<()> {
28 if let Some(parent) = Path::new(path).parent() {
29 fs::create_dir_all(parent).ok();
30 }
31 let s = serde_json::to_string_pretty(self)?;
32 fs::write(path, s).with_context(|| format!("write baseline {path}"))?;
33 Ok(())
34 }
35
36 pub(crate) fn from_results(results: &[WorkloadResult]) -> Self {
37 let workloads = results
38 .iter()
39 .map(|r| (r.name.clone(), r.metrics.iter().cloned().collect()))
40 .collect();
41 Self {
42 machine: hostname(),
43 captured_at: now_rfc3339(),
44 shop_commit: git_head(),
45 workloads,
46 }
47 }
48
49 pub(crate) fn metric(&self, workload: &str, metric: &str) -> Option<&f64> {
50 self.workloads.get(workload).and_then(|m| m.get(metric))
51 }
52
53 /// Enforce the Phase-0 gates from `shop-typed-protocol.md`. Additive:
54 /// unknown metrics or unknown workloads are not gated.
55 pub(crate) fn check_gate(&self, r: &WorkloadResult) -> Result<(), String> {
56 for gate in GATES {
57 if gate.workload != r.name {
58 continue;
59 }
60 let Some(&prev) = self.metric(&r.name, gate.metric) else {
61 continue;
62 };
63 let Some(&new) = r
64 .metrics
65 .iter()
66 .find(|(k, _)| k == gate.metric)
67 .map(|(_, v)| v)
68 else {
69 continue;
70 };
71 gate.check(prev, new)?;
72 }
73 Ok(())
74 }
75 }
76
77 struct Gate {
78 workload: &'static str,
79 metric: &'static str,
80 kind: GateKind,
81 }
82
83 enum GateKind {
84 /// New value must be at least `prev * (1 - tol)`. For throughput.
85 ThroughputRegressionPct(f64),
86 /// New value must be at most `prev * (1 + tol)`. For latency / frame time.
87 LatencyRegressionPct(f64),
88 /// Absolute cap on regression, in the metric's units. For raw latency.
89 LatencyRegressionAbs(f64),
90 }
91
92 impl Gate {
93 fn check(&self, prev: f64, new: f64) -> Result<(), String> {
94 match self.kind {
95 GateKind::ThroughputRegressionPct(tol) => {
96 let floor = prev * (1.0 - tol);
97 if new < floor {
98 return Err(format!(
99 "{} throughput {new:.4} below floor {floor:.4} (prev {prev:.4}, tol {:.2}%)",
100 self.metric,
101 tol * 100.0
102 ));
103 }
104 }
105 GateKind::LatencyRegressionPct(tol) => {
106 let ceil = prev * (1.0 + tol);
107 if new > ceil {
108 return Err(format!(
109 "{} {new:.4} above ceil {ceil:.4} (prev {prev:.4}, tol {:.2}%)",
110 self.metric,
111 tol * 100.0
112 ));
113 }
114 }
115 GateKind::LatencyRegressionAbs(bound) => {
116 if new - prev > bound {
117 return Err(format!(
118 "{} {new:.4} exceeds prev {prev:.4} by more than {bound}",
119 self.metric
120 ));
121 }
122 }
123 }
124 Ok(())
125 }
126 }
127
128 // Gate table sourced from shop-typed-protocol.md Phase 0. Table-specific
129 // metrics (typed-*) are recorded, not gated, until their respective phase.
130 const GATES: &[Gate] = &[
131 Gate {
132 workload: "vtebench-cat",
133 metric: "cell_throughput_mb_s",
134 kind: GateKind::ThroughputRegressionPct(0.01),
135 },
136 Gate {
137 workload: "vtebench-scrolling",
138 metric: "frame_time_p99_ms",
139 kind: GateKind::LatencyRegressionPct(0.02),
140 },
141 Gate {
142 workload: "vtebench-unicode",
143 metric: "cell_throughput_mb_s",
144 kind: GateKind::ThroughputRegressionPct(0.01),
145 },
146 Gate {
147 workload: "kitty-image-flood",
148 metric: "throughput_mb_s",
149 kind: GateKind::ThroughputRegressionPct(0.0),
150 },
151 Gate {
152 workload: "pty-to-pixel",
153 metric: "latency_p50_us",
154 kind: GateKind::LatencyRegressionAbs(200.0),
155 },
156 ];
157
158 fn hostname() -> String {
159 std::env::var("HOSTNAME")
160 .or_else(|_| std::env::var("HOST"))
161 .unwrap_or_else(|_| "unknown".into())
162 }
163
164 fn now_rfc3339() -> String {
165 // Minimal RFC3339 formatter to avoid a chrono dep for a stamp field.
166 let secs = std::time::SystemTime::now()
167 .duration_since(std::time::UNIX_EPOCH)
168 .map_or(0, |d| d.as_secs());
169 format!("@unix:{secs}")
170 }
171
172 fn git_head() -> Option<String> {
173 std::process::Command::new("git")
174 .args(["rev-parse", "--short", "HEAD"])
175 .output()
176 .ok()
177 .and_then(|o| String::from_utf8(o.stdout).ok())
178 .map(|s| s.trim().to_string())
179 .filter(|s| !s.is_empty())
180 }
181