Skip to main content

max / shop

3.3 KB · 125 lines History Blame Raw
1 //! shop-bench: perf harness driving external workloads (vtebench) and
2 //! internal microbenches, comparing results against a stored per-machine
3 //! baseline. See `docs/typed-protocol.md` Phase 0 for the design.
4 //!
5 //! <!-- wiki: shop-typed-protocol -->
6
7 use anyhow::Result;
8 use clap::{Parser, Subcommand};
9
10 mod baseline;
11 mod workloads;
12
13 use baseline::Baseline;
14 use workloads::{Registry, WorkloadResult};
15
16 #[derive(Parser)]
17 #[command(name = "shop-bench", about = "shop perf harness")]
18 struct Cli {
19 #[command(subcommand)]
20 cmd: Cmd,
21 }
22
23 #[derive(Subcommand)]
24 enum Cmd {
25 /// Run one or all workloads; compare against the machine baseline.
26 Run {
27 /// Workload name; omit to run all.
28 workload: Option<String>,
29 /// Path to baseline JSON (defaults to
30 /// `_private/docs/shop/bench-baselines/<hostname>-latest.json`).
31 #[arg(long)]
32 baseline: Option<String>,
33 /// Skip the gate check; report numbers only.
34 #[arg(long)]
35 no_gate: bool,
36 },
37 /// Run all workloads and write the results as a new baseline.
38 Save {
39 #[arg(long)]
40 out: String,
41 },
42 /// List registered workloads.
43 List,
44 }
45
46 fn main() -> Result<()> {
47 let cli = Cli::parse();
48 let reg = Registry::default_set();
49
50 match cli.cmd {
51 Cmd::List => {
52 for w in reg.iter() {
53 println!("{}\t{}", w.name(), w.description());
54 }
55 Ok(())
56 }
57 Cmd::Run {
58 workload,
59 baseline,
60 no_gate,
61 } => {
62 let results = run(&reg, workload.as_deref())?;
63 let base = baseline.as_deref().map(Baseline::load).transpose()?;
64 report(&results, base.as_ref(), !no_gate)
65 }
66 Cmd::Save { out } => {
67 let results = run(&reg, None)?;
68 let base = Baseline::from_results(&results);
69 base.save(&out)?;
70 eprintln!("wrote baseline to {out}");
71 Ok(())
72 }
73 }
74 }
75
76 fn run(reg: &Registry, only: Option<&str>) -> Result<Vec<WorkloadResult>> {
77 let mut out = Vec::new();
78 for w in reg.iter() {
79 if let Some(name) = only {
80 if w.name() != name {
81 continue;
82 }
83 }
84 eprintln!("running {}...", w.name());
85 out.push(w.run()?);
86 }
87 if out.is_empty() {
88 anyhow::bail!("no workload matched selector");
89 }
90 Ok(out)
91 }
92
93 fn report(results: &[WorkloadResult], base: Option<&Baseline>, gate: bool) -> Result<()> {
94 let mut breach = false;
95 for r in results {
96 println!("== {} ==", r.name);
97 for (k, v) in &r.metrics {
98 let cmp = base
99 .and_then(|b| b.metric(&r.name, k))
100 .map(|prev| format!(" (baseline {prev:.4}, delta {:+.2}%)", pct(*v, *prev)));
101 println!(" {k} = {v:.4}{}", cmp.unwrap_or_default());
102 }
103 if gate {
104 if let Some(b) = base {
105 if let Err(msg) = b.check_gate(r) {
106 eprintln!("GATE BREACH [{}]: {msg}", r.name);
107 breach = true;
108 }
109 }
110 }
111 }
112 if breach {
113 anyhow::bail!("one or more gates breached");
114 }
115 Ok(())
116 }
117
118 fn pct(new: f64, old: f64) -> f64 {
119 if old == 0.0 {
120 0.0
121 } else {
122 (new - old) / old * 100.0
123 }
124 }
125