Skip to main content

max / shop

Add shop-bench crate and justfile with bench/build/check recipes
Author: Max Johnson <me@maxj.phd> · 2026-07-26 20:07 UTC
Signed with PGP, not checked
Commit: 018cdd22e23b9526b690d0d923fecb10714cf5db
Parent: b8c3d72
10 files changed, +680 insertions, -0 deletions
M Cargo.lock +191
@@ -32,6 +32,56 @@
32 32 "libc",
33 33 ]
34 34
35 + [[package]]
36 + name = "anstream"
37 + version = "1.0.0"
38 + source = "registry+https://github.com/rust-lang/crates.io-index"
39 + checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
40 + dependencies = [
41 + "anstyle",
42 + "anstyle-parse",
43 + "anstyle-query",
44 + "anstyle-wincon",
45 + "colorchoice",
46 + "is_terminal_polyfill",
47 + "utf8parse",
48 + ]
49 +
50 + [[package]]
51 + name = "anstyle"
52 + version = "1.0.14"
53 + source = "registry+https://github.com/rust-lang/crates.io-index"
54 + checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
55 +
56 + [[package]]
57 + name = "anstyle-parse"
58 + version = "1.0.0"
59 + source = "registry+https://github.com/rust-lang/crates.io-index"
60 + checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
61 + dependencies = [
62 + "utf8parse",
63 + ]
64 +
65 + [[package]]
66 + name = "anstyle-query"
67 + version = "1.1.5"
68 + source = "registry+https://github.com/rust-lang/crates.io-index"
69 + checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
70 + dependencies = [
71 + "windows-sys",
72 + ]
73 +
74 + [[package]]
75 + name = "anstyle-wincon"
76 + version = "3.0.11"
77 + source = "registry+https://github.com/rust-lang/crates.io-index"
78 + checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
79 + dependencies = [
80 + "anstyle",
81 + "once_cell_polyfill",
82 + "windows-sys",
83 + ]
84 +
35 85 [[package]]
36 86 name = "anyhow"
37 87 version = "1.0.104"
@@ -168,6 +218,46 @@
168 218 source = "registry+https://github.com/rust-lang/crates.io-index"
169 219 checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
170 220
221 + [[package]]
222 + name = "clap"
223 + version = "4.6.4"
224 + source = "registry+https://github.com/rust-lang/crates.io-index"
225 + checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
226 + dependencies = [
227 + "clap_builder",
228 + "clap_derive",
229 + ]
230 +
231 + [[package]]
232 + name = "clap_builder"
233 + version = "4.6.2"
234 + source = "registry+https://github.com/rust-lang/crates.io-index"
235 + checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
236 + dependencies = [
237 + "anstream",
238 + "anstyle",
239 + "clap_lex",
240 + "strsim",
241 + ]
242 +
243 + [[package]]
244 + name = "clap_derive"
245 + version = "4.6.4"
246 + source = "registry+https://github.com/rust-lang/crates.io-index"
247 + checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
248 + dependencies = [
249 + "heck",
250 + "proc-macro2",
251 + "quote",
252 + "syn 3.0.3",
253 + ]
254 +
255 + [[package]]
256 + name = "clap_lex"
257 + version = "1.1.0"
258 + source = "registry+https://github.com/rust-lang/crates.io-index"
259 + checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
260 +
171 261 [[package]]
172 262 name = "codespan-reporting"
173 263 version = "0.13.1"
@@ -177,6 +267,12 @@
177 267 "unicode-width",
178 268 ]
179 269
270 + [[package]]
271 + name = "colorchoice"
272 + version = "1.0.5"
273 + source = "registry+https://github.com/rust-lang/crates.io-index"
274 + checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
275 +
180 276 [[package]]
181 277 name = "concurrent-queue"
182 278 version = "2.5.0"
@@ -366,6 +462,12 @@
366 462 "foldhash",
367 463 ]
368 464
465 + [[package]]
466 + name = "heck"
467 + version = "0.5.0"
468 + source = "registry+https://github.com/rust-lang/crates.io-index"
469 + checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
470 +
369 471 [[package]]
370 472 name = "hermit-abi"
371 473 version = "0.5.2"
@@ -395,6 +497,18 @@
395 497 "hashbrown 0.17.1",
396 498 ]
397 499
500 + [[package]]
501 + name = "is_terminal_polyfill"
502 + version = "1.70.2"
503 + source = "registry+https://github.com/rust-lang/crates.io-index"
504 + checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
505 +
506 + [[package]]
507 + name = "itoa"
508 + version = "1.0.18"
509 + source = "registry+https://github.com/rust-lang/crates.io-index"
510 + checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
511 +
398 512 [[package]]
399 513 name = "kittygfx"
400 514 version = "0.0.0"
@@ -663,6 +777,12 @@
663 777 source = "registry+https://github.com/rust-lang/crates.io-index"
664 778 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
665 779
780 + [[package]]
781 + name = "once_cell_polyfill"
782 + version = "1.70.2"
783 + source = "registry+https://github.com/rust-lang/crates.io-index"
784 + checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
785 +
666 786 [[package]]
667 787 name = "ordered-float"
668 788 version = "5.3.0"
@@ -892,6 +1012,49 @@
892 1012 source = "registry+https://github.com/rust-lang/crates.io-index"
893 1013 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
894 1014
1015 + [[package]]
1016 + name = "serde"
1017 + version = "1.0.229"
1018 + source = "registry+https://github.com/rust-lang/crates.io-index"
1019 + checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
1020 + dependencies = [
1021 + "serde_core",
1022 + "serde_derive",
1023 + ]
1024 +
1025 + [[package]]
1026 + name = "serde_core"
1027 + version = "1.0.229"
1028 + source = "registry+https://github.com/rust-lang/crates.io-index"
1029 + checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
1030 + dependencies = [
1031 + "serde_derive",
1032 + ]
1033 +
1034 + [[package]]
1035 + name = "serde_derive"
1036 + version = "1.0.229"
1037 + source = "registry+https://github.com/rust-lang/crates.io-index"
1038 + checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
1039 + dependencies = [
1040 + "proc-macro2",
1041 + "quote",
1042 + "syn 3.0.3",
1043 + ]
1044 +
1045 + [[package]]
1046 + name = "serde_json"
1047 + version = "1.0.151"
1048 + source = "registry+https://github.com/rust-lang/crates.io-index"
1049 + checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
1050 + dependencies = [
1051 + "itoa",
1052 + "memchr",
1053 + "serde",
1054 + "serde_core",
1055 + "zmij",
1056 + ]
1057 +
895 1058 [[package]]
896 1059 name = "sharded-slab"
897 1060 version = "0.1.7"
@@ -929,6 +1092,16 @@
929 1092 "wgpu",
930 1093 ]
931 1094
1095 + [[package]]
1096 + name = "shop-bench"
1097 + version = "0.0.0"
1098 + dependencies = [
1099 + "anyhow",
1100 + "clap",
1101 + "serde",
1102 + "serde_json",
1103 + ]
1104 +
932 1105 [[package]]
933 1106 name = "shop-grid"
934 1107 version = "0.0.0"
@@ -1044,6 +1217,12 @@
1044 1217 source = "registry+https://github.com/rust-lang/crates.io-index"
1045 1218 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
1046 1219
1220 + [[package]]
1221 + name = "strsim"
1222 + version = "0.11.1"
1223 + source = "registry+https://github.com/rust-lang/crates.io-index"
1224 + checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
1225 +
1047 1226 [[package]]
1048 1227 name = "swash"
1049 1228 version = "0.2.10"
@@ -1180,6 +1359,12 @@
1180 1359 source = "registry+https://github.com/rust-lang/crates.io-index"
1181 1360 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
1182 1361
1362 + [[package]]
1363 + name = "utf8parse"
1364 + version = "0.2.2"
1365 + source = "registry+https://github.com/rust-lang/crates.io-index"
1366 + checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
1367 +
1183 1368 [[package]]
1184 1369 name = "valuable"
1185 1370 version = "0.1.1"
@@ -1625,3 +1810,9 @@
1625 1810 "quote",
1626 1811 "syn 2.0.119",
1627 1812 ]
1813 +
1814 + [[package]]
1815 + name = "zmij"
1816 + version = "1.0.23"
1817 + source = "registry+https://github.com/rust-lang/crates.io-index"
1818 + checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
M Cargo.toml +1
@@ -8,6 +8,7 @@
8 8 "crates/shop-pty",
9 9 "crates/shop-grid",
10 10 "crates/shop-vt",
11 + "crates/shop-bench",
11 12 ]
12 13
13 14 [workspace.dependencies]
A crates/shop-bench/Cargo.toml +23
@@ -1,0 +1,23 @@
1 + [package]
2 + name = "shop-bench"
3 + version = "0.0.0"
4 + description = "Perf harness for shop: drives vtebench + internal workloads and gates against a stored baseline"
5 + edition.workspace = true
6 + rust-version.workspace = true
7 + authors.workspace = true
8 + repository.workspace = true
9 + license.workspace = true
10 + publish = false
11 +
12 + [lints]
13 + workspace = true
14 +
15 + [[bin]]
16 + name = "shop-bench"
17 + path = "src/main.rs"
18 +
19 + [dependencies]
20 + anyhow = { workspace = true }
21 + clap = { version = "4", features = ["derive"] }
22 + serde = { version = "1", features = ["derive"] }
23 + serde_json = "1"
A crates/shop-bench/src/baseline.rs +176
@@ -1,0 +1,176 @@
1 + use std::collections::BTreeMap;
2 + use std::fs;
3 + use std::path::Path;
4 +
5 + use anyhow::{Context, Result};
6 + use serde::{Deserialize, Serialize};
7 +
8 + use crate::workloads::WorkloadResult;
9 +
10 + /// A frozen set of per-workload metrics captured on one machine. Not committed
11 + /// to git — regenerable via `shop-bench save`.
12 + #[derive(Debug, Serialize, Deserialize)]
13 + pub struct Baseline {
14 + pub machine: String,
15 + pub captured_at: String,
16 + pub shop_commit: Option<String>,
17 + pub workloads: BTreeMap<String, BTreeMap<String, f64>>,
18 + }
19 +
20 + impl Baseline {
21 + pub fn load(path: &str) -> Result<Self> {
22 + let s = fs::read_to_string(path).with_context(|| format!("read baseline {path}"))?;
23 + let b: Self = serde_json::from_str(&s).with_context(|| format!("parse baseline {path}"))?;
24 + Ok(b)
25 + }
26 +
27 + pub fn save(&self, path: &str) -> Result<()> {
28 + if let Some(parent) = Path::new(path).parent() {
29 + fs::create_dir_all(parent).ok();
30 + }
31 + let s = serde_json::to_string_pretty(self)?;
32 + fs::write(path, s).with_context(|| format!("write baseline {path}"))?;
33 + Ok(())
34 + }
35 +
36 + pub fn from_results(results: &[WorkloadResult]) -> Self {
37 + let workloads = results
38 + .iter()
39 + .map(|r| (r.name.clone(), r.metrics.iter().cloned().collect()))
40 + .collect();
41 + Self {
42 + machine: hostname(),
43 + captured_at: now_rfc3339(),
44 + shop_commit: git_head(),
45 + workloads,
46 + }
47 + }
48 +
49 + pub fn metric(&self, workload: &str, metric: &str) -> Option<&f64> {
50 + self.workloads.get(workload).and_then(|m| m.get(metric))
51 + }
52 +
53 + /// Enforce the Phase-0 gates from `shop-typed-protocol.md`. Additive:
54 + /// unknown metrics or unknown workloads are not gated.
55 + pub fn check_gate(&self, r: &WorkloadResult) -> Result<(), String> {
56 + for gate in GATES {
57 + if gate.workload != r.name {
58 + continue;
59 + }
60 + let Some(&prev) = self.metric(&r.name, gate.metric) else {
61 + continue;
62 + };
63 + let Some(&new) = r.metrics.iter().find(|(k, _)| k == gate.metric).map(|(_, v)| v)
64 + else {
65 + continue;
66 + };
67 + gate.check(prev, new)?;
68 + }
69 + Ok(())
70 + }
71 + }
72 +
73 + struct Gate {
74 + workload: &'static str,
75 + metric: &'static str,
76 + kind: GateKind,
77 + }
78 +
79 + enum GateKind {
80 + /// New value must be at least `prev * (1 - tol)`. For throughput.
81 + ThroughputRegressionPct(f64),
82 + /// New value must be at most `prev * (1 + tol)`. For latency / frame time.
83 + LatencyRegressionPct(f64),
84 + /// Absolute cap on regression, in the metric's units. For raw latency.
85 + LatencyRegressionAbs(f64),
86 + }
87 +
88 + impl Gate {
89 + fn check(&self, prev: f64, new: f64) -> Result<(), String> {
90 + match self.kind {
91 + GateKind::ThroughputRegressionPct(tol) => {
92 + let floor = prev * (1.0 - tol);
93 + if new < floor {
94 + return Err(format!(
95 + "{} throughput {new:.4} below floor {floor:.4} (prev {prev:.4}, tol {:.2}%)",
96 + self.metric,
97 + tol * 100.0
98 + ));
99 + }
100 + }
101 + GateKind::LatencyRegressionPct(tol) => {
102 + let ceil = prev * (1.0 + tol);
103 + if new > ceil {
104 + return Err(format!(
105 + "{} {new:.4} above ceil {ceil:.4} (prev {prev:.4}, tol {:.2}%)",
106 + self.metric,
107 + tol * 100.0
108 + ));
109 + }
110 + }
111 + GateKind::LatencyRegressionAbs(bound) => {
112 + if new - prev > bound {
113 + return Err(format!(
114 + "{} {new:.4} exceeds prev {prev:.4} by more than {bound}",
115 + self.metric
116 + ));
117 + }
118 + }
119 + }
120 + Ok(())
121 + }
122 + }
123 +
124 + // Gate table sourced from shop-typed-protocol.md Phase 0. Table-specific
125 + // metrics (typed-*) are recorded, not gated, until their respective phase.
126 + const GATES: &[Gate] = &[
127 + Gate {
128 + workload: "vtebench-cat",
129 + metric: "cell_throughput_mb_s",
130 + kind: GateKind::ThroughputRegressionPct(0.01),
131 + },
132 + Gate {
133 + workload: "vtebench-scrolling",
134 + metric: "frame_time_p99_ms",
135 + kind: GateKind::LatencyRegressionPct(0.02),
136 + },
137 + Gate {
138 + workload: "vtebench-unicode",
139 + metric: "cell_throughput_mb_s",
140 + kind: GateKind::ThroughputRegressionPct(0.01),
141 + },
142 + Gate {
143 + workload: "kitty-image-flood",
144 + metric: "throughput_mb_s",
145 + kind: GateKind::ThroughputRegressionPct(0.0),
146 + },
147 + Gate {
148 + workload: "pty-to-pixel",
149 + metric: "latency_p50_us",
150 + kind: GateKind::LatencyRegressionAbs(200.0),
151 + },
152 + ];
153 +
154 + fn hostname() -> String {
155 + std::env::var("HOSTNAME")
156 + .or_else(|_| std::env::var("HOST"))
157 + .unwrap_or_else(|_| "unknown".into())
158 + }
159 +
160 + fn now_rfc3339() -> String {
161 + // Minimal RFC3339 formatter to avoid a chrono dep for a stamp field.
162 + let secs = std::time::SystemTime::now()
163 + .duration_since(std::time::UNIX_EPOCH)
164 + .map_or(0, |d| d.as_secs());
165 + format!("@unix:{secs}")
166 + }
167 +
168 + fn git_head() -> Option<String> {
169 + std::process::Command::new("git")
170 + .args(["rev-parse", "--short", "HEAD"])
171 + .output()
172 + .ok()
173 + .and_then(|o| String::from_utf8(o.stdout).ok())
174 + .map(|s| s.trim().to_string())
175 + .filter(|s| !s.is_empty())
176 + }
A crates/shop-bench/src/main.rs +119
@@ -1,0 +1,119 @@
1 + //! shop-bench: perf harness driving external workloads (vtebench) and
2 + //! internal microbenches, comparing results against a stored per-machine
3 + //! baseline. See `docs/typed-protocol.md` Phase 0 for the design.
4 + //!
5 + //! <!-- wiki: shop-typed-protocol -->
6 +
7 + use anyhow::Result;
8 + use clap::{Parser, Subcommand};
9 +
10 + mod baseline;
11 + mod workloads;
12 +
13 + use baseline::Baseline;
14 + use workloads::{Registry, WorkloadResult};
15 +
16 + #[derive(Parser)]
17 + #[command(name = "shop-bench", about = "shop perf harness")]
18 + struct Cli {
19 + #[command(subcommand)]
20 + cmd: Cmd,
21 + }
22 +
23 + #[derive(Subcommand)]
24 + enum Cmd {
25 + /// Run one or all workloads; compare against the machine baseline.
26 + Run {
27 + /// Workload name; omit to run all.
28 + workload: Option<String>,
29 + /// Path to baseline JSON (defaults to
30 + /// `_private/docs/shop/bench-baselines/<hostname>-latest.json`).
31 + #[arg(long)]
32 + baseline: Option<String>,
33 + /// Skip the gate check; report numbers only.
34 + #[arg(long)]
35 + no_gate: bool,
36 + },
37 + /// Run all workloads and write the results as a new baseline.
38 + Save {
39 + #[arg(long)]
40 + out: String,
41 + },
42 + /// List registered workloads.
43 + List,
44 + }
45 +
46 + fn main() -> Result<()> {
47 + let cli = Cli::parse();
48 + let reg = Registry::default_set();
49 +
50 + match cli.cmd {
51 + Cmd::List => {
52 + for w in reg.iter() {
53 + println!("{}\t{}", w.name(), w.description());
54 + }
55 + Ok(())
56 + }
57 + Cmd::Run { workload, baseline, no_gate } => {
58 + let results = run(&reg, workload.as_deref())?;
59 + let base = baseline
60 + .as_deref()
61 + .map(Baseline::load)
62 + .transpose()?;
63 + report(&results, base.as_ref(), !no_gate)
64 + }
65 + Cmd::Save { out } => {
66 + let results = run(&reg, None)?;
67 + let base = Baseline::from_results(&results);
68 + base.save(&out)?;
69 + eprintln!("wrote baseline to {out}");
70 + Ok(())
71 + }
72 + }
73 + }
74 +
75 + fn run(reg: &Registry, only: Option<&str>) -> Result<Vec<WorkloadResult>> {
76 + let mut out = Vec::new();
77 + for w in reg.iter() {
78 + if let Some(name) = only {
79 + if w.name() != name {
80 + continue;
81 + }
82 + }
83 + eprintln!("running {}...", w.name());
84 + out.push(w.run()?);
85 + }
86 + if out.is_empty() {
87 + anyhow::bail!("no workload matched selector");
88 + }
89 + Ok(out)
90 + }
91 +
92 + fn report(results: &[WorkloadResult], base: Option<&Baseline>, gate: bool) -> Result<()> {
93 + let mut breach = false;
94 + for r in results {
95 + println!("== {} ==", r.name);
96 + for (k, v) in &r.metrics {
97 + let cmp = base
98 + .and_then(|b| b.metric(&r.name, k))
99 + .map(|prev| format!(" (baseline {prev:.4}, delta {:+.2}%)", pct(*v, *prev)));
100 + println!(" {k} = {v:.4}{}", cmp.unwrap_or_default());
101 + }
102 + if gate {
103 + if let Some(b) = base {
104 + if let Err(msg) = b.check_gate(r) {
105 + eprintln!("GATE BREACH [{}]: {msg}", r.name);
106 + breach = true;
107 + }
108 + }
109 + }
110 + }
111 + if breach {
112 + anyhow::bail!("one or more gates breached");
113 + }
114 + Ok(())
115 + }
116 +
117 + fn pct(new: f64, old: f64) -> f64 {
118 + if old == 0.0 { 0.0 } else { (new - old) / old * 100.0 }
119 + }
A crates/shop-bench/src/workloads/kitty_image_flood.rs +26
@@ -1,0 +1,26 @@
1 + use anyhow::{bail, Result};
2 +
3 + use super::{Workload, WorkloadResult};
4 +
5 + pub struct KittyImageFlood;
6 +
7 + impl Workload for KittyImageFlood {
8 + fn name(&self) -> &'static str { "kitty-image-flood" }
9 + fn description(&self) -> &'static str {
10 + "kitty-graphics decode+upload throughput under a stream of transmissions"
11 + }
12 + fn run(&self) -> Result<WorkloadResult> {
13 + // fw13-only wiring:
14 + // 1. Generate a corpus of N kitty-graphics transmissions (mix of
15 + // chunked / one-shot, RGB / RGBA, PNG / raw).
16 + // 2. Pipe through a running shop, sample:
17 + // - throughput_mb_s: raw bytes / wall time
18 + // - decode_ms_p99: from bench-hooks span
19 + // - upload_ms_p99: from bench-hooks span
20 + // 3. Emit metrics.
21 + bail!(
22 + "kitty-image-flood is fw13-only and not yet wired; \
23 + see crates/shop-bench/src/workloads/kitty_image_flood.rs"
24 + )
25 + }
26 + }
A crates/shop-bench/src/workloads/mod.rs +35
@@ -1,0 +1,35 @@
1 + use anyhow::Result;
2 +
3 + mod kitty_image_flood;
4 + mod pty_to_pixel;
5 + mod vtebench;
6 +
7 + pub struct WorkloadResult {
8 + pub name: String,
9 + /// Ordered so report output is stable.
10 + pub metrics: Vec<(String, f64)>,
11 + }
12 +
13 + pub trait Workload: Send + Sync {
14 + fn name(&self) -> &'static str;
15 + fn description(&self) -> &'static str;
16 + fn run(&self) -> Result<WorkloadResult>;
17 + }
18 +
19 + pub struct Registry(Vec<Box<dyn Workload>>);
20 +
21 + impl Registry {
22 + pub fn default_set() -> Self {
23 + Self(vec![
24 + Box::new(vtebench::Cat),
25 + Box::new(vtebench::Scrolling),
26 + Box::new(vtebench::Unicode),
27 + Box::new(kitty_image_flood::KittyImageFlood),
28 + Box::new(pty_to_pixel::PtyToPixel),
29 + ])
30 + }
31 +
32 + pub fn iter(&self) -> impl Iterator<Item = &dyn Workload> {
33 + self.0.iter().map(AsRef::as_ref)
34 + }
35 + }
A crates/shop-bench/src/workloads/pty_to_pixel.rs +25
@@ -1,0 +1,25 @@
1 + use anyhow::{bail, Result};
2 +
3 + use super::{Workload, WorkloadResult};
4 +
5 + pub struct PtyToPixel;
6 +
7 + impl Workload for PtyToPixel {
8 + fn name(&self) -> &'static str { "pty-to-pixel" }
9 + fn description(&self) -> &'static str {
10 + "end-to-end input-to-pixel latency via wp-presentation-time"
11 + }
12 + fn run(&self) -> Result<WorkloadResult> {
13 + // fw13-only wiring:
14 + // 1. Bring shop up under a real Wayland compositor with
15 + // wp_presentation_time supported (Sway, KWin).
16 + // 2. Emit a distinctive marker byte down the PTY at t_write.
17 + // 3. Watch the frame that first contains the mark (bench-hooks
18 + // tag) and read its wp_presentation_time feedback.
19 + // 4. latency_p50_us / latency_p99_us over N=1000 samples.
20 + bail!(
21 + "pty-to-pixel is fw13-only, needs wp-presentation-time hook + bench-hooks feature; \
22 + see crates/shop-bench/src/workloads/pty_to_pixel.rs"
23 + )
24 + }
25 + }
A crates/shop-bench/src/workloads/vtebench.rs +59
@@ -1,0 +1,59 @@
1 + // vtebench-driven cell path workloads. On fw13 these shell out to the
2 + // vtebench binary (https://github.com/alacritty/vtebench) and time shop's
3 + // consumption of the corpus. The runner spawns shop, pipes the corpus to its
4 + // stdin (or through the PTY primary once shop-bench can drive one), and
5 + // samples PTY-drain throughput + wl-presentation-time frame stamps.
6 +
7 + use anyhow::{bail, Result};
8 +
9 + use super::{Workload, WorkloadResult};
10 +
11 + pub struct Cat;
12 + pub struct Scrolling;
13 + pub struct Unicode;
14 +
15 + impl Workload for Cat {
16 + fn name(&self) -> &'static str { "vtebench-cat" }
17 + fn description(&self) -> &'static str {
18 + "vtebench --dat cat: raw byte throughput, hot cell path"
19 + }
20 + fn run(&self) -> Result<WorkloadResult> {
21 + run_vtebench("cat", &["cell_throughput_mb_s", "frame_time_p99_ms"])
22 + }
23 + }
24 +
25 + impl Workload for Scrolling {
26 + fn name(&self) -> &'static str { "vtebench-scrolling" }
27 + fn description(&self) -> &'static str {
28 + "vtebench --dat scrolling: scroll+render frame time"
29 + }
30 + fn run(&self) -> Result<WorkloadResult> {
31 + run_vtebench("scrolling", &["frame_time_p99_ms", "frame_time_p50_ms"])
32 + }
33 + }
34 +
35 + impl Workload for Unicode {
36 + fn name(&self) -> &'static str { "vtebench-unicode" }
37 + fn description(&self) -> &'static str {
38 + "vtebench --dat unicode: grapheme + width fast path"
39 + }
40 + fn run(&self) -> Result<WorkloadResult> {
41 + run_vtebench("unicode", &["cell_throughput_mb_s"])
42 + }
43 + }
44 +
45 + fn run_vtebench(_dat: &str, _metrics: &[&str]) -> Result<WorkloadResult> {
46 + // fw13-only wiring lives here. Steps:
47 + // 1. Resolve the vtebench binary (env SHOP_BENCH_VTEBENCH or PATH).
48 + // 2. Spawn shop under a headless-Wayland fixture (weston --backend=headless
49 + // or sway --no-outputs, TBD; see docs/typed-protocol.md phase 0).
50 + // 3. Open a PTY, launch `vtebench --dat <_dat>` inside shop, and time
51 + // the read-drain via a wrapper shell that timestamps EOF.
52 + // 4. Sample presentation-time timestamps from shop via a debug socket
53 + // (added under `--feature bench-hooks` on the shop binary).
54 + // 5. Emit each name in `_metrics` as an f64.
55 + bail!(
56 + "vtebench workloads are fw13-only and not yet wired; \
57 + see crates/shop-bench/src/workloads/vtebench.rs for the checklist"
58 + )
59 + }
A justfile +25
@@ -1,0 +1,25 @@
1 + # shop developer recipes
2 +
3 + # Build every crate.
4 + build:
5 + cargo build --workspace --all-targets
6 +
7 + # Format, lint, deny.
8 + check:
9 + cargo fmt --all -- --check
10 + cargo clippy --workspace --all-targets -- -D warnings
11 + cargo deny check
12 +
13 + # Run the perf harness against the machine baseline. Baseline path:
14 + # _private/docs/shop/bench-baselines/<hostname>-latest.json
15 + # Gates fail with nonzero exit — do not merge on breach.
16 + bench baseline="_private/docs/shop/bench-baselines/latest.json":
17 + cargo run --profile profiling -p shop-bench -- run --baseline {{baseline}}
18 +
19 + # Capture a fresh baseline. Overwrites the target path.
20 + bench-save out="_private/docs/shop/bench-baselines/latest.json":
21 + cargo run --profile profiling -p shop-bench -- save --out {{out}}
22 +
23 + # List registered workloads.
24 + bench-list:
25 + cargo run -p shop-bench -- list