Skip to main content

max / shop

Kitty graphics protocol MVP: yazi / timg / image previews kitty-graphics crate: - Parser for one APC body per call, returning Some(Command) once a chunked transmission completes. - Control field parsing for a, f, t, i, I, p, s, v, c, r, C, q, m. - Chunk reassembly keyed on image id / number / anon. - Base64-decode of the payload; formats emitted as Rgb / Rgba / Png (host decodes). - Commands: Transmit (with control + payload), Delete. - Deferred from MVP: animation, virtual placements, file/temp/shm media, query, placement IDs, z-order. shop-render: - New ImageRenderer with an RGBA image pipeline separate from the text atlas (sRGB textures, alpha-blend, one draw call per image). - set_image / drop_image / drop_all / resize / draw API. shop binary: - ApcScanner: stateful byte-stream extractor for `\e_G<body>\e\` and `\e_G<body>\a`. Runs before vte::Parser.advance to work around vte's silent-drop of APC content. Followup: replace with a native APC-dispatching shop-vt. - handle_kitty: routes Transmit to PNG decode (via `image` crate), RGBA passthrough, or RGB->RGBA expansion; uploads to ImageRenderer; records the placement at the cursor cell sized by control's `c`,`r`. Delete drops all images. - Render path draws the tracked image on top of the text layer. Single-image MVP — sufficient for yazi's one-preview-pane use case. Multi- image and delete-by-id-selector wait for real workloads that push on it.
Co-Authored-By
Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Author: Max Johnson <me@maxj.phd> · 2026-07-23 20:35 UTC
Signed with PGP, not checked
Commit: bd989e4f9b24f6b541a11b19b665665e72f2065e
Parent: 37dabef
8 files changed, +906 insertions, -13 deletions
M Cargo.lock +110
@@ -2,6 +2,12 @@
2 2 # It is not intended for manual editing.
3 3 version = 4
4 4
5 + [[package]]
6 + name = "adler2"
7 + version = "2.0.1"
8 + source = "registry+https://github.com/rust-lang/crates.io-index"
9 + checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
10 +
5 11 [[package]]
6 12 name = "aho-corasick"
7 13 version = "1.1.4"
@@ -53,6 +59,12 @@
53 59 source = "registry+https://github.com/rust-lang/crates.io-index"
54 60 checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
55 61
62 + [[package]]
63 + name = "base64"
64 + version = "0.23.0"
65 + source = "registry+https://github.com/rust-lang/crates.io-index"
66 + checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9"
67 +
56 68 [[package]]
57 69 name = "bit-set"
58 70 version = "0.10.0"
@@ -103,6 +115,12 @@
103 115 "syn 2.0.119",
104 116 ]
105 117
118 + [[package]]
119 + name = "byteorder-lite"
120 + version = "0.1.0"
121 + source = "registry+https://github.com/rust-lang/crates.io-index"
122 + checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
123 +
106 124 [[package]]
107 125 name = "calloop"
108 126 version = "0.14.4"
@@ -168,6 +186,15 @@
168 186 "crossbeam-utils",
169 187 ]
170 188
189 + [[package]]
190 + name = "crc32fast"
191 + version = "1.5.0"
192 + source = "registry+https://github.com/rust-lang/crates.io-index"
193 + checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
194 + dependencies = [
195 + "cfg-if",
196 + ]
197 +
171 198 [[package]]
172 199 name = "crossbeam-utils"
173 200 version = "0.8.22"
@@ -245,12 +272,31 @@
245 272 "num-traits",
246 273 ]
247 274
275 + [[package]]
276 + name = "fdeflate"
277 + version = "0.3.7"
278 + source = "registry+https://github.com/rust-lang/crates.io-index"
279 + checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
280 + dependencies = [
281 + "simd-adler32",
282 + ]
283 +
248 284 [[package]]
249 285 name = "find-msvc-tools"
250 286 version = "0.1.9"
251 287 source = "registry+https://github.com/rust-lang/crates.io-index"
252 288 checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
253 289
290 + [[package]]
291 + name = "flate2"
292 + version = "1.1.9"
293 + source = "registry+https://github.com/rust-lang/crates.io-index"
294 + checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
295 + dependencies = [
296 + "crc32fast",
297 + "miniz_oxide",
298 + ]
299 +
254 300 [[package]]
255 301 name = "foldhash"
256 302 version = "0.2.0"
@@ -326,6 +372,19 @@
326 372 source = "registry+https://github.com/rust-lang/crates.io-index"
327 373 checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
328 374
375 + [[package]]
376 + name = "image"
377 + version = "0.25.10"
378 + source = "registry+https://github.com/rust-lang/crates.io-index"
379 + checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
380 + dependencies = [
381 + "bytemuck",
382 + "byteorder-lite",
383 + "moxcms",
384 + "num-traits",
385 + "png",
386 + ]
387 +
329 388 [[package]]
330 389 name = "indexmap"
331 390 version = "2.14.0"
@@ -339,6 +398,10 @@
339 398 [[package]]
340 399 name = "kitty-graphics"
341 400 version = "0.0.0"
401 + dependencies = [
402 + "base64",
403 + "tracing",
404 + ]
342 405
343 406 [[package]]
344 407 name = "lazy_static"
@@ -419,6 +482,26 @@
419 482 "libc",
420 483 ]
421 484
485 + [[package]]
486 + name = "miniz_oxide"
487 + version = "0.8.9"
488 + source = "registry+https://github.com/rust-lang/crates.io-index"
489 + checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
490 + dependencies = [
491 + "adler2",
492 + "simd-adler32",
493 + ]
494 +
495 + [[package]]
496 + name = "moxcms"
497 + version = "0.8.1"
498 + source = "registry+https://github.com/rust-lang/crates.io-index"
499 + checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
500 + dependencies = [
501 + "num-traits",
502 + "pxfm",
503 + ]
504 +
422 505 [[package]]
423 506 name = "naga"
424 507 version = "30.0.0"
@@ -624,6 +707,19 @@
624 707 source = "registry+https://github.com/rust-lang/crates.io-index"
625 708 checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
626 709
710 + [[package]]
711 + name = "png"
712 + version = "0.18.1"
713 + source = "registry+https://github.com/rust-lang/crates.io-index"
714 + checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
715 + dependencies = [
716 + "bitflags",
717 + "crc32fast",
718 + "fdeflate",
719 + "flate2",
720 + "miniz_oxide",
721 + ]
722 +
627 723 [[package]]
628 724 name = "polling"
629 725 version = "3.11.0"
@@ -680,6 +776,12 @@
680 776 source = "registry+https://github.com/rust-lang/crates.io-index"
681 777 checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5"
682 778
779 + [[package]]
780 + name = "pxfm"
781 + version = "0.1.30"
782 + source = "registry+https://github.com/rust-lang/crates.io-index"
783 + checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
784 +
683 785 [[package]]
684 786 name = "quick-xml"
685 787 version = "0.41.0"
@@ -812,6 +914,8 @@
812 914 "anyhow",
813 915 "calloop",
814 916 "calloop-wayland-source",
917 + "image",
918 + "kitty-graphics",
815 919 "pollster",
816 920 "shop-grid",
817 921 "shop-pty",
@@ -865,6 +969,12 @@
865 969 "wayland-client",
866 970 ]
867 971
972 + [[package]]
973 + name = "simd-adler32"
974 + version = "0.3.10"
975 + source = "registry+https://github.com/rust-lang/crates.io-index"
976 + checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
977 +
868 978 [[package]]
869 979 name = "skrifa"
870 980 version = "0.44.0"
M Cargo.toml +2
@@ -25,6 +25,8 @@
25 25 bytemuck = { version = "1", features = ["derive"] }
26 26 nix = { version = "0.31", features = ["term", "process", "fs", "ioctl", "signal"] }
27 27 vte = "0.15"
28 + base64 = "0.23"
29 + image = { version = "0.25", default-features = false, features = ["png"] }
28 30
29 31 [workspace.package]
30 32 edition = "2024"
M crates/kitty-graphics/Cargo.toml +2
@@ -15,3 +15,5 @@
15 15 workspace = true
16 16
17 17 [dependencies]
18 + base64 = { workspace = true }
19 + tracing = { workspace = true }
M crates/shop/Cargo.toml +2
@@ -20,6 +20,8 @@
20 20 shop-render = { path = "../shop-render" }
21 21 shop-pty = { path = "../shop-pty" }
22 22 shop-grid = { path = "../shop-grid" }
23 + kitty-graphics = { path = "../kitty-graphics" }
24 + image = { workspace = true }
23 25 smithay-client-toolkit.workspace = true
24 26 wayland-client.workspace = true
25 27 calloop.workspace = true
M crates/kitty-graphics/src/lib.rs +241 -11
@@ -1,20 +1,250 @@
1 1 //! Rendering-agnostic implementation of the [Kitty terminal graphics protocol].
2 2 //!
3 - //! This crate parses APC-payload bytes from a VT stream and emits structured
4 - //! commands (transmit, place, delete, animate, query). It does not decode
5 - //! images, does not render pixels, and does not own a grid — the host does all
6 - //! three. The crate manages command parsing, chunk reassembly, image-ID
7 - //! lifecycle, and placement metadata.
3 + //! Parses APC-payload bytes from a VT stream and emits structured commands
4 + //! (transmit, place, delete). Does not decode images, does not render pixels,
5 + //! does not own a grid — the host does all three. The crate manages command
6 + //! parsing, chunk reassembly, image-ID lifecycle, and placement metadata.
8 7 //!
9 8 //! [Kitty terminal graphics protocol]: https://sw.kovidgoyal.net/kitty/graphics-protocol/
10 9 //!
11 - //! Bring your own image decoder (`image` crate pairs naturally), your own
12 - //! renderer (wgpu, GL, softbuffer — the crate does not care), and your own
13 - //! scrollback (placement lifecycle events tell you when to move or drop
14 - //! images).
10 + //! MVP scope: `a=T` transmit+display, `a=t` transmit, `a=d` delete, formats
11 + //! `f=24` (RGB), `f=32` (RGBA), `f=100` (PNG), medium `t=d` (base64 inline),
12 + //! chunked payloads (`m=1`/`m=0`), placement in cells (`c=`, `r=`), don't-
13 + //! move-cursor (`C=1`).
15 14 //!
16 - //! Status: skeleton. See `~/Wiki/shop-overview.md` for the crate design.
15 + //! Not covered yet: file/temp/shm media, animations, virtual placements
16 + //! (Unicode placeholders), placement IDs, z-order, query.
17 + //!
18 + //! Reference read: rio's `rio-backend/src/ansi/kitty_graphics_protocol.rs`
19 + //! (MIT) — architecture consulted, no direct code copied.
17 20
18 21 #![deny(unsafe_code)]
19 22
20 - // Parser, commands, state — filled in during shop development.
23 + use std::collections::HashMap;
24 +
25 + use base64::{Engine, engine::general_purpose::STANDARD as B64};
26 +
27 + /// Pixel format of a transmit payload.
28 + #[derive(Copy, Clone, Debug, PartialEq, Eq)]
29 + pub enum Format {
30 + Rgb,
31 + Rgba,
32 + Png,
33 + }
34 +
35 + /// Transmission medium. MVP only handles `Direct` (base64 inline).
36 + #[derive(Copy, Clone, Debug, PartialEq, Eq)]
37 + pub enum Medium {
38 + Direct,
39 + File,
40 + TempFile,
41 + Shm,
42 + }
43 +
44 + /// Parsed control fields from one kitty-graphics APC command.
45 + #[derive(Clone, Debug, Default)]
46 + pub struct Control {
47 + pub action: char,
48 + pub format: Option<Format>,
49 + pub medium: Option<Medium>,
50 + /// Image ID (`i=`) — server-assigned identity.
51 + pub id: Option<u32>,
52 + /// Image number (`I=`) — client-assigned identity.
53 + pub number: Option<u32>,
54 + /// Placement ID (`p=`).
55 + pub placement: Option<u32>,
56 + /// Width in pixels (`s=`, required for raw formats).
57 + pub width_px: Option<u32>,
58 + /// Height in pixels (`v=`, required for raw formats).
59 + pub height_px: Option<u32>,
60 + /// Display width in cells (`c=`).
61 + pub cell_cols: Option<u32>,
62 + /// Display height in cells (`r=`).
63 + pub cell_rows: Option<u32>,
64 + /// Don't move cursor after placement (`C=1`).
65 + pub no_cursor_move: bool,
66 + /// `q=` quiet mode — 0 = verbose, 1 = suppress OK, 2 = suppress errors.
67 + pub quiet: u8,
68 + /// Set when the chunk carried `m=1` (more chunks follow).
69 + pub more_chunks: bool,
70 + /// Set when this chunk is `m=0` — either last of a series or a standalone
71 + /// command.
72 + pub last_chunk: bool,
73 + }
74 +
75 + /// One high-level protocol event ready for the host to act on. Payload bytes
76 + /// are already base64-decoded and chunks are already reassembled.
77 + #[derive(Clone, Debug)]
78 + pub enum Command {
79 + /// Transmit and (if control.action == 'T') display an image.
80 + Transmit {
81 + control: Control,
82 + payload: Vec<u8>,
83 + },
84 + /// Delete images. MVP only supports "delete all placements" (`a=d,d=A`) —
85 + /// the sub-selector variants are stored but not filtered.
86 + Delete { control: Control },
87 + }
88 +
89 + /// Parser for kitty-graphics APC payloads. Feed one APC body at a time via
90 + /// [`Parser::feed`]. Returns `Some(Command)` once a chunked transmission
91 + /// completes; returns `None` while more chunks are expected or for a chunk
92 + /// that couldn't be parsed.
93 + #[derive(Debug, Default)]
94 + pub struct Parser {
95 + partial: HashMap<PartialKey, Partial>,
96 + }
97 +
98 + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
99 + enum PartialKey {
100 + ById(u32),
101 + ByNumber(u32),
102 + /// Fallback when the client didn't send an id or number; single-in-flight.
103 + Anon,
104 + }
105 +
106 + #[derive(Debug)]
107 + struct Partial {
108 + control: Control,
109 + payload: Vec<u8>,
110 + }
111 +
112 + impl Parser {
113 + pub fn new() -> Self {
114 + Self::default()
115 + }
116 +
117 + /// Consume one APC body (the bytes between `\e_G` and the ST/BEL
118 + /// terminator). Bodies have the form `<control>;<base64-payload>` or
119 + /// `<control>` alone (delete/query with no payload).
120 + pub fn feed(&mut self, body: &[u8]) -> Option<Command> {
121 + // The protocol always uses `G` as the introducer. Callers may or may
122 + // not have stripped it; tolerate both.
123 + let body = body.strip_prefix(b"G").unwrap_or(body);
124 +
125 + let (control_bytes, payload_b64) = split_once_byte(body, b';');
126 + let control = parse_control(control_bytes)?;
127 + let payload = if payload_b64.is_empty() {
128 + Vec::new()
129 + } else {
130 + B64.decode(payload_b64).ok()?
131 + };
132 +
133 + match control.action {
134 + 'T' | 't' | 'a' => self.accept_transmit(control, payload),
135 + 'd' => Some(Command::Delete { control }),
136 + _ => {
137 + tracing::trace!("kitty-graphics: unhandled action {}", control.action);
138 + None
139 + }
140 + }
141 + }
142 +
143 + fn accept_transmit(&mut self, control: Control, payload: Vec<u8>) -> Option<Command> {
144 + let key = if let Some(id) = control.id {
145 + PartialKey::ById(id)
146 + } else if let Some(n) = control.number {
147 + PartialKey::ByNumber(n)
148 + } else {
149 + PartialKey::Anon
150 + };
151 +
152 + if !control.more_chunks && !self.partial.contains_key(&key) {
153 + // Single-shot — no reassembly needed.
154 + return Some(Command::Transmit { control, payload });
155 + }
156 +
157 + let entry = self.partial.entry(key).or_insert_with(|| Partial {
158 + control: control.clone(),
159 + payload: Vec::new(),
160 + });
161 + entry.payload.extend_from_slice(&payload);
162 + // Merge in fields the first chunk didn't carry.
163 + merge_control(&mut entry.control, &control);
164 +
165 + if control.more_chunks {
166 + None
167 + } else {
168 + let Partial { control, payload } = self.partial.remove(&key)?;
169 + Some(Command::Transmit { control, payload })
170 + }
171 + }
172 + }
173 +
174 + fn split_once_byte(bytes: &[u8], sep: u8) -> (&[u8], &[u8]) {
175 + match bytes.iter().position(|&b| b == sep) {
176 + Some(i) => (&bytes[..i], &bytes[i + 1..]),
177 + None => (bytes, &[]),
178 + }
179 + }
180 +
181 + fn parse_control(bytes: &[u8]) -> Option<Control> {
182 + let mut c = Control {
183 + action: 'T',
184 + ..Control::default()
185 + };
186 + let text = std::str::from_utf8(bytes).ok()?;
187 + for kv in text.split(',') {
188 + let (k, v) = kv.split_once('=')?;
189 + match k {
190 + "a" => c.action = v.chars().next()?,
191 + "f" => {
192 + c.format = match v {
193 + "24" => Some(Format::Rgb),
194 + "32" => Some(Format::Rgba),
195 + "100" => Some(Format::Png),
196 + _ => None,
197 + }
198 + }
199 + "t" => {
200 + c.medium = match v {
201 + "d" => Some(Medium::Direct),
202 + "f" => Some(Medium::File),
203 + "t" => Some(Medium::TempFile),
204 + "s" => Some(Medium::Shm),
205 + _ => None,
206 + }
207 + }
208 + "i" => c.id = v.parse().ok(),
209 + "I" => c.number = v.parse().ok(),
210 + "p" => c.placement = v.parse().ok(),
211 + "s" => c.width_px = v.parse().ok(),
212 + "v" => c.height_px = v.parse().ok(),
213 + "c" => c.cell_cols = v.parse().ok(),
214 + "r" => c.cell_rows = v.parse().ok(),
215 + "C" => c.no_cursor_move = v == "1",
216 + "q" => c.quiet = v.parse().unwrap_or(0),
217 + "m" => match v {
218 + "0" => c.last_chunk = true,
219 + "1" => c.more_chunks = true,
220 + _ => {}
221 + },
222 + _ => {}
223 + }
224 + }
225 + if !c.more_chunks {
226 + c.last_chunk = true;
227 + }
228 + Some(c)
229 + }
230 +
231 + fn merge_control(into: &mut Control, from: &Control) {
232 + if into.format.is_none() {
233 + into.format = from.format;
234 + }
235 + if into.width_px.is_none() {
236 + into.width_px = from.width_px;
237 + }
238 + if into.height_px.is_none() {
239 + into.height_px = from.height_px;
240 + }
241 + if into.cell_cols.is_none() {
242 + into.cell_cols = from.cell_cols;
243 + }
244 + if into.cell_rows.is_none() {
245 + into.cell_rows = from.cell_rows;
246 + }
247 + if from.no_cursor_move {
248 + into.no_cursor_move = true;
249 + }
250 + }
M crates/shop-render/src/lib.rs +2
@@ -9,7 +9,9 @@
9 9 //! and cluster tracking arrive with the grid milestone.
10 10
11 11 mod atlas;
12 + mod image;
12 13 mod pipeline;
13 14 mod shaper;
14 15
16 + pub use image::{ImagePlacement, ImageRenderer};
15 17 pub use pipeline::{BgFill, CellDraw, TextRenderer};
M crates/shop/src/main.rs +193 -2
@@ -12,9 +12,10 @@
12 12 use calloop::EventLoop;
13 13 use calloop::generic::{FdWrapper, Generic};
14 14 use calloop_wayland_source::WaylandSource;
15 + use kitty_graphics as kgp;
15 16 use shop_grid::{Color as GridColor, CursorShape, Grid};
16 17 use shop_pty::{Pty, PtySize};
17 - use shop_render::{BgFill, CellDraw, TextRenderer};
18 + use shop_render::{BgFill, CellDraw, ImagePlacement, ImageRenderer, TextRenderer};
18 19 use shop_wayland::{
19 20 Capability, CompositorHandler, CompositorState, Connection, OutputHandler, OutputState,
20 21 Pending, ProvidesRegistryState, QueueHandle, RegistryState, SeatHandler, SeatState,
@@ -152,6 +153,7 @@
152 153 present_mode: wgpu::PresentMode::Mailbox,
153 154 };
154 155 let text = TextRenderer::new(&device, &queue, format, font_data, FONT_PX)?;
156 + let images = ImageRenderer::new(&device, format);
155 157
156 158 let grid = Grid::new(cols_initial, rows_initial);
157 159 let parser = vte::Parser::new();
@@ -163,9 +165,14 @@
163 165 device: Arc::new(device),
164 166 queue: Arc::new(queue),
165 167 text,
168 + images,
166 169 pty,
167 170 grid,
168 171 parser,
172 + apc_scanner: ApcScanner::default(),
173 + kitty: kgp::Parser::new(),
174 + image_placement: None,
175 + next_anon_image_id: 1,
169 176 font_data: FONT_BYTES.to_vec(),
170 177 scale: 1,
171 178 keyboard: None,
@@ -205,7 +212,14 @@
205 212 break;
206 213 }
207 214 Ok(n) => {
208 - app.parser.advance(&mut app.grid, &buf[..n]);
215 + let mut for_vte = Vec::with_capacity(n);
216 + let apc_bodies = app.apc_scanner.scan(&buf[..n], &mut for_vte);
217 + app.parser.advance(&mut app.grid, &for_vte);
218 + for body in apc_bodies {
219 + if let Some(cmd) = app.kitty.feed(&body) {
220 + handle_kitty(&mut *app, cmd);
221 + }
222 + }
209 223 if let Some(title) = app.grid.take_pending_title() {
210 224 app.chrome.xdg_window.set_title(title);
211 225 }
@@ -240,6 +254,7 @@
240 254 app.surface_config.height = physical_h;
241 255 app.surface.configure(&app.device, &app.surface_config);
242 256 app.text.resize(&app.queue, physical_w, physical_h);
257 + app.images.resize(&app.queue, physical_w, physical_h);
243 258 // Tell the compositor our buffer is N logical px per buffer px.
244 259 app.chrome.xdg_window.wl_surface().set_buffer_scale(s as i32);
245 260 let cols = grid_cols(logical_w);
@@ -268,6 +283,165 @@
268 283 Ok(())
269 284 }
270 285
286 + /// Stateful byte-stream scanner that separates kitty-graphics APC bodies
287 + /// (`\e_G<body>\e\` or `\e_G<body>\a`) from the rest of the vt stream.
288 + #[derive(Debug, Default)]
289 + struct ApcScanner {
290 + state: ApcScanState,
291 + body: Vec<u8>,
292 + }
293 +
294 + #[derive(Debug, Default, Clone, Copy, PartialEq)]
295 + enum ApcScanState {
296 + #[default]
297 + Normal,
298 + SawEsc,
299 + InApc,
300 + InApcSawEsc,
301 + }
302 +
303 + impl ApcScanner {
304 + /// Consume `bytes`, appending non-APC bytes to `for_vte` and returning
305 + /// completed APC bodies (without the `\e_G` intro or the terminator).
306 + fn scan(&mut self, bytes: &[u8], for_vte: &mut Vec<u8>) -> Vec<Vec<u8>> {
307 + let mut bodies = Vec::new();
308 + for &b in bytes {
309 + match self.state {
310 + ApcScanState::Normal => {
311 + if b == 0x1b {
312 + self.state = ApcScanState::SawEsc;
313 + } else {
314 + for_vte.push(b);
315 + }
316 + }
317 + ApcScanState::SawEsc => {
318 + if b == b'_' {
319 + self.state = ApcScanState::InApc;
320 + self.body.clear();
321 + } else {
322 + // Not APC — replay the ESC and the follow byte.
323 + for_vte.push(0x1b);
324 + for_vte.push(b);
325 + self.state = ApcScanState::Normal;
326 + }
327 + }
328 + ApcScanState::InApc => {
329 + if b == 0x1b {
330 + self.state = ApcScanState::InApcSawEsc;
331 + } else if b == 0x07 {
332 + bodies.push(std::mem::take(&mut self.body));
333 + self.state = ApcScanState::Normal;
334 + } else {
335 + self.body.push(b);
336 + }
337 + }
338 + ApcScanState::InApcSawEsc => {
339 + if b == b'\\' {
340 + bodies.push(std::mem::take(&mut self.body));
341 + self.state = ApcScanState::Normal;
342 + } else {
343 + self.body.push(0x1b);
344 + self.body.push(b);
345 + self.state = ApcScanState::InApc;
346 + }
347 + }
348 + }
349 + }
350 + bodies
351 + }
352 + }
353 +
354 + fn handle_kitty(app: &mut App, cmd: kgp::Command) {
355 + match cmd {
356 + kgp::Command::Transmit { control, payload } => {
357 + let Some(format) = control.format else {
358 + warn!("kitty: transmit missing format");
359 + return;
360 + };
361 + let rgba = match format {
362 + kgp::Format::Png => match image::load_from_memory_with_format(
363 + &payload,
364 + image::ImageFormat::Png,
365 + ) {
366 + Ok(img) => {
367 + let rgba = img.to_rgba8();
368 + let (w, h) = (rgba.width(), rgba.height());
369 + Some((rgba.into_raw(), w, h))
370 + }
371 + Err(e) => {
372 + warn!("kitty: png decode: {e}");
373 + None
374 + }
375 + },
376 + kgp::Format::Rgba => {
377 + let (w, h) =
378 + (control.width_px.unwrap_or(0), control.height_px.unwrap_or(0));
379 + if w == 0 || h == 0 {
380 + warn!("kitty: rgba transmit missing s= / v=");
381 + None
382 + } else {
383 + Some((payload, w, h))
384 + }
385 + }
386 + kgp::Format::Rgb => {
387 + let (w, h) =
388 + (control.width_px.unwrap_or(0), control.height_px.unwrap_or(0));
389 + if w == 0 || h == 0 {
390 + warn!("kitty: rgb transmit missing s= / v=");
391 + None
392 + } else {
393 + let mut rgba = Vec::with_capacity((w * h * 4) as usize);
394 + for chunk in payload.chunks_exact(3) {
395 + rgba.extend_from_slice(chunk);
396 + rgba.push(0xff);
397 + }
398 + Some((rgba, w, h))
399 + }
400 + }
401 + };
402 + let Some((rgba, w, h)) = rgba else {
403 + return;
404 + };
405 + let image_id = control.id.unwrap_or_else(|| {
406 + let id = app.next_anon_image_id;
407 + app.next_anon_image_id = app.next_anon_image_id.wrapping_add(1);
408 + id
409 + });
410 + app.images
411 + .set_image(&app.device, &app.queue, image_id, &rgba, w, h);
412 +
413 + if control.action == 'T' {
414 + let s = app.scale as f32;
415 + let cursor = app.grid.cursor();
416 + let x = (PAD_X + cursor.col as f32 * CELL_ADVANCE) * s;
417 + let y = (PAD_Y + cursor.row as f32 * CELL_HEIGHT) * s;
418 + // Prefer the cell-sized placement (`c=`, `r=`) if given; fall
419 + // back to the source pixel dims scaled up. yazi always sends
420 + // `c` and `r`.
421 + let placement_w = control
422 + .cell_cols
423 + .map(|c| c as f32 * CELL_ADVANCE * s)
424 + .unwrap_or(w as f32);
425 + let placement_h = control
426 + .cell_rows
427 + .map(|r| r as f32 * CELL_HEIGHT * s)
428 + .unwrap_or(h as f32);
429 + app.image_placement = Some(ImagePlacement {
430 + image_id,
431 + x,
432 + y,
433 + w: placement_w,
434 + h: placement_h,
435 + });
436 + }
437 + }
438 + kgp::Command::Delete { .. } => {
439 + app.images.drop_all();
440 + app.image_placement = None;
441 + }
442 + }
443 + }
444 +
271 445 fn grid_cols(px_w: u32) -> u16 {
272 446 let usable = (px_w as f32 - 2.0 * PAD_X).max(CELL_ADVANCE);
273 447 (usable / CELL_ADVANCE) as u16
@@ -286,9 +460,21 @@
286 460 device: Arc<wgpu::Device>,
287 461 queue: Arc<wgpu::Queue>,
288 462 text: TextRenderer,
463 + images: ImageRenderer,
289 464 pty: Pty,
290 465 grid: Grid,
291 466 parser: vte::Parser,
467 + /// Kitty-graphics uses APC escapes (`\e_G…\e\`) which `vte` recognizes but
468 + /// doesn't dispatch. The scanner peels them off before the byte stream
469 + /// reaches vte, hands the bodies to the kitty parser, and forwards the
470 + /// rest untouched. Followup: replace with a proper shop-vt that dispatches
471 + /// APC natively.
472 + apc_scanner: ApcScanner,
473 + kitty: kgp::Parser,
474 + /// Single-image MVP: track the most recently displayed image + where.
475 + /// Multi-image / z-order / delete-selectors are follow-ups.
476 + image_placement: Option<ImagePlacement>,
477 + next_anon_image_id: u32,
292 478 font_data: Vec<u8>,
293 479 /// HiDPI integer scale from `wl_surface.enter` outputs.
294 480 scale: u32,
@@ -409,6 +595,11 @@
409 595 app.text
410 596 .draw_frame(&app.device, &app.queue, &view, &mut encoder, &fills, &cells);
411 597 }
598 + // Draw any live kitty-graphics image on top of the text layer.
599 + if let Some(placement) = app.image_placement {
600 + app.images
601 + .draw(&app.queue, &view, &mut encoder, std::slice::from_ref(&placement));
602 + }
412 603 app.queue.submit(std::iter::once(encoder.finish()));
413 604 app.queue.present(frame);
414 605 Ok(())
A crates/shop-render/src/image.rs +354
@@ -1,0 +1,354 @@
1 + //! RGBA image pipeline: one wgpu texture per image, one draw call per image.
2 + //!
3 + //! MVP scope: a fixed pool of image slots, each a `wgpu::Texture` and a
4 + //! precomputed bind group. `set_image` uploads or replaces a slot's texture;
5 + //! `draw_frame` draws all live slots at their placement rectangles. Instance
6 + //! batching within a single pipeline is not worth it while we're expecting
7 + //! ~1 image on screen at a time (yazi preview pane).
8 +
9 + use std::collections::HashMap;
10 +
11 + use bytemuck::{Pod, Zeroable};
12 + use wgpu::util::DeviceExt;
13 +
14 + /// Placement of an image on the screen, in physical pixels.
15 + #[derive(Debug, Clone, Copy)]
16 + pub struct ImagePlacement {
17 + pub image_id: u32,
18 + pub x: f32,
19 + pub y: f32,
20 + pub w: f32,
21 + pub h: f32,
22 + }
23 +
24 + #[repr(C)]
25 + #[derive(Clone, Copy, Pod, Zeroable)]
26 + struct ImageUniforms {
27 + screen: [f32; 2],
28 + _pad: [f32; 2],
29 + }
30 +
31 + const IMAGE_SHADER: &str = r#"
32 + struct Uniforms {
33 + screen: vec2<f32>,
34 + _pad: vec2<f32>,
35 + };
36 +
37 + @group(0) @binding(0) var<uniform> u: Uniforms;
38 + @group(0) @binding(1) var tex: texture_2d<f32>;
39 + @group(0) @binding(2) var samp: sampler;
40 +
41 + struct VOut {
42 + @builtin(position) pos: vec4<f32>,
43 + @location(0) uv: vec2<f32>,
44 + };
45 +
46 + @vertex
47 + fn vs(
48 + @location(0) corner: vec2<f32>,
49 + @location(1) inst_pos: vec2<f32>,
50 + @location(2) inst_size: vec2<f32>,
51 + ) -> VOut {
52 + let px = inst_pos + corner * inst_size;
53 + let ndc = vec2<f32>(
54 + (px.x / u.screen.x) * 2.0 - 1.0,
55 + 1.0 - (px.y / u.screen.y) * 2.0,
56 + );
57 + var out: VOut;
58 + out.pos = vec4<f32>(ndc, 0.0, 1.0);
59 + out.uv = corner;
60 + return out;
61 + }
62 +
63 + @fragment
64 + fn fs(in: VOut) -> @location(0) vec4<f32> {
65 + return textureSample(tex, samp, in.uv);
66 + }
67 + "#;
68 +
69 + #[repr(C)]
70 + #[derive(Clone, Copy, Pod, Zeroable)]
71 + struct ImgInstance {
72 + pos: [f32; 2],
73 + size: [f32; 2],
74 + }
75 +
76 + const QUAD: [[f32; 2]; 4] = [[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [1.0, 1.0]];
77 + const INDICES: [u16; 6] = [0, 1, 2, 2, 1, 3];
78 +
79 + struct ImageSlot {
80 + _texture: wgpu::Texture,
81 + bind_group: wgpu::BindGroup,
82 + }
83 +
84 + pub struct ImageRenderer {
85 + pipeline: wgpu::RenderPipeline,
86 + bind_group_layout: wgpu::BindGroupLayout,
87 + sampler: wgpu::Sampler,
88 + uniforms_buf: wgpu::Buffer,
89 + quad_buf: wgpu::Buffer,
90 + index_buf: wgpu::Buffer,
91 + instance_buf: wgpu::Buffer,
92 + images: HashMap<u32, ImageSlot>,
93 + }
94 +
95 + impl ImageRenderer {
96 + pub fn new(device: &wgpu::Device, format: wgpu::TextureFormat) -> Self {
97 + let uniforms_buf = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
98 + label: Some("shop.image.uniforms"),
99 + contents: bytemuck::bytes_of(&ImageUniforms {
100 + screen: [1.0, 1.0],
101 + _pad: [0.0, 0.0],
102 + }),
103 + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
104 + });
105 + let quad_buf = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
106 + label: Some("shop.image.quad"),
107 + contents: bytemuck::cast_slice(&QUAD),
108 + usage: wgpu::BufferUsages::VERTEX,
109 + });
110 + let index_buf = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
111 + label: Some("shop.image.index"),
112 + contents: bytemuck::cast_slice(&INDICES),
113 + usage: wgpu::BufferUsages::INDEX,
114 + });
115 + let instance_buf = device.create_buffer(&wgpu::BufferDescriptor {
116 + label: Some("shop.image.instance"),
117 + size: std::mem::size_of::<ImgInstance>() as u64,
118 + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,
119 + mapped_at_creation: false,
120 + });
121 +
122 + let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
123 + label: Some("shop.image.sampler"),
124 + mag_filter: wgpu::FilterMode::Linear,
125 + min_filter: wgpu::FilterMode::Linear,
126 + ..Default::default()
127 + });
128 +
129 + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
130 + label: Some("shop.image.bgl"),
131 + entries: &[
132 + wgpu::BindGroupLayoutEntry {
133 + binding: 0,
134 + visibility: wgpu::ShaderStages::VERTEX,
135 + ty: wgpu::BindingType::Buffer {
136 + ty: wgpu::BufferBindingType::Uniform,
137 + has_dynamic_offset: false,
138 + min_binding_size: None,
139 + },
140 + count: None,
141 + },
142 + wgpu::BindGroupLayoutEntry {
143 + binding: 1,
144 + visibility: wgpu::ShaderStages::FRAGMENT,
145 + ty: wgpu::BindingType::Texture {
146 + sample_type: wgpu::TextureSampleType::Float { filterable: true },
147 + view_dimension: wgpu::TextureViewDimension::D2,
148 + multisampled: false,
149 + },
150 + count: None,
151 + },
152 + wgpu::BindGroupLayoutEntry {
153 + binding: 2,
154 + visibility: wgpu::ShaderStages::FRAGMENT,
155 + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
156 + count: None,
157 + },
158 + ],
159 + });
160 +
161 + let module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
162 + label: Some("shop.image.shader"),
163 + source: wgpu::ShaderSource::Wgsl(IMAGE_SHADER.into()),
164 + });
165 + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
166 + label: Some("shop.image.pl"),
167 + bind_group_layouts: &[Some(&bind_group_layout)],
168 + immediate_size: 0,
169 + });
170 + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
171 + label: Some("shop.image.pipeline"),
172 + layout: Some(&pipeline_layout),
173 + vertex: wgpu::VertexState {
174 + module: &module,
175 + entry_point: Some("vs"),
176 + compilation_options: Default::default(),
177 + buffers: &[
178 + Some(wgpu::VertexBufferLayout {
179 + array_stride: 8,
180 + step_mode: wgpu::VertexStepMode::Vertex,
181 + attributes: &wgpu::vertex_attr_array![0 => Float32x2],
182 + }),
183 + Some(wgpu::VertexBufferLayout {
184 + array_stride: std::mem::size_of::<ImgInstance>() as u64,
185 + step_mode: wgpu::VertexStepMode::Instance,
186 + attributes: &wgpu::vertex_attr_array![
187 + 1 => Float32x2,
188 + 2 => Float32x2,
189 + ],
190 + }),
191 + ],
192 + },
193 + fragment: Some(wgpu::FragmentState {
194 + module: &module,
195 + entry_point: Some("fs"),
196 + compilation_options: Default::default(),
197 + targets: &[Some(wgpu::ColorTargetState {
198 + format,
199 + blend: Some(wgpu::BlendState::ALPHA_BLENDING),
200 + write_mask: wgpu::ColorWrites::ALL,
201 + })],
202 + }),
203 + primitive: wgpu::PrimitiveState::default(),
204 + depth_stencil: None,
205 + multisample: wgpu::MultisampleState::default(),
206 + multiview_mask: None,
207 + cache: None,
208 + });
209 +
210 + Self {
211 + pipeline,
212 + bind_group_layout,
213 + sampler,
214 + uniforms_buf,
215 + quad_buf,
216 + index_buf,
217 + instance_buf,
218 + images: HashMap::new(),
219 + }
220 + }
221 +
222 + pub fn resize(&self, queue: &wgpu::Queue, width: u32, height: u32) {
223 + let u = ImageUniforms {
224 + screen: [width as f32, height as f32],
225 + _pad: [0.0, 0.0],
226 + };
227 + queue.write_buffer(&self.uniforms_buf, 0, bytemuck::bytes_of(&u));
228 + }
229 +
230 + /// Upload an RGBA image and register it under `id`. Replaces any existing
231 + /// image at the same id.
232 + pub fn set_image(
233 + &mut self,
234 + device: &wgpu::Device,
235 + queue: &wgpu::Queue,
236 + id: u32,
237 + rgba: &[u8],
238 + width: u32,
239 + height: u32,
240 + ) {
241 + let texture = device.create_texture(&wgpu::TextureDescriptor {
242 + label: Some("shop.image.tex"),
243 + size: wgpu::Extent3d {
244 + width,
245 + height,
246 + depth_or_array_layers: 1,
247 + },
248 + mip_level_count: 1,
249 + sample_count: 1,
250 + dimension: wgpu::TextureDimension::D2,
251 + format: wgpu::TextureFormat::Rgba8UnormSrgb,
252 + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
253 + view_formats: &[],
254 + });
255 + queue.write_texture(
256 + wgpu::TexelCopyTextureInfo {
257 + texture: &texture,
258 + mip_level: 0,
259 + origin: wgpu::Origin3d::ZERO,
260 + aspect: wgpu::TextureAspect::All,
261 + },
262 + rgba,
263 + wgpu::TexelCopyBufferLayout {
264 + offset: 0,
265 + bytes_per_row: Some(width * 4),
266 + rows_per_image: Some(height),
267 + },
268 + wgpu::Extent3d {
269 + width,
270 + height,
271 + depth_or_array_layers: 1,
272 + },
273 + );
274 + let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
275 + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
276 + label: Some("shop.image.bg"),
277 + layout: &self.bind_group_layout,
278 + entries: &[
279 + wgpu::BindGroupEntry {
280 + binding: 0,
281 + resource: self.uniforms_buf.as_entire_binding(),
282 + },
283 + wgpu::BindGroupEntry {
284 + binding: 1,
285 + resource: wgpu::BindingResource::TextureView(&view),
286 + },
287 + wgpu::BindGroupEntry {
288 + binding: 2,
289 + resource: wgpu::BindingResource::Sampler(&self.sampler),
290 + },
291 + ],
292 + });
293 + self.images.insert(
294 + id,
295 + ImageSlot {
296 + _texture: texture,
297 + bind_group,
298 + },
299 + );
300 + }
301 +
302 + pub fn drop_image(&mut self, id: u32) {
303 + self.images.remove(&id);
304 + }
305 +
306 + pub fn drop_all(&mut self) {
307 + self.images.clear();
308 + }
309 +
310 + /// Draw each placement in order (one texture bind + draw call per image).
311 + pub fn draw(
312 + &self,
313 + queue: &wgpu::Queue,
314 + view: &wgpu::TextureView,
315 + encoder: &mut wgpu::CommandEncoder,
316 + placements: &[ImagePlacement],
317 + ) {
318 + if placements.is_empty() {
319 + return;
320 + }
321 + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
322 + label: Some("shop.image.pass"),
323 + color_attachments: &[Some(wgpu::RenderPassColorAttachment {
324 + view,
325 + resolve_target: None,
326 + ops: wgpu::Operations {
327 + load: wgpu::LoadOp::Load,
328 + store: wgpu::StoreOp::Store,
329 + },
330 + depth_slice: None,
331 + })],
332 + ..Default::default()
333 + });
334 + pass.set_pipeline(&self.pipeline);
335 + pass.set_vertex_buffer(0, self.quad_buf.slice(..));
336 + pass.set_index_buffer(self.index_buf.slice(..), wgpu::IndexFormat::Uint16);
337 + for placement in placements {
338 + let Some(slot) = self.images.get(&placement.image_id) else {
339 + continue;
340 + };
341 + queue.write_buffer(
342 + &self.instance_buf,
343 + 0,
344 + bytemuck::bytes_of(&ImgInstance {
345 + pos: [placement.x, placement.y],
346 + size: [placement.w, placement.h],
347 + }),
348 + );
349 + pass.set_vertex_buffer(1, self.instance_buf.slice(..));
350 + pass.set_bind_group(0, &slot.bind_group, &[]);
351 + pass.draw_indexed(0..6, 0, 0..1);
352 + }
353 + }
354 + }