Skip to main content

max / docengine

Add render_runs: markdown to text runs that keep their inline marks render_plain throws the emphasis away, which is right for a preview line and wrong for a destination that has no markup but does have bold. A terminal cell is the case that asked for it. render_runs is the same walk one step earlier, and render_plain is now its runs concatenated, so the two cannot drift on what a table row or a nested list is worth in newlines. Asserted as an invariant. Code blocks are marked as code as well as code spans: pulldown emits a block's body as Text and reserves Code for a span, so the fence has to be tracked to know.
Author: Max Johnson <me@maxj.phd> · 2026-08-12 16:39 UTC
Signed with PGP, not checked
Commit: 2e115532df2bc859014d523790b67542d3496237
Parent: ae55a5a
3 files changed, +395 insertions, -29 deletions
M Cargo.toml +1 -1
@@ -1,6 +1,6 @@
1 1 [package]
2 2 name = "docengine"
3 - version = "0.5.0"
3 + version = "0.6.0"
4 4 edition = "2024"
5 5 license = "MIT"
6 6
M src/lib.rs +4 -2
@@ -13,7 +13,9 @@
13 13 //!
14 14 //! [`render_plain`] goes the other way, to text rather than to markup, for the
15 15 //! places that cannot take markup at all: a list row's preview, a notification,
16 - //! an `alt` attribute.
16 + //! an `alt` attribute. [`render_runs`] is the same walk with the emphasis left
17 + //! on, for a destination with no markup that can still paint a run bold: a
18 + //! terminal cell, an `egui` galley.
17 19 //!
18 20 //! Optional features add document loading, TOML frontmatter, @mention resolution,
19 21 //! and quote attribution post-processing.
@@ -41,7 +43,7 @@
41 43 mod quotes;
42 44
43 45 // Re-export core types
44 - pub use plain::render_plain;
46 + pub use plain::{Emphasis, TextRun, render_plain, render_runs};
45 47 pub use render::{RenderResult, Renderer};
46 48 pub use sanitize::SanitizePreset;
47 49 pub use text::{extract_title, reading_time_minutes, strip_first_heading, word_count};
M src/plain.rs +390 -26
@@ -1,4 +1,4 @@
1 - //! Markdown to plain text.
1 + //! Markdown to text.
2 2 //!
3 3 //! The other direction from the presets. They all answer "how do I show this
4 4 //! markdown", and this answers "what does this markdown say", which is what a
@@ -9,8 +9,84 @@
9 9 //! Without it every such caller does the same wrong thing, which is to print
10 10 //! the source and let the syntax through. `**bold**` in a preview is not
11 11 //! emphasis and is not the word the author wrote either.
12 + //!
13 + //! [`render_runs`] is the same walk, stopping one step earlier: it hands back
14 + //! the text in runs that still say which inline marks were over them, for a
15 + //! caller whose destination has no markup but does have bold. A terminal cell
16 + //! is the case that asked for it. [`render_plain`] is the runs concatenated,
17 + //! and the two are one function so they cannot drift on what a table row or a
18 + //! nested list is worth in newlines.
12 19
13 20 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
21 + use serde::{Deserialize, Serialize};
22 +
23 + /// The inline marks in force over a run of text.
24 + ///
25 + /// Markdown's own names: `strong` is `**this**`, `italic` is `*this*`. All four
26 + /// are independent and any combination can hold at once, which is why this is a
27 + /// set of flags rather than one kind.
28 + ///
29 + /// Block structure is deliberately not here. A heading's level, a list's
30 + /// nesting and a quote's depth are gone by this point for the same reason they
31 + /// are gone from [`render_plain`]: the destination is a place with no room to
32 + /// spend on them. What survives is exactly what a caller can paint on a run of
33 + /// text without moving it.
34 + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
35 + pub struct Emphasis {
36 + /// `**strong**`.
37 + pub strong: bool,
38 + /// `*italic*`.
39 + pub italic: bool,
40 + /// `~~struck~~`.
41 + pub struck: bool,
42 + /// A code span or the body of a code block.
43 + pub code: bool,
44 + }
45 +
46 + impl Emphasis {
47 + /// Whether nothing is in force, so a caller can take its own default style
48 + /// rather than building one that says "no marks".
49 + #[must_use]
50 + pub const fn is_plain(self) -> bool {
51 + !self.strong && !self.italic && !self.struck && !self.code
52 + }
53 + }
54 +
55 + /// A stretch of text under one set of marks.
56 + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
57 + pub struct TextRun {
58 + /// The words, with none of the syntax that shaped them.
59 + pub text: String,
60 + /// What was over them.
61 + pub emphasis: Emphasis,
62 + }
63 +
64 + /// How deep we are inside each inline mark.
65 + ///
66 + /// Counters and not booleans: `**a *b* a**` closes the inner mark without
67 + /// ending the outer one, and markdown lets that nest as far as an author cares
68 + /// to go.
69 + #[derive(Clone, Copy, Default)]
70 + struct Marks {
71 + strong: u16,
72 + italic: u16,
73 + struck: u16,
74 + /// A fenced or indented block. Its body arrives as `Event::Text` and not as
75 + /// `Event::Code`, which pulldown reserves for a span, so the only way to
76 + /// know the words are code is to have seen the fence.
77 + code_block: u16,
78 + }
79 +
80 + impl Marks {
81 + fn emphasis(self) -> Emphasis {
82 + Emphasis {
83 + strong: self.strong > 0,
84 + italic: self.italic > 0,
85 + struck: self.struck > 0,
86 + code: self.code_block > 0,
87 + }
88 + }
89 + }
14 90
15 91 /// Render markdown as plain text: the words, without the syntax that shapes
16 92 /// them.
@@ -34,8 +110,37 @@
34 110 /// assert_eq!(plain, "Title\n\nSee the docs.");
35 111 /// ```
36 112 pub fn render_plain(markdown: &str) -> String {
113 + render_runs(markdown)
114 + .into_iter()
115 + .map(|run| run.text)
116 + .collect()
117 + }
118 +
119 + /// Render markdown as text runs that keep the inline marks that were over them.
120 + ///
121 + /// [`render_plain`] with the emphasis left on. Everything that function says
122 + /// about block structure, links, raw HTML and GFM holds here unchanged, and the
123 + /// concatenation of these runs is exactly what it returns.
124 + ///
125 + /// For a caller whose destination has no markup but is not flat either: a
126 + /// terminal cell can be bold, an `egui` galley can carry a text format, and
127 + /// both were previously handed the words with the emphasis already thrown away.
128 + /// A caller that only wants the words should keep using [`render_plain`].
129 + ///
130 + /// Adjacent text under the same marks arrives as one run, so a paragraph with
131 + /// no emphasis in it is one run rather than one per parsed event.
132 + ///
133 + /// ```
134 + /// let runs = docengine::render_runs("plain **bold**");
135 + /// assert_eq!(runs.len(), 2);
136 + /// assert_eq!(runs[0].text, "plain ");
137 + /// assert!(runs[0].emphasis.is_plain());
138 + /// assert_eq!(runs[1].text, "bold");
139 + /// assert!(runs[1].emphasis.strong);
140 + /// ```
141 + pub fn render_runs(markdown: &str) -> Vec<TextRun> {
37 142 if markdown.is_empty() {
38 - return String::new();
143 + return Vec::new();
39 144 }
40 145
41 146 let mut options = Options::empty();
@@ -44,42 +149,70 @@
44 149 options.insert(Options::ENABLE_FOOTNOTES);
45 150 options.insert(Options::ENABLE_TASKLISTS);
46 151
47 - let mut out = String::with_capacity(markdown.len());
152 + let mut runs: Vec<TextRun> = Vec::new();
153 + let mut marks = Marks::default();
48 154
49 155 for event in Parser::new_ext(markdown, options) {
50 156 match event {
51 157 // A code span's text is the text. A code block's is too: it is
52 158 // usually the least readable thing in a preview, and dropping it
53 159 // would silently empty out a message that is nothing but a snippet.
54 - Event::Text(text) | Event::Code(text) => out.push_str(&text),
160 + Event::Code(text) => push(
161 + &mut runs,
162 + &text,
163 + Emphasis {
164 + code: true,
165 + ..marks.emphasis()
166 + },
167 + ),
168 + Event::Text(text) => push(&mut runs, &text, marks.emphasis()),
55 169
56 - // A line the author broke on purpose.
57 - Event::HardBreak => out.push('\n'),
170 + // Inline marks, tracked rather than passed over. This is the whole
171 + // difference between the two functions above; the rest of the walk
172 + // is the same one it always was.
173 + Event::Start(Tag::Strong) => marks.strong += 1,
174 + Event::End(TagEnd::Strong) => marks.strong = marks.strong.saturating_sub(1),
175 + Event::Start(Tag::Emphasis) => marks.italic += 1,
176 + Event::End(TagEnd::Emphasis) => marks.italic = marks.italic.saturating_sub(1),
177 + Event::Start(Tag::Strikethrough) => marks.struck += 1,
178 + Event::End(TagEnd::Strikethrough) => marks.struck = marks.struck.saturating_sub(1),
179 + Event::Start(Tag::CodeBlock(_)) => marks.code_block += 1,
58 180
59 - // A line the source wrapped. The author wrote one line, so this
60 - // gives them one line back rather than a break they never typed.
61 - Event::SoftBreak => out.push(' '),
181 + // A line the author broke on purpose, and a line the source
182 + // wrapped: the author wrote one line there, so that one gives them
183 + // one line back rather than a break they never typed. Both carry
184 + // the marks in force, so a break inside `**...**` does not cut the
185 + // run in three over a character nobody can see the style of.
186 + Event::HardBreak => push(&mut runs, "\n", marks.emphasis()),
187 + Event::SoftBreak => push(&mut runs, " ", marks.emphasis()),
62 188
63 189 // A prose block ends with a blank line after it, because two
64 190 // paragraphs run together read as one sentence that does not parse.
191 + // Inline marks are closed by now, so these are unmarked by
192 + // construction rather than by choice.
193 + Event::End(TagEnd::CodeBlock) => {
194 + marks.code_block = marks.code_block.saturating_sub(1);
195 + push(&mut runs, "\n\n", Emphasis::default());
196 + }
65 197 Event::End(
66 198 TagEnd::Paragraph
67 199 | TagEnd::Heading(_)
68 - | TagEnd::CodeBlock
69 200 | TagEnd::BlockQuote(_)
70 201 | TagEnd::List(_)
71 202 | TagEnd::FootnoteDefinition,
72 - ) => out.push_str("\n\n"),
203 + ) => push(&mut runs, "\n\n", Emphasis::default()),
73 204
74 205 // A member of a block ends a line and no more: a list is one item
75 206 // per line, and a table is one row per line.
76 - Event::End(TagEnd::Item | TagEnd::TableRow | TagEnd::TableHead) => out.push('\n'),
207 + Event::End(TagEnd::Item | TagEnd::TableRow | TagEnd::TableHead) => {
208 + push(&mut runs, "\n", Emphasis::default());
209 + }
77 210
78 211 // Cells sit in a row, so they want a separator rather than a break.
79 - Event::End(TagEnd::TableCell) => out.push('\t'),
212 + Event::End(TagEnd::TableCell) => push(&mut runs, "\t", Emphasis::default()),
80 213
81 214 // A rule is a boundary with nothing to say.
82 - Event::Rule => out.push('\n'),
215 + Event::Rule => push(&mut runs, "\n", Emphasis::default()),
83 216
84 217 // An image's alt text is the only thing here a reader can use, and
85 218 // pulldown emits it as the tag's inner text, so the tag itself is
@@ -93,7 +226,22 @@
93 226 }
94 227 }
95 228
96 - tidy(&out)
229 + tidy(runs)
230 + }
231 +
232 + /// Append `text` under `emphasis`, joining the run before it when the marks
233 + /// match, so a paragraph with no emphasis in it stays one run.
234 + fn push(runs: &mut Vec<TextRun>, text: &str, emphasis: Emphasis) {
235 + if text.is_empty() {
236 + return;
237 + }
238 + match runs.last_mut() {
239 + Some(last) if last.emphasis == emphasis => last.text.push_str(text),
240 + _ => runs.push(TextRun {
241 + text: text.to_string(),
242 + emphasis,
243 + }),
244 + }
97 245 }
98 246
99 247 /// Collapse what the walk above leaves behind: trailing spaces on a line, and
@@ -102,32 +250,96 @@
102 250 /// Nesting is the reason this is a pass rather than care taken inline. A list
103 251 /// inside a blockquote ends three blocks at the same point and each one is
104 252 /// right to end a line; only together are they wrong.
105 - fn tidy(text: &str) -> String {
106 - let mut out = String::with_capacity(text.len());
107 - let mut blank_run = 0;
253 + ///
254 + /// It works a line at a time and a run can span several lines, so the first
255 + /// move is to cut the runs at every newline. What comes back out is re-joined
256 + /// and re-merged, so cutting costs nothing a caller can see.
257 + fn tidy(runs: Vec<TextRun>) -> Vec<TextRun> {
258 + let mut lines = split_lines(runs);
259 + for line in &mut lines {
260 + trim_end(line);
261 + }
108 262
109 - for line in text.lines() {
110 - let line = line.trim_end();
263 + let mut kept: Vec<Vec<TextRun>> = Vec::new();
264 + let mut blank_run = 0;
265 + for line in lines {
111 266 if line.is_empty() {
112 267 blank_run += 1;
113 268 // One blank line separates two blocks. A second says nothing the
114 - // first did not.
115 - if blank_run > 1 || out.is_empty() {
269 + // first did not, and neither does one before anything at all.
270 + if blank_run > 1 || kept.is_empty() {
116 271 continue;
117 272 }
118 273 } else {
119 274 blank_run = 0;
120 275 }
121 - out.push_str(line);
122 - out.push('\n');
276 + kept.push(line);
277 + }
278 + // The last block ended a line too, and there is nothing after it to
279 + // separate from.
280 + while kept.last().is_some_and(Vec::is_empty) {
281 + kept.pop();
123 282 }
124 283
125 - out.trim_end().to_string()
284 + let mut out: Vec<TextRun> = Vec::new();
285 + for (index, line) in kept.into_iter().enumerate() {
286 + if index > 0 {
287 + push(&mut out, "\n", Emphasis::default());
288 + }
289 + for run in line {
290 + push(&mut out, &run.text, run.emphasis);
291 + }
292 + }
293 + out
294 + }
295 +
296 + /// Cut `runs` at every newline, into one group of runs per line.
297 + fn split_lines(runs: Vec<TextRun>) -> Vec<Vec<TextRun>> {
298 + let mut lines: Vec<Vec<TextRun>> = vec![Vec::new()];
299 + for run in runs {
300 + for (index, piece) in run.text.split('\n').enumerate() {
301 + if index > 0 {
302 + lines.push(Vec::new());
303 + }
304 + if !piece.is_empty() {
305 + // A cut cannot make two adjacent runs mergeable that were not
306 + // already, so this only ever appends.
307 + let line = lines.last_mut().expect("a line was just pushed");
308 + line.push(TextRun {
309 + text: piece.to_string(),
310 + emphasis: run.emphasis,
311 + });
312 + }
313 + }
314 + }
315 + lines
316 + }
317 +
318 + /// Drop trailing whitespace from a line, across as many runs as it spans.
319 + fn trim_end(line: &mut Vec<TextRun>) {
320 + while let Some(last) = line.last_mut() {
321 + let trimmed = last.text.trim_end();
322 + if trimmed.is_empty() {
323 + line.pop();
324 + } else {
325 + last.text.truncate(trimmed.len());
326 + break;
327 + }
328 + }
126 329 }
127 330
128 331 #[cfg(test)]
129 332 mod tests {
130 - use super::render_plain;
333 + use super::{Emphasis, render_plain, render_runs};
334 +
335 + /// The marks over each run, for asserting shape without spelling out four
336 + /// booleans a run.
337 + fn marks(markdown: &str) -> Vec<(String, Emphasis)> {
338 + render_runs(markdown)
339 + .into_iter()
340 + .map(|run| (run.text, run.emphasis))
341 + .collect()
342 + }
131 343
132 344 #[test]
133 345 fn inline_syntax_becomes_the_words_it_wrapped() {
@@ -226,6 +438,8 @@
226 438 fn empty_in_empty_out() {
227 439 assert_eq!(render_plain(""), "");
228 440 assert_eq!(render_plain("\n\n"), "");
441 + assert_eq!(render_runs(""), Vec::new());
442 + assert_eq!(render_runs("\n\n"), Vec::new());
229 443 }
230 444
231 445 #[test]
@@ -237,4 +451,196 @@
237 451 "the *args and **kwargs conventions"
238 452 );
239 453 }
454 +
455 + #[test]
456 + fn runs_concatenate_to_the_plain_render() {
457 + // The invariant the two functions exist as one walk to keep. If this
458 + // ever fails, a caller that switched from one to the other has silently
459 + // changed what its rows say.
460 + let sources = [
461 + "**bold**, *italic*, `code` and ~~struck~~",
462 + "# Title\n\nBody with **weight** in it.",
463 + "> - one\n> - **two**\n\nAfter.",
464 + "| a | **b** |\n|---|---|\n| 1 | 2 |",
465 + "```rust\nlet x = 1;\n```",
466 + "Read [the **announcement**](https://example.com).",
467 + "before <b>bold</b> after",
468 + "the *args and **kwargs conventions",
469 + "one \ntwo",
470 + "",
471 + ];
472 + for source in sources {
473 + let joined: String = render_runs(source)
474 + .into_iter()
475 + .map(|run| run.text)
476 + .collect();
477 + assert_eq!(joined, render_plain(source), "source: {source:?}");
478 + }
479 + }
480 +
481 + #[test]
482 + fn each_mark_is_carried() {
483 + assert_eq!(
484 + marks("**b** *i* `c` ~~s~~"),
485 + vec![
486 + (
487 + "b".to_string(),
488 + Emphasis {
489 + strong: true,
490 + ..Emphasis::default()
491 + }
492 + ),
493 + (" ".to_string(), Emphasis::default()),
494 + (
495 + "i".to_string(),
496 + Emphasis {
497 + italic: true,
498 + ..Emphasis::default()
499 + }
500 + ),
501 + (" ".to_string(), Emphasis::default()),
502 + (
503 + "c".to_string(),
504 + Emphasis {
505 + code: true,
506 + ..Emphasis::default()
507 + }
508 + ),
509 + (" ".to_string(), Emphasis::default()),
510 + (
511 + "s".to_string(),
512 + Emphasis {
513 + struck: true,
514 + ..Emphasis::default()
515 + }
516 + ),
517 + ]
518 + );
519 + }
520 +
521 + #[test]
522 + fn marks_nest_and_combine() {
523 + // The counters earn themselves here: the inner mark closes and the
524 + // outer one is still in force over the text after it.
525 + assert_eq!(
526 + marks("**a *b* c**"),
527 + vec![
528 + (
529 + "a ".to_string(),
530 + Emphasis {
531 + strong: true,
532 + ..Emphasis::default()
533 + }
534 + ),
535 + (
536 + "b".to_string(),
537 + Emphasis {
538 + strong: true,
539 + italic: true,
540 + ..Emphasis::default()
541 + }
542 + ),
543 + (
544 + " c".to_string(),
545 + Emphasis {
546 + strong: true,
547 + ..Emphasis::default()
548 + }
549 + ),
550 + ]
551 + );
552 + }
553 +
554 + #[test]
555 + fn a_code_span_inside_emphasis_carries_both() {
556 + assert_eq!(
557 + marks("**`cargo build`**"),
558 + vec![(
559 + "cargo build".to_string(),
560 + Emphasis {
561 + strong: true,
562 + code: true,
563 + ..Emphasis::default()
564 + }
565 + )]
566 + );
567 + }
568 +
569 + #[test]
570 + fn a_code_block_is_marked_as_code() {
571 + let runs = render_runs("```rust\nlet x = 1;\n```");
572 + assert_eq!(runs.len(), 1);
573 + assert!(runs[0].emphasis.code);
574 + assert_eq!(runs[0].text, "let x = 1;");
575 + }
576 +
577 + #[test]
578 + fn adjacent_text_under_the_same_marks_is_one_run() {
579 + // A link's text and the prose around it are separate events and one
580 + // run: a caller styling per run should not see a seam where the source
581 + // had a bracket.
582 + assert_eq!(
583 + marks("Read [the docs](https://example.com) now."),
584 + vec![("Read the docs now.".to_string(), Emphasis::default())]
585 + );
586 + }
587 +
588 + #[test]
589 + fn a_soft_break_inside_emphasis_does_not_cut_the_run() {
590 + // The space stands where the source wrapped, and giving it the marks in
591 + // force is what keeps `**a b**` one run rather than three.
592 + assert_eq!(
593 + marks("**one\ntwo**"),
594 + vec![(
595 + "one two".to_string(),
596 + Emphasis {
597 + strong: true,
598 + ..Emphasis::default()
599 + }
600 + )]
601 + );
602 + }
603 +
Lines truncated