//! Markdown to text. //! //! The other direction from the presets. They all answer "how do I show this //! markdown", and this answers "what does this markdown say", which is what a //! caller needs whenever it has markdown and a place that cannot take markup: //! a list row's preview line, a notification body, a search snippet, a window //! title, an `alt` attribute. //! //! Without it every such caller does the same wrong thing, which is to print //! the source and let the syntax through. `**bold**` in a preview is not //! emphasis and is not the word the author wrote either. //! //! [`render_runs`] is the same walk, stopping one step earlier: it hands back //! the text in runs that still say which inline marks were over them, for a //! caller whose destination has no markup but does have bold. A terminal cell //! is the case that asked for it. [`render_plain`] is the runs concatenated, //! and the two are one function so they cannot drift on what a table row or a //! nested list is worth in newlines. use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd}; use serde::{Deserialize, Serialize}; /// The inline marks in force over a run of text. /// /// Markdown's own names: `strong` is `**this**`, `italic` is `*this*`. All four /// are independent and any combination can hold at once, which is why this is a /// set of flags rather than one kind. /// /// What kind of block the run sits in is [`Block`], on the run beside this. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct Emphasis { /// `**strong**`. pub strong: bool, /// `*italic*`. pub italic: bool, /// `~~struck~~`. pub struck: bool, /// A code span or the body of a code block. pub code: bool, } impl Emphasis { /// Whether nothing is in force, so a caller can take its own default style /// rather than building one that says "no marks". #[must_use] pub const fn is_plain(self) -> bool { !self.strong && !self.italic && !self.struck && !self.code } } /// The kind of block a run sits in. /// /// Emphasis is what a caller paints on the words. This is what the words *are*, /// and a destination with no markup can still answer it: a heading takes weight, /// an item takes a bullet, a quote takes a marker in the margin. /// /// The innermost block wins and nesting is not carried. An item inside a quote /// reads as an item, because a caller that has one line to draw has to pick one /// of the two anyway and the inner one is the one holding the words. A caller /// that needs the tree wants a markup preset rather than this. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum Block { /// A paragraph, a table cell, or anything else with nothing to say about /// itself. #[default] Prose, /// A heading, at its markdown level: 1 for `#` through 6 for `######`. Heading(u8), /// A list item, ordered or not. The marker is not in the text, because what /// a bullet looks like is the caller's answer and a number would be wrong /// for half of them. Item, /// A line inside a block quote. Quote, } /// A stretch of text under one set of marks, in one kind of block. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct TextRun { /// The words, with none of the syntax that shaped them. pub text: String, /// What was over them. pub emphasis: Emphasis, /// What they are part of. pub block: Block, } /// How deep we are inside each inline mark. /// /// Counters and not booleans: `**a *b* a**` closes the inner mark without /// ending the outer one, and markdown lets that nest as far as an author cares /// to go. #[derive(Clone, Copy, Default)] struct Marks { strong: u16, italic: u16, struck: u16, /// A fenced or indented block. Its body arrives as `Event::Text` and not as /// `Event::Code`, which pulldown reserves for a span, so the only way to /// know the words are code is to have seen the fence. code_block: u16, } /// Which block a run is in, innermost last. /// /// A stack and not a field, because the blocks that matter here nest: an item /// inside a quote inside an item is ordinary markdown, and closing the inner one /// has to put the outer one back rather than fall to prose. #[derive(Default)] struct Blocks(Vec); impl Blocks { fn current(&self) -> Block { self.0.last().copied().unwrap_or_default() } fn open(&mut self, block: Block) { self.0.push(block); } fn close(&mut self) { self.0.pop(); } } /// A markdown heading level as the number the author typed. fn level_of(level: pulldown_cmark::HeadingLevel) -> u8 { use pulldown_cmark::HeadingLevel::{H1, H2, H3, H4, H5, H6}; match level { H1 => 1, H2 => 2, H3 => 3, H4 => 4, H5 => 5, H6 => 6, } } impl Marks { fn emphasis(self) -> Emphasis { Emphasis { strong: self.strong > 0, italic: self.italic > 0, struck: self.struck > 0, code: self.code_block > 0, } } } /// Render markdown as plain text: the words, without the syntax that shapes /// them. /// /// Block structure survives as newlines, because a preview that runs two /// paragraphs together reads as one sentence that does not parse. Inline /// structure does not survive at all: emphasis, code spans and link text all /// come through as their text, and a link's URL is dropped, since a reader /// looking at a one-line preview cannot follow it and it crowds out the words /// that would have told them whether they want to. /// /// Raw HTML is dropped rather than unescaped. This is not a sanitizer and must /// not be used as one: it produces text, and text put into a document still has /// to be escaped by whoever puts it there. /// /// The GFM options match the render presets, so a table or a task list /// contributes its cell and item text rather than its pipes and brackets. /// /// ``` /// let plain = docengine::render_plain("# Title\n\nSee **the [docs](https://example.com)**."); /// assert_eq!(plain, "Title\n\nSee the docs."); /// ``` pub fn render_plain(markdown: &str) -> String { render_runs(markdown) .into_iter() .map(|run| run.text) .collect() } /// Render markdown as text runs that keep the inline marks that were over them. /// /// [`render_plain`] with the emphasis left on. Everything that function says /// about block structure, links, raw HTML and GFM holds here unchanged, and the /// concatenation of these runs is exactly what it returns. /// /// For a caller whose destination has no markup but is not flat either: a /// terminal cell can be bold, an `egui` galley can carry a text format. A caller /// that only wants the words should use [`render_plain`]. /// /// Adjacent text under the same marks arrives as one run, so a paragraph with /// no emphasis in it is one run rather than one per parsed event. /// /// ``` /// let runs = docengine::render_runs("plain **bold**"); /// assert_eq!(runs.len(), 2); /// assert_eq!(runs[0].text, "plain "); /// assert!(runs[0].emphasis.is_plain()); /// assert_eq!(runs[1].text, "bold"); /// assert!(runs[1].emphasis.strong); /// ``` pub fn render_runs(markdown: &str) -> Vec { if markdown.is_empty() { return Vec::new(); } let mut options = Options::empty(); options.insert(Options::ENABLE_TABLES); options.insert(Options::ENABLE_STRIKETHROUGH); options.insert(Options::ENABLE_FOOTNOTES); options.insert(Options::ENABLE_TASKLISTS); let mut runs: Vec = Vec::new(); let mut marks = Marks::default(); let mut blocks = Blocks::default(); for event in Parser::new_ext(markdown, options) { match event { // A code span's text is the text. A code block's is too: it is // usually the least readable thing in a preview, and dropping it // would silently empty out a message that is nothing but a snippet. Event::Code(text) => push( &mut runs, &text, Emphasis { code: true, ..marks.emphasis() }, blocks.current(), ), Event::Text(text) => push(&mut runs, &text, marks.emphasis(), blocks.current()), // What the words are, as opposed to what is painted on them. A // heading has an answer on any destination that can set one line // heavier than the next, and a terminal is one of them. Event::Start(Tag::Heading { level, .. }) => { blocks.open(Block::Heading(level_of(level))); } Event::Start(Tag::Item) => blocks.open(Block::Item), Event::Start(Tag::BlockQuote(_)) => blocks.open(Block::Quote), // Inline marks, tracked rather than passed over. This is the whole // difference between the two functions above; the rest of the walk // is the same one it always was. Event::Start(Tag::Strong) => marks.strong += 1, Event::End(TagEnd::Strong) => marks.strong = marks.strong.saturating_sub(1), Event::Start(Tag::Emphasis) => marks.italic += 1, Event::End(TagEnd::Emphasis) => marks.italic = marks.italic.saturating_sub(1), Event::Start(Tag::Strikethrough) => marks.struck += 1, Event::End(TagEnd::Strikethrough) => marks.struck = marks.struck.saturating_sub(1), Event::Start(Tag::CodeBlock(_)) => marks.code_block += 1, // A line the author broke on purpose, and a line the source // wrapped: the author wrote one line there, so that one gives them // one line back rather than a break they never typed. Both carry // the marks in force, so a break inside `**...**` does not cut the // run in three over a character nobody can see the style of. Event::HardBreak => push(&mut runs, "\n", marks.emphasis(), blocks.current()), Event::SoftBreak => push(&mut runs, " ", marks.emphasis(), blocks.current()), // A prose block ends with a blank line after it, because two // paragraphs run together read as one sentence that does not parse. // // Every separator below is prose under no marks, and by // construction rather than by choice: it stands between two blocks // and belongs to neither, and the block it closes is closed before // it is written. That is also what keeps two adjacent items from // merging into one run. Event::End(TagEnd::CodeBlock) => { marks.code_block = marks.code_block.saturating_sub(1); push(&mut runs, "\n\n", Emphasis::default(), Block::Prose); } Event::End(TagEnd::Heading(_) | TagEnd::BlockQuote(_)) => { blocks.close(); push(&mut runs, "\n\n", Emphasis::default(), Block::Prose); } Event::End(TagEnd::Paragraph | TagEnd::List(_) | TagEnd::FootnoteDefinition) => { push(&mut runs, "\n\n", Emphasis::default(), Block::Prose); } // A member of a block ends a line and no more: a list is one item // per line, and a table is one row per line. Event::End(TagEnd::Item) => { blocks.close(); push(&mut runs, "\n", Emphasis::default(), Block::Prose); } Event::End(TagEnd::TableRow | TagEnd::TableHead) => { push(&mut runs, "\n", Emphasis::default(), Block::Prose); } // Cells sit in a row, so they want a separator rather than a break. Event::End(TagEnd::TableCell) => { push(&mut runs, "\t", Emphasis::default(), Block::Prose); } // A rule is a boundary with nothing to say. Event::Rule => push(&mut runs, "\n", Emphasis::default(), Block::Prose), // An image's alt text is the only thing here a reader can use, and // pulldown emits it as the tag's inner text, so the tag itself is // simply passed over. Event::Start(Tag::Image { .. }) | Event::End(TagEnd::Image) => {} // Markup, markers and references: nothing a reader would have read // aloud. Raw HTML included, and see the note above about why that // does not make this a sanitizer. _ => {} } } tidy(runs) } /// Append `text` under `emphasis` and `block`, joining the run before it when /// both match, so a paragraph with no emphasis in it stays one run. fn push(runs: &mut Vec, text: &str, emphasis: Emphasis, block: Block) { if text.is_empty() { return; } match runs.last_mut() { Some(last) if last.emphasis == emphasis && last.block == block => { last.text.push_str(text); } _ => runs.push(TextRun { text: text.to_string(), emphasis, block, }), } } /// Collapse what the walk above leaves behind: trailing spaces on a line, and /// runs of blank lines from nested blocks that each ended one. /// /// Nesting is the reason this is a pass rather than care taken inline. A list /// inside a blockquote ends three blocks at the same point and each one is /// right to end a line; only together are they wrong. /// /// It works a line at a time and a run can span several lines, so the first /// move is to cut the runs at every newline. What comes back out is re-joined /// and re-merged, so cutting costs nothing a caller can see. fn tidy(runs: Vec) -> Vec { let mut lines = split_lines(runs); for line in &mut lines { trim_end(line); } let mut kept: Vec> = Vec::new(); let mut blank_run = 0; for line in lines { if line.is_empty() { blank_run += 1; // One blank line separates two blocks. A second says nothing the // first did not, and neither does one before anything at all. if blank_run > 1 || kept.is_empty() { continue; } } else { blank_run = 0; } kept.push(line); } // The last block ended a line too, and there is nothing after it to // separate from. while kept.last().is_some_and(Vec::is_empty) { kept.pop(); } let mut out: Vec = Vec::new(); for (index, line) in kept.into_iter().enumerate() { if index > 0 { push(&mut out, "\n", Emphasis::default(), Block::Prose); } for run in line { push(&mut out, &run.text, run.emphasis, run.block); } } out } /// Cut `runs` at every newline, into one group of runs per line. fn split_lines(runs: Vec) -> Vec> { let mut lines: Vec> = vec![Vec::new()]; for run in runs { for (index, piece) in run.text.split('\n').enumerate() { if index > 0 { lines.push(Vec::new()); } if !piece.is_empty() { // A cut cannot make two adjacent runs mergeable that were not // already, so this only ever appends. let line = lines.last_mut().expect("a line was just pushed"); line.push(TextRun { text: piece.to_string(), emphasis: run.emphasis, block: run.block, }); } } } lines } /// Drop trailing whitespace from a line, across as many runs as it spans. fn trim_end(line: &mut Vec) { while let Some(last) = line.last_mut() { let trimmed = last.text.trim_end(); if trimmed.is_empty() { line.pop(); } else { last.text.truncate(trimmed.len()); break; } } } #[cfg(test)] mod tests { use super::{Block, Emphasis, render_plain, render_runs}; /// The marks over each run, for asserting shape without spelling out four /// booleans a run. fn marks(markdown: &str) -> Vec<(String, Emphasis)> { render_runs(markdown) .into_iter() .map(|run| (run.text, run.emphasis)) .collect() } /// The block each run sits in. fn blocks(markdown: &str) -> Vec<(String, Block)> { render_runs(markdown) .into_iter() .map(|run| (run.text, run.block)) .collect() } #[test] fn inline_syntax_becomes_the_words_it_wrapped() { assert_eq!( render_plain("**bold**, *italic*, `code` and ~~struck~~"), "bold, italic, code and struck" ); } #[test] fn a_link_keeps_its_text_and_drops_its_url() { // The preview case that motivated this: pter turns every anchor in an // HTML email into `[text](url)`, and a row showing the source shows // more URL than words. assert_eq!( render_plain("Read [the announcement](https://example.com/a/very/long/path)."), "Read the announcement." ); } #[test] fn blocks_are_separated_and_soft_wraps_are_not() { // A soft break is where the source wrapped, not where the author did. assert_eq!( render_plain("First paragraph\nwrapped in the source.\n\nSecond."), "First paragraph wrapped in the source.\n\nSecond." ); } #[test] fn a_hard_break_is_kept() { assert_eq!(render_plain("one \ntwo"), "one\ntwo"); } #[test] fn nested_blocks_do_not_pile_up_blank_lines() { // A list inside a quote ends three blocks in one place. Each is right // to end a line and together they would leave a hole. let md = "> - one\n> - two\n\nAfter."; assert_eq!(render_plain(md), "one\ntwo\n\nAfter."); } #[test] fn headings_are_their_text() { assert_eq!(render_plain("# Title\n\nBody."), "Title\n\nBody."); } #[test] fn a_list_is_one_item_per_line() { assert_eq!(render_plain("- one\n- two\n- three"), "one\ntwo\nthree"); } #[test] fn a_code_block_keeps_its_contents() { // Least readable thing in a preview, and dropping it would empty out a // message that is nothing but a snippet. assert_eq!(render_plain("```rust\nlet x = 1;\n```"), "let x = 1;"); } #[test] fn a_table_reads_as_cells_rather_than_pipes() { let md = "| a | b |\n|---|---|\n| 1 | 2 |"; assert_eq!(render_plain(md), "a\tb\n1\t2"); } #[test] fn raw_html_is_dropped_not_unescaped() { // An inline tag goes and the prose around it stays, which is the case // that matters: a sender's stray `` should not cost the sentence. assert_eq!( render_plain("before bold after"), "before bold after" ); // A whole block of raw HTML goes with its contents, because pulldown // hands it over as one opaque event and there is no text to pick out of // it. Worth knowing rather than worth fixing here: markdown that is // really an HTML document wants a converter, not this. assert_eq!(render_plain("
inside
"), ""); // And this is not a sanitizer. What comes back is text, and text put // into a document still has to be escaped by whoever puts it there. assert_eq!( render_plain("before after"), "before alert(1) after" ); } #[test] fn an_image_contributes_its_alt_text() { assert_eq!( render_plain("![a diagram](/img/d.png) explains it"), "a diagram explains it" ); } #[test] fn empty_in_empty_out() { assert_eq!(render_plain(""), ""); assert_eq!(render_plain("\n\n"), ""); assert_eq!(render_runs(""), Vec::new()); assert_eq!(render_runs("\n\n"), Vec::new()); } #[test] fn plain_text_survives_unchanged() { // The other half of the format question in a caller that has both: text // that is not markdown must come through as it was written. assert_eq!( render_plain("the *args and **kwargs conventions"), "the *args and **kwargs conventions" ); } #[test] fn runs_concatenate_to_the_plain_render() { // The invariant the two functions exist as one walk to keep. If this // ever fails, a caller that switched from one to the other has silently // changed what its rows say. let sources = [ "**bold**, *italic*, `code` and ~~struck~~", "# Title\n\nBody with **weight** in it.", "> - one\n> - **two**\n\nAfter.", "| a | **b** |\n|---|---|\n| 1 | 2 |", "```rust\nlet x = 1;\n```", "Read [the **announcement**](https://example.com).", "before bold after", "the *args and **kwargs conventions", "one \ntwo", "", ]; for source in sources { let joined: String = render_runs(source) .into_iter() .map(|run| run.text) .collect(); assert_eq!(joined, render_plain(source), "source: {source:?}"); } } #[test] fn each_mark_is_carried() { assert_eq!( marks("**b** *i* `c` ~~s~~"), vec![ ( "b".to_string(), Emphasis { strong: true, ..Emphasis::default() } ), (" ".to_string(), Emphasis::default()), ( "i".to_string(), Emphasis { italic: true, ..Emphasis::default() } ), (" ".to_string(), Emphasis::default()), ( "c".to_string(), Emphasis { code: true, ..Emphasis::default() } ), (" ".to_string(), Emphasis::default()), ( "s".to_string(), Emphasis { struck: true, ..Emphasis::default() } ), ] ); } #[test] fn marks_nest_and_combine() { // The counters earn themselves here: the inner mark closes and the // outer one is still in force over the text after it. assert_eq!( marks("**a *b* c**"), vec![ ( "a ".to_string(), Emphasis { strong: true, ..Emphasis::default() } ), ( "b".to_string(), Emphasis { strong: true, italic: true, ..Emphasis::default() } ), ( " c".to_string(), Emphasis { strong: true, ..Emphasis::default() } ), ] ); } #[test] fn a_code_span_inside_emphasis_carries_both() { assert_eq!( marks("**`cargo build`**"), vec![( "cargo build".to_string(), Emphasis { strong: true, code: true, ..Emphasis::default() } )] ); } #[test] fn a_code_block_is_marked_as_code() { let runs = render_runs("```rust\nlet x = 1;\n```"); assert_eq!(runs.len(), 1); assert!(runs[0].emphasis.code); assert_eq!(runs[0].text, "let x = 1;"); } #[test] fn adjacent_text_under_the_same_marks_is_one_run() { // A link's text and the prose around it are separate events and one // run: a caller styling per run should not see a seam where the source // had a bracket. assert_eq!( marks("Read [the docs](https://example.com) now."), vec![("Read the docs now.".to_string(), Emphasis::default())] ); } #[test] fn a_soft_break_inside_emphasis_does_not_cut_the_run() { // The space stands where the source wrapped, and giving it the marks in // force is what keeps `**a b**` one run rather than three. assert_eq!( marks("**one\ntwo**"), vec![( "one two".to_string(), Emphasis { strong: true, ..Emphasis::default() } )] ); } #[test] fn emphasis_survives_the_tidy_pass() { // The pass cuts runs at newlines to work a line at a time. What it // re-joins has to carry the same marks it took apart, and a list inside // a quote is where it does the most cutting. assert_eq!( marks("> - **one**\n> - two\n\nAfter."), vec![ ( "one".to_string(), Emphasis { strong: true, ..Emphasis::default() } ), ("\n".to_string(), Emphasis::default()), ("two".to_string(), Emphasis::default()), ("\n\nAfter.".to_string(), Emphasis::default()), ] ); } #[test] fn a_heading_says_which_level_it_was() { // The one this was written for: a destination with no markup can still // set one line heavier than the next, and until now it was not told // which line. assert_eq!( blocks("# Title\n\nBody.\n\n### Deeper"), vec![ ("Title".to_string(), Block::Heading(1)), ("\n\nBody.\n\n".to_string(), Block::Prose), ("Deeper".to_string(), Block::Heading(3)), ] ); } #[test] fn an_item_carries_no_marker_of_its_own() { // What a bullet looks like is the caller's answer, and a number would be // wrong for half of them. The separator between two items is prose, // which is also what keeps them from merging into one run. assert_eq!( blocks("- one\n- two"), vec![ ("one".to_string(), Block::Item), ("\n".to_string(), Block::Prose), ("two".to_string(), Block::Item), ] ); assert_eq!(render_plain("1. one\n2. two"), "one\ntwo"); } #[test] fn the_innermost_block_wins_and_the_outer_one_comes_back() { // An item inside a quote reads as an item: a caller with one line to // draw has to pick one of the two, and the inner one holds the words. // What the stack is for is the line after it, where the quote is still // open and prose would be wrong. assert_eq!( blocks("> - one\n>\n> after"), vec![ ("one".to_string(), Block::Item), ("\n\n".to_string(), Block::Prose), ("after".to_string(), Block::Quote), ] ); } #[test] fn a_block_role_does_not_disturb_the_words() { // The invariant again, over the sources that carry blocks. Adding a // reason to split a run must not add or drop a character. for source in [ "# Title\n\nBody.", "- one\n- two\n\nAfter.", "> quoted\n\nafter", "> - **one**\n> - two\n\nAfter.", "1. one\n2. two", ] { let joined: String = render_runs(source) .into_iter() .map(|run| run.text) .collect(); assert_eq!(joined, render_plain(source), "source: {source:?}"); } } #[test] fn a_line_ending_in_its_own_run_of_whitespace_loses_it() { // A row ends every cell with a tab, including the last one, so each // line ends in a run that is nothing but whitespace and `tidy` has to // drop the run rather than trim inside it. The marked cell next to it // is what makes the case worth a test: popping the wrong run would take // the emphasis with it. assert_eq!( marks("| a | **b** |\n|---|---|\n| 1 | 2 |"), vec![ ("a\t".to_string(), Emphasis::default()), ( "b".to_string(), Emphasis { strong: true, ..Emphasis::default() } ), ("\n1\t2".to_string(), Emphasis::default()), ] ); } }