max / docengine
- Co-Authored-By
- Claude Opus 5 (1M context) <noreply@anthropic.com>
6 files changed,
+474 insertions,
-3 deletions
| @@ -1,6 +1,6 @@ | |||
| 1 | 1 | [package] | |
| 2 | 2 | name = "docengine" | |
| 3 | - | version = "0.4.0" | |
| 3 | + | version = "0.5.0" | |
| 4 | 4 | edition = "2024" | |
| 5 | 5 | license = "MIT" | |
| 6 | 6 |
| @@ -21,8 +21,17 @@ | |||
| 21 | 21 | | **Chat** | Chat messages | N | N | N | Y | nofollow on links, inline tags only | | |
| 22 | 22 | | **Sanitize-only** | External HTML (RSS feeds) | -- | -- | -- | -- | Default ammonia, no markdown parsing | | |
| 23 | 23 | ||
| 24 | + | ## Plain text | |
| 25 | + | ||
| 26 | + | The presets all answer "how do I show this markdown". `render_plain` answers | |
| 27 | + | "what does this markdown say", for the places that cannot take markup at all: a | |
| 28 | + | list row's preview line, a notification body, a search snippet, an `alt` | |
| 29 | + | attribute. Block structure survives as newlines; emphasis, code spans and link | |
| 30 | + | URLs do not. It produces text, not safe HTML, so whatever puts that text in a | |
| 31 | + | document still escapes it. | |
| 32 | + | ||
| 24 | 33 | ```rust | |
| 25 | - | use docengine::{render_chat, render_permissive, render_standard, render_strict, sanitize_html}; | |
| 34 | + | use docengine::{render_chat, render_permissive, render_plain, render_standard, render_strict, sanitize_html}; | |
| 26 | 35 | ||
| 27 | 36 | // Convenience functions | |
| 28 | 37 | let html = render_permissive("# Hello\n\n**Bold** text"); | |
| @@ -31,6 +40,9 @@ | |||
| 31 | 40 | let html = render_chat("A chat message with a [link](https://example.com)"); | |
| 32 | 41 | let html = sanitize_html("<p>Pre-rendered</p><script>stripped</script>"); | |
| 33 | 42 | ||
| 43 | + | // Markdown to text, for a preview line rather than a document | |
| 44 | + | let text = render_plain("See **the [docs](https://example.com)**."); // "See the docs." | |
| 45 | + | ||
| 34 | 46 | // Builder pattern for custom configurations | |
| 35 | 47 | use docengine::{Renderer, SanitizePreset}; | |
| 36 | 48 |
| @@ -3,19 +3,25 @@ | |||
| 3 | 3 | //! Design + roadmap: maintainer wiki. | |
| 4 | 4 | //! <!-- wiki: docengine-overview --> | |
| 5 | 5 | //! | |
| 6 | - | //! Provides five rendering presets for different trust levels: | |
| 6 | + | //! Provides six rendering presets for different trust levels and shapes: | |
| 7 | 7 | //! - **Permissive** -- full GFM (tables, footnotes, images, raw HTML). For trusted content. | |
| 8 | 8 | //! - **Standard** -- GFM without images. For app text fields. | |
| 9 | 9 | //! - **Strict** -- no images, no raw HTML, dangerous scheme filtering, nofollow. For UGC. | |
| 10 | 10 | //! - **Chat** -- Strict, plus block structure flattened to inline. For chat messages. | |
| 11 | + | //! - **Phrase** -- Chat, minus paragraphs and links. For one line inside something else. | |
| 11 | 12 | //! - **Sanitize-only** -- ammonia cleaning without markdown parsing. For external HTML. | |
| 12 | 13 | //! | |
| 14 | + | //! [`render_plain`] goes the other way, to text rather than to markup, for the | |
| 15 | + | //! places that cannot take markup at all: a list row's preview, a notification, | |
| 16 | + | //! an `alt` attribute. | |
| 17 | + | //! | |
| 13 | 18 | //! Optional features add document loading, TOML frontmatter, @mention resolution, | |
| 14 | 19 | //! and quote attribution post-processing. | |
| 15 | 20 | ||
| 16 | 21 | #[cfg(any(feature = "mentions", test))] | |
| 17 | 22 | mod code_spans; | |
| 18 | 23 | mod escape; | |
| 24 | + | mod plain; | |
| 19 | 25 | mod render; | |
| 20 | 26 | mod sanitize; | |
| 21 | 27 | mod text; | |
| @@ -35,6 +41,7 @@ | |||
| 35 | 41 | mod quotes; | |
| 36 | 42 | ||
| 37 | 43 | // Re-export core types | |
| 44 | + | pub use plain::render_plain; | |
| 38 | 45 | pub use render::{RenderResult, Renderer}; | |
| 39 | 46 | pub use sanitize::SanitizePreset; | |
| 40 | 47 | pub use text::{extract_title, reading_time_minutes, strip_first_heading, word_count}; | |
| @@ -81,6 +88,16 @@ | |||
| 81 | 88 | Renderer::chat().render(markdown) | |
| 82 | 89 | } | |
| 83 | 90 | ||
| 91 | + | /// Render one line of prose: inline emphasis only, no blocks, no links. | |
| 92 | + | /// | |
| 93 | + | /// For markdown in a slot one line tall and usually already clickable: a list | |
| 94 | + | /// row's supporting text, a card subtitle, a control's label. The step between | |
| 95 | + | /// [`render_chat`] and [`render_plain`] -- it keeps the emphasis a reader can | |
| 96 | + | /// see at a glance and drops everything that needs room. | |
| 97 | + | pub fn render_phrase(markdown: &str) -> String { | |
| 98 | + | Renderer::phrase().render(markdown) | |
| 99 | + | } | |
| 100 | + | ||
| 84 | 101 | /// Sanitize HTML without markdown parsing. | |
| 85 | 102 | pub fn sanitize_html(html: &str) -> String { | |
| 86 | 103 | Renderer::sanitize_only().sanitize_html(html) |
| @@ -36,6 +36,7 @@ | |||
| 36 | 36 | smart_punctuation: bool, | |
| 37 | 37 | tasklists: bool, | |
| 38 | 38 | strip_images: bool, | |
| 39 | + | strip_links: bool, | |
| 39 | 40 | strip_raw_html: bool, | |
| 40 | 41 | dangerous_scheme_filter: bool, | |
| 41 | 42 | heading_ids: bool, | |
| @@ -91,6 +92,7 @@ | |||
| 91 | 92 | smart_punctuation: true, | |
| 92 | 93 | tasklists: true, | |
| 93 | 94 | strip_images: false, | |
| 95 | + | strip_links: false, | |
| 94 | 96 | strip_raw_html: false, | |
| 95 | 97 | dangerous_scheme_filter: false, | |
| 96 | 98 | heading_ids: false, | |
| @@ -109,6 +111,7 @@ | |||
| 109 | 111 | smart_punctuation: true, | |
| 110 | 112 | tasklists: true, | |
| 111 | 113 | strip_images: true, | |
| 114 | + | strip_links: false, | |
| 112 | 115 | strip_raw_html: false, | |
| 113 | 116 | dangerous_scheme_filter: false, | |
| 114 | 117 | heading_ids: false, | |
| @@ -127,6 +130,7 @@ | |||
| 127 | 130 | smart_punctuation: false, | |
| 128 | 131 | tasklists: false, | |
| 129 | 132 | strip_images: true, | |
| 133 | + | strip_links: false, | |
| 130 | 134 | strip_raw_html: true, | |
| 131 | 135 | dangerous_scheme_filter: true, | |
| 132 | 136 | heading_ids: false, | |
| @@ -162,6 +166,7 @@ | |||
| 162 | 166 | smart_punctuation: false, | |
| 163 | 167 | tasklists: false, | |
| 164 | 168 | strip_images: true, | |
| 169 | + | strip_links: false, | |
| 165 | 170 | strip_raw_html: true, | |
| 166 | 171 | dangerous_scheme_filter: true, | |
| 167 | 172 | heading_ids: false, | |
| @@ -170,6 +175,32 @@ | |||
| 170 | 175 | } | |
| 171 | 176 | } | |
| 172 | 177 | ||
| 178 | + | /// One line of prose: [`chat`](Self::chat) with paragraphs and links gone | |
| 179 | + | /// too. | |
| 180 | + | /// | |
| 181 | + | /// For markdown in a slot that is one line tall and usually already a | |
| 182 | + | /// target: a list row's supporting text, a card subtitle, a control's label, | |
| 183 | + | /// a notification title. `chat` keeps `<p>`, correctly, because a chat | |
| 184 | + | /// message *is* a paragraph; a row's second line is not, and a block there | |
| 185 | + | /// is structure the slot cannot hold. | |
| 186 | + | /// | |
| 187 | + | /// Links are stripped rather than neutralized, keeping their text. Those | |
| 188 | + | /// slots are usually clickable as a whole, so an anchor inside is a second | |
| 189 | + | /// target inside the first, and a URL is not followable from a summary | |
| 190 | + | /// anyway. A phrase somewhere genuinely not clickable can say | |
| 191 | + | /// `.with_strip_links(false)`. | |
| 192 | + | /// | |
| 193 | + | /// Emphasis, code spans and strikethrough survive, which is the whole | |
| 194 | + | /// reason to render rather than to flatten with | |
| 195 | + | /// [`render_plain`](crate::render_plain). | |
| 196 | + | pub fn phrase() -> Self { | |
| 197 | + | Self { | |
| 198 | + | strip_links: true, | |
| 199 | + | sanitize: SanitizePreset::Phrase, | |
| 200 | + | ..Self::chat() | |
| 201 | + | } | |
| 202 | + | } | |
| 203 | + | ||
| 173 | 204 | /// No markdown parsing, only ammonia sanitization. Suitable for HTML from | |
| 174 | 205 | /// external sources (RSS feeds). | |
| 175 | 206 | pub fn sanitize_only() -> Self { | |
| @@ -180,6 +211,7 @@ | |||
| 180 | 211 | smart_punctuation: false, | |
| 181 | 212 | tasklists: false, | |
| 182 | 213 | strip_images: false, | |
| 214 | + | strip_links: false, | |
| 183 | 215 | strip_raw_html: false, | |
| 184 | 216 | dangerous_scheme_filter: false, | |
| 185 | 217 | heading_ids: false, | |
| @@ -224,6 +256,23 @@ | |||
| 224 | 256 | self | |
| 225 | 257 | } | |
| 226 | 258 | ||
| 259 | + | /// Drop links, keeping the text they wrapped. | |
| 260 | + | /// | |
| 261 | + | /// For prose rendered somewhere that is already one target: a list row that | |
| 262 | + | /// selects something when clicked, a card, a button's own label. An anchor | |
| 263 | + | /// there is a second target inside the first, which is ambiguous to click | |
| 264 | + | /// and worse to reach by keyboard, and the URL is not followable from a | |
| 265 | + | /// one-line summary anyway. | |
| 266 | + | /// | |
| 267 | + | /// The text survives, the way alt text survives `with_strip_images`. This is | |
| 268 | + | /// not a security control: a dangerous scheme is | |
| 269 | + | /// `with_dangerous_scheme_filter`'s job and stays on regardless of this. | |
| 270 | + | #[must_use] | |
| 271 | + | pub fn with_strip_links(mut self, enabled: bool) -> Self { | |
| 272 | + | self.strip_links = enabled; | |
| 273 | + | self | |
| 274 | + | } | |
| 275 | + | ||
| 227 | 276 | #[must_use] | |
| 228 | 277 | pub fn with_strip_raw_html(mut self, enabled: bool) -> Self { | |
| 229 | 278 | self.strip_raw_html = enabled; | |
| @@ -322,12 +371,18 @@ | |||
| 322 | 371 | let parser = Parser::new_ext(input, options); | |
| 323 | 372 | ||
| 324 | 373 | let strip_images = self.strip_images; | |
| 374 | + | let strip_links = self.strip_links; | |
| 325 | 375 | let strip_raw_html = self.strip_raw_html; | |
| 326 | 376 | let scheme_filter = self.dangerous_scheme_filter; | |
| 327 | 377 | ||
| 328 | 378 | let filtered = parser.filter_map(move |event| match event { | |
| 329 | 379 | // Strip raw HTML events | |
| 330 | 380 | Event::Html(_) | Event::InlineHtml(_) if strip_raw_html => None, | |
| 381 | + | // Strip the anchor, keep what it wrapped, same as an image's alt. | |
| 382 | + | // Ahead of the scheme filter deliberately: with links stripped there | |
| 383 | + | // is no anchor left to neutralize, and running the filter first | |
| 384 | + | // would leave a dangerous link behind as a live `href="#"`. | |
| 385 | + | Event::Start(Tag::Link { .. }) | Event::End(TagEnd::Link) if strip_links => None, | |
| 331 | 386 | // Neutralize dangerous schemes on links | |
| 332 | 387 | Event::Start(Tag::Link { | |
| 333 | 388 | link_type, | |
| @@ -354,6 +409,13 @@ | |||
| 354 | 409 | Event::End(TagEnd::Item) if inline_only => { | |
| 355 | 410 | vec![Event::End(TagEnd::Item), Event::SoftBreak] | |
| 356 | 411 | } | |
| 412 | + | // Same for `</p>` wherever the sanitize preset unwraps that too | |
| 413 | + | // (`phrase` does, `chat` does not). Without it two paragraphs fuse | |
| 414 | + | // into one word across the join. Harmless where `p` is kept: it is | |
| 415 | + | // whitespace between two block tags. | |
| 416 | + | Event::End(TagEnd::Paragraph) if inline_only => { | |
| 417 | + | vec![Event::End(TagEnd::Paragraph), Event::SoftBreak] | |
| 418 | + | } | |
| 357 | 419 | other => vec![other], | |
| 358 | 420 | }); | |
| 359 | 421 | ||
| @@ -818,8 +880,122 @@ | |||
| 818 | 880 | assert!(!html.contains("<script>")); | |
| 819 | 881 | } | |
| 820 | 882 | ||
| 883 | + | // --- Phrase preset | |
| 884 | + | ||
| 885 | + | #[test] | |
| 886 | + | fn phrase_keeps_inline_emphasis() { | |
| 887 | + | let html = Renderer::phrase().render("**bold** *italic* `code()` ~~struck~~"); | |
| 888 | + | assert!(html.contains("<strong>bold</strong>"), "got: {html}"); | |
| 889 | + | assert!(html.contains("<em>italic</em>"), "got: {html}"); | |
| 890 | + | assert!(html.contains("<code>code()</code>"), "got: {html}"); | |
| 891 | + | assert!(html.contains("<del>struck</del>"), "got: {html}"); | |
| 892 | + | } | |
| 893 | + | ||
| 894 | + | #[test] | |
| 895 | + | fn phrase_carries_no_block_at_all() { | |
| 896 | + | // The difference from chat, which keeps `<p>` because a message is a | |
| 897 | + | // paragraph. A row's second line is not one. | |
| 898 | + | let html = Renderer::phrase().render("# Goal\n\n> ship it\n\n- one\n- two"); | |
| 899 | + | for block in ["<p", "<h1", "<blockquote", "<ul", "<li"] { | |
| 900 | + | assert!(!html.contains(block), "no {block} in a phrase: {html}"); | |
| 901 | + | } | |
| 902 | + | for word in ["Goal", "ship it", "one", "two"] { | |
| 903 | + | assert!(html.contains(word), "{word} survives: {html}"); | |
| 904 | + | } | |
| 905 | + | } | |
| 906 | + | ||
| 907 | + | #[test] | |
| 908 | + | fn phrase_paragraphs_do_not_fuse() { | |
| 909 | + | // Unwrapping `<p>` would run "first" and "second" together. | |
| 910 | + | let html = Renderer::phrase().render("first\n\nsecond"); | |
| 911 | + | assert!(!html.contains("firstsecond"), "got: {html}"); | |
| 912 | + | assert!(html.contains("first"), "got: {html}"); | |
| 913 | + | assert!(html.contains("second"), "got: {html}"); | |
| 914 | + | } | |
| 915 | + | ||
| 916 | + | #[test] | |
| 917 | + | fn phrase_drops_links_and_keeps_their_text() { | |
| 918 | + | let html = Renderer::phrase().render("see [the brief](https://example.com/long/path)"); | |
| 919 | + | assert!(!html.contains("<a "), "got: {html}"); | |
| 920 | + | assert!(!html.contains("example.com"), "got: {html}"); | |
| 921 | + | assert!(html.contains("see the brief"), "got: {html}"); | |
| 922 | + | } | |
| 923 | + | ||
| 924 | + | #[test] | |
| 925 | + | fn phrase_strips_images_and_raw_html() { | |
| 926 | + | let r = Renderer::phrase(); | |
| 927 | + | assert!( | |
| 928 | + | !r.render("") | |
| 929 | + | .contains("<img") | |
| 930 | + | ); | |
| 931 | + | let html = r.render("hi <script>alert(1)</script> <img src=x onerror=alert(1)>"); | |
| 932 | + | assert!(!html.contains("<script"), "got: {html}"); | |
| 933 | + | assert!(!html.contains("onerror"), "got: {html}"); | |
| 934 | + | } | |
| 935 | + | ||
| 936 | + | #[test] | |
| 937 | + | fn phrase_can_have_its_links_back() { | |
| 938 | + | // The one slot that is not itself a target can ask. | |
| 939 | + | let html = Renderer::phrase() | |
| 940 | + | .with_strip_links(false) | |
| 941 | + | .render("see [the brief](https://example.com)"); | |
| 942 | + | assert!(html.contains("<a "), "got: {html}"); | |
| 943 | + | assert!(html.contains("nofollow"), "got: {html}"); | |
| 944 | + | } | |
| 945 | + | ||
| 946 | + | #[test] | |
| 947 | + | fn phrase_empty_input() { | |
| 948 | + | assert_eq!(Renderer::phrase().render(""), ""); | |
| 949 | + | } | |
| 950 | + | ||
| 821 | 951 | // --- builder methods | |
| 822 | 952 | ||
| 953 | + | #[test] | |
| 954 | + | fn strip_links_keeps_the_text_and_drops_the_anchor() { | |
| 955 | + | // For prose in something already clickable: a row, a card, a label. | |
| 956 | + | let html = Renderer::chat() | |
| 957 | + | .with_strip_links(true) | |
| 958 | + | .render("see [the brief](https://example.com/a/long/path)"); | |
| 959 | + | assert!(!html.contains("<a "), "no anchor: {html}"); | |
| 960 | + | assert!(!html.contains("example.com"), "no href either: {html}"); | |
| 961 | + | assert!(html.contains("see the brief"), "the text survives: {html}"); | |
| 962 | + | } | |
| 963 | + | ||
| 964 | + | #[test] | |
| 965 | + | fn strip_links_leaves_other_inline_markup_alone() { | |
| 966 | + | let html = Renderer::chat() | |
| 967 | + | .with_strip_links(true) | |
| 968 | + | .render("**bold** and [a link](https://example.com) and `code`"); | |
| 969 | + | assert!(html.contains("<strong>bold</strong>"), "got: {html}"); | |
| 970 | + | assert!(html.contains("<code>code</code>"), "got: {html}"); | |
| 971 | + | assert!(html.contains("a link"), "got: {html}"); | |
| 972 | + | } | |
| 973 | + | ||
| 974 | + | #[test] | |
| 975 | + | fn strip_links_beats_the_scheme_filter_rather_than_racing_it() { | |
| 976 | + | // The scheme filter rewrites a dangerous link to `href="#"`, which is | |
| 977 | + | // still an anchor. With links stripped there should be no anchor at all, | |
| 978 | + | // which only holds if this arm runs first. | |
| 979 | + | let html = Renderer::chat() | |
| 980 | + | .with_strip_links(true) | |
| 981 | + | .render("[x](javascript:alert(1))"); | |
| 982 | + | assert!(!html.contains("<a "), "no anchor at all: {html}"); | |
| 983 | + | assert!(!html.contains("javascript:"), "got: {html}"); | |
| 984 | + | assert!(html.contains('x'), "the text survives: {html}"); | |
| 985 | + | } | |
| 986 | + | ||
| 987 | + | #[test] | |
| 988 | + | fn strip_links_is_off_by_default_in_every_preset() { | |
| 989 | + | for html in [ | |
| 990 | + | Renderer::permissive().render("[x](https://example.com)"), | |
| 991 | + | Renderer::standard().render("[x](https://example.com)"), | |
| 992 | + | Renderer::strict().render("[x](https://example.com)"), | |
| 993 | + | Renderer::chat().render("[x](https://example.com)"), | |
| 994 | + | ] { | |
| 995 | + | assert!(html.contains("<a "), "presets are unchanged: {html}"); | |
| 996 | + | } | |
| 997 | + | } | |
| 998 | + | ||
| 823 | 999 | #[test] | |
| 824 | 1000 | fn builder_override() { | |
| 825 | 1001 | let r = Renderer::strict().with_strip_images(false); |
| @@ -14,6 +14,11 @@ | |||
| 14 | 14 | /// messages, where block structure is not wanted and a link must never | |
| 15 | 15 | /// become anything the server has to fetch. | |
| 16 | 16 | Chat, | |
| 17 | + | /// [`Chat`](Self::Chat) with no `p`. For one line of prose sitting inside | |
| 18 | + | /// something else, where even a paragraph is more structure than the slot | |
| 19 | + | /// has room for. Whether links survive is `Renderer::with_strip_links`'s | |
| 20 | + | /// question, not this one. | |
| 21 | + | Phrase, | |
| 17 | 22 | } | |
| 18 | 23 | ||
| 19 | 24 | /// Tags a chat message may keep. Everything else is unwrapped to its text, so a | |
| @@ -22,6 +27,17 @@ | |||
| 22 | 27 | /// most a one-line message should carry. | |
| 23 | 28 | const CHAT_TAGS: [&str; 6] = ["a", "code", "del", "em", "p", "strong"]; | |
| 24 | 29 | ||
| 30 | + | /// Tags one line of prose may keep: [`CHAT_TAGS`] without `p`, because a phrase | |
| 31 | + | /// sits inside a row, a card subtitle or a label, and a paragraph inside one of | |
| 32 | + | /// those is a block the slot has no room for. | |
| 33 | + | /// | |
| 34 | + | /// `a` stays on the list even though `Renderer::phrase` drops every link before | |
| 35 | + | /// sanitizing. The two are different questions and keeping them apart is what | |
| 36 | + | /// makes the preset adjustable: this says what markup is *possible*, and | |
| 37 | + | /// `with_strip_links` is the policy on top of it, so the rare phrase that is not | |
| 38 | + | /// itself a click target can turn links back on and get them. | |
| 39 | + | const PHRASE_TAGS: [&str; 5] = ["a", "code", "del", "em", "strong"]; | |
| 40 | + | ||
| 25 | 41 | /// The single in-repo authority for permissive (creator long-form) sanitization. | |
| 26 | 42 | /// | |
| 27 | 43 | /// `ammonia::clean` is exactly `Builder::default().clean()`, so the XSS guarantee | |
| @@ -75,6 +91,16 @@ | |||
| 75 | 91 | b.link_rel(Some("noopener noreferrer nofollow")); | |
| 76 | 92 | b | |
| 77 | 93 | } | |
| 94 | + | SanitizePreset::Phrase => { | |
| 95 | + | let tags: std::collections::HashSet<&str> = PHRASE_TAGS.iter().copied().collect(); | |
| 96 | + | let mut b = ammonia::Builder::default(); | |
| 97 | + | b.tags(tags); | |
| 98 | + | // Belt and braces with the missing `a`: if one ever gets added | |
| 99 | + | // back, it arrives already carrying the same rel as everywhere | |
| 100 | + | // else rather than as the one unmarked link in the codebase. | |
| 101 | + | b.link_rel(Some("noopener noreferrer nofollow")); | |
| 102 | + | b | |
| 103 | + | } | |
| 78 | 104 | }; | |
| 79 | 105 | if allow_heading_ids { | |
| 80 | 106 | for tag in ["h1", "h2", "h3", "h4", "h5", "h6"] { |
| @@ -1,0 +1,240 @@ | |||
| 1 | + | //! Markdown to plain text. | |
| 2 | + | //! | |
| 3 | + | //! The other direction from the presets. They all answer "how do I show this | |
| 4 | + | //! markdown", and this answers "what does this markdown say", which is what a | |
| 5 | + | //! caller needs whenever it has markdown and a place that cannot take markup: | |
| 6 | + | //! a list row's preview line, a notification body, a search snippet, a window | |
| 7 | + | //! title, an `alt` attribute. | |
| 8 | + | //! | |
| 9 | + | //! Without it every such caller does the same wrong thing, which is to print | |
| 10 | + | //! the source and let the syntax through. `**bold**` in a preview is not | |
| 11 | + | //! emphasis and is not the word the author wrote either. | |
| 12 | + | ||
| 13 | + | use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd}; | |
| 14 | + | ||
| 15 | + | /// Render markdown as plain text: the words, without the syntax that shapes | |
| 16 | + | /// them. | |
| 17 | + | /// | |
| 18 | + | /// Block structure survives as newlines, because a preview that runs two | |
| 19 | + | /// paragraphs together reads as one sentence that does not parse. Inline | |
| 20 | + | /// structure does not survive at all: emphasis, code spans and link text all | |
| 21 | + | /// come through as their text, and a link's URL is dropped, since a reader | |
| 22 | + | /// looking at a one-line preview cannot follow it and it crowds out the words | |
| 23 | + | /// that would have told them whether they want to. | |
| 24 | + | /// | |
| 25 | + | /// Raw HTML is dropped rather than unescaped. This is not a sanitizer and must | |
| 26 | + | /// not be used as one: it produces text, and text put into a document still has | |
| 27 | + | /// to be escaped by whoever puts it there. | |
| 28 | + | /// | |
| 29 | + | /// The GFM options match the render presets, so a table or a task list | |
| 30 | + | /// contributes its cell and item text rather than its pipes and brackets. | |
| 31 | + | /// | |
| 32 | + | /// ``` | |
| 33 | + | /// let plain = docengine::render_plain("# Title\n\nSee **the [docs](https://example.com)**."); | |
| 34 | + | /// assert_eq!(plain, "Title\n\nSee the docs."); | |
| 35 | + | /// ``` | |
| 36 | + | pub fn render_plain(markdown: &str) -> String { | |
| 37 | + | if markdown.is_empty() { | |
| 38 | + | return String::new(); | |
| 39 | + | } | |
| 40 | + | ||
| 41 | + | let mut options = Options::empty(); | |
| 42 | + | options.insert(Options::ENABLE_TABLES); | |
| 43 | + | options.insert(Options::ENABLE_STRIKETHROUGH); | |
| 44 | + | options.insert(Options::ENABLE_FOOTNOTES); | |
| 45 | + | options.insert(Options::ENABLE_TASKLISTS); | |
| 46 | + | ||
| 47 | + | let mut out = String::with_capacity(markdown.len()); | |
| 48 | + | ||
| 49 | + | for event in Parser::new_ext(markdown, options) { | |
| 50 | + | match event { | |
| 51 | + | // A code span's text is the text. A code block's is too: it is | |
| 52 | + | // usually the least readable thing in a preview, and dropping it | |
| 53 | + | // would silently empty out a message that is nothing but a snippet. | |
| 54 | + | Event::Text(text) | Event::Code(text) => out.push_str(&text), | |
| 55 | + | ||
| 56 | + | // A line the author broke on purpose. | |
| 57 | + | Event::HardBreak => out.push('\n'), | |
| 58 | + | ||
| 59 | + | // A line the source wrapped. The author wrote one line, so this | |
| 60 | + | // gives them one line back rather than a break they never typed. | |
| 61 | + | Event::SoftBreak => out.push(' '), | |
| 62 | + | ||
| 63 | + | // A prose block ends with a blank line after it, because two | |
| 64 | + | // paragraphs run together read as one sentence that does not parse. | |
| 65 | + | Event::End( | |
| 66 | + | TagEnd::Paragraph | |
| 67 | + | | TagEnd::Heading(_) | |
| 68 | + | | TagEnd::CodeBlock | |
| 69 | + | | TagEnd::BlockQuote(_) | |
| 70 | + | | TagEnd::List(_) | |
| 71 | + | | TagEnd::FootnoteDefinition, | |
| 72 | + | ) => out.push_str("\n\n"), | |
| 73 | + | ||
| 74 | + | // A member of a block ends a line and no more: a list is one item | |
| 75 | + | // per line, and a table is one row per line. | |
| 76 | + | Event::End(TagEnd::Item | TagEnd::TableRow | TagEnd::TableHead) => out.push('\n'), | |
| 77 | + | ||
| 78 | + | // Cells sit in a row, so they want a separator rather than a break. | |
| 79 | + | Event::End(TagEnd::TableCell) => out.push('\t'), | |
| 80 | + | ||
| 81 | + | // A rule is a boundary with nothing to say. | |
| 82 | + | Event::Rule => out.push('\n'), | |
| 83 | + | ||
| 84 | + | // An image's alt text is the only thing here a reader can use, and | |
| 85 | + | // pulldown emits it as the tag's inner text, so the tag itself is | |
| 86 | + | // simply passed over. | |
| 87 | + | Event::Start(Tag::Image { .. }) | Event::End(TagEnd::Image) => {} | |
| 88 | + | ||
| 89 | + | // Markup, markers and references: nothing a reader would have read | |
| 90 | + | // aloud. Raw HTML included, and see the note above about why that | |
| 91 | + | // does not make this a sanitizer. | |
| 92 | + | _ => {} | |
| 93 | + | } | |
| 94 | + | } | |
| 95 | + | ||
| 96 | + | tidy(&out) | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | /// Collapse what the walk above leaves behind: trailing spaces on a line, and | |
| 100 | + | /// runs of blank lines from nested blocks that each ended one. | |
| 101 | + | /// | |
| 102 | + | /// Nesting is the reason this is a pass rather than care taken inline. A list | |
| 103 | + | /// inside a blockquote ends three blocks at the same point and each one is | |
| 104 | + | /// right to end a line; only together are they wrong. | |
| 105 | + | fn tidy(text: &str) -> String { | |
| 106 | + | let mut out = String::with_capacity(text.len()); | |
| 107 | + | let mut blank_run = 0; | |
| 108 | + | ||
| 109 | + | for line in text.lines() { | |
| 110 | + | let line = line.trim_end(); | |
| 111 | + | if line.is_empty() { | |
| 112 | + | blank_run += 1; | |
| 113 | + | // One blank line separates two blocks. A second says nothing the | |
| 114 | + | // first did not. | |
| 115 | + | if blank_run > 1 || out.is_empty() { | |
| 116 | + | continue; | |
| 117 | + | } | |
| 118 | + | } else { | |
| 119 | + | blank_run = 0; | |
| 120 | + | } | |
| 121 | + | out.push_str(line); | |
| 122 | + | out.push('\n'); | |
| 123 | + | } | |
| 124 | + | ||
| 125 | + | out.trim_end().to_string() | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | #[cfg(test)] | |
| 129 | + | mod tests { | |
| 130 | + | use super::render_plain; | |
| 131 | + | ||
| 132 | + | #[test] | |
| 133 | + | fn inline_syntax_becomes_the_words_it_wrapped() { | |
| 134 | + | assert_eq!( | |
| 135 | + | render_plain("**bold**, *italic*, `code` and ~~struck~~"), | |
| 136 | + | "bold, italic, code and struck" | |
| 137 | + | ); | |
| 138 | + | } | |
| 139 | + | ||
| 140 | + | #[test] | |
| 141 | + | fn a_link_keeps_its_text_and_drops_its_url() { | |
| 142 | + | // The preview case that motivated this: pter turns every anchor in an | |
| 143 | + | // HTML email into `[text](url)`, and a row showing the source shows | |
| 144 | + | // more URL than words. | |
| 145 | + | assert_eq!( | |
| 146 | + | render_plain("Read [the announcement](https://example.com/a/very/long/path)."), | |
| 147 | + | "Read the announcement." | |
| 148 | + | ); | |
| 149 | + | } | |
| 150 | + | ||
| 151 | + | #[test] | |
| 152 | + | fn blocks_are_separated_and_soft_wraps_are_not() { | |
| 153 | + | // A soft break is where the source wrapped, not where the author did. | |
| 154 | + | assert_eq!( | |
| 155 | + | render_plain("First paragraph\nwrapped in the source.\n\nSecond."), | |
| 156 | + | "First paragraph wrapped in the source.\n\nSecond." | |
| 157 | + | ); | |
| 158 | + | } | |
| 159 | + | ||
| 160 | + | #[test] | |
| 161 | + | fn a_hard_break_is_kept() { | |
| 162 | + | assert_eq!(render_plain("one \ntwo"), "one\ntwo"); | |
| 163 | + | } | |
| 164 | + | ||
| 165 | + | #[test] | |
| 166 | + | fn nested_blocks_do_not_pile_up_blank_lines() { | |
| 167 | + | // A list inside a quote ends three blocks in one place. Each is right | |
| 168 | + | // to end a line and together they would leave a hole. | |
| 169 | + | let md = "> - one\n> - two\n\nAfter."; | |
| 170 | + | assert_eq!(render_plain(md), "one\ntwo\n\nAfter."); | |
| 171 | + | } | |
| 172 | + | ||
| 173 | + | #[test] | |
| 174 | + | fn headings_are_their_text() { | |
| 175 | + | assert_eq!(render_plain("# Title\n\nBody."), "Title\n\nBody."); | |
| 176 | + | } | |
| 177 | + | ||
| 178 | + | #[test] | |
| 179 | + | fn a_list_is_one_item_per_line() { | |
| 180 | + | assert_eq!(render_plain("- one\n- two\n- three"), "one\ntwo\nthree"); | |
| 181 | + | } | |
| 182 | + | ||
| 183 | + | #[test] | |
| 184 | + | fn a_code_block_keeps_its_contents() { | |
| 185 | + | // Least readable thing in a preview, and dropping it would empty out a | |
| 186 | + | // message that is nothing but a snippet. | |
| 187 | + | assert_eq!(render_plain("```rust\nlet x = 1;\n```"), "let x = 1;"); | |
| 188 | + | } | |
| 189 | + | ||
| 190 | + | #[test] | |
| 191 | + | fn a_table_reads_as_cells_rather_than_pipes() { | |
| 192 | + | let md = "| a | b |\n|---|---|\n| 1 | 2 |"; | |
| 193 | + | assert_eq!(render_plain(md), "a\tb\n1\t2"); | |
| 194 | + | } | |
| 195 | + | ||
| 196 | + | #[test] | |
| 197 | + | fn raw_html_is_dropped_not_unescaped() { | |
| 198 | + | // An inline tag goes and the prose around it stays, which is the case | |
| 199 | + | // that matters: a sender's stray `<b>` should not cost the sentence. | |
| 200 | + | assert_eq!( | |
| 201 | + | render_plain("before <b>bold</b> after"), | |
| 202 | + | "before bold after" | |
| 203 | + | ); | |
| 204 | + | // A whole block of raw HTML goes with its contents, because pulldown | |
| 205 | + | // hands it over as one opaque event and there is no text to pick out of | |
| 206 | + | // it. Worth knowing rather than worth fixing here: markdown that is | |
| 207 | + | // really an HTML document wants a converter, not this. | |
| 208 | + | assert_eq!(render_plain("<div>inside</div>"), ""); | |
| 209 | + | // And this is not a sanitizer. What comes back is text, and text put | |
| 210 | + | // into a document still has to be escaped by whoever puts it there. | |
| 211 | + | assert_eq!( | |
| 212 | + | render_plain("before <script>alert(1)</script> after"), | |
| 213 | + | "before alert(1) after" | |
| 214 | + | ); | |
| 215 | + | } | |
| 216 | + | ||
| 217 | + | #[test] | |
| 218 | + | fn an_image_contributes_its_alt_text() { | |
| 219 | + | assert_eq!( | |
| 220 | + | render_plain(" explains it"), | |
| 221 | + | "a diagram explains it" | |
| 222 | + | ); | |
| 223 | + | } | |
| 224 | + | ||
| 225 | + | #[test] | |
| 226 | + | fn empty_in_empty_out() { | |
| 227 | + | assert_eq!(render_plain(""), ""); | |
| 228 | + | assert_eq!(render_plain("\n\n"), ""); | |
| 229 | + | } | |
| 230 | + | ||
| 231 | + | #[test] | |
| 232 | + | fn plain_text_survives_unchanged() { | |
| 233 | + | // The other half of the format question in a caller that has both: text | |
| 234 | + | // that is not markdown must come through as it was written. | |
| 235 | + | assert_eq!( | |
| 236 | + | render_plain("the *args and **kwargs conventions"), | |
| 237 | + | "the *args and **kwargs conventions" | |
| 238 | + | ); | |
| 239 | + | } | |
| 240 | + | } |