max / pter

23.1 KB · 780 lines History Blame Raw

1	use scraper::node::Node;
2	use scraper::{ElementRef, Html};
3
4	use crate::elements::{self, BlockKind, ElementAction, InlineKind};
5	use crate::replies;
6	use crate::tables;
7	use crate::whitespace;
8
9	/// Convert an HTML email body into readable markdown.
10	///
11	/// This is the main entry point for pter. Pass in an HTML string
12	/// (just the body, not MIME structure) and get back clean markdown.
13	///
14	/// ```
15	/// let md = pter::convert("<p>Hello <strong>world</strong></p>");
16	/// assert_eq!(md, "Hello world");
17	/// ```
18	pub fn convert(html: &str) -> String {
19	if html.is_empty() {
20	return String::new();
21	}
22
23	let document = Html::parse_document(html);
24	let mut ctx = Context::new();
25	walk_children(document.root_element(), &mut ctx);
26	whitespace::normalize(&ctx.output)
27	}
28
29	/// Maximum DOM nesting depth walked before children are dropped. html5ever
30	/// imposes no depth limit of its own, so a flat but deeply-nested body (e.g.
31	/// `<div>`×500k) would otherwise recurse `walk_children`/`handle_element` once
32	/// per level and overflow the stack — an uncatchable `abort()`. 512 is the
33	/// browser-conventional nesting cap; no legitimate email nests remotely that
34	/// deep, and 512 frames stay well within even a 2 MB worker stack.
35	const MAX_DEPTH: u32 = 512;
36
37	/// Conversion state threaded through the tree walk.
38	struct Context {
39	output: String,
40	/// Current DOM recursion depth (guards against pathological nesting).
41	depth: u32,
42	/// Current list nesting depth (for indentation).
43	list_depth: u32,
44	/// Whether we're inside a <pre> block (preserve whitespace).
45	in_pre: bool,
46	/// Whether we're inside an <a> tag (don't nest links).
47	in_link: bool,
48	/// Stack of list types for proper ordered/unordered rendering.
49	list_stack: Vec<ListType>,
50	}
51
52	#[derive(Clone, Copy)]
53	enum ListType {
54	Unordered,
55	Ordered(u32), // current item number
56	}
57
58	impl Context {
59	fn new() -> Self {
60	Self {
61	output: String::with_capacity(4096),
62	depth: 0,
63	list_depth: 0,
64	in_pre: false,
65	in_link: false,
66	list_stack: Vec::new(),
67	}
68	}
69
70	fn push(&mut self, s: &str) {
71	self.output.push_str(s);
72	}
73
74	fn push_char(&mut self, c: char) {
75	self.output.push(c);
76	}
77
78	fn ensure_blank_line(&mut self) {
79	let trimmed = self.output.trim_end_matches(' ');
80	if trimmed.is_empty() {
81	return;
82	}
83	if trimmed.ends_with("\n\n") {
84	return;
85	}
86	self.output.truncate(trimmed.len());
87	self.output.push_str("\n\n");
88	}
89
90	fn ensure_newline(&mut self) {
91	if !self.output.is_empty() && !self.output.ends_with('\n') {
92	self.output.push('\n');
93	}
94	}
95
96	fn list_indent(&self) -> String {
97	if self.list_depth <= 1 {
98	return String::new();
99	}
100	" ".repeat((self.list_depth - 1) as usize)
101	}
102	}
103
104	/// Walk all children of a node, converting each to markdown.
105	fn walk_children(parent: ElementRef, ctx: &mut Context) {
106	// Bound recursion depth. Beyond MAX_DEPTH we stop descending — the content
107	// is either pathological or hostile, and dropping it is vastly preferable to
108	// a stack-overflow abort that would wedge the caller (a hostile email would
109	// otherwise re-crash on every subsequent sync).
110	if ctx.depth >= MAX_DEPTH {
111	return;
112	}
113	ctx.depth += 1;
114	for child in parent.children() {
115	match child.value() {
116	Node::Text(text) => {
117	handle_text(&text.text, ctx);
118	}
119	Node::Element(_) => {
120	if let Some(el_ref) = ElementRef::wrap(child) {
121	handle_element(el_ref, ctx);
122	}
123	}
124	_ => {}
125	}
126	}
127	ctx.depth -= 1;
128	}
129
130	/// Handle a text node.
131	fn handle_text(text: &str, ctx: &mut Context) {
132	if ctx.in_pre {
133	ctx.push(text);
134	return;
135	}
136
137	// Collapse whitespace in normal flow
138	let mut last_was_space = ctx.output.ends_with(' ') \|\| ctx.output.ends_with('\n');
139	for ch in text.chars() {
140	if ch.is_ascii_whitespace() {
141	if !last_was_space {
142	ctx.push_char(' ');
143	last_was_space = true;
144	}
145	} else {
146	ctx.push_char(ch);
147	last_was_space = false;
148	}
149	}
150	}
151
152	/// Handle an element node — classify it and render accordingly.
153	fn handle_element(el: ElementRef, ctx: &mut Context) {
154	let element = el.value();
155
156	// Check hidden elements
157	if elements::is_hidden(element) {
158	return;
159	}
160
161	// Check for reply boundaries before normal classification.
162	// Reply boundaries (gmail_quote, type=cite, etc.) get rendered
163	// as blockquotes regardless of their actual element type.
164	if replies::is_reply_boundary(el) {
165	render_reply_block(el, ctx);
166	return;
167	}
168
169	// Check for Outlook-style "From: ... Sent: ..." separator blocks.
170	// These introduce quoted content that follows them.
171	if replies::is_outlook_separator(el) {
172	ctx.ensure_blank_line();
173	// Render the separator header as attribution
174	let text: String = el.text().collect();
175	let trimmed = text.split_whitespace().collect::<Vec<_>>().join(" ");
176	ctx.push(&trimmed);
177	ctx.ensure_blank_line();
178	return;
179	}
180
181	match elements::classify(element) {
182	ElementAction::Skip => {}
183	ElementAction::Transparent => walk_children(el, ctx),
184	ElementAction::Block(kind) => handle_block(el, ctx, kind),
185	ElementAction::Inline(kind) => handle_inline(el, ctx, kind),
186	}
187	}
188
189	fn handle_block(el: ElementRef, ctx: &mut Context, kind: BlockKind) {
190	match kind {
191	BlockKind::Paragraph => {
192	ctx.ensure_blank_line();
193	walk_children(el, ctx);
194	ctx.ensure_blank_line();
195	}
196
197	BlockKind::Heading(level) => {
198	ctx.ensure_blank_line();
199	let prefix = "#".repeat(level as usize);
200	ctx.push(&prefix);
201	ctx.push_char(' ');
202	walk_children(el, ctx);
203	ctx.ensure_blank_line();
204	}
205
206	BlockKind::Blockquote => {
207	ctx.ensure_blank_line();
208	// Render children into a temporary buffer, then prefix each line with >
209	let mut inner_ctx = Context::new();
210	inner_ctx.in_pre = ctx.in_pre;
211	inner_ctx.in_link = ctx.in_link;
212	walk_children(el, &mut inner_ctx);
213	let inner = whitespace::normalize(&inner_ctx.output);
214	for line in inner.lines() {
215	ctx.push("> ");
216	ctx.push(line);
217	ctx.push_char('\n');
218	}
219	ctx.push_char('\n');
220	}
221
222	BlockKind::UnorderedList => {
223	ctx.ensure_blank_line();
224	ctx.list_depth += 1;
225	ctx.list_stack.push(ListType::Unordered);
226	walk_children(el, ctx);
227	ctx.list_stack.pop();
228	ctx.list_depth -= 1;
229	ctx.ensure_blank_line();
230	}
231
232	BlockKind::OrderedList => {
233	ctx.ensure_blank_line();
234	ctx.list_depth += 1;
235	ctx.list_stack.push(ListType::Ordered(0));
236	walk_children(el, ctx);
237	ctx.list_stack.pop();
238	ctx.list_depth -= 1;
239	ctx.ensure_blank_line();
240	}
241
242	BlockKind::ListItem => {
243	ctx.ensure_newline();
244	let indent = ctx.list_indent();
245	ctx.push(&indent);
246
247	// Determine bullet or number
248	let marker = match ctx.list_stack.last_mut() {
249	Some(ListType::Unordered) => "- ".to_string(),
250	Some(ListType::Ordered(n)) => {
251	*n += 1;
252	format!("{}. ", *n)
253	}
254	None => "- ".to_string(),
255	};
256	ctx.push(&marker);
257	walk_children(el, ctx);
258	ctx.ensure_newline();
259	}
260
261	BlockKind::PreFormatted => {
262	ctx.ensure_blank_line();
263	ctx.push("```\n");
264	ctx.in_pre = true;
265	walk_children(el, ctx);
266	ctx.in_pre = false;
267	ctx.ensure_newline();
268	ctx.push("```");
269	ctx.ensure_blank_line();
270	}
271
272	BlockKind::HorizontalRule => {
273	ctx.ensure_blank_line();
274	ctx.push("---");
275	ctx.ensure_blank_line();
276	}
277
278	BlockKind::Table => {
279	ctx.ensure_blank_line();
280	if tables::is_data_table(el) {
281	let (headers, rows) = tables::extract_table_data(el);
282	let md = tables::render_markdown_table(&headers, &rows);
283	if !md.is_empty() {
284	ctx.push(&md);
285	}
286	} else {
287	// Layout table — unwrap and render cell contents directly
288	render_layout_table(el, ctx);
289	}
290	ctx.ensure_blank_line();
291	}
292
293	BlockKind::Div => {
294	// Divs act as block separators but don't add their own markup
295	ctx.ensure_blank_line();
296	walk_children(el, ctx);
297	ctx.ensure_blank_line();
298	}
299	}
300	}
301
302	fn handle_inline(el: ElementRef, ctx: &mut Context, kind: InlineKind) {
303	match kind {
304	InlineKind::Bold => {
305	ctx.push("**");
306	walk_children(el, ctx);
307	ctx.push("**");
308	}
309
310	InlineKind::Italic => {
311	ctx.push("*");
312	walk_children(el, ctx);
313	ctx.push("*");
314	}
315
316	InlineKind::Strikethrough => {
317	ctx.push("~~");
318	walk_children(el, ctx);
319	ctx.push("~~");
320	}
321
322	InlineKind::Code => {
323	if ctx.in_pre {
324	// Inside a <pre>, don't double-wrap
325	walk_children(el, ctx);
326	} else {
327	ctx.push("`");
328	walk_children(el, ctx);
329	ctx.push("`");
330	}
331	}
332
333	InlineKind::Link => {
334	if ctx.in_link {
335	// Don't nest links
336	walk_children(el, ctx);
337	return;
338	}
339
340	let href = el.value().attr("href").unwrap_or("");
341
342	if href.is_empty() \|\| href == "#" {
343	walk_children(el, ctx);
344	return;
345	}
346
347	// Collect the link text
348	let mut text_ctx = Context::new();
349	text_ctx.in_link = true;
350	walk_children(el, &mut text_ctx);
351	let text = text_ctx.output.trim().to_string();
352
353	if text.is_empty() {
354	// Link with no text — just show the URL
355	ctx.push(href);
356	} else if text == href {
357	// Link text matches URL — no need for markdown link syntax
358	ctx.push(href);
359	} else {
360	ctx.push("[");
361	ctx.push(&text);
362	ctx.push("](");
363	ctx.push(href);
364	ctx.push(")");
365	}
366	}
367
368	InlineKind::Image => {
369	let element = el.value();
370	if elements::is_tracking_pixel(element) {
371	return;
372	}
373
374	let alt = element.attr("alt").unwrap_or("");
375	let src = element.attr("src").unwrap_or("");
376
377	if src.is_empty() {
378	return;
379	}
380
381	ctx.push("![");
382	ctx.push(alt);
383	ctx.push("](");
384	ctx.push(src);
385	ctx.push(")");
386	}
387
388	InlineKind::LineBreak => {
389	ctx.push_char('\n');
390	}
391
392	InlineKind::Superscript => {
393	ctx.push("^");
394	walk_children(el, ctx);
395	}
396
397	InlineKind::Subscript => {
398	ctx.push("~");
399	walk_children(el, ctx);
400	}
401	}
402	}
403
404	/// Render a reply boundary as a quoted block.
405	///
406	/// This is the same rendering logic as `<blockquote>` — children are
407	/// rendered into a temporary buffer and each line gets `> ` prefixed.
408	/// Attribution lines (e.g. "On ... wrote:") are rendered above the quote.
409	fn render_reply_block(el: ElementRef, ctx: &mut Context) {
410	ctx.ensure_blank_line();
411
412	// Look for attribution text
413	if let Some(attribution) = replies::find_attribution(el) {
414	ctx.push(&attribution);
415	ctx.push_char('\n');
416	}
417
418	// Render children into temp buffer, then prefix with >
419	let mut inner_ctx = Context::new();
420	inner_ctx.in_pre = ctx.in_pre;
421	inner_ctx.in_link = ctx.in_link;
422	walk_children(el, &mut inner_ctx);
423	let inner = whitespace::normalize(&inner_ctx.output);
424
425	if !inner.is_empty() {
426	for line in inner.lines() {
427	ctx.push("> ");
428	ctx.push(line);
429	ctx.push_char('\n');
430	}
431	ctx.push_char('\n');
432	}
433	}
434
435	/// Unwrap a layout table by rendering cell contents sequentially.
436	///
437	/// Walks through rows and cells, rendering each cell's content as if
438	/// the table wrapper didn't exist. This handles the common email pattern
439	/// of wrapping everything in `<table><tr><td>...</td></tr></table>`.
440	fn render_layout_table(table: ElementRef, ctx: &mut Context) {
441	for descendant in table.descendants() {
442	if let Some(el_ref) = ElementRef::wrap(descendant) {
443	let name = el_ref.value().name();
444	if name == "td" \|\| name == "th" {
445	// Check if the cell itself is hidden
446	if !elements::is_hidden(el_ref.value()) {
447	walk_children(el_ref, ctx);
448	ctx.ensure_blank_line();
449	}
450	}
451	}
452	}
453	}
454
455	#[cfg(test)]
456	mod tests {
457	use super::*;
458
459	// -- Basic elements --
460
461	#[test]
462	fn empty_input() {
463	assert_eq!(convert(""), "");
464	}
465
466	#[test]
467	fn plain_text() {
468	assert_eq!(convert("hello world"), "hello world");
469	}
470
471	#[test]
472	fn pathological_nesting_does_not_overflow() {
473	// A flat but deeply-nested body (far past MAX_DEPTH) must not recurse
474	// the tree walk into a stack-overflow abort — it returns bounded output.
475	// Guards the hostile-email DoS: without the depth cap this aborts the
476	// process and every subsequent sync re-crashes on the same message.
477	let n = (MAX_DEPTH as usize) + 5_000;
478	let deep = format!("text{}{}", "<div>".repeat(n), "</div>".repeat(n));
479	let md = convert(&deep);
480	// Reaching this line at all proves no stack-overflow abort; the shallow
481	// "text" (depth 1, within the cap) still renders.
482	assert!(md.contains("text"), "content within the cap still renders");
483	}
484
485	#[test]
486	fn paragraph() {
487	assert_eq!(convert("<p>one</p><p>two</p>"), "one\n\ntwo");
488	}
489
490	#[test]
491	fn headings() {
492	assert_eq!(convert("<h1>Title</h1>"), "# Title");
493	assert_eq!(convert("<h3>Sub</h3>"), "### Sub");
494	}
495
496	#[test]
497	fn bold_and_italic() {
498	assert_eq!(
499	convert("<p><strong>bold</strong> and <em>italic</em></p>"),
500	"bold and italic"
501	);
502	}
503
504	#[test]
505	fn link() {
506	assert_eq!(
507	convert(r#"<a href="https://example.com">click</a>"#),
508	"[click](https://example.com)"
509	);
510	}
511
512	#[test]
513	fn link_text_matches_url() {
514	assert_eq!(
515	convert(r#"<a href="https://example.com">https://example.com</a>"#),
516	"https://example.com"
517	);
518	}
519
520	#[test]
521	fn link_empty_href() {
522	assert_eq!(convert(r#"<a href="">click</a>"#), "click");
523	}
524
525	#[test]
526	fn image() {
527	assert_eq!(
528	convert(r#"<img src="photo.jpg" alt="A photo">"#),
529	"![A photo](photo.jpg)"
530	);
531	}
532
533	#[test]
534	fn tracking_pixel_skipped() {
535	assert_eq!(convert(r#"<img src="track.gif" width="1" height="1">"#), "");
536	}
537
538	#[test]
539	fn unordered_list() {
540	assert_eq!(
541	convert("<ul><li>one</li><li>two</li></ul>"),
542	"- one\n- two"
543	);
544	}
545
546	#[test]
547	fn ordered_list() {
548	assert_eq!(
549	convert("<ol><li>first</li><li>second</li></ol>"),
550	"1. first\n2. second"
551	);
552	}
553
554	#[test]
555	fn nested_list() {
556	let html = "<ul><li>outer<ul><li>inner</li></ul></li></ul>";
557	let md = convert(html);
558	assert!(md.contains("- outer"));
559	assert!(md.contains(" - inner"));
560	}
561
562	#[test]
563	fn nested_list_exact_indent_depth_2() {
564	// At depth 2, `list_indent` returns `" "` (exactly two spaces, one indent level).
565	// Catches `list_indent` mutations:
566	// - `(depth - 1)` → `(depth + 1)`: would produce 3 indent levels (6 spaces).
567	// - `(depth - 1)` → `(depth / 1)`: would produce 2 indent levels (4 spaces).
568	// Either makes this exact-match assertion fail.
569	// (The converter emits a blank line before each nested list — that's a
570	// separate stylistic question; the indent is what we're pinning down here.)
571	assert_eq!(
572	convert("<ul><li>A<ul><li>B</li></ul></li></ul>"),
573	"- A\n\n - B"
574	);
575	}
576
577	#[test]
578	fn triple_nested_list_exact_indent_depth_3() {
579	// At depth 3, indent is exactly `" "` (four spaces).
580	assert_eq!(
581	convert("<ul><li>A<ul><li>B<ul><li>C</li></ul></li></ul></li></ul>"),
582	"- A\n\n - B\n\n - C"
583	);
584	}
585
586	#[test]
587	fn sibling_top_level_lists_have_no_indent_after_nesting() {
588	// After a nested <ul> closes, `list_depth -= 1` must execute to return
589	// to outer scope. If mutated to `+= 1` or `/= 1`, list_depth stays
590	// elevated and the SECOND top-level list ends up incorrectly indented.
591	let md = convert(
592	"<ul><li>A<ul><li>B</li></ul></li></ul><ul><li>C</li></ul>",
593	);
594	// The second list's "C" item must appear at column 0, not indented.
595	// We check the exact substring "\n- C" (newline then no leading whitespace).
596	assert!(
597	md.contains("\n- C"),
598	"second top-level list must not be indented after a nested list closes; got: {md:?}"
599	);
600	// And explicitly: it must NOT appear with leading spaces.
601	assert!(
602	!md.contains("\n - C"),
603	"second list incorrectly indented; got: {md:?}"
604	);
605	}
606
607	#[test]
608	fn ordered_list_decrements_depth_after_nesting() {
609	// Same shape but with <ol> — exercises the L218 `-= 1` mutation in the
610	// OrderedList block, distinct from UnorderedList's L208.
611	let md = convert(
612	"<ol><li>A<ol><li>B</li></ol></li></ol><ol><li>C</li></ol>",
613	);
614	assert!(md.contains("\n1. C"), "second ol must restart at depth 1: {md:?}");
615	assert!(!md.contains("\n 1. C"), "second ol indented incorrectly: {md:?}");
616	}
617
618	#[test]
619	fn blockquote() {
620	assert_eq!(convert("<blockquote>quoted text</blockquote>"), "> quoted text");
621	}
622
623	#[test]
624	fn nested_blockquote() {
625	let html = "<blockquote>outer<blockquote>inner</blockquote></blockquote>";
626	let md = convert(html);
627	assert!(md.contains("> outer"));
628	assert!(md.contains("> > inner"));
629	}
630
631	#[test]
632	fn preformatted() {
633	let html = "<pre><code>fn main() {\n println!(\"hi\");\n}</code></pre>";
634	let md = convert(html);
635	assert!(md.starts_with("```\n"));
636	assert!(md.contains("fn main()"));
637	assert!(md.ends_with("\n```"));
638	}
639
640	#[test]
641	fn horizontal_rule() {
642	assert_eq!(convert("<p>above</p><hr><p>below</p>"), "above\n\n---\n\nbelow");
643	}
644
645	#[test]
646	fn br_tag() {
647	assert_eq!(convert("line one<br>line two"), "line one\nline two");
648	}
649
650	#[test]
651	fn strikethrough() {
652	assert_eq!(convert("<del>removed</del>"), "~~removed~~");
653	}
654
655	#[test]
656	fn inline_code() {
657	assert_eq!(convert("use <code>pter</code> here"), "use `pter` here");
658	}
659
660	#[test]
661	fn script_and_style_stripped() {
662	assert_eq!(
663	convert("<p>text</p><script>alert('x')</script><style>.x{}</style>"),
664	"text"
665	);
666	}
667
668	#[test]
669	fn unknown_elements_transparent() {
670	assert_eq!(convert("<span>hello</span>"), "hello");
671	}
672
673	#[test]
674	fn hidden_element_skipped() {
675	assert_eq!(
676	convert(r#"<p>visible</p><div style="display:none">hidden</div>"#),
677	"visible"
678	);
679	}
680
681	#[test]
682	fn whitespace_collapsed() {
683	assert_eq!(convert(" lots of space "), "lots of space");
684	}
685
686	#[test]
687	fn entities_decoded() {
688	// html5ever decodes entities during parsing
689	assert_eq!(convert("<p>& < > "</p>"), "& < > \"");
690	}
691
692	#[test]
693	fn sup_and_sub() {
694	assert_eq!(convert("x<sup>2</sup>"), "x^2");
695	assert_eq!(convert("H<sub>2</sub>O"), "H~2O");
696	}
697
698	// -- Div / section as block separator --
699
700	#[test]
701	fn div_separates_blocks() {
702	assert_eq!(convert("<div>one</div><div>two</div>"), "one\n\ntwo");
703	}
704
705	// -- Tables --
706
707	#[test]
708	fn layout_table_single_cell_unwrapped() {
709	let html = "<table><tr><td><p>Hello world</p></td></tr></table>";
710	assert_eq!(convert(html), "Hello world");
711	}
712
713	#[test]
714	fn layout_table_multi_column_linearized() {
715	let html = "<table><tr><td>Left</td><td>Right</td></tr></table>";
716	let md = convert(html);
717	assert!(md.contains("Left"));
718	assert!(md.contains("Right"));
719	}
720
721	#[test]
722	fn data_table_rendered_as_markdown() {
723	let html = "<table><tr><th>Name</th><th>Age</th></tr>\
724	<tr><td>Alice</td><td>30</td></tr>\
725	<tr><td>Bob</td><td>25</td></tr></table>";
726	let md = convert(html);
727	assert!(md.contains("\| Name \| Age \|"));
728	assert!(md.contains("\| --- \| --- \|"));
729	assert!(md.contains("\| Alice \| 30 \|"));
730	assert!(md.contains("\| Bob \| 25 \|"));
731	}
732
733	#[test]
734	fn nested_layout_tables_unwrapped() {
735	let html = "<table><tr><td>\
736	<table><tr><td>Inner content</td></tr></table>\
737	</td></tr></table>";
738	let md = convert(html);
739	assert!(md.contains("Inner content"));
740	assert!(!md.contains("\|"));
741	}
742
743	#[test]
744	fn presentation_role_is_layout() {
745	let html = r#"<table role="presentation"><tr><td>Content</td><td> </td></tr></table>"#;
746	let md = convert(html);
747	assert!(md.contains("Content"));
748	assert!(!md.contains("\|"));
749	}
750
751	#[test]
752	fn spacer_element_hidden() {
753	let html = r#"<p>real</p><div style="font-size:0">spacer</div><p>also real</p>"#;
754	let md = convert(html);
755	assert!(md.contains("real"));
756	assert!(!md.contains("spacer"));
757	assert!(md.contains("also real"));
758	}
759
760	// -- Combined --
761
762	#[test]
763	fn mixed_content() {
764	let html = r#"
765	<h1>Subject</h1>
766	<p>Hello <strong>Max</strong>,</p>
767	<p>Check out <a href="https://example.com">this link</a>.</p>
768	<ul>
769	<li>Item one</li>
770	<li>Item two</li>
771	</ul>
772	"#;
773	let md = convert(html);
774	assert!(md.starts_with("# Subject"));
775	assert!(md.contains("Hello Max,"));
776	assert!(md.contains("[this link](https://example.com)"));
777	assert!(md.contains("- Item one\n- Item two"));
778	}
779	}
780