max / pter

23.1 KB · 784 lines History Blame Raw

1	use scraper::node::Node;
2	use scraper::{ElementRef, Html};
3
4	use crate::elements::{self, BlockKind, ElementAction, InlineKind};
5	use crate::replies;
6	use crate::tables;
7	use crate::whitespace;
8
9	/// Convert an HTML email body into readable markdown.
10	///
11	/// This is the main entry point for pter. Pass in an HTML string
12	/// (just the body, not MIME structure) and get back clean markdown.
13	///
14	/// ```
15	/// let md = pter::convert("<p>Hello <strong>world</strong></p>");
16	/// assert_eq!(md, "Hello world");
17	/// ```
18	pub fn convert(html: &str) -> String {
19	if html.is_empty() {
20	return String::new();
21	}
22
23	let document = Html::parse_document(html);
24	let mut ctx = Context::new();
25	walk_children(document.root_element(), &mut ctx);
26	whitespace::normalize(&ctx.output)
27	}
28
29	/// Maximum DOM nesting depth walked before children are dropped. html5ever
30	/// imposes no depth limit of its own, so a flat but deeply-nested body (e.g.
31	/// `<div>`×500k) would otherwise recurse `walk_children`/`handle_element` once
32	/// per level and overflow the stack — an uncatchable `abort()`. 512 is the
33	/// browser-conventional nesting cap; no legitimate email nests remotely that
34	/// deep, and 512 frames stay well within even a 2 MB worker stack.
35	const MAX_DEPTH: u32 = 512;
36
37	/// Conversion state threaded through the tree walk.
38	struct Context {
39	output: String,
40	/// Current DOM recursion depth (guards against pathological nesting).
41	depth: u32,
42	/// Current list nesting depth (for indentation).
43	list_depth: u32,
44	/// Whether we're inside a <pre> block (preserve whitespace).
45	in_pre: bool,
46	/// Whether we're inside an <a> tag (don't nest links).
47	in_link: bool,
48	/// Stack of list types for proper ordered/unordered rendering.
49	list_stack: Vec<ListType>,
50	}
51
52	#[derive(Clone, Copy)]
53	enum ListType {
54	Unordered,
55	Ordered(u32), // current item number
56	}
57
58	impl Context {
59	fn new() -> Self {
60	Self {
61	output: String::with_capacity(4096),
62	depth: 0,
63	list_depth: 0,
64	in_pre: false,
65	in_link: false,
66	list_stack: Vec::new(),
67	}
68	}
69
70	fn push(&mut self, s: &str) {
71	self.output.push_str(s);
72	}
73
74	fn push_char(&mut self, c: char) {
75	self.output.push(c);
76	}
77
78	fn ensure_blank_line(&mut self) {
79	let trimmed = self.output.trim_end_matches(' ');
80	if trimmed.is_empty() {
81	return;
82	}
83	if trimmed.ends_with("\n\n") {
84	return;
85	}
86	self.output.truncate(trimmed.len());
87	self.output.push_str("\n\n");
88	}
89
90	fn ensure_newline(&mut self) {
91	if !self.output.is_empty() && !self.output.ends_with('\n') {
92	self.output.push('\n');
93	}
94	}
95
96	fn list_indent(&self) -> String {
97	if self.list_depth <= 1 {
98	return String::new();
99	}
100	" ".repeat((self.list_depth - 1) as usize)
101	}
102	}
103
104	/// Walk all children of a node, converting each to markdown.
105	fn walk_children(parent: ElementRef, ctx: &mut Context) {
106	// Bound recursion depth. Beyond MAX_DEPTH we stop descending — the content
107	// is either pathological or hostile, and dropping it is vastly preferable to
108	// a stack-overflow abort that would wedge the caller (a hostile email would
109	// otherwise re-crash on every subsequent sync).
110	if ctx.depth >= MAX_DEPTH {
111	return;
112	}
113	ctx.depth += 1;
114	for child in parent.children() {
115	match child.value() {
116	Node::Text(text) => {
117	handle_text(&text.text, ctx);
118	}
119	Node::Element(_) => {
120	if let Some(el_ref) = ElementRef::wrap(child) {
121	handle_element(el_ref, ctx);
122	}
123	}
124	_ => {}
125	}
126	}
127	ctx.depth -= 1;
128	}
129
130	/// Handle a text node.
131	fn handle_text(text: &str, ctx: &mut Context) {
132	if ctx.in_pre {
133	ctx.push(text);
134	return;
135	}
136
137	// Collapse whitespace in normal flow
138	let mut last_was_space = ctx.output.ends_with(' ') \|\| ctx.output.ends_with('\n');
139	for ch in text.chars() {
140	if ch.is_ascii_whitespace() {
141	if !last_was_space {
142	ctx.push_char(' ');
143	last_was_space = true;
144	}
145	} else {
146	ctx.push_char(ch);
147	last_was_space = false;
148	}
149	}
150	}
151
152	/// Handle an element node — classify it and render accordingly.
153	fn handle_element(el: ElementRef, ctx: &mut Context) {
154	let element = el.value();
155
156	// Check hidden elements
157	if elements::is_hidden(element) {
158	return;
159	}
160
161	// Check for reply boundaries before normal classification.
162	// Reply boundaries (gmail_quote, type=cite, etc.) get rendered
163	// as blockquotes regardless of their actual element type.
164	if replies::is_reply_boundary(el) {
165	render_reply_block(el, ctx);
166	return;
167	}
168
169	// Check for Outlook-style "From: ... Sent: ..." separator blocks.
170	// These introduce quoted content that follows them.
171	if replies::is_outlook_separator(el) {
172	ctx.ensure_blank_line();
173	// Render the separator header as attribution
174	let text: String = el.text().collect();
175	let trimmed = text.split_whitespace().collect::<Vec<_>>().join(" ");
176	ctx.push(&trimmed);
177	ctx.ensure_blank_line();
178	return;
179	}
180
181	match elements::classify(element) {
182	ElementAction::Skip => {}
183	ElementAction::Transparent => walk_children(el, ctx),
184	ElementAction::Block(kind) => handle_block(el, ctx, kind),
185	ElementAction::Inline(kind) => handle_inline(el, ctx, kind),
186	}
187	}
188
189	fn handle_block(el: ElementRef, ctx: &mut Context, kind: BlockKind) {
190	match kind {
191	BlockKind::Paragraph => {
192	ctx.ensure_blank_line();
193	walk_children(el, ctx);
194	ctx.ensure_blank_line();
195	}
196
197	BlockKind::Heading(level) => {
198	ctx.ensure_blank_line();
199	let prefix = "#".repeat(level as usize);
200	ctx.push(&prefix);
201	ctx.push_char(' ');
202	walk_children(el, ctx);
203	ctx.ensure_blank_line();
204	}
205
206	BlockKind::Blockquote => {
207	ctx.ensure_blank_line();
208	// Render children into a temporary buffer, then prefix each line with >
209	let mut inner_ctx = Context::new();
210	inner_ctx.in_pre = ctx.in_pre;
211	inner_ctx.in_link = ctx.in_link;
212	walk_children(el, &mut inner_ctx);
213	let inner = whitespace::normalize(&inner_ctx.output);
214	for line in inner.lines() {
215	ctx.push("> ");
216	ctx.push(line);
217	ctx.push_char('\n');
218	}
219	ctx.push_char('\n');
220	}
221
222	BlockKind::UnorderedList => {
223	ctx.ensure_blank_line();
224	ctx.list_depth += 1;
225	ctx.list_stack.push(ListType::Unordered);
226	walk_children(el, ctx);
227	ctx.list_stack.pop();
228	ctx.list_depth -= 1;
229	ctx.ensure_blank_line();
230	}
231
232	BlockKind::OrderedList => {
233	ctx.ensure_blank_line();
234	ctx.list_depth += 1;
235	ctx.list_stack.push(ListType::Ordered(0));
236	walk_children(el, ctx);
237	ctx.list_stack.pop();
238	ctx.list_depth -= 1;
239	ctx.ensure_blank_line();
240	}
241
242	BlockKind::ListItem => {
243	ctx.ensure_newline();
244	let indent = ctx.list_indent();
245	ctx.push(&indent);
246
247	// Determine bullet or number
248	let marker = match ctx.list_stack.last_mut() {
249	Some(ListType::Ordered(n)) => {
250	*n += 1;
251	format!("{}. ", *n)
252	}
253	Some(ListType::Unordered) \| None => "- ".to_string(),
254	};
255	ctx.push(&marker);
256	walk_children(el, ctx);
257	ctx.ensure_newline();
258	}
259
260	BlockKind::PreFormatted => {
261	ctx.ensure_blank_line();
262	ctx.push("```\n");
263	ctx.in_pre = true;
264	walk_children(el, ctx);
265	ctx.in_pre = false;
266	ctx.ensure_newline();
267	ctx.push("```");
268	ctx.ensure_blank_line();
269	}
270
271	BlockKind::HorizontalRule => {
272	ctx.ensure_blank_line();
273	ctx.push("---");
274	ctx.ensure_blank_line();
275	}
276
277	BlockKind::Table => {
278	ctx.ensure_blank_line();
279	if tables::is_data_table(el) {
280	let (headers, rows) = tables::extract_table_data(el);
281	let md = tables::render_markdown_table(&headers, &rows);
282	if !md.is_empty() {
283	ctx.push(&md);
284	}
285	} else {
286	// Layout table — unwrap and render cell contents directly
287	render_layout_table(el, ctx);
288	}
289	ctx.ensure_blank_line();
290	}
291
292	BlockKind::Div => {
293	// Divs act as block separators but don't add their own markup
294	ctx.ensure_blank_line();
295	walk_children(el, ctx);
296	ctx.ensure_blank_line();
297	}
298	}
299	}
300
301	fn handle_inline(el: ElementRef, ctx: &mut Context, kind: InlineKind) {
302	match kind {
303	InlineKind::Bold => {
304	ctx.push("**");
305	walk_children(el, ctx);
306	ctx.push("**");
307	}
308
309	InlineKind::Italic => {
310	ctx.push("*");
311	walk_children(el, ctx);
312	ctx.push("*");
313	}
314
315	InlineKind::Strikethrough => {
316	ctx.push("~~");
317	walk_children(el, ctx);
318	ctx.push("~~");
319	}
320
321	InlineKind::Code => {
322	if ctx.in_pre {
323	// Inside a <pre>, don't double-wrap
324	walk_children(el, ctx);
325	} else {
326	ctx.push("`");
327	walk_children(el, ctx);
328	ctx.push("`");
329	}
330	}
331
332	InlineKind::Link => {
333	if ctx.in_link {
334	// Don't nest links
335	walk_children(el, ctx);
336	return;
337	}
338
339	let href = el.value().attr("href").unwrap_or("");
340
341	if href.is_empty() \|\| href == "#" {
342	walk_children(el, ctx);
343	return;
344	}
345
346	// Collect the link text
347	let mut text_ctx = Context::new();
348	text_ctx.in_link = true;
349	walk_children(el, &mut text_ctx);
350	let text = text_ctx.output.trim().to_string();
351
352	if text.is_empty() {
353	// Link with no text — just show the URL
354	ctx.push(href);
355	} else if text == href {
356	// Link text matches URL — no need for markdown link syntax
357	ctx.push(href);
358	} else {
359	ctx.push("[");
360	ctx.push(&text);
361	ctx.push("](");
362	ctx.push(href);
363	ctx.push(")");
364	}
365	}
366
367	InlineKind::Image => {
368	let element = el.value();
369	if elements::is_tracking_pixel(element) {
370	return;
371	}
372
373	let alt = element.attr("alt").unwrap_or("");
374	let src = element.attr("src").unwrap_or("");
375
376	if src.is_empty() {
377	return;
378	}
379
380	ctx.push("![");
381	ctx.push(alt);
382	ctx.push("](");
383	ctx.push(src);
384	ctx.push(")");
385	}
386
387	InlineKind::LineBreak => {
388	ctx.push_char('\n');
389	}
390
391	InlineKind::Superscript => {
392	ctx.push("^");
393	walk_children(el, ctx);
394	}
395
396	InlineKind::Subscript => {
397	ctx.push("~");
398	walk_children(el, ctx);
399	}
400	}
401	}
402
403	/// Render a reply boundary as a quoted block.
404	///
405	/// This is the same rendering logic as `<blockquote>` — children are
406	/// rendered into a temporary buffer and each line gets `> ` prefixed.
407	/// Attribution lines (e.g. "On ... wrote:") are rendered above the quote.
408	fn render_reply_block(el: ElementRef, ctx: &mut Context) {
409	ctx.ensure_blank_line();
410
411	// Look for attribution text
412	if let Some(attribution) = replies::find_attribution(el) {
413	ctx.push(&attribution);
414	ctx.push_char('\n');
415	}
416
417	// Render children into temp buffer, then prefix with >
418	let mut inner_ctx = Context::new();
419	inner_ctx.in_pre = ctx.in_pre;
420	inner_ctx.in_link = ctx.in_link;
421	walk_children(el, &mut inner_ctx);
422	let inner = whitespace::normalize(&inner_ctx.output);
423
424	if !inner.is_empty() {
425	for line in inner.lines() {
426	ctx.push("> ");
427	ctx.push(line);
428	ctx.push_char('\n');
429	}
430	ctx.push_char('\n');
431	}
432	}
433
434	/// Unwrap a layout table by rendering cell contents sequentially.
435	///
436	/// Walks through rows and cells, rendering each cell's content as if
437	/// the table wrapper didn't exist. This handles the common email pattern
438	/// of wrapping everything in `<table><tr><td>...</td></tr></table>`.
439	fn render_layout_table(table: ElementRef, ctx: &mut Context) {
440	for descendant in table.descendants() {
441	if let Some(el_ref) = ElementRef::wrap(descendant) {
442	let name = el_ref.value().name();
443	if name == "td" \|\| name == "th" {
444	// Check if the cell itself is hidden
445	if !elements::is_hidden(el_ref.value()) {
446	walk_children(el_ref, ctx);
447	ctx.ensure_blank_line();
448	}
449	}
450	}
451	}
452	}
453
454	#[cfg(test)]
455	mod tests {
456	use super::*;
457
458	// -- Basic elements --
459
460	#[test]
461	fn empty_input() {
462	assert_eq!(convert(""), "");
463	}
464
465	#[test]
466	fn plain_text() {
467	assert_eq!(convert("hello world"), "hello world");
468	}
469
470	#[test]
471	fn pathological_nesting_does_not_overflow() {
472	// A flat but deeply-nested body (far past MAX_DEPTH) must not recurse
473	// the tree walk into a stack-overflow abort — it returns bounded output.
474	// Guards the hostile-email DoS: without the depth cap this aborts the
475	// process and every subsequent sync re-crashes on the same message.
476	let n = (MAX_DEPTH as usize) + 5_000;
477	let deep = format!("text{}{}", "<div>".repeat(n), "</div>".repeat(n));
478	let md = convert(&deep);
479	// Reaching this line at all proves no stack-overflow abort; the shallow
480	// "text" (depth 1, within the cap) still renders.
481	assert!(md.contains("text"), "content within the cap still renders");
482	}
483
484	#[test]
485	fn paragraph() {
486	assert_eq!(convert("<p>one</p><p>two</p>"), "one\n\ntwo");
487	}
488
489	#[test]
490	fn headings() {
491	assert_eq!(convert("<h1>Title</h1>"), "# Title");
492	assert_eq!(convert("<h3>Sub</h3>"), "### Sub");
493	}
494
495	#[test]
496	fn bold_and_italic() {
497	assert_eq!(
498	convert("<p><strong>bold</strong> and <em>italic</em></p>"),
499	"bold and italic"
500	);
501	}
502
503	#[test]
504	fn link() {
505	assert_eq!(
506	convert(r#"<a href="https://example.com">click</a>"#),
507	"[click](https://example.com)"
508	);
509	}
510
511	#[test]
512	fn link_text_matches_url() {
513	assert_eq!(
514	convert(r#"<a href="https://example.com">https://example.com</a>"#),
515	"https://example.com"
516	);
517	}
518
519	#[test]
520	fn link_empty_href() {
521	assert_eq!(convert(r#"<a href="">click</a>"#), "click");
522	}
523
524	#[test]
525	fn image() {
526	assert_eq!(
527	convert(r#"<img src="photo.jpg" alt="A photo">"#),
528	"![A photo](photo.jpg)"
529	);
530	}
531
532	#[test]
533	fn tracking_pixel_skipped() {
534	assert_eq!(convert(r#"<img src="track.gif" width="1" height="1">"#), "");
535	}
536
537	#[test]
538	fn unordered_list() {
539	assert_eq!(convert("<ul><li>one</li><li>two</li></ul>"), "- one\n- two");
540	}
541
542	#[test]
543	fn ordered_list() {
544	assert_eq!(
545	convert("<ol><li>first</li><li>second</li></ol>"),
546	"1. first\n2. second"
547	);
548	}
549
550	#[test]
551	fn nested_list() {
552	let html = "<ul><li>outer<ul><li>inner</li></ul></li></ul>";
553	let md = convert(html);
554	assert!(md.contains("- outer"));
555	assert!(md.contains(" - inner"));
556	}
557
558	#[test]
559	fn nested_list_exact_indent_depth_2() {
560	// At depth 2, `list_indent` returns `" "` (exactly two spaces, one indent level).
561	// Catches `list_indent` mutations:
562	// - `(depth - 1)` → `(depth + 1)`: would produce 3 indent levels (6 spaces).
563	// - `(depth - 1)` → `(depth / 1)`: would produce 2 indent levels (4 spaces).
564	// Either makes this exact-match assertion fail.
565	// (The converter emits a blank line before each nested list — that's a
566	// separate stylistic question; the indent is what we're pinning down here.)
567	assert_eq!(
568	convert("<ul><li>A<ul><li>B</li></ul></li></ul>"),
569	"- A\n\n - B"
570	);
571	}
572
573	#[test]
574	fn triple_nested_list_exact_indent_depth_3() {
575	// At depth 3, indent is exactly `" "` (four spaces).
576	assert_eq!(
577	convert("<ul><li>A<ul><li>B<ul><li>C</li></ul></li></ul></li></ul>"),
578	"- A\n\n - B\n\n - C"
579	);
580	}
581
582	#[test]
583	fn sibling_top_level_lists_have_no_indent_after_nesting() {
584	// After a nested <ul> closes, `list_depth -= 1` must execute to return
585	// to outer scope. If mutated to `+= 1` or `/= 1`, list_depth stays
586	// elevated and the SECOND top-level list ends up incorrectly indented.
587	let md = convert("<ul><li>A<ul><li>B</li></ul></li></ul><ul><li>C</li></ul>");
588	// The second list's "C" item must appear at column 0, not indented.
589	// We check the exact substring "\n- C" (newline then no leading whitespace).
590	assert!(
591	md.contains("\n- C"),
592	"second top-level list must not be indented after a nested list closes; got: {md:?}"
593	);
594	// And explicitly: it must NOT appear with leading spaces.
595	assert!(
596	!md.contains("\n - C"),
597	"second list incorrectly indented; got: {md:?}"
598	);
599	}
600
601	#[test]
602	fn ordered_list_decrements_depth_after_nesting() {
603	// Same shape but with <ol> — exercises the L218 `-= 1` mutation in the
604	// OrderedList block, distinct from UnorderedList's L208.
605	let md = convert("<ol><li>A<ol><li>B</li></ol></li></ol><ol><li>C</li></ol>");
606	assert!(
607	md.contains("\n1. C"),
608	"second ol must restart at depth 1: {md:?}"
609	);
610	assert!(
611	!md.contains("\n 1. C"),
612	"second ol indented incorrectly: {md:?}"
613	);
614	}
615
616	#[test]
617	fn blockquote() {
618	assert_eq!(
619	convert("<blockquote>quoted text</blockquote>"),
620	"> quoted text"
621	);
622	}
623
624	#[test]
625	fn nested_blockquote() {
626	let html = "<blockquote>outer<blockquote>inner</blockquote></blockquote>";
627	let md = convert(html);
628	assert!(md.contains("> outer"));
629	assert!(md.contains("> > inner"));
630	}
631
632	#[test]
633	fn preformatted() {
634	let html = "<pre><code>fn main() {\n println!(\"hi\");\n}</code></pre>";
635	let md = convert(html);
636	assert!(md.starts_with("```\n"));
637	assert!(md.contains("fn main()"));
638	assert!(md.ends_with("\n```"));
639	}
640
641	#[test]
642	fn horizontal_rule() {
643	assert_eq!(
644	convert("<p>above</p><hr><p>below</p>"),
645	"above\n\n---\n\nbelow"
646	);
647	}
648
649	#[test]
650	fn br_tag() {
651	assert_eq!(convert("line one<br>line two"), "line one\nline two");
652	}
653
654	#[test]
655	fn strikethrough() {
656	assert_eq!(convert("<del>removed</del>"), "~~removed~~");
657	}
658
659	#[test]
660	fn inline_code() {
661	assert_eq!(convert("use <code>pter</code> here"), "use `pter` here");
662	}
663
664	#[test]
665	fn script_and_style_stripped() {
666	assert_eq!(
667	convert("<p>text</p><script>alert('x')</script><style>.x{}</style>"),
668	"text"
669	);
670	}
671
672	#[test]
673	fn unknown_elements_transparent() {
674	assert_eq!(convert("<span>hello</span>"), "hello");
675	}
676
677	#[test]
678	fn hidden_element_skipped() {
679	assert_eq!(
680	convert(r#"<p>visible</p><div style="display:none">hidden</div>"#),
681	"visible"
682	);
683	}
684
685	#[test]
686	fn whitespace_collapsed() {
687	assert_eq!(convert(" lots of space "), "lots of space");
688	}
689
690	#[test]
691	fn entities_decoded() {
692	// html5ever decodes entities during parsing
693	assert_eq!(convert("<p>& < > "</p>"), "& < > \"");
694	}
695
696	#[test]
697	fn sup_and_sub() {
698	assert_eq!(convert("x<sup>2</sup>"), "x^2");
699	assert_eq!(convert("H<sub>2</sub>O"), "H~2O");
700	}
701
702	// -- Div / section as block separator --
703
704	#[test]
705	fn div_separates_blocks() {
706	assert_eq!(convert("<div>one</div><div>two</div>"), "one\n\ntwo");
707	}
708
709	// -- Tables --
710
711	#[test]
712	fn layout_table_single_cell_unwrapped() {
713	let html = "<table><tr><td><p>Hello world</p></td></tr></table>";
714	assert_eq!(convert(html), "Hello world");
715	}
716
717	#[test]
718	fn layout_table_multi_column_linearized() {
719	let html = "<table><tr><td>Left</td><td>Right</td></tr></table>";
720	let md = convert(html);
721	assert!(md.contains("Left"));
722	assert!(md.contains("Right"));
723	}
724
725	#[test]
726	fn data_table_rendered_as_markdown() {
727	let html = "<table><tr><th>Name</th><th>Age</th></tr>\
728	<tr><td>Alice</td><td>30</td></tr>\
729	<tr><td>Bob</td><td>25</td></tr></table>";
730	let md = convert(html);
731	assert!(md.contains("\| Name \| Age \|"));
732	assert!(md.contains("\| --- \| --- \|"));
733	assert!(md.contains("\| Alice \| 30 \|"));
734	assert!(md.contains("\| Bob \| 25 \|"));
735	}
736
737	#[test]
738	fn nested_layout_tables_unwrapped() {
739	let html = "<table><tr><td>\
740	<table><tr><td>Inner content</td></tr></table>\
741	</td></tr></table>";
742	let md = convert(html);
743	assert!(md.contains("Inner content"));
744	assert!(!md.contains('\|'));
745	}
746
747	#[test]
748	fn presentation_role_is_layout() {
749	let html = r#"<table role="presentation"><tr><td>Content</td><td> </td></tr></table>"#;
750	let md = convert(html);
751	assert!(md.contains("Content"));
752	assert!(!md.contains('\|'));
753	}
754
755	#[test]
756	fn spacer_element_hidden() {
757	let html = r#"<p>real</p><div style="font-size:0">spacer</div><p>also real</p>"#;
758	let md = convert(html);
759	assert!(md.contains("real"));
760	assert!(!md.contains("spacer"));
761	assert!(md.contains("also real"));
762	}
763
764	// -- Combined --
765
766	#[test]
767	fn mixed_content() {
768	let html = r#"
769	<h1>Subject</h1>
770	<p>Hello <strong>Max</strong>,</p>
771	<p>Check out <a href="https://example.com">this link</a>.</p>
772	<ul>
773	<li>Item one</li>
774	<li>Item two</li>
775	</ul>
776	"#;
777	let md = convert(html);
778	assert!(md.starts_with("# Subject"));
779	assert!(md.contains("Hello Max,"));
780	assert!(md.contains("[this link](https://example.com)"));
781	assert!(md.contains("- Item one\n- Item two"));
782	}
783	}
784