Skip to main content

max / pter

21.3 KB · 622 lines History Blame Raw
1 use scraper::ElementRef;
2 use scraper::node::Node;
3
4 /// Check if an element marks the beginning of a quoted reply.
5 ///
6 /// This is the central abstraction for reply detection. Rather than
7 /// building per-client logic throughout the converter, all client-specific
8 /// knowledge lives here behind a single predicate.
9 ///
10 /// An element is a reply boundary if it's a container that wraps quoted
11 /// content from a previous message in the thread. The converter treats
12 /// these identically to `<blockquote>` — children get `>` prefixed.
13 pub(crate) fn is_reply_boundary(el: ElementRef) -> bool {
14 let element = el.value();
15 let name = element.name();
16
17 // <blockquote> is already handled by the element classifier.
18 // This function catches non-blockquote reply wrappers.
19
20 // Structural: elements with type="cite" (Apple Mail, some webmail)
21 if element.attr("type") == Some("cite") {
22 return true;
23 }
24
25 // Class/ID-based detection — thin per-client checks
26 if element.attr("class").is_some_and(is_reply_class) {
27 return true;
28 }
29
30 if element.attr("id").is_some_and(is_reply_id) {
31 return true;
32 }
33
34 // Heuristic: a <div> whose first meaningful text child matches
35 // an attribution pattern ("On ... wrote:") followed by a blockquote
36 if name == "div" && has_attribution_then_quote(el) {
37 return true;
38 }
39
40 false
41 }
42
43 /// Extract an attribution line from just before or at the start of a reply boundary.
44 ///
45 /// Returns the attribution text (e.g. "On Mon, Jan 5, Alice wrote:") if found,
46 /// so the converter can render it above the quoted block.
47 pub(crate) fn find_attribution(el: ElementRef) -> Option<String> {
48 // Check the element's own leading text for attribution patterns
49 for child in el.children() {
50 match child.value() {
51 Node::Text(text) => {
52 let trimmed = text.text.trim();
53 if is_attribution_text(trimmed) {
54 return Some(trimmed.to_string());
55 }
56 }
57 Node::Element(_) => {
58 // Stop at the first child element — attribution is leading text
59 break;
60 }
61 _ => {}
62 }
63 }
64
65 // Check for a preceding sibling text node or element with attribution
66 if let Some(prev) = previous_sibling_text(el) {
67 let trimmed = prev.trim().to_string();
68 if is_attribution_text(&trimmed) {
69 return Some(trimmed);
70 }
71 }
72
73 None
74 }
75
76 /// Check if text matches common email attribution patterns.
77 ///
78 /// These patterns are cross-client — every email client generates some
79 /// variant of "On [date], [person] wrote:" or "--- Forwarded message ---".
80 fn is_attribution_text(text: &str) -> bool {
81 let t = text.trim();
82
83 // "On ... wrote:" (Gmail, Apple Mail, Thunderbird, most clients)
84 if t.starts_with("On ") && t.ends_with("wrote:") {
85 return true;
86 }
87 // Localized variants: "Le ... a écrit :" (French), "Am ... schrieb" (German)
88 if (t.starts_with("Le ") || t.starts_with("El "))
89 && (t.ends_with("crit :") || t.ends_with("crit:"))
90 {
91 return true;
92 }
93 if t.starts_with("Am ") && (t.ends_with("schrieb:") || t.ends_with("schrieb :")) {
94 return true;
95 }
96
97 // Forwarded message separators
98 if t.contains("Forwarded message")
99 || t.contains("Begin forwarded message")
100 || t.contains("Original Message")
101 {
102 return true;
103 }
104
105 false
106 }
107
108 /// Thin per-client class checks. Each is one line — easy to add new clients.
109 fn is_reply_class(class: &str) -> bool {
110 // Split on whitespace to check individual class names
111 class.split_whitespace().any(|c| {
112 matches!(
113 c,
114 "gmail_quote"
115 | "gmail_extra"
116 | "yahoo_quoted"
117 | "protonmail_quote"
118 | "tutanota_quote"
119 | "moz-cite-prefix" // Thunderbird
120 | "zmail_extra" // Zoho
121 | "WordSection1" // Outlook (sometimes wraps replies)
122 )
123 })
124 }
125
126 /// Thin per-client ID checks.
127 fn is_reply_id(id: &str) -> bool {
128 matches!(
129 id,
130 "divRplyFwdMsg" // Outlook
131 | "reply-message" // Generic
132 | "OLK_SRC_BODY_SECTION" // Outlook Mac
133 )
134 }
135
136 /// Check if a div contains attribution text followed by a blockquote.
137 ///
138 /// This catches the common pattern where no class/id is present but
139 /// the structure is: `<div>On ... wrote:<br><blockquote>...</blockquote></div>`
140 fn has_attribution_then_quote(el: ElementRef) -> bool {
141 let mut found_attribution = false;
142
143 for child in el.children() {
144 match child.value() {
145 Node::Text(text) if is_attribution_text(text.text.trim()) => {
146 found_attribution = true;
147 }
148 Node::Element(e) => {
149 if found_attribution && e.name() == "blockquote" {
150 return true;
151 }
152 // Skip <br> tags between attribution and blockquote
153 if e.name() != "br" {
154 // If we hit a non-br element before finding attribution, stop
155 if !found_attribution {
156 return false;
157 }
158 }
159 }
160 _ => {}
161 }
162 }
163
164 false
165 }
166
167 /// Get text from the previous sibling, if it exists and is a text or inline element.
168 fn previous_sibling_text(el: ElementRef) -> Option<String> {
169 let prev = el.prev_sibling()?;
170
171 match prev.value() {
172 Node::Text(text) => Some(text.text.to_string()),
173 Node::Element(e) => {
174 // Check inline elements like <span>, <font> that might wrap attribution
175 if matches!(e.name(), "span" | "font" | "b" | "i" | "div" | "p") {
176 let el_ref = ElementRef::wrap(prev)?;
177 let text: String = el_ref.text().collect();
178 if !text.trim().is_empty() {
179 return Some(text);
180 }
181 }
182 None
183 }
184 _ => None,
185 }
186 }
187
188 /// Check if a separator element marks the boundary between original
189 /// content and a forwarded/replied message.
190 ///
191 /// This catches `<hr>` or styled divs that act as visual separators
192 /// before reply content (common in Outlook "From: ... Sent: ..." blocks).
193 pub(crate) fn is_outlook_separator(el: ElementRef) -> bool {
194 let element = el.value();
195
196 // Outlook uses a specific pattern: a div containing
197 // "From: ... Sent: ... To: ... Subject: ..." as a reply header
198 if element.name() == "div" || element.name() == "p" {
199 let text: String = el.text().collect();
200 let t = text.trim();
201
202 // Must have at least From + Sent/Date or Subject
203 let has_from = t.contains("From:");
204 let has_sent = t.contains("Sent:") || t.contains("Date:");
205 let has_subject = t.contains("Subject:");
206
207 if has_from && (has_sent || has_subject) {
208 return true;
209 }
210 }
211
212 false
213 }
214
215 #[cfg(test)]
216 mod tests {
217 use super::*;
218 use scraper::{Html, Selector};
219
220 fn parse_and_select(html: &str, selector: &str) -> (Html, Selector) {
221 let doc = Html::parse_document(html);
222 let sel = Selector::parse(selector).unwrap();
223 (doc, sel)
224 }
225
226 // -- Attribution detection --
227
228 #[test]
229 fn attribution_on_wrote() {
230 assert!(is_attribution_text(
231 "On Mon, Jan 5, 2026 at 3:00 PM Alice <alice@example.com> wrote:"
232 ));
233 }
234
235 #[test]
236 fn attribution_forwarded() {
237 assert!(is_attribution_text(
238 "---------- Forwarded message ----------"
239 ));
240 }
241
242 #[test]
243 fn attribution_original_message() {
244 assert!(is_attribution_text("-----Original Message-----"));
245 }
246
247 #[test]
248 fn attribution_begin_forwarded() {
249 assert!(is_attribution_text("Begin forwarded message:"));
250 }
251
252 #[test]
253 fn not_attribution() {
254 assert!(!is_attribution_text("Hello, how are you?"));
255 assert!(!is_attribution_text("On the other hand, this is fine."));
256 }
257
258 // -- Reply class detection --
259
260 #[test]
261 fn gmail_quote_class() {
262 assert!(is_reply_class("gmail_quote"));
263 }
264
265 #[test]
266 fn multiple_classes_with_reply() {
267 assert!(is_reply_class("some-class gmail_quote another"));
268 }
269
270 #[test]
271 fn non_reply_class() {
272 assert!(!is_reply_class("regular-div content-wrapper"));
273 }
274
275 // -- Reply boundary detection --
276
277 #[test]
278 fn type_cite_is_boundary() {
279 let html = r#"<div type="cite"><p>quoted</p></div>"#;
280 let (doc, sel) = parse_and_select(html, r#"div[type="cite"]"#);
281 let el = doc.select(&sel).next().unwrap();
282 assert!(is_reply_boundary(el));
283 }
284
285 #[test]
286 fn gmail_quote_is_boundary() {
287 let html = r#"<div class="gmail_quote"><p>quoted</p></div>"#;
288 let (doc, sel) = parse_and_select(html, "div.gmail_quote");
289 let el = doc.select(&sel).next().unwrap();
290 assert!(is_reply_boundary(el));
291 }
292
293 #[test]
294 fn outlook_id_is_boundary() {
295 let html = r#"<div id="divRplyFwdMsg"><p>quoted</p></div>"#;
296 let (doc, sel) = parse_and_select(html, "#divRplyFwdMsg");
297 let el = doc.select(&sel).next().unwrap();
298 assert!(is_reply_boundary(el));
299 }
300
301 #[test]
302 fn plain_div_not_boundary() {
303 let html = r#"<div class="content"><p>not quoted</p></div>"#;
304 let (doc, sel) = parse_and_select(html, "div.content");
305 let el = doc.select(&sel).next().unwrap();
306 assert!(!is_reply_boundary(el));
307 }
308
309 // -- Outlook separator --
310
311 #[test]
312 fn outlook_from_sent_subject() {
313 let html = "<div>From: Alice\nSent: Monday\nTo: Bob\nSubject: Hello</div>";
314 let (doc, sel) = parse_and_select(html, "div");
315 let el = doc.select(&sel).next().unwrap();
316 assert!(is_outlook_separator(el));
317 }
318
319 #[test]
320 fn regular_div_not_separator() {
321 let html = "<div>Just a normal paragraph.</div>";
322 let (doc, sel) = parse_and_select(html, "div");
323 let el = doc.select(&sel).next().unwrap();
324 assert!(!is_outlook_separator(el));
325 }
326
327 // -- Boundary tests for `is_attribution_text`: each arm needs both sides --
328
329 #[test]
330 fn attribution_on_without_wrote_is_false() {
331 // "On ..." without "wrote:" — catches mutating && to ||
332 assert!(!is_attribution_text("On the bright side, this is fine."));
333 }
334
335 #[test]
336 fn attribution_wrote_without_on_is_false() {
337 // "... wrote:" without leading "On " — catches mutating && to ||
338 assert!(!is_attribution_text("Alice wrote:"));
339 }
340
341 #[test]
342 fn attribution_french_le_with_colon_space() {
343 assert!(is_attribution_text(
344 "Le lundi 5 janvier 2026, Alice a écrit :"
345 ));
346 }
347
348 #[test]
349 fn attribution_french_le_no_space_before_colon() {
350 // "écrit:" without space — covers L89 || mutation between the two ending forms
351 assert!(is_attribution_text("Le lundi, Alice a écrit:"));
352 }
353
354 #[test]
355 fn attribution_spanish_el_with_colon_space() {
356 assert!(is_attribution_text("El lunes 5 de enero, Alice a escrit :"));
357 }
358
359 #[test]
360 fn attribution_spanish_el_no_space_before_colon() {
361 assert!(is_attribution_text("El lunes, Alice a escrit:"));
362 }
363
364 #[test]
365 fn attribution_french_le_without_wrote_ending_is_false() {
366 // "Le X" without "écrit" — catches L89 mutating || to &&
367 assert!(!is_attribution_text("Le lundi, Alice est ici."));
368 }
369
370 #[test]
371 fn attribution_starts_with_le_but_not_french_pattern() {
372 // Word starts with "Le" but isn't the French attribution form.
373 assert!(!is_attribution_text("Le sigh."));
374 }
375
376 #[test]
377 fn attribution_german_am_with_colon() {
378 assert!(is_attribution_text("Am Montag, 5. Januar 2026, schrieb:"));
379 }
380
381 #[test]
382 fn attribution_german_am_with_space_colon() {
383 assert!(is_attribution_text("Am Montag schrieb :"));
384 }
385
386 #[test]
387 fn attribution_german_am_without_schrieb_is_false() {
388 // "Am X" without "schrieb" — catches L93 && mutation
389 assert!(!is_attribution_text("Am very fine, thanks."));
390 }
391
392 #[test]
393 fn attribution_german_schrieb_without_am_is_false() {
394 // "schrieb:" without leading "Am " — catches L93 && mutation
395 assert!(!is_attribution_text("Bob schrieb:"));
396 }
397
398 #[test]
399 fn attribution_begin_forwarded_only() {
400 // Only "Begin forwarded message" present — catches the || chain mutating to &&
401 assert!(is_attribution_text("Begin forwarded message"));
402 }
403
404 #[test]
405 fn attribution_original_message_only() {
406 // Only "Original Message" present — catches the || chain mutating to &&
407 assert!(is_attribution_text("-----Original Message-----"));
408 }
409
410 // -- Boundary tests for `is_reply_id` --
411
412 #[test]
413 fn reply_id_reply_message() {
414 assert!(is_reply_id("reply-message"));
415 }
416
417 #[test]
418 fn reply_id_olk_src_body_section() {
419 assert!(is_reply_id("OLK_SRC_BODY_SECTION"));
420 }
421
422 #[test]
423 fn reply_id_unknown_is_false() {
424 // Catches `replace is_reply_id -> bool with true` mutant
425 assert!(!is_reply_id("main-content"));
426 assert!(!is_reply_id(""));
427 assert!(!is_reply_id("reply"));
428 }
429
430 // -- Boundary tests for `find_attribution` --
431
432 #[test]
433 fn find_attribution_in_leading_text() {
434 let html = r"<div>On Mon, Alice wrote:<blockquote>quoted</blockquote></div>";
435 let (doc, sel) = parse_and_select(html, "div");
436 let el = doc.select(&sel).next().unwrap();
437 let attr = find_attribution(el);
438 assert!(attr.is_some());
439 assert!(attr.unwrap().contains("wrote:"));
440 }
441
442 #[test]
443 fn find_attribution_none_when_no_match() {
444 let html = r"<div>Just regular text here, nothing fancy.</div>";
445 let (doc, sel) = parse_and_select(html, "div");
446 let el = doc.select(&sel).next().unwrap();
447 assert!(find_attribution(el).is_none());
448 }
449
450 #[test]
451 fn find_attribution_stops_at_first_element_child() {
452 // Element-then-text: the Text(_) arm should still match leading text BEFORE
453 // hitting any element. With a leading element, the loop should `break`
454 // out without inspecting later text. Catches "delete match arm Node::Element(_)".
455 let html = r"<div><span>hi</span>On Mon, Alice wrote:</div>";
456 let (doc, sel) = parse_and_select(html, "div");
457 let el = doc.select(&sel).next().unwrap();
458 // Leading content is an element, not text — and the later text falls outside
459 // the leading-text scan. So no attribution should be found from leading text.
460 // Also, no preceding sibling. → None.
461 assert!(find_attribution(el).is_none());
462 }
463
464 #[test]
465 fn find_attribution_in_preceding_sibling() {
466 let html = r#"<div><p>On Mon, Alice wrote:</p><div class="quote">body</div></div>"#;
467 let (doc, sel) = parse_and_select(html, "div.quote");
468 let el = doc.select(&sel).next().unwrap();
469 let attr = find_attribution(el);
470 assert!(attr.is_some(), "expected attribution from preceding <p>");
471 }
472
473 // -- Boundary tests for `has_attribution_then_quote` --
474 // These exercise the function via `is_reply_boundary` since it's private.
475
476 #[test]
477 fn boundary_div_with_attribution_then_blockquote() {
478 let html = r"<div>On Mon, Alice wrote:<blockquote>quoted</blockquote></div>";
479 let (doc, sel) = parse_and_select(html, "div");
480 let el = doc.select(&sel).next().unwrap();
481 assert!(is_reply_boundary(el));
482 }
483
484 #[test]
485 fn boundary_div_blockquote_without_attribution_is_false() {
486 // A bare blockquote-wrapping div without attribution text is not a boundary.
487 // Catches "replace has_attribution_then_quote -> bool with false" (would
488 // make this still pass, but the positive case above would fail).
489 let html = r"<div><blockquote>quoted</blockquote></div>";
490 let (doc, sel) = parse_and_select(html, "div");
491 let el = doc.select(&sel).next().unwrap();
492 assert!(!is_reply_boundary(el));
493 }
494
495 #[test]
496 fn boundary_div_attribution_no_blockquote_is_false() {
497 // Attribution text but no blockquote → not a boundary.
498 // Catches the L151 == mutation (would treat any element as blockquote).
499 let html = r"<div>On Mon, Alice wrote:<p>not a quote</p></div>";
500 let (doc, sel) = parse_and_select(html, "div");
501 let el = doc.select(&sel).next().unwrap();
502 assert!(!is_reply_boundary(el));
503 }
504
505 #[test]
506 fn boundary_div_attribution_br_blockquote() {
507 // Attribution → <br> → blockquote. The <br> must be skipped.
508 // Catches the L155 != mutation in br-handling.
509 let html = r"<div>On Mon, Alice wrote:<br><blockquote>quoted</blockquote></div>";
510 let (doc, sel) = parse_and_select(html, "div");
511 let el = doc.select(&sel).next().unwrap();
512 assert!(is_reply_boundary(el));
513 }
514
515 #[test]
516 fn boundary_div_non_br_element_before_attribution_is_false() {
517 // Non-br element BEFORE finding attribution → early return false.
518 // Catches the L157 `!` deletion.
519 let html = r"<div><p>preface</p>On Mon, Alice wrote:<blockquote>q</blockquote></div>";
520 let (doc, sel) = parse_and_select(html, "div");
521 let el = doc.select(&sel).next().unwrap();
522 assert!(!is_reply_boundary(el));
523 }
524
525 // -- Boundary tests for `previous_sibling_text` --
526 // Exercised via find_attribution since the function is private.
527
528 #[test]
529 fn prev_sibling_text_node() {
530 // Raw Text node as preceding sibling. Inside a parent <div>, a leading
531 // text run followed by a child <div class="q"> means the inner div's
532 // `prev_sibling()` is a `Node::Text`. Catches `delete match arm Node::Text(text)`.
533 let html = r#"<div>On Mon, Alice wrote:<div class="q">body</div></div>"#;
534 let (doc, sel) = parse_and_select(html, "div.q");
535 let el = doc.select(&sel).next().unwrap();
536 assert!(find_attribution(el).is_some());
537 }
538
539 #[test]
540 fn prev_sibling_inline_span_with_attribution() {
541 let html = r#"<div><span>On Mon, Alice wrote:</span><div class="q">body</div></div>"#;
542 let (doc, sel) = parse_and_select(html, "div.q");
543 let el = doc.select(&sel).next().unwrap();
544 assert!(find_attribution(el).is_some());
545 }
546
547 #[test]
548 fn prev_sibling_inline_font_with_attribution() {
549 // <font> is also inline-treated; covers a different arm in the matches!.
550 let html = r#"<div><font>On Mon, Alice wrote:</font><div class="q">body</div></div>"#;
551 let (doc, sel) = parse_and_select(html, "div.q");
552 let el = doc.select(&sel).next().unwrap();
553 assert!(find_attribution(el).is_some());
554 }
555
556 #[test]
557 fn prev_sibling_non_inline_element_returns_none() {
558 // <table> is not in the inline whitelist → preceding-sibling lookup fails.
559 let html = r#"<div><table><tr><td>On Mon, Alice wrote:</td></tr></table><div class="q">body</div></div>"#;
560 let (doc, sel) = parse_and_select(html, "div.q");
561 let el = doc.select(&sel).next().unwrap();
562 assert!(find_attribution(el).is_none());
563 }
564
565 #[test]
566 fn prev_sibling_empty_inline_returns_none() {
567 let html = r#"<div><span> </span><div class="q">body</div></div>"#;
568 let (doc, sel) = parse_and_select(html, "div.q");
569 let el = doc.select(&sel).next().unwrap();
570 // Whitespace-only preceding span → no attribution match.
571 assert!(find_attribution(el).is_none());
572 }
573
574 // -- Boundary tests for `is_outlook_separator` --
575
576 #[test]
577 fn outlook_from_date_subject_is_separator() {
578 // Date instead of Sent → covers L206 || (Sent || Date) mutation
579 let html = "<div>From: Alice\nDate: Monday\nSubject: Hello</div>";
580 let (doc, sel) = parse_and_select(html, "div");
581 let el = doc.select(&sel).next().unwrap();
582 assert!(is_outlook_separator(el));
583 }
584
585 #[test]
586 fn outlook_from_sent_no_subject_is_separator() {
587 // From + Sent, no Subject → catches L209 mutating || to &&
588 let html = "<div>From: Alice\nSent: Monday</div>";
589 let (doc, sel) = parse_and_select(html, "div");
590 let el = doc.select(&sel).next().unwrap();
591 assert!(is_outlook_separator(el));
592 }
593
594 #[test]
595 fn outlook_from_subject_no_sent_is_separator() {
596 // From + Subject, no Sent/Date → catches L209 mutating || to &&
597 let html = "<div>From: Alice\nSubject: Hello</div>";
598 let (doc, sel) = parse_and_select(html, "div");
599 let el = doc.select(&sel).next().unwrap();
600 assert!(is_outlook_separator(el));
601 }
602
603 #[test]
604 fn outlook_from_only_is_not_separator() {
605 // From alone (no Sent/Date/Subject) → must be false.
606 // Catches L209 && mutation to ||.
607 let html = "<div>From: Alice</div>";
608 let (doc, sel) = parse_and_select(html, "div");
609 let el = doc.select(&sel).next().unwrap();
610 assert!(!is_outlook_separator(el));
611 }
612
613 #[test]
614 fn outlook_sent_subject_no_from_is_not_separator() {
615 // No From → must be false regardless of Sent/Subject presence.
616 let html = "<div>Sent: Monday\nSubject: Hello</div>";
617 let (doc, sel) = parse_and_select(html, "div");
618 let el = doc.select(&sel).next().unwrap();
619 assert!(!is_outlook_separator(el));
620 }
621 }
622