| 2 |
2 |
|
|
| 3 |
3 |
|
use async_imap::Client;
|
| 4 |
4 |
|
use base64::{engine::general_purpose::STANDARD as BASE64, Engine};
|
| 5 |
|
- |
use chrono::{DateTime, TimeZone, Utc};
|
|
5 |
+ |
use chrono::{DateTime, Utc};
|
| 6 |
6 |
|
use futures_util::StreamExt;
|
| 7 |
7 |
|
use goingson_core::{EmailAccount, FolderSyncState};
|
| 8 |
8 |
|
use tokio::net::TcpStream;
|
| 9 |
9 |
|
use tokio_native_tls::native_tls::TlsConnector as NativeTlsConnector;
|
| 10 |
10 |
|
use tokio_native_tls::TlsConnector;
|
| 11 |
11 |
|
|
|
12 |
+ |
use super::mime_parse::{build_parsed_email, mime_nesting_within_limit};
|
|
13 |
+ |
|
| 12 |
14 |
|
type ImapSession = async_imap::Session<tokio_native_tls::TlsStream<TcpStream>>;
|
| 13 |
15 |
|
|
| 14 |
16 |
|
/// Maximum email size to download (25 MB). Emails exceeding this are skipped during sync.
|
| 15 |
17 |
|
const MAX_EMAIL_SIZE: u32 = 25 * 1024 * 1024;
|
| 16 |
18 |
|
|
| 17 |
|
- |
/// Maximum number of `multipart/...` markers permitted in one message. `mailparse`
|
| 18 |
|
- |
/// recurses once per nesting level with no depth cap, so a deeply nested message —
|
| 19 |
|
- |
/// cheap to encode and well within `MAX_EMAIL_SIZE` — can overflow the stack, which
|
| 20 |
|
- |
/// is an uncatchable abort that kills the whole process. Each nesting level needs a
|
| 21 |
|
- |
/// `multipart/...` Content-Type, so capping their count bounds the recursion before
|
| 22 |
|
- |
/// `parse_mail` runs. Real emails use a handful; 256 is far above any legitimate value.
|
| 23 |
|
- |
const MAX_MIME_MULTIPART_PARTS: usize = 256;
|
| 24 |
|
- |
|
| 25 |
|
- |
/// Depth bound for our own MIME-tree walks (belt-and-braces with the pre-parse
|
| 26 |
|
- |
/// guard above; the parsed tree can be no deeper than the multipart-marker count).
|
| 27 |
|
- |
const MAX_MIME_DEPTH: usize = 256;
|
| 28 |
|
- |
|
| 29 |
|
- |
/// Rejects pathologically nested multipart content before `parse_mail` recurses.
|
| 30 |
|
- |
/// Counts `multipart/` markers (case-insensitive) in the raw message.
|
| 31 |
|
- |
fn mime_nesting_within_limit(body: &[u8]) -> bool {
|
| 32 |
|
- |
const NEEDLE: &[u8] = b"multipart/";
|
| 33 |
|
- |
let count = body
|
| 34 |
|
- |
.windows(NEEDLE.len())
|
| 35 |
|
- |
.filter(|w| (w[0] | 0x20) == b'm' && w.eq_ignore_ascii_case(NEEDLE))
|
| 36 |
|
- |
.count();
|
| 37 |
|
- |
count <= MAX_MIME_MULTIPART_PARTS
|
| 38 |
|
- |
}
|
| 39 |
19 |
|
|
| 40 |
20 |
|
/// How many UIDs to body-fetch per IMAP command. Bounds the command length and
|
| 41 |
21 |
|
/// the in-flight fetch pipeline so a large mailbox doesn't issue one enormous
|
| 300 |
280 |
|
continue;
|
| 301 |
281 |
|
}
|
| 302 |
282 |
|
match mailparse::parse_mail(body) {
|
| 303 |
|
- |
Ok(parsed) => emails.push(Self::build_parsed_email(&parsed, uid, &folder_name, is_read)),
|
|
283 |
+ |
Ok(parsed) => emails.push(build_parsed_email(&parsed, uid, &folder_name, is_read)),
|
| 304 |
284 |
|
Err(e) => {
|
| 305 |
285 |
|
tracing::debug!(uid, folder = %folder_name, error = %e, "Failed to parse email");
|
| 306 |
286 |
|
parse_errors += 1;
|
| 504 |
484 |
|
}
|
| 505 |
485 |
|
match mailparse::parse_mail(body) {
|
| 506 |
486 |
|
Ok(parsed) => {
|
| 507 |
|
- |
emails.push(Self::build_parsed_email(&parsed, uid, &folder_name, is_read));
|
|
487 |
+ |
emails.push(build_parsed_email(&parsed, uid, &folder_name, is_read));
|
| 508 |
488 |
|
// Advance only once the message is saved, so a parse
|
| 509 |
489 |
|
// failure below is retried rather than silently lost.
|
| 510 |
490 |
|
max_uid = Some(max_uid.map_or(uid, |m: u32| m.max(uid)));
|
| 647 |
627 |
|
Ok(format!("Folder '{}': exists={}, recent={}, search_all={}", folder, exists, recent, count))
|
| 648 |
628 |
|
}
|
| 649 |
629 |
|
|
| 650 |
|
- |
/// Build a `ParsedEmail` from a parsed message and its IMAP metadata.
|
| 651 |
|
- |
///
|
| 652 |
|
- |
/// Shared by both fetch loops (full-folder and incremental) so the header
|
| 653 |
|
- |
/// extraction lives in one place.
|
| 654 |
|
- |
fn build_parsed_email(
|
| 655 |
|
- |
parsed: &mailparse::ParsedMail,
|
| 656 |
|
- |
uid: u32,
|
| 657 |
|
- |
folder_name: &str,
|
| 658 |
|
- |
is_read: bool,
|
| 659 |
|
- |
) -> ParsedEmail {
|
| 660 |
|
- |
let header = |key: &str| {
|
| 661 |
|
- |
parsed
|
| 662 |
|
- |
.headers
|
| 663 |
|
- |
.iter()
|
| 664 |
|
- |
.find(|h| h.get_key().to_lowercase() == key)
|
| 665 |
|
- |
.map(|h| h.get_value())
|
| 666 |
|
- |
};
|
| 667 |
|
- |
|
| 668 |
|
- |
let date = header("date")
|
| 669 |
|
- |
.and_then(|s| mailparse::dateparse(&s).ok())
|
| 670 |
|
- |
.and_then(|ts| Utc.timestamp_opt(ts, 0).single())
|
| 671 |
|
- |
.unwrap_or(DateTime::UNIX_EPOCH);
|
| 672 |
|
- |
|
| 673 |
|
- |
let (body_text, html_body) = Self::extract_body_with_html(parsed);
|
| 674 |
|
- |
let attachments = extract_attachment_parts(parsed);
|
| 675 |
|
- |
|
| 676 |
|
- |
ParsedEmail {
|
| 677 |
|
- |
message_id: header("message-id"),
|
| 678 |
|
- |
in_reply_to: header("in-reply-to"),
|
| 679 |
|
- |
references_root: extract_references_root(&parsed.headers),
|
| 680 |
|
- |
imap_uid: uid,
|
| 681 |
|
- |
source_folder: folder_name.to_string(),
|
| 682 |
|
- |
from: header("from").unwrap_or_default(),
|
| 683 |
|
- |
to: header("to").unwrap_or_default(),
|
| 684 |
|
- |
subject: header("subject").unwrap_or_default(),
|
| 685 |
|
- |
body: body_text,
|
| 686 |
|
- |
html_body,
|
| 687 |
|
- |
date,
|
| 688 |
|
- |
is_read,
|
| 689 |
|
- |
attachments,
|
| 690 |
|
- |
}
|
| 691 |
|
- |
}
|
| 692 |
|
- |
|
| 693 |
|
- |
/// Extracts body text and optionally HTML from a parsed email.
|
| 694 |
|
- |
/// Returns (plain_text_body, optional_html_body).
|
| 695 |
|
- |
fn extract_body_with_html(mail: &mailparse::ParsedMail) -> (String, Option<String>) {
|
| 696 |
|
- |
let mut plain_text: Option<String> = None;
|
| 697 |
|
- |
let mut html_body: Option<String> = None;
|
| 698 |
|
- |
|
| 699 |
|
- |
Self::collect_body_parts(mail, &mut plain_text, &mut html_body, 0);
|
| 700 |
|
- |
|
| 701 |
|
- |
// Build final result - prefer plain text, fall back to pter markdown conversion
|
| 702 |
|
- |
let body_text = if let Some(ref plain) = plain_text {
|
| 703 |
|
- |
// If we have plain text but it looks like it contains HTML tags,
|
| 704 |
|
- |
// we should convert them (some emails have incorrect content-types)
|
| 705 |
|
- |
if plain.contains("<html") || plain.contains("<body") || plain.contains("<div") {
|
| 706 |
|
- |
pter::convert(plain)
|
| 707 |
|
- |
} else {
|
| 708 |
|
- |
plain.clone()
|
| 709 |
|
- |
}
|
| 710 |
|
- |
} else if let Some(ref html) = html_body {
|
| 711 |
|
- |
pter::convert(html)
|
| 712 |
|
- |
} else {
|
| 713 |
|
- |
// Fallback to whatever body is available
|
| 714 |
|
- |
let body = mail.get_body().unwrap_or_default();
|
| 715 |
|
- |
if body.contains("<html") || body.contains("<body") || body.contains("<div") {
|
| 716 |
|
- |
pter::convert(&body)
|
| 717 |
|
- |
} else {
|
| 718 |
|
- |
body
|
| 719 |
|
- |
}
|
| 720 |
|
- |
};
|
| 721 |
|
- |
|
| 722 |
|
- |
(body_text, html_body)
|
| 723 |
|
- |
}
|
| 724 |
|
- |
|
| 725 |
|
- |
/// Recursively collects text/plain and text/html parts from a mail structure.
|
| 726 |
|
- |
fn collect_body_parts(
|
| 727 |
|
- |
mail: &mailparse::ParsedMail,
|
| 728 |
|
- |
plain_text: &mut Option<String>,
|
| 729 |
|
- |
html_body: &mut Option<String>,
|
| 730 |
|
- |
depth: usize,
|
| 731 |
|
- |
) {
|
| 732 |
|
- |
if depth > MAX_MIME_DEPTH {
|
| 733 |
|
- |
return;
|
| 734 |
|
- |
}
|
| 735 |
|
- |
let mime_type = mail.ctype.mimetype.to_lowercase();
|
| 736 |
|
- |
|
| 737 |
|
- |
if mail.subparts.is_empty() {
|
| 738 |
|
- |
// Leaf node - check content type
|
| 739 |
|
- |
if mime_type == "text/plain" && plain_text.is_none() {
|
| 740 |
|
- |
*plain_text = Some(mail.get_body().unwrap_or_default());
|
| 741 |
|
- |
} else if mime_type == "text/html" && html_body.is_none() {
|
| 742 |
|
- |
*html_body = Some(mail.get_body().unwrap_or_default());
|
| 743 |
|
- |
}
|
| 744 |
|
- |
} else {
|
| 745 |
|
- |
// Multipart - recurse into all subparts
|
| 746 |
|
- |
for part in &mail.subparts {
|
| 747 |
|
- |
Self::collect_body_parts(part, plain_text, html_body, depth + 1);
|
| 748 |
|
- |
// Stop early if we've found both
|
| 749 |
|
- |
if plain_text.is_some() && html_body.is_some() {
|
| 750 |
|
- |
break;
|
| 751 |
|
- |
}
|
| 752 |
|
- |
}
|
| 753 |
|
- |
}
|
| 754 |
|
- |
}
|
| 755 |
|
- |
|
| 756 |
|
- |
// strip_html, extract_href, strip_tags_simple, decode_html_entities
|
| 757 |
|
- |
// removed — replaced by pter::convert().
|
| 758 |
|
- |
|
| 759 |
|
- |
}
|
| 760 |
|
- |
|
| 761 |
|
- |
/// Recursively extract attachment parts from a MIME tree.
|
| 762 |
|
- |
///
|
| 763 |
|
- |
/// Extracts the first message-ID from the References header (the thread root).
|
| 764 |
|
- |
fn extract_references_root(headers: &[mailparse::MailHeader]) -> Option<String> {
|
| 765 |
|
- |
headers
|
| 766 |
|
- |
.iter()
|
| 767 |
|
- |
.find(|h| h.get_key().to_lowercase() == "references")
|
| 768 |
|
- |
.and_then(|h| {
|
| 769 |
|
- |
h.get_value()
|
| 770 |
|
- |
.split_whitespace()
|
| 771 |
|
- |
.find(|s| s.starts_with('<') && s.ends_with('>'))
|
| 772 |
|
- |
.map(|s| s.to_string())
|
| 773 |
|
- |
})
|
| 774 |
|
- |
}
|
| 775 |
|
- |
|
| 776 |
|
- |
/// Walks the MIME structure and collects non-text leaf parts as attachments.
|
| 777 |
|
- |
/// Skips text/plain and text/html (those are body parts), and parts with empty bodies.
|
| 778 |
|
- |
fn extract_attachment_parts(mail: &mailparse::ParsedMail) -> Vec<AttachmentPart> {
|
| 779 |
|
- |
let mut parts = Vec::new();
|
| 780 |
|
- |
collect_attachment_parts(mail, &mut parts, 0);
|
| 781 |
|
- |
parts
|
| 782 |
|
- |
}
|
| 783 |
|
- |
|
| 784 |
|
- |
fn collect_attachment_parts(mail: &mailparse::ParsedMail, parts: &mut Vec<AttachmentPart>, depth: usize) {
|
| 785 |
|
- |
if depth > MAX_MIME_DEPTH {
|
| 786 |
|
- |
return;
|
| 787 |
|
- |
}
|
| 788 |
|
- |
let mime_type = mail.ctype.mimetype.to_lowercase();
|
| 789 |
|
- |
|
| 790 |
|
- |
if mail.subparts.is_empty() {
|
| 791 |
|
- |
// Leaf node — skip text/plain and text/html (body parts)
|
| 792 |
|
- |
if mime_type == "text/plain" || mime_type == "text/html" {
|
| 793 |
|
- |
return;
|
| 794 |
|
- |
}
|
| 795 |
|
- |
|
| 796 |
|
- |
// Get the raw decoded body
|
| 797 |
|
- |
let data = match mail.get_body_raw() {
|
| 798 |
|
- |
Ok(d) if !d.is_empty() => d,
|
| 799 |
|
- |
_ => return, // Skip parts with empty body
|
| 800 |
|
- |
};
|
| 801 |
|
- |
|
| 802 |
|
- |
// Extract filename: Content-Disposition filename, then Content-Type name, then fallback
|
| 803 |
|
- |
let filename = mail.get_content_disposition()
|
| 804 |
|
- |
.params
|
| 805 |
|
- |
.get("filename")
|
| 806 |
|
- |
.cloned()
|
| 807 |
|
- |
.or_else(|| mail.ctype.params.get("name").cloned())
|
| 808 |
|
- |
.unwrap_or_else(|| "attachment".to_string());
|
| 809 |
|
- |
|
| 810 |
|
- |
parts.push(AttachmentPart {
|
| 811 |
|
- |
filename,
|
| 812 |
|
- |
mime_type: mail.ctype.mimetype.clone(),
|
| 813 |
|
- |
data,
|
| 814 |
|
- |
});
|
| 815 |
|
- |
} else {
|
| 816 |
|
- |
// Multipart — recurse into subparts
|
| 817 |
|
- |
for part in &mail.subparts {
|
| 818 |
|
- |
collect_attachment_parts(part, parts, depth + 1);
|
| 819 |
|
- |
}
|
| 820 |
|
- |
}
|
| 821 |
630 |
|
}
|
| 822 |
631 |
|
|
| 823 |
632 |
|
struct XOAuth2Authenticator(String);
|
| 830 |
639 |
|
std::mem::take(&mut self.0)
|
| 831 |
640 |
|
}
|
| 832 |
641 |
|
}
|
| 833 |
|
- |
|
| 834 |
|
- |
#[cfg(test)]
|
| 835 |
|
- |
mod tests {
|
| 836 |
|
- |
// strip_tags_simple, extract_href, decode_html_entities, and strip_html
|
| 837 |
|
- |
// tests removed — these functions were replaced by pter::convert().
|
| 838 |
|
- |
|
| 839 |
|
- |
#[test]
|
| 840 |
|
- |
fn rejects_pathologically_nested_mime() {
|
| 841 |
|
- |
// Many more multipart markers than any real email: rejected before parse.
|
| 842 |
|
- |
let mut body = Vec::new();
|
| 843 |
|
- |
for _ in 0..(super::MAX_MIME_MULTIPART_PARTS + 10) {
|
| 844 |
|
- |
body.extend_from_slice(b"Content-Type: multipart/mixed; boundary=b\r\n");
|
| 845 |
|
- |
}
|
| 846 |
|
- |
assert!(!super::mime_nesting_within_limit(&body));
|
| 847 |
|
- |
|
| 848 |
|
- |
// A normal message with a couple of multiparts passes (case-insensitive).
|
| 849 |
|
- |
let ok = b"Content-Type: Multipart/Alternative; boundary=b\r\n\r\nhi";
|
| 850 |
|
- |
assert!(super::mime_nesting_within_limit(ok));
|
| 851 |
|
- |
}
|
| 852 |
|
- |
|
| 853 |
|
- |
#[test]
|
| 854 |
|
- |
fn pter_replaces_strip_html() {
|
| 855 |
|
- |
// Verify pter handles what strip_html used to do
|
| 856 |
|
- |
let html = "<p>Hello <strong>world</strong></p>";
|
| 857 |
|
- |
let result = pter::convert(html);
|
| 858 |
|
- |
assert!(result.contains("Hello"));
|
| 859 |
|
- |
assert!(result.contains("**world**"));
|
| 860 |
|
- |
}
|
| 861 |
|
- |
|
| 862 |
|
- |
// --- extract_attachment_parts ---
|
| 863 |
|
- |
|
| 864 |
|
- |
#[test]
|
| 865 |
|
- |
fn extract_attachments_simple() {
|
| 866 |
|
- |
// Multipart email with text/plain + application/pdf
|
| 867 |
|
- |
let raw = b"MIME-Version: 1.0\r\n\
|
| 868 |
|
- |
Content-Type: multipart/mixed; boundary=\"boundary1\"\r\n\
|
| 869 |
|
- |
\r\n\
|
| 870 |
|
- |
--boundary1\r\n\
|
| 871 |
|
- |
Content-Type: text/plain\r\n\
|
| 872 |
|
- |
\r\n\
|
| 873 |
|
- |
Hello world\r\n\
|
| 874 |
|
- |
--boundary1\r\n\
|
| 875 |
|
- |
Content-Type: application/pdf\r\n\
|
| 876 |
|
- |
Content-Disposition: attachment; filename=\"report.pdf\"\r\n\
|
| 877 |
|
- |
Content-Transfer-Encoding: base64\r\n\
|
| 878 |
|
- |
\r\n\
|
| 879 |
|
- |
JVBERi0xLjQK\r\n\
|
| 880 |
|
- |
--boundary1--\r\n";
|
| 881 |
|
- |
|
| 882 |
|
- |
let parsed = mailparse::parse_mail(raw).unwrap();
|
| 883 |
|
- |
let parts = super::extract_attachment_parts(&parsed);
|
| 884 |
|
- |
assert_eq!(parts.len(), 1);
|
| 885 |
|
- |
assert_eq!(parts[0].filename, "report.pdf");
|
| 886 |
|
- |
assert_eq!(parts[0].mime_type, "application/pdf");
|
| 887 |
|
- |
assert!(!parts[0].data.is_empty());
|
| 888 |
|
- |
}
|
| 889 |
|
- |
|
| 890 |
|
- |
#[test]
|
| 891 |
|
- |
fn extract_attachments_none_text_only() {
|
| 892 |
|
- |
// Plain text email with no attachments
|
| 893 |
|
- |
let raw = b"MIME-Version: 1.0\r\n\
|
| 894 |
|
- |
Content-Type: text/plain\r\n\
|
| 895 |
|
- |
\r\n\
|
| 896 |
|
- |
Just a plain text message.\r\n";
|
| 897 |
|
- |
|
| 898 |
|
- |
let parsed = mailparse::parse_mail(raw).unwrap();
|
| 899 |
|
- |
let parts = super::extract_attachment_parts(&parsed);
|
| 900 |
|
- |
assert!(parts.is_empty());
|
| 901 |
|
- |
}
|
| 902 |
|
- |
|
| 903 |
|
- |
#[test]
|
| 904 |
|
- |
fn extract_attachments_multipart_mixed() {
|
| 905 |
|
- |
// Multipart with text + image + pdf
|
| 906 |
|
- |
let raw = b"MIME-Version: 1.0\r\n\
|
| 907 |
|
- |
Content-Type: multipart/mixed; boundary=\"outer\"\r\n\
|
| 908 |
|
- |
\r\n\
|
| 909 |
|
- |
--outer\r\n\
|
| 910 |
|
- |
Content-Type: text/plain\r\n\
|
| 911 |
|
- |
\r\n\
|
| 912 |
|
- |
Body text\r\n\
|
| 913 |
|
- |
--outer\r\n\
|
| 914 |
|
- |
Content-Type: image/png; name=\"photo.png\"\r\n\
|
| 915 |
|
- |
Content-Transfer-Encoding: base64\r\n\
|
| 916 |
|
- |
\r\n\
|
| 917 |
|
- |
iVBORw0KGgo=\r\n\
|
| 918 |
|
- |
--outer\r\n\
|
| 919 |
|
- |
Content-Type: application/pdf\r\n\
|
| 920 |
|
- |
Content-Disposition: attachment; filename=\"doc.pdf\"\r\n\
|
| 921 |
|
- |
Content-Transfer-Encoding: base64\r\n\
|
| 922 |
|
- |
\r\n\
|
| 923 |
|
- |
JVBERi0xLjQK\r\n\
|
| 924 |
|
- |
--outer--\r\n";
|
| 925 |
|
- |
|
| 926 |
|
- |
let parsed = mailparse::parse_mail(raw).unwrap();
|
| 927 |
|
- |
let parts = super::extract_attachment_parts(&parsed);
|
| 928 |
|
- |
assert_eq!(parts.len(), 2);
|
| 929 |
|
- |
|
| 930 |
|
- |
// Image with name from Content-Type
|
| 931 |
|
- |
assert_eq!(parts[0].filename, "photo.png");
|
| 932 |
|
- |
assert_eq!(parts[0].mime_type, "image/png");
|
| 933 |
|
- |
|
| 934 |
|
- |
// PDF with name from Content-Disposition
|
| 935 |
|
- |
assert_eq!(parts[1].filename, "doc.pdf");
|
| 936 |
|
- |
assert_eq!(parts[1].mime_type, "application/pdf");
|
| 937 |
|
- |
}
|
| 938 |
|
- |
}
|