//! IMAP client for fetching and parsing email messages. use async_imap::Client; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use chrono::{DateTime, TimeZone, Utc}; use futures_util::StreamExt; use goingson_core::{EmailAccount, FolderSyncState}; use tokio::net::TcpStream; use tokio_native_tls::native_tls::TlsConnector as NativeTlsConnector; use tokio_native_tls::TlsConnector; type ImapSession = async_imap::Session>; /// Maximum email size to download (25 MB). Emails exceeding this are skipped during sync. const MAX_EMAIL_SIZE: u32 = 25 * 1024 * 1024; /// Raw attachment data extracted during IMAP RFC822 parse. #[derive(Debug, Clone)] pub struct AttachmentPart { pub filename: String, pub mime_type: String, pub data: Vec, } #[derive(Debug, Clone)] pub struct ParsedEmail { pub message_id: Option, pub in_reply_to: Option, /// First entry from the References header (thread root). pub references_root: Option, pub imap_uid: u32, pub source_folder: String, pub from: String, pub to: String, pub subject: String, pub body: String, /// Original HTML body for "Open in Browser" feature. pub html_body: Option, pub date: DateTime, pub is_read: bool, /// Attachment parts extracted from MIME structure. pub attachments: Vec, } /// Result of fetching from a single IMAP folder, including sync state metadata. pub struct FolderFetchResult { pub emails: Vec, pub uid_validity: Option, pub max_uid_fetched: Option, pub debug_info: String, } /// Authentication method for IMAP #[derive(Debug, Clone)] pub enum ImapAuth { /// Traditional username/password Password { username: String, password: String }, /// OAuth2 XOAUTH2 mechanism XOAuth2 { email: String, access_token: String, }, } pub struct ImapClient { server: String, port: u16, auth: ImapAuth, } impl ImapClient { /// Create an IMAP client with explicit password authentication. /// /// Use this when retrieving credentials from secure storage (keychain). pub fn with_password(account: &EmailAccount, password: &str) -> Self { Self { server: account.imap_server.trim().to_string(), port: account.imap_port as u16, auth: ImapAuth::Password { username: account.username.trim().to_string(), password: password.to_string(), }, } } /// Create an IMAP client with OAuth2 XOAUTH2 authentication pub fn with_oauth(server: &str, port: u16, email: &str, access_token: &str) -> Self { Self { server: server.to_string(), port, auth: ImapAuth::XOAuth2 { email: email.to_string(), access_token: access_token.to_string(), }, } } /// Connect to IMAP server and return an authenticated session async fn connect(&self) -> Result { let addr = format!("{}:{}", self.server, self.port); let tcp_stream = tokio::time::timeout( std::time::Duration::from_secs(30), TcpStream::connect(&addr), ) .await .map_err(|_| format!("Connection to {} timed out after 30s", addr))? .map_err(|e| format!("Connection error: {}", e))?; let tls_connector = NativeTlsConnector::new() .map_err(|e| format!("TLS setup error: {}", e))?; let tls = TlsConnector::from(tls_connector); let tls_stream = tokio::time::timeout( std::time::Duration::from_secs(30), tls.connect(&self.server, tcp_stream), ) .await .map_err(|_| format!("TLS handshake with {} timed out after 30s", self.server))? .map_err(|e| format!("TLS error: {}", e))?; let client = Client::new(tls_stream); match &self.auth { ImapAuth::Password { username, password } => { let session = client .login(username, password) .await .map_err(|e| format!("Login error: {}", e.0))?; Ok(session) } ImapAuth::XOAuth2 { email, access_token } => { // XOAUTH2 format: base64("user=" + email + "\x01auth=Bearer " + token + "\x01\x01") let auth_string = format!("user={}\x01auth=Bearer {}\x01\x01", email, access_token); let auth_base64 = BASE64.encode(auth_string.as_bytes()); let session = client .authenticate("XOAUTH2", XOAuth2Authenticator(auth_base64)) .await .map_err(|e| format!("XOAUTH2 login error: {}", e.0))?; Ok(session) } } } /// Fetch emails from a specific folder with debug info pub async fn fetch_emails_from_folder_debug( &self, folder: &str, since: Option>, ) -> Result<(Vec, String), String> { let mut debug = Vec::new(); let mut session = self.connect().await?; session .select(folder) .await .map_err(|e| format!("Select {} error: {}", folder, e))?; // Build search query let search_query = if let Some(since_date) = since { format!("SINCE {}", since_date.format("%d-%b-%Y")) } else { "ALL".to_string() }; debug.push(format!("search: {}", search_query)); let search_result = session .search(&search_query) .await .map_err(|e| format!("Search error: {}", e))?; // Limit to last 50 if no date filter let sequence_nums: Vec = if since.is_none() { let mut nums: Vec = search_result.into_iter().collect(); nums.sort(); nums.into_iter().rev().take(50).collect() } else { search_result.into_iter().collect() }; debug.push(format!("seq_nums: {}", sequence_nums.len())); if sequence_nums.is_empty() { session.logout().await.ok(); return Ok((Vec::new(), debug.join(", "))); } let sequence_set = sequence_nums .iter() .map(|n| n.to_string()) .collect::>() .join(","); // Pre-filter oversized emails let mut size_stream = session .fetch(&sequence_set, "(UID RFC822.SIZE)") .await .map_err(|e| format!("Size fetch error: {}", e))?; let mut safe_seqs: Vec = Vec::new(); let mut skipped_large = 0usize; // Build a set of UIDs that are safe to fetch while let Some(result) = size_stream.next().await { if let Ok(msg) = result { let over_limit = msg.size.map_or(false, |s| s > MAX_EMAIL_SIZE); if over_limit { skipped_large += 1; tracing::warn!(uid = ?msg.uid, size = ?msg.size, folder = %folder, "Skipping oversized email"); continue; } // Use the sequence number (message index in stream matches input order) // Re-collect the UIDs we want, then re-fetch by UID if let Some(uid) = msg.uid { safe_seqs.push(uid); } } } drop(size_stream); if skipped_large > 0 { debug.push(format!("skipped_large: {}", skipped_large)); } if safe_seqs.is_empty() { session.logout().await.ok(); return Ok((Vec::new(), debug.join(", "))); } let safe_uid_set = safe_seqs.iter().map(|n| n.to_string()).collect::>().join(","); // Fetch full bodies only for safe-sized emails let mut messages = session .uid_fetch(&safe_uid_set, "(UID FLAGS RFC822)") .await .map_err(|e| format!("Fetch error: {}", e))?; let mut emails = Vec::new(); let folder_name = folder.to_string(); let mut msg_count = 0; let mut body_count = 0; let mut parse_errors = 0; while let Some(result) = messages.next().await { msg_count += 1; let message = match result { Ok(m) => m, Err(e) => { debug.push(format!("msg_err: {}", e)); continue; } }; let uid = match message.uid { Some(u) => u, None => { tracing::warn!(folder = %folder_name, "IMAP message missing UID, skipping"); continue; } }; // Check if \Seen flag is present (email has been read) let is_read = message.flags().any(|f| { matches!(f, async_imap::types::Flag::Seen) }); if let Some(body) = message.body() { body_count += 1; match mailparse::parse_mail(body) { Ok(parsed) => { let message_id = parsed .headers .iter() .find(|h| h.get_key().to_lowercase() == "message-id") .map(|h| h.get_value()); let in_reply_to = parsed .headers .iter() .find(|h| h.get_key().to_lowercase() == "in-reply-to") .map(|h| h.get_value()); let references_root = extract_references_root(&parsed.headers); let from = parsed .headers .iter() .find(|h| h.get_key().to_lowercase() == "from") .map(|h| h.get_value()) .unwrap_or_default(); let to = parsed .headers .iter() .find(|h| h.get_key().to_lowercase() == "to") .map(|h| h.get_value()) .unwrap_or_default(); let subject = parsed .headers .iter() .find(|h| h.get_key().to_lowercase() == "subject") .map(|h| h.get_value()) .unwrap_or_default(); let date_str = parsed .headers .iter() .find(|h| h.get_key().to_lowercase() == "date") .map(|h| h.get_value()); let date = date_str .and_then(|s| mailparse::dateparse(&s).ok()) .and_then(|ts| Utc.timestamp_opt(ts, 0).single()) .unwrap_or(DateTime::UNIX_EPOCH); let (body_text, html_body) = Self::extract_body_with_html(&parsed); let attachments = extract_attachment_parts(&parsed); emails.push(ParsedEmail { message_id, in_reply_to, references_root, imap_uid: uid, source_folder: folder_name.clone(), from, to, subject, body: body_text, html_body, date, is_read, attachments, }); } Err(e) => { tracing::debug!(uid, folder = %folder_name, error = %e, "Failed to parse email"); parse_errors += 1; } } } } debug.push(format!("msgs: {}, bodies: {}, parsed: {}, errs: {}", msg_count, body_count, emails.len(), parse_errors)); drop(messages); session.logout().await.ok(); Ok((emails, debug.join(", "))) } /// Fetch emails incrementally using IMAP UIDs when sync state is available, /// falling back to SINCE-based search otherwise. pub async fn fetch_emails_incremental( &self, folder: &str, sync_state: Option<&FolderSyncState>, since_fallback: Option>, ) -> Result { let mut debug = Vec::new(); let mut session = self.connect().await?; let mailbox = session .select(folder) .await .map_err(|e| format!("Select {} error: {}", folder, e))?; let server_uid_validity = mailbox.uid_validity; // Decide whether to use UID-based or SINCE-based search let use_uid = match (sync_state, server_uid_validity) { (Some(state), Some(validity)) if validity == state.uid_validity => { debug.push(format!("uid_mode: UID {}:* (validity {})", state.last_seen_uid + 1, validity)); true } (Some(state), Some(validity)) => { debug.push(format!("uid_validity_mismatch: stored={} server={}, falling back to SINCE", state.uid_validity, validity)); false } (Some(_), None) => { debug.push("server_no_uidvalidity, falling back to SINCE".to_string()); false } (None, _) => { debug.push("no_sync_state, using SINCE fallback".to_string()); false } }; let uids: Vec = if use_uid { let state = sync_state.unwrap(); let search_query = format!("UID {}:*", state.last_seen_uid.saturating_add(1)); debug.push(format!("uid_search: {}", search_query)); let search_result = session .uid_search(&search_query) .await .map_err(|e| format!("UID search error: {}", e))?; // Filter out the last_seen_uid itself (IMAP UID ranges are inclusive, // and `n:*` returns n even if it's the only match) search_result .into_iter() .filter(|&uid| uid > state.last_seen_uid) .collect() } else { // Fall back to SINCE-based search, then get UIDs via uid_search let search_query = if let Some(since_date) = since_fallback { format!("SINCE {}", since_date.format("%d-%b-%Y")) } else { "ALL".to_string() }; debug.push(format!("since_search: {}", search_query)); let search_result = session .uid_search(&search_query) .await .map_err(|e| format!("Search error: {}", e))?; let mut nums: Vec = search_result.into_iter().collect(); if since_fallback.is_none() { nums.sort(); nums = nums.into_iter().rev().take(50).collect(); } nums }; debug.push(format!("uids_to_fetch: {}", uids.len())); if uids.is_empty() { session.logout().await.ok(); return Ok(FolderFetchResult { emails: Vec::new(), uid_validity: server_uid_validity, max_uid_fetched: None, debug_info: debug.join(", "), }); } // Pre-filter oversized emails by fetching sizes first let uid_set = uids.iter().map(|n| n.to_string()).collect::>().join(","); let mut size_stream = session .uid_fetch(&uid_set, "(UID RFC822.SIZE)") .await .map_err(|e| format!("UID size fetch error: {}", e))?; let mut safe_uids: Vec = Vec::new(); let mut skipped_large = 0usize; while let Some(result) = size_stream.next().await { if let Ok(msg) = result { if let Some(uid) = msg.uid { if let Some(size) = msg.size { if size > MAX_EMAIL_SIZE { skipped_large += 1; tracing::warn!(uid, size, folder = %folder, "Skipping oversized email ({} bytes)", size); continue; } } safe_uids.push(uid); } } } drop(size_stream); if skipped_large > 0 { debug.push(format!("skipped_large: {}", skipped_large)); } if safe_uids.is_empty() { session.logout().await.ok(); return Ok(FolderFetchResult { emails: Vec::new(), uid_validity: server_uid_validity, max_uid_fetched: uids.iter().copied().max(), debug_info: debug.join(", "), }); } let safe_uid_set = safe_uids.iter().map(|n| n.to_string()).collect::>().join(","); let mut messages = session .uid_fetch(&safe_uid_set, "(UID FLAGS RFC822)") .await .map_err(|e| format!("UID fetch error: {}", e))?; let mut emails = Vec::new(); let folder_name = folder.to_string(); let mut msg_count = 0; let mut body_count = 0; let mut parse_errors = 0; // Seed max_uid from all UIDs (including skipped large ones) so they aren't re-fetched let mut max_uid: Option = uids.iter().copied().max(); while let Some(result) = messages.next().await { msg_count += 1; let message = match result { Ok(m) => m, Err(e) => { debug.push(format!("msg_err: {}", e)); continue; } }; let uid = match message.uid { Some(u) => u, None => { tracing::warn!(folder = %folder_name, "IMAP message missing UID, skipping"); continue; } }; max_uid = Some(max_uid.map_or(uid, |m: u32| m.max(uid))); let is_read = message.flags().any(|f| matches!(f, async_imap::types::Flag::Seen)); if let Some(body) = message.body() { body_count += 1; match mailparse::parse_mail(body) { Ok(parsed) => { let message_id = parsed.headers.iter() .find(|h| h.get_key().to_lowercase() == "message-id") .map(|h| h.get_value()); let in_reply_to = parsed.headers.iter() .find(|h| h.get_key().to_lowercase() == "in-reply-to") .map(|h| h.get_value()); let references_root = extract_references_root(&parsed.headers); let from = parsed.headers.iter() .find(|h| h.get_key().to_lowercase() == "from") .map(|h| h.get_value()).unwrap_or_default(); let to = parsed.headers.iter() .find(|h| h.get_key().to_lowercase() == "to") .map(|h| h.get_value()).unwrap_or_default(); let subject = parsed.headers.iter() .find(|h| h.get_key().to_lowercase() == "subject") .map(|h| h.get_value()).unwrap_or_default(); let date_str = parsed.headers.iter() .find(|h| h.get_key().to_lowercase() == "date") .map(|h| h.get_value()); let date = date_str .and_then(|s| mailparse::dateparse(&s).ok()) .and_then(|ts| Utc.timestamp_opt(ts, 0).single()) .unwrap_or(DateTime::UNIX_EPOCH); let (body_text, html_body) = Self::extract_body_with_html(&parsed); let attachments = extract_attachment_parts(&parsed); emails.push(ParsedEmail { message_id, in_reply_to, references_root, imap_uid: uid, source_folder: folder_name.clone(), from, to, subject, body: body_text, html_body, date, is_read, attachments, }); } Err(e) => { tracing::debug!(uid, folder = %folder_name, error = %e, "Failed to parse email"); parse_errors += 1; } } } } debug.push(format!("msgs: {}, bodies: {}, parsed: {}, errs: {}", msg_count, body_count, emails.len(), parse_errors)); drop(messages); session.logout().await.ok(); Ok(FolderFetchResult { emails, uid_validity: server_uid_validity, max_uid_fetched: max_uid, debug_info: debug.join(", "), }) } /// Move a message between folders using IMAP UID MOVE or COPY+DELETE pub async fn move_message( &self, uid: u32, from_folder: &str, to_folder: &str, ) -> Result<(), String> { let mut session = self.connect().await?; // Select source folder session .select(from_folder) .await .map_err(|e| format!("Failed to select {}: {}", from_folder, e))?; let uid_str = uid.to_string(); // Try MOVE first (RFC 6851) - more efficient and atomic let move_result = session.uid_mv(&uid_str, to_folder).await; match move_result { Ok(_) => { session.logout().await.ok(); Ok(()) } Err(_) => { // Fallback to COPY + DELETE + EXPUNGE session .uid_copy(&uid_str, to_folder) .await .map_err(|e| format!("Copy failed: {}", e))?; // Mark as deleted - drop the stream since operation is done let _ = session .uid_store(&uid_str, "+FLAGS (\\Deleted)") .await .map_err(|e| format!("Delete flag failed: {}", e))?; // Expunge - drop the stream since operation is done let _ = session .expunge() .await .map_err(|e| format!("Expunge failed: {}", e))?; session.logout().await.ok(); Ok(()) } } } /// Archive a message (move from INBOX to Archive folder) pub async fn archive_message(&self, uid: u32, archive_folder: &str) -> Result<(), String> { self.move_message(uid, "INBOX", archive_folder).await } /// Unarchive a message (move from Archive folder to INBOX) pub async fn unarchive_message(&self, uid: u32, archive_folder: &str) -> Result<(), String> { self.move_message(uid, archive_folder, "INBOX").await } pub async fn test_connection(&self) -> Result<(), String> { let mut session = self.connect().await?; session.logout().await.ok(); Ok(()) } /// List all available folders on the IMAP server pub async fn list_folders(&self) -> Result, String> { let mut session = self.connect().await?; let folders_stream = session .list(Some(""), Some("*")) .await .map_err(|e| format!("List folders error: {}", e))?; use futures_util::StreamExt; let folders: Vec = folders_stream .filter_map(|result| async { result.ok().map(|name| name.name().to_string()) }) .collect() .await; session.logout().await.ok(); Ok(folders) } /// Debug fetch - returns diagnostic info about what's in a folder pub async fn debug_folder(&self, folder: &str) -> Result { let mut session = self.connect().await?; let mailbox = session .select(folder) .await .map_err(|e| format!("Select {} error: {}", folder, e))?; let exists = mailbox.exists; let recent = mailbox.recent; // Try a simple search let search_result = session .search("ALL") .await .map_err(|e| format!("Search error: {}", e))?; let count: usize = search_result.len(); session.logout().await.ok(); Ok(format!("Folder '{}': exists={}, recent={}, search_all={}", folder, exists, recent, count)) } /// Extracts body text and optionally HTML from a parsed email. /// Returns (plain_text_body, optional_html_body). fn extract_body_with_html(mail: &mailparse::ParsedMail) -> (String, Option) { let mut plain_text: Option = None; let mut html_body: Option = None; Self::collect_body_parts(mail, &mut plain_text, &mut html_body); // Build final result - prefer plain text, fall back to pter markdown conversion let body_text = if let Some(ref plain) = plain_text { // If we have plain text but it looks like it contains HTML tags, // we should convert them (some emails have incorrect content-types) if plain.contains(", html_body: &mut Option, ) { let mime_type = mail.ctype.mimetype.to_lowercase(); if mail.subparts.is_empty() { // Leaf node - check content type if mime_type == "text/plain" && plain_text.is_none() { *plain_text = Some(mail.get_body().unwrap_or_default()); } else if mime_type == "text/html" && html_body.is_none() { *html_body = Some(mail.get_body().unwrap_or_default()); } } else { // Multipart - recurse into all subparts for part in &mail.subparts { Self::collect_body_parts(part, plain_text, html_body); // Stop early if we've found both if plain_text.is_some() && html_body.is_some() { break; } } } } // strip_html, extract_href, strip_tags_simple, decode_html_entities // removed — replaced by pter::convert(). } /// Recursively extract attachment parts from a MIME tree. /// /// Extracts the first message-ID from the References header (the thread root). fn extract_references_root(headers: &[mailparse::MailHeader]) -> Option { headers .iter() .find(|h| h.get_key().to_lowercase() == "references") .and_then(|h| { h.get_value() .split_whitespace() .find(|s| s.starts_with('<') && s.ends_with('>')) .map(|s| s.to_string()) }) } /// Walks the MIME structure and collects non-text leaf parts as attachments. /// Skips text/plain and text/html (those are body parts), and parts with empty bodies. fn extract_attachment_parts(mail: &mailparse::ParsedMail) -> Vec { let mut parts = Vec::new(); collect_attachment_parts(mail, &mut parts); parts } fn collect_attachment_parts(mail: &mailparse::ParsedMail, parts: &mut Vec) { let mime_type = mail.ctype.mimetype.to_lowercase(); if mail.subparts.is_empty() { // Leaf node — skip text/plain and text/html (body parts) if mime_type == "text/plain" || mime_type == "text/html" { return; } // Get the raw decoded body let data = match mail.get_body_raw() { Ok(d) if !d.is_empty() => d, _ => return, // Skip parts with empty body }; // Extract filename: Content-Disposition filename, then Content-Type name, then fallback let filename = mail.get_content_disposition() .params .get("filename") .cloned() .or_else(|| mail.ctype.params.get("name").cloned()) .unwrap_or_else(|| "attachment".to_string()); parts.push(AttachmentPart { filename, mime_type: mail.ctype.mimetype.clone(), data, }); } else { // Multipart — recurse into subparts for part in &mail.subparts { collect_attachment_parts(part, parts); } } } struct XOAuth2Authenticator(String); impl async_imap::Authenticator for XOAuth2Authenticator { type Response = String; fn process(&mut self, _challenge: &[u8]) -> Self::Response { // Return the pre-computed base64 XOAUTH2 string std::mem::take(&mut self.0) } } #[cfg(test)] mod tests { use super::ImapClient; // strip_tags_simple, extract_href, decode_html_entities, and strip_html // tests removed — these functions were replaced by pter::convert(). #[test] fn pter_replaces_strip_html() { // Verify pter handles what strip_html used to do let html = "

Hello world

"; let result = pter::convert(html); assert!(result.contains("Hello")); assert!(result.contains("**world**")); } // --- extract_attachment_parts --- #[test] fn extract_attachments_simple() { // Multipart email with text/plain + application/pdf let raw = b"MIME-Version: 1.0\r\n\ Content-Type: multipart/mixed; boundary=\"boundary1\"\r\n\ \r\n\ --boundary1\r\n\ Content-Type: text/plain\r\n\ \r\n\ Hello world\r\n\ --boundary1\r\n\ Content-Type: application/pdf\r\n\ Content-Disposition: attachment; filename=\"report.pdf\"\r\n\ Content-Transfer-Encoding: base64\r\n\ \r\n\ JVBERi0xLjQK\r\n\ --boundary1--\r\n"; let parsed = mailparse::parse_mail(raw).unwrap(); let parts = super::extract_attachment_parts(&parsed); assert_eq!(parts.len(), 1); assert_eq!(parts[0].filename, "report.pdf"); assert_eq!(parts[0].mime_type, "application/pdf"); assert!(!parts[0].data.is_empty()); } #[test] fn extract_attachments_none_text_only() { // Plain text email with no attachments let raw = b"MIME-Version: 1.0\r\n\ Content-Type: text/plain\r\n\ \r\n\ Just a plain text message.\r\n"; let parsed = mailparse::parse_mail(raw).unwrap(); let parts = super::extract_attachment_parts(&parsed); assert!(parts.is_empty()); } #[test] fn extract_attachments_multipart_mixed() { // Multipart with text + image + pdf let raw = b"MIME-Version: 1.0\r\n\ Content-Type: multipart/mixed; boundary=\"outer\"\r\n\ \r\n\ --outer\r\n\ Content-Type: text/plain\r\n\ \r\n\ Body text\r\n\ --outer\r\n\ Content-Type: image/png; name=\"photo.png\"\r\n\ Content-Transfer-Encoding: base64\r\n\ \r\n\ iVBORw0KGgo=\r\n\ --outer\r\n\ Content-Type: application/pdf\r\n\ Content-Disposition: attachment; filename=\"doc.pdf\"\r\n\ Content-Transfer-Encoding: base64\r\n\ \r\n\ JVBERi0xLjQK\r\n\ --outer--\r\n"; let parsed = mailparse::parse_mail(raw).unwrap(); let parts = super::extract_attachment_parts(&parsed); assert_eq!(parts.len(), 2); // Image with name from Content-Type assert_eq!(parts[0].filename, "photo.png"); assert_eq!(parts[0].mime_type, "image/png"); // PDF with name from Content-Disposition assert_eq!(parts[1].filename, "doc.pdf"); assert_eq!(parts[1].mime_type, "application/pdf"); } }