refactor!: replace Tantivy search engine with DuckDB

src/modules/envelope/extractor.rs

@@ -20,124 +20,55 @@ use crate::modules::common::AddrVec;
 use crate::modules::envelope::utils::normalize_subject;
 use crate::modules::error::code::ErrorCode;
 use crate::modules::error::BichonResult;
-use crate::modules::utils::create_hash;
+use crate::modules::message::content::AttachmentInfo;
+use crate::modules::utils::create_hash2;
 use crate::modules::utils::html::extract_text;
 use crate::{calculate_hash, raise_error, utc_now};
 use crate::{id, modules::indexer::envelope::Envelope};
 use async_imap::types::Fetch;
-use mail_parser::{HeaderName, Message, MessageParser, MimeHeaders};
+use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders};
 
-pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> BichonResult<Envelope> {
+pub fn extract_envelope(
+    fetch: &Fetch,
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<(Envelope, Vec<AttachmentInfo>)> {
     let internal_date = fetch
         .internal_date()
         .map(|d| d.timestamp_millis())
         .unwrap_or(0);
     let uid = fetch.uid.unwrap_or(0);
-
     let body = fetch
         .body()
         .ok_or_else(|| raise_error!("No body available".into(), ErrorCode::InternalError))?;
-
     let size = fetch.size.unwrap_or(body.len() as u32);
-    let message = MessageParser::new().parse(body).ok_or_else(|| {
-        raise_error!(
-            "Email header parse result is not available".into(),
-            ErrorCode::InternalError
-        )
-    })?;
 
-    let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
-        text
-    } else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
-        extract_text(html)
-    } else {
-        String::new()
-    };
-
-    let message_id = message
-        .message_id()
-        .map(String::from)
-        .unwrap_or(generate_message_id());
-    let in_reply_to = message.in_reply_to().as_text().map(String::from);
-    let references = extract_references(&message);
-    let thread_id = compute_thread_id(in_reply_to, references, &message_id);
-
-    let mut subject = message.subject().map(String::from).unwrap_or_default();
-
-    if subject.contains('\u{FFFD}') {
-        subject = normalize_subject(message.header_raw(HeaderName::Subject));
-    }
-
-    let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
-    let bcc: Option<Vec<String>> = message.bcc().map(|addr| {
-        AddrVec::from(addr)
-            .0
-            .into_iter()
-            .filter_map(|a| a.address)
-            .collect()
-    });
-    let cc: Option<Vec<String>> = message.cc().map(|addr| {
-        AddrVec::from(addr)
-            .0
-            .into_iter()
-            .filter_map(|a| a.address)
-            .collect()
-    });
-    let to: Option<Vec<String>> = message.to().map(|addr| {
-        AddrVec::from(addr)
-            .0
-            .into_iter()
-            .filter_map(|a| a.address)
-            .collect()
-    });
-    let from = message
-        .from()
-        .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
-        .and_then(|add| add.address)
-        .unwrap_or_else(|| "unknown".to_string());
-
-    let attachments: Vec<String> = message
-        .attachments()
-        .filter(|att| {
-            let disp = att.content_disposition();
-            let is_inline = disp.map(|d| d.is_inline()).unwrap_or(false);
-            let has_filename = att.attachment_name().is_some();
-            has_filename && !is_inline
-        })
-        .filter_map(|att| att.attachment_name())
-        .map(|name| name.to_string())
-        .collect();
-    let envelope = Envelope {
-        id: create_hash(account_id, &message_id),
-        message_id,
-        account_id,
-        mailbox_id,
-        uid,
-        subject,
-        text,
-        from,
-        to: to.unwrap_or_default(),
-        cc: cc.unwrap_or_default(),
-        bcc: bcc.unwrap_or_default(),
-        date,
-        internal_date,
-        size,
-        thread_id,
-        attachments,
-        tags: None,
-        account_email: None,
-        mailbox_name: None,
-    };
-    Ok(envelope)
+    extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id)
 }
 
 pub fn extract_envelope_from_eml(
     body: &[u8],
     account_id: u64,
     mailbox_id: u64,
-) -> BichonResult<Envelope> {
-    let uid = 0;
-    let size = body.len() as u32;
+) -> BichonResult<(Envelope, Vec<AttachmentInfo>)> {
+    extract_envelope_core(body, 0, body.len() as u32, 0, account_id, mailbox_id).map(
+        |(mut env, att)| {
+            if env.internal_date == 0 {
+                env.internal_date = env.date;
+            }
+            (env, att)
+        },
+    )
+}
+
+fn extract_envelope_core(
+    body: &[u8],
+    uid: u32,
+    size: u32,
+    internal_date: i64,
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<(Envelope, Vec<AttachmentInfo>)> {
     let message = MessageParser::new().parse(body).ok_or_else(|| {
         raise_error!(
             "Email header parse result is not available".into(),
@@ -156,7 +87,8 @@ pub fn extract_envelope_from_eml(
     let message_id = message
         .message_id()
         .map(String::from)
-        .unwrap_or(generate_message_id());
+        .unwrap_or_else(generate_message_id);
+
     let in_reply_to = message.in_reply_to().as_text().map(String::from);
     let references = extract_references(&message);
     let thread_id = compute_thread_id(in_reply_to, references, &message_id);
@@ -167,46 +99,66 @@ pub fn extract_envelope_from_eml(
     }
 
     let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
-    let bcc: Option<Vec<String>> = message.bcc().map(|addr| {
-        AddrVec::from(addr)
-            .0
-            .into_iter()
-            .filter_map(|a| a.address)
-            .collect()
-    });
-    let cc: Option<Vec<String>> = message.cc().map(|addr| {
-        AddrVec::from(addr)
-            .0
-            .into_iter()
-            .filter_map(|a| a.address)
-            .collect()
-    });
-    let to: Option<Vec<String>> = message.to().map(|addr| {
-        AddrVec::from(addr)
-            .0
-            .into_iter()
-            .filter_map(|a| a.address)
-            .collect()
-    });
+
+    let parse_addrs = |addrs: Option<&Address<'_>>| {
+        addrs
+            .map(|addr| {
+                AddrVec::from(addr)
+                    .0
+                    .into_iter()
+                    .filter_map(|a| a.address)
+                    .collect()
+            })
+            .unwrap_or_default()
+    };
+
+    let bcc = parse_addrs(message.bcc());
+    let cc = parse_addrs(message.cc());
+    let to = parse_addrs(message.to());
+
     let from = message
         .from()
         .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
         .and_then(|add| add.address)
         .unwrap_or_else(|| "unknown".to_string());
-
-    let attachments: Vec<String> = message
+    let attachments: Vec<AttachmentInfo> = message
         .attachments()
-        .filter(|att| {
-            let disp = att.content_disposition();
-            let is_inline = disp.map(|d| d.is_inline()).unwrap_or(false);
-            let has_filename = att.attachment_name().is_some();
-            has_filename && !is_inline
+        .filter_map(|attachment| {
+            let content_id = attachment.content_id().map(Into::into);
+            let inline = attachment
+                .content_disposition()
+                .map(|d| d.is_inline())
+                .unwrap_or(false);
+            if inline && content_id.is_some() {
+                return None;
+            }
+
+            let file_type = attachment
+                .content_type()
+                .map(|ct| {
+                    format!(
+                        "{}/{}",
+                        ct.c_type.as_ref(),
+                        ct.c_subtype.as_deref().unwrap_or("")
+                    )
+                })
+                .unwrap_or_else(|| "application/octet-stream".to_string());
+
+            Some(AttachmentInfo {
+                filename: attachment
+                    .attachment_name()
+                    .map(|name| name.to_string())
+                    .unwrap_or_default(),
+                size: attachment.contents().len(),
+                inline,
+                file_type,
+                content_id,
+            })
         })
-        .filter_map(|att| att.attachment_name())
-        .map(|name| name.to_string())
         .collect();
+
     let envelope = Envelope {
-        id: create_hash(account_id, &message_id),
+        id: create_hash2(account_id, mailbox_id, &message_id),
         message_id,
         account_id,
         mailbox_id,
@@ -214,19 +166,20 @@ pub fn extract_envelope_from_eml(
         subject,
         text,
         from,
-        to: to.unwrap_or_default(),
-        cc: cc.unwrap_or_default(),
-        bcc: bcc.unwrap_or_default(),
+        to,
+        cc,
+        bcc,
         date,
-        internal_date: date,
+        internal_date,
         size,
         thread_id,
-        attachments,
+        attachment_count: attachments.len(),
         tags: None,
         account_email: None,
         mailbox_name: None,
     };
-    Ok(envelope)
+
+    Ok((envelope, attachments))
 }
 
 pub fn compute_thread_id(