feat(cli): add interactive email import tool for EML, MBOX, and Thunderbird

- Implement `bichonctl` interactive CLI using `dialoguer`. - Support recursive EML directory scanning with folder structure preservation. - Support single MBOX file streaming import. - Support Thunderbird profile import with automatic `.sbd` hierarchy detection. - Add batch processing (Base64 encoding & batch API requests) for improved performance.
2026-08-03 07:48:34 +02:00 · 2026-01-05 14:31:04 +08:00
parent 09375ee11c
commit 3f4b37be17
15 changed files with 1089 additions and 11 deletions
@@ -0,0 +1,43 @@
+use std::collections::HashSet;
+
+pub fn determine_folder(labels_raw: &str) -> String {
+    let mut status_blacklist = HashSet::new();
+    status_blacklist.insert("Opened");
+    status_blacklist.insert("Unread");
+    status_blacklist.insert("Archived");
+
+    let all_labels: Vec<&str> = labels_raw
+        .split(',')
+        .map(|s| s.trim())
+        .filter(|s| !s.is_empty())
+        .collect();
+
+    if all_labels.is_empty() {
+        return "Unknown".to_string();
+    }
+
+    let filtered: Vec<&str> = all_labels
+        .iter()
+        .filter(|&&l| !status_blacklist.contains(l))
+        .cloned()
+        .collect();
+
+    match filtered.len() {
+        // Case A: If all labels were status labels, fallback to the first original label
+        0 => all_labels[0].to_string(),
+        // Case B: If only one label remains, that's our target destination
+        1 => filtered[0].to_string(),
+        // Case C: Multiple labels remain (e.g., ["Inbox", "medium"])
+        _ => {
+            // Prioritize custom business labels by excluding generic locations like "Inbox" or "Sent"
+            let business_label = filtered.iter().find(|&&l| l != "Inbox" && l != "Sent");
+
+            match business_label {
+                // Return the first non-generic label found
+                Some(label) => label.to_string(),
+                // If only generic labels remain (e.g., ["Sent", "Inbox"]), pick the first available
+                None => filtered[0].to_string(),
+            }
+        }
+    }
+}
@@ -0,0 +1,145 @@
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+use crate::base64_encode_url_safe;
+use crate::modules::cli::mbox::gmail::determine_folder;
+use crate::modules::cli::mbox::reader::MboxFile;
+use crate::modules::cli::sender::send_batch_request;
+use crate::modules::cli::BichonCtlConfig;
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Input};
+use dialoguer::{Confirm, Select};
+use mail_parser::MessageParser;
+use reqwest::Client;
+
+pub mod gmail;
+pub mod reader;
+
+pub async fn handle_mbox_single_file_import(
+    config: &BichonCtlConfig,
+    account_id: u64,
+    theme: &ColorfulTheme,
+) {
+    let path_str: String = Input::with_theme(theme)
+        .with_prompt("Enter the path to your SINGLE .mbox file")
+        .validate_with(|input: &String| {
+            let p = std::path::Path::new(input);
+            if !p.exists() {
+                return Err("The specified path does not exist.");
+            }
+            if !p.is_file() {
+                return Err("MBOX mode requires a SINGLE file, not a directory.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let mbox_path = PathBuf::from(path_str);
+
+    let options = vec![
+        "Use labels from mail headers (X-Gmail-Labels)",
+        "Specify a single target folder for all emails",
+    ];
+
+    let selection = Select::with_theme(theme)
+        .with_prompt("How should we determine the target folder?")
+        .items(&options)
+        .default(0)
+        .interact()
+        .unwrap();
+
+    let target_folder: Option<String> = match selection {
+        0 => None,
+        1 => {
+            let folder: String = Input::with_theme(theme)
+                .with_prompt("Target folder name")
+                .default("INBOX".into())
+                .interact_text()
+                .unwrap();
+            Some(folder)
+        }
+        _ => unreachable!(),
+    };
+
+    if let Some(ref folder) = target_folder {
+        println!(
+            "{}",
+            style(format!("Mode: Fixed folder ({})", folder)).dim()
+        );
+    } else {
+        println!("{}", style("Mode: Dynamic (header-based)").dim());
+    }
+
+    println!(
+        "\n{} Ready to process MBOX file: {}",
+        style("✔").green(),
+        style(mbox_path.display()).cyan()
+    );
+
+    if let Ok(meta) = std::fs::metadata(&mbox_path) {
+        let size_mb = meta.len() as f64 / 1024.0 / 1024.0;
+        println!(
+            "{}",
+            style(format!("Processing file: {:.1} MB", size_mb)).dim()
+        );
+    }
+
+    if Confirm::with_theme(theme)
+        .with_prompt("Start importing?")
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        run_import(account_id, &mbox_path, config, target_folder).await
+    }
+}
+
+pub async fn run_import(
+    account_id: u64,
+    mbox_path: &PathBuf,
+    config: &BichonCtlConfig,
+    target_folder: Option<String>,
+) {
+    let client = Client::new();
+    let mbox = MboxFile::from_file(mbox_path).unwrap();
+
+    let mut folder_buffers: HashMap<String, Vec<String>> = HashMap::new();
+    let batch_limit = 50;
+
+    println!("Starting import process...");
+
+    for e in mbox.iter() {
+        let body = e.data;
+        let message = MessageParser::new().parse(body).unwrap();
+
+        let folder_name = match target_folder {
+            Some(ref folder_name) => folder_name.clone(),
+            None => {
+                let labels = message
+                    .header("X-Gmail-Labels")
+                    .and_then(|h| h.as_text())
+                    .unwrap_or("Inbox");
+                determine_folder(labels)
+            }
+        };
+        let b64_eml = base64_encode_url_safe!(&body);
+        let buffer = folder_buffers
+            .entry(folder_name.clone())
+            .or_insert_with(|| Vec::new());
+        buffer.push(b64_eml);
+
+        if buffer.len() >= batch_limit {
+            let emls_to_send = folder_buffers.remove(&folder_name).unwrap();
+            send_batch_request(&client, config, account_id, &folder_name, emls_to_send).await;
+        }
+    }
+
+    for (folder_name, emls) in folder_buffers {
+        if !emls.is_empty() {
+            send_batch_request(&client, config, account_id, &folder_name, emls).await;
+        }
+    }
+
+    println!("{}", style("Import completed successfully!").green().bold());
+}
@@ -0,0 +1,210 @@
+use memmap2::Mmap;
+use std::fs;
+use std::io;
+use std::path::Path;
+
+pub struct MboxFile {
+    map: Mmap,
+}
+
+impl MboxFile {
+    pub fn from_file(name: &Path) -> io::Result<Self> {
+        let file = fs::File::open(name)?;
+        let metadata = file.metadata()?;
+        if metadata.len() == 0 {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidData,
+                "Empty MBOX file",
+            ));
+        }
+        let map = unsafe { Mmap::map(&file)? };
+        Ok(Self { map })
+    }
+
+    pub fn iter(&self) -> MboxReader<'_> {
+        MboxReader::new(&self.map)
+    }
+}
+
+pub struct Entry<'a> {
+    pub offset: usize,
+    pub data: &'a [u8],
+}
+
+pub struct MboxReader<'a> {
+    data: &'a [u8],
+    len: usize,
+    scan_pos: usize,
+    body_start: Option<usize>,
+}
+
+impl<'a> MboxReader<'a> {
+    fn new(data: &'a [u8]) -> Self {
+        Self {
+            data,
+            len: data.len(),
+            scan_pos: 0,
+            body_start: None,
+        }
+    }
+
+    fn is_from_line(&self, i: usize) -> bool {
+        if i + 5 > self.len {
+            return false;
+        }
+
+        if i == 0 {
+            &self.data[0..5] == b"From "
+        } else {
+            self.data[i - 1] == b'\n' && &self.data[i..i + 5] == b"From "
+        }
+    }
+
+    fn skip_from_line(&self, mut i: usize) -> usize {
+        while i < self.len && self.data[i] != b'\n' {
+            i += 1;
+        }
+        if i < self.len {
+            i += 1;
+        }
+        i
+    }
+}
+
+impl<'a> Iterator for MboxReader<'a> {
+    type Item = Entry<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        while self.scan_pos < self.len {
+            if self.is_from_line(self.scan_pos) {
+                let from_pos = self.scan_pos;
+                let body_pos = self.skip_from_line(from_pos);
+
+                if let Some(start) = self.body_start {
+                    let entry = Entry {
+                        offset: start,
+                        data: &self.data[start..from_pos],
+                    };
+                    self.body_start = Some(body_pos);
+                    self.scan_pos = body_pos;
+                    return Some(entry);
+                } else {
+                    self.body_start = Some(body_pos);
+                    self.scan_pos = body_pos;
+                    continue;
+                }
+            }
+
+            self.scan_pos += 1;
+        }
+        if let Some(start) = self.body_start.take() {
+            return Some(Entry {
+                offset: start,
+                data: &self.data[start..self.len],
+            });
+        }
+        None
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use mail_parser::MessageParser;
+
+    use crate::modules::cli::mbox::gmail::determine_folder;
+
+    use super::*;
+
+    fn collect_entries(data: &[u8]) -> Vec<&[u8]> {
+        let reader = MboxReader::new(data);
+        reader.map(|e| e.data).collect()
+    }
+
+    #[test]
+    fn two_mails() {
+        let data = b"From a\nmail1\nFrom b\nmail2\n";
+        let e = collect_entries(data);
+        assert_eq!(e, vec![b"mail1\n", b"mail2\n"]);
+    }
+    #[test]
+    fn no_trailing_newline() {
+        let data = b"From a\nmail1";
+        let e = collect_entries(data);
+        assert_eq!(e, vec![b"mail1"]);
+    }
+
+    #[test]
+    fn from_inside_body() {
+        let data = b"From a\nhello\nFrom is here\nbye\n";
+        let e = collect_entries(data);
+        assert_eq!(e.len(), 2);
+    }
+
+    #[test]
+    fn inline_from_not_separator() {
+        let data = b"From a\nhello From world\n";
+        let e = collect_entries(data);
+        assert_eq!(e.len(), 1);
+    }
+
+    #[test]
+    fn realistic_mbox() {
+        let data = b"From a\nH:1\n\nbody1\nFrom b\nH:2\n\nbody2\n";
+        let e = collect_entries(data);
+        assert_eq!(e.len(), 2);
+    }
+
+    #[test]
+    fn empty_body() {
+        let data = b"From a\nFrom b\nbody\n";
+        let e = collect_entries(data);
+        assert_eq!(e[0], b"");
+        assert_eq!(e[1], b"body\n");
+    }
+
+    #[test]
+    fn only_from_line() {
+        let data = b"From a\n";
+        let e = collect_entries(data);
+        assert_eq!(e.len(), 1);
+        assert_eq!(e[0], b"");
+    }
+
+    #[test]
+    fn windows_newlines() {
+        let data = b"From a\r\nbody\r\nFrom b\r\nbody2\r\n";
+        let e = collect_entries(data);
+        assert_eq!(e.len(), 2);
+    }
+
+    #[test]
+    fn many_small_mails() {
+        let mut data = Vec::new();
+        for i in 0..1000 {
+            data.extend_from_slice(b"From a\nx\n");
+        }
+        let e = collect_entries(&data);
+        assert_eq!(e.len(), 1000);
+    }
+
+    #[test]
+    fn test11() {
+        let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap();
+
+        for e in mbox.iter() {
+            let body = e.data;
+
+            let message = MessageParser::new().parse(body).unwrap();
+            let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap();
+            //println!("offset={} X-Gmail-Labels={:?}", e.offset, labels);
+            println!(
+                "X-Gmail-Labels={:?}, determine_folder={}",
+                labels,
+                determine_folder(labels)
+            )
+
+
+            
+        }
+    }
+}