From 3f4b37be172a6358022d7a79e67aa06b3b92148c Mon Sep 17 00:00:00 2001 From: rustmailer Date: Mon, 5 Jan 2026 14:31:04 +0800 Subject: [PATCH] feat(cli): add interactive email import tool for EML, MBOX, and Thunderbird - Implement `bichonctl` interactive CLI using `dialoguer`. - Support recursive EML directory scanning with folder structure preservation. - Support single MBOX file streaming import. - Support Thunderbird profile import with automatic `.sbd` hierarchy detection. - Add batch processing (Base64 encoding & batch API requests) for improved performance. --- Cargo.lock | 81 ++++++++++- Cargo.toml | 8 +- config.toml | 2 + src/bin/bichon_cli.rs | 1 - src/bin/bichonctl.rs | 95 +++++++++++++ src/modules/account/migration.rs | 6 +- src/modules/cli/auth.rs | 169 +++++++++++++++++++++++ src/modules/cli/eml/mod.rs | 133 ++++++++++++++++++ src/modules/cli/mbox/gmail.rs | 43 ++++++ src/modules/cli/mbox/mod.rs | 145 ++++++++++++++++++++ src/modules/cli/mbox/reader.rs | 210 +++++++++++++++++++++++++++++ src/modules/cli/mod.rs | 34 +++++ src/modules/cli/sender.rs | 54 ++++++++ src/modules/cli/thunderbird/mod.rs | 114 ++++++++++++++++ src/modules/rest/api/account.rs | 5 +- 15 files changed, 1089 insertions(+), 11 deletions(-) create mode 100644 config.toml delete mode 100644 src/bin/bichon_cli.rs create mode 100644 src/bin/bichonctl.rs create mode 100644 src/modules/cli/auth.rs create mode 100644 src/modules/cli/eml/mod.rs create mode 100644 src/modules/cli/mbox/gmail.rs create mode 100644 src/modules/cli/mbox/mod.rs create mode 100644 src/modules/cli/mbox/reader.rs create mode 100644 src/modules/cli/sender.rs create mode 100644 src/modules/cli/thunderbird/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 0e1c156..2af6737 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -435,7 +435,9 @@ dependencies = [ "cacache", "chrono", "clap", + "console", "dashmap", + "dialoguer", "email_address", "encoding_rs", "futures", @@ -449,6 +451,7 @@ dependencies = [ "itoa", "lru 0.16.2", "mail-parser", + "memmap2 0.9.9", "mimalloc", "mime_guess", "murmur3", @@ -480,6 +483,7 @@ dependencies = [ "tokio-io-timeout", "tokio-rustls", "tokio-socks", + "toml", "tracing", "tracing-appender", "tracing-subscriber", @@ -815,6 +819,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width 0.2.2", + "windows-sys 0.61.2", +] + [[package]] name = "const_fn" version = "0.4.11" @@ -1089,6 +1106,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "dialoguer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f104b501bf2364e78d0d3974cbc774f738f5865306ed128e1e0d7499c0ad96" +dependencies = [ + "console", + "shell-words", + "tempfile", + "zeroize", +] + [[package]] name = "digest" version = "0.9.0" @@ -1152,6 +1181,12 @@ dependencies = [ "serde", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -3700,6 +3735,15 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3795,6 +3839,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + [[package]] name = "shlex" version = "1.3.0" @@ -4557,10 +4607,25 @@ dependencies = [ ] [[package]] -name = "toml_datetime" -version = "0.7.3" +name = "toml" +version = "0.9.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +checksum = "0825052159284a1a8b4d6c0c86cbc801f2da5afd2b225fa548c72f2e74002f48" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] @@ -4579,13 +4644,19 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" dependencies = [ "winnow", ] +[[package]] +name = "toml_writer" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" + [[package]] name = "tower" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index e07190e..f05e77e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,8 @@ name = "bichon" path = "src/main.rs" [[bin]] -name = "bichon-cli" -path = "src/bin/bichon_cli.rs" +name = "bichonctl" +path = "src/bin/bichonctl.rs" [features] default = [] @@ -111,6 +111,10 @@ tantivy = { version = "0.25.0", features = ["quickwit", "zstd-compression"] } itoa = "1.0.17" html2text = "0.16.5" bytes = "1.11.0" +dialoguer = "0.12.0" +console = "0.16.2" +toml = "0.9.8" +memmap2 = "0.9.9" [dev-dependencies] #bincode = "1.3.3" #secret-lib = "1.0.0" diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..4af6015 --- /dev/null +++ b/config.toml @@ -0,0 +1,2 @@ +base_url = "http://localhost:15630" +api_token = "UvkGJO0Mn1tO6igGZwQnTtI2" diff --git a/src/bin/bichon_cli.rs b/src/bin/bichon_cli.rs deleted file mode 100644 index f328e4d..0000000 --- a/src/bin/bichon_cli.rs +++ /dev/null @@ -1 +0,0 @@ -fn main() {} diff --git a/src/bin/bichonctl.rs b/src/bin/bichonctl.rs new file mode 100644 index 0000000..9b86993 --- /dev/null +++ b/src/bin/bichonctl.rs @@ -0,0 +1,95 @@ +use bichon::modules::cli::{ + auth::verify_user_and_get_account, eml::handle_eml_directory_import, + mbox::handle_mbox_single_file_import, thunderbird::handle_thunderbird_import, BichonCli, + BichonCtlConfig, +}; +use clap::Parser; +use console::style; +use dialoguer::{theme::ColorfulTheme, Confirm, Input, Select}; +use std::fs; + +#[tokio::main] +async fn main() { + let cli = BichonCli::parse(); + let theme = ColorfulTheme::default(); + let config_path = &cli.config; + let mut current_config: Option = None; + + if config_path.exists() { + if let Ok(content) = fs::read_to_string(config_path) { + if let Ok(config) = toml::from_str::(&content) { + println!("{}", style("✔ Existing configuration found:").green()); + println!(" Base URL: {}", style(&config.base_url).yellow()); + println!(" API Token: {}", style(&config.api_token).yellow()); + + // Confirm with user + if Confirm::with_theme(&theme) + .with_prompt("Do you want to use this configuration?") + .default(true) + .interact() + .unwrap() + { + current_config = Some(config); + } + } + } + } + + let final_config = match current_config { + Some(conf) => conf, + None => { + println!("\n{}", style("Please enter Bichon service details:").bold()); + + let url: String = Input::with_theme(&theme) + .with_prompt("Bichon Base URL") + .default("http://localhost:15630".into()) + .interact_text() + .unwrap(); + + let token: String = Input::with_theme(&theme) + .with_prompt("API Token") + .interact_text() + .unwrap(); + + let conf = BichonCtlConfig { + base_url: url, + api_token: token, + }; + + // 3. Offer to save the new configuration + if Confirm::with_theme(&theme) + .with_prompt("Save this configuration for future use?") + .default(true) + .interact() + .unwrap() + { + let toml_str = toml::to_string(&conf).unwrap(); + fs::write(config_path, toml_str).expect("Failed to save config file"); + println!("{}", style("Configuration saved successfully!").green()); + } + conf + } + }; + + let target_account_id = verify_user_and_get_account(&final_config, &theme).await; + + let import_modes = &[ + "EML: Scan directory recursively (Maintains folder structure)", + "MBOX: Single archive file (Stream from one file)", + "Thunderbird: Import from local profile directory", + ]; + + let mode_idx = Select::with_theme(&theme) + .with_prompt("Select import method") + .items(import_modes) + .default(0) + .interact() + .unwrap(); + + match mode_idx { + 0 => handle_eml_directory_import(&final_config, target_account_id, &theme).await, + 1 => handle_mbox_single_file_import(&final_config, target_account_id, &theme).await, + 2 => handle_thunderbird_import(&final_config, target_account_id, &theme).await, + _ => unreachable!(), + } +} diff --git a/src/modules/account/migration.rs b/src/modules/account/migration.rs index 002265b..dae82bd 100644 --- a/src/modules/account/migration.rs +++ b/src/modules/account/migration.rs @@ -363,11 +363,13 @@ impl AccountV3 { list_all_impl(DB_MANAGER.meta_db()).await } - pub async fn minimal_list() -> BichonResult> { + pub async fn minimal_list(only_nosync: bool) -> BichonResult> { let result = list_all_impl(DB_MANAGER.meta_db()) .await? .into_iter() - //.filter(|a: &AccountModel| a.enabled) + .filter(|account: &AccountModel| { + !only_nosync || matches!(account.account_type, AccountType::NoSync) + }) .map(|account: AccountModel| MinimalAccount { id: account.id, email: account.email, diff --git a/src/modules/cli/auth.rs b/src/modules/cli/auth.rs new file mode 100644 index 0000000..cfdca80 --- /dev/null +++ b/src/modules/cli/auth.rs @@ -0,0 +1,169 @@ +use std::process; + +use console::style; +use dialoguer::{theme::ColorfulTheme, Select}; +use reqwest::Client; + +use crate::modules::{ + account::payload::MinimalAccount, + cli::BichonCtlConfig, + users::{permissions::Permission, view::UserView}, +}; + +pub async fn verify_user_and_get_account(config: &BichonCtlConfig, theme: &ColorfulTheme) -> u64 { + let client = Client::new(); + let url = format!("{}/api/v1/current-user", config.base_url); + + let response = match client + .get(&url) + .header("Authorization", format!("Bearer {}", config.api_token)) + .send() + .await + { + Ok(res) => res, + Err(e) => { + eprintln!( + "\n{} {}", + style("✘ Network Error:").red().bold(), + "Could not connect to Bichon service." + ); + eprintln!("{} {}", style("Details:").dim(), e); + eprintln!( + "\n{} Please check if the Base URL is correct and the server is running.", + style("Tip:").cyan() + ); + process::exit(1); + } + }; + + if !response.status().is_success() { + let status = response.status(); + let error_body = response + .text() + .await + .unwrap_or_else(|_| "No error detail provided".to_string()); + + eprintln!( + "\n{} Server returned an error (Status: {})", + style("✘ API Error:").red().bold(), + style(status).yellow() + ); + + if status == 401 { + eprintln!( + "{} Your API Token seems to be invalid or expired.", + style("Context:").dim() + ); + } else if status == 404 { + eprintln!( + "{} The endpoint was not found. Please check your Base URL.", + style("Context:").dim() + ); + } + + eprintln!("{} {}", style("Response:").dim(), error_body); + process::exit(1); + } + + let user: UserView = response.json().await.expect("Failed to parse user data"); + println!("Welcome, {}!", style(&user.username).cyan()); + + let account_list_url = format!( + "{}/api/v1/minimal-account-list?only_nosync=true", + config.base_url + ); + let acc_response = client + .get(&account_list_url) + .header("Authorization", format!("Bearer {}", config.api_token)) + .send() + .await + .expect("Failed to fetch account list"); + + if !acc_response.status().is_success() { + panic!( + "Failed to retrieve accounts. Status: {}", + acc_response.status() + ); + } + + let accounts: Vec = acc_response + .json() + .await + .expect("Failed to parse minimal account list"); + + if accounts.is_empty() { + println!( + "\n{}", + style("Error: No 'nosync' accounts found.").red().bold() + ); + println!( + "{}", + style("Mail import is only supported for 'nosync' type accounts.").dim() + ); + println!( + "Please create a new {} account in the Bichon web interface first.", + style("Nosync").bold().yellow() + ); + process::exit(1); + } + let required_permission = Permission::DATA_IMPORT_BATCH; + let mut selectable_accounts = Vec::new(); + let mut options = Vec::new(); + + for acc in accounts { + let has_permission = if let Some(perms) = user.account_permissions.get(&acc.id) { + perms.iter().any(|p| p == required_permission) + } else { + user.global_permissions + .iter() + .any(|p| p == Permission::DATA_MANAGE_ALL || p == Permission::ROOT) + }; + + let status_prefix = if has_permission { + style(" [READY] ").green() + } else { + style(" [NO PERMISSION] ").red() + }; + + options.push(format!( + "{}{} - {}", + status_prefix, + style(&acc.email).bold(), + style(format!("ID: {}", acc.id)).dim() + )); + + selectable_accounts.push((acc, has_permission)); + } + + let selection = Select::with_theme(theme) + .with_prompt("Select the target account for import") + .items(&options) + .default(0) + .max_length(10) + .interact() + .unwrap(); + + let (selected_acc, can_import) = &selectable_accounts[selection]; + + if !*can_import { + eprintln!( + "\n{} You do not have '{}' permission for account {}.", + style("✘ Permission Denied:").red().bold(), + style(required_permission).yellow(), + style(&selected_acc.email).cyan() + ); + eprintln!( + "{} Please contact your administrator to upgrade your role for this account.", + style("Tip:").dim() + ); + process::exit(1); + } + + println!( + "{} Targeting account: {}", + style("✔").green(), + style(&selected_acc.email).cyan().bold() + ); + + selected_acc.id +} diff --git a/src/modules/cli/eml/mod.rs b/src/modules/cli/eml/mod.rs new file mode 100644 index 0000000..bf8a6a5 --- /dev/null +++ b/src/modules/cli/eml/mod.rs @@ -0,0 +1,133 @@ +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; + +use console::style; +use dialoguer::{theme::ColorfulTheme, Input}; +use mail_parser::MessageParser; +use reqwest::Client; + +use crate::{ + base64_encode_url_safe, + modules::cli::{sender::send_batch_request, BichonCtlConfig}, +}; + +pub async fn handle_eml_directory_import( + config: &BichonCtlConfig, + account_id: u64, + theme: &ColorfulTheme, +) { + let root_str: String = Input::with_theme(theme) + .with_prompt("Enter the ROOT directory to scan for .eml files") + .validate_with(|input: &String| { + let p = std::path::Path::new(input); + if p.exists() && p.is_dir() { + Ok(()) + } else { + Err("Directory not found.") + } + }) + .interact_text() + .unwrap(); + + let root_path = std::path::PathBuf::from(root_str); + let mut tasks: HashMap> = HashMap::new(); + println!( + "{}", + style("🔍 Scanning recursively using std::fs...").dim() + ); + if let Err(e) = scan_dir(&root_path, &root_path, &mut tasks) { + eprintln!("Error scanning directory: {}", e); + return; + } + + if tasks.is_empty() { + println!("{}", style("No .eml files found.").yellow()); + } else { + process_and_upload(config, account_id, tasks).await; + } +} + +fn scan_dir( + root: &Path, + current: &Path, + tasks: &mut HashMap>, +) -> std::io::Result<()> { + if current.is_dir() { + for entry in fs::read_dir(current)? { + let entry = entry?; + let path = entry.path(); + + if path.is_dir() { + scan_dir(root, &path, tasks)?; + } else if path.is_file() { + if path.extension().and_then(|s| s.to_str()) == Some("eml") { + let rel_path = path.strip_prefix(root).unwrap_or(Path::new("")); + let mailbox_name = rel_path + .parent() + .map(|p| p.to_string_lossy().replace('\\', "/")) + .unwrap_or_default(); + let folder = if mailbox_name.is_empty() { + "Inbox".to_string() + } else { + mailbox_name + }; + tasks.entry(folder).or_insert_with(|| Vec::new()).push(path); + } + } + } + } + Ok(()) +} + +async fn process_and_upload( + config: &BichonCtlConfig, + account_id: u64, + tasks: HashMap>, +) { + let client = Client::new(); + let batch_size = 50; + + for (mailbox, files) in tasks { + println!("\n🚀 Processing mailbox: {}", style(&mailbox).cyan().bold()); + + let mut current_batch = Vec::new(); + + for file_path in files { + let body = match fs::read(&file_path) { + Ok(b) => b, + Err(e) => { + eprintln!( + " {} Failed to read file {:?}: {}", + style("✘").red(), + file_path, + e + ); + continue; + } + }; + + if MessageParser::new().parse(&body).is_some() { + let b64_content = base64_encode_url_safe!(&body); + current_batch.push(b64_content); + + if current_batch.len() >= batch_size { + let to_send = current_batch; + current_batch = Vec::with_capacity(batch_size); + send_batch_request(&client, config, account_id, &mailbox, to_send).await; + } + } else { + eprintln!( + " {} Invalid format, skipping: {:?}", + style("⚠").yellow(), + file_path + ); + } + } + if !current_batch.is_empty() { + send_batch_request(&client, config, account_id, &mailbox, current_batch).await; + } + } +} diff --git a/src/modules/cli/mbox/gmail.rs b/src/modules/cli/mbox/gmail.rs new file mode 100644 index 0000000..8869bd9 --- /dev/null +++ b/src/modules/cli/mbox/gmail.rs @@ -0,0 +1,43 @@ +use std::collections::HashSet; + +pub fn determine_folder(labels_raw: &str) -> String { + let mut status_blacklist = HashSet::new(); + status_blacklist.insert("Opened"); + status_blacklist.insert("Unread"); + status_blacklist.insert("Archived"); + + let all_labels: Vec<&str> = labels_raw + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect(); + + if all_labels.is_empty() { + return "Unknown".to_string(); + } + + let filtered: Vec<&str> = all_labels + .iter() + .filter(|&&l| !status_blacklist.contains(l)) + .cloned() + .collect(); + + match filtered.len() { + // Case A: If all labels were status labels, fallback to the first original label + 0 => all_labels[0].to_string(), + // Case B: If only one label remains, that's our target destination + 1 => filtered[0].to_string(), + // Case C: Multiple labels remain (e.g., ["Inbox", "medium"]) + _ => { + // Prioritize custom business labels by excluding generic locations like "Inbox" or "Sent" + let business_label = filtered.iter().find(|&&l| l != "Inbox" && l != "Sent"); + + match business_label { + // Return the first non-generic label found + Some(label) => label.to_string(), + // If only generic labels remain (e.g., ["Sent", "Inbox"]), pick the first available + None => filtered[0].to_string(), + } + } + } +} diff --git a/src/modules/cli/mbox/mod.rs b/src/modules/cli/mbox/mod.rs new file mode 100644 index 0000000..a7fb52f --- /dev/null +++ b/src/modules/cli/mbox/mod.rs @@ -0,0 +1,145 @@ +use std::collections::HashMap; +use std::path::PathBuf; + +use crate::base64_encode_url_safe; +use crate::modules::cli::mbox::gmail::determine_folder; +use crate::modules::cli::mbox::reader::MboxFile; +use crate::modules::cli::sender::send_batch_request; +use crate::modules::cli::BichonCtlConfig; +use console::style; +use dialoguer::{theme::ColorfulTheme, Input}; +use dialoguer::{Confirm, Select}; +use mail_parser::MessageParser; +use reqwest::Client; + +pub mod gmail; +pub mod reader; + +pub async fn handle_mbox_single_file_import( + config: &BichonCtlConfig, + account_id: u64, + theme: &ColorfulTheme, +) { + let path_str: String = Input::with_theme(theme) + .with_prompt("Enter the path to your SINGLE .mbox file") + .validate_with(|input: &String| { + let p = std::path::Path::new(input); + if !p.exists() { + return Err("The specified path does not exist."); + } + if !p.is_file() { + return Err("MBOX mode requires a SINGLE file, not a directory."); + } + Ok(()) + }) + .interact_text() + .unwrap(); + + let mbox_path = PathBuf::from(path_str); + + let options = vec![ + "Use labels from mail headers (X-Gmail-Labels)", + "Specify a single target folder for all emails", + ]; + + let selection = Select::with_theme(theme) + .with_prompt("How should we determine the target folder?") + .items(&options) + .default(0) + .interact() + .unwrap(); + + let target_folder: Option = match selection { + 0 => None, + 1 => { + let folder: String = Input::with_theme(theme) + .with_prompt("Target folder name") + .default("INBOX".into()) + .interact_text() + .unwrap(); + Some(folder) + } + _ => unreachable!(), + }; + + if let Some(ref folder) = target_folder { + println!( + "{}", + style(format!("Mode: Fixed folder ({})", folder)).dim() + ); + } else { + println!("{}", style("Mode: Dynamic (header-based)").dim()); + } + + println!( + "\n{} Ready to process MBOX file: {}", + style("✔").green(), + style(mbox_path.display()).cyan() + ); + + if let Ok(meta) = std::fs::metadata(&mbox_path) { + let size_mb = meta.len() as f64 / 1024.0 / 1024.0; + println!( + "{}", + style(format!("Processing file: {:.1} MB", size_mb)).dim() + ); + } + + if Confirm::with_theme(theme) + .with_prompt("Start importing?") + .default(true) + .interact() + .unwrap() + { + run_import(account_id, &mbox_path, config, target_folder).await + } +} + +pub async fn run_import( + account_id: u64, + mbox_path: &PathBuf, + config: &BichonCtlConfig, + target_folder: Option, +) { + let client = Client::new(); + let mbox = MboxFile::from_file(mbox_path).unwrap(); + + let mut folder_buffers: HashMap> = HashMap::new(); + let batch_limit = 50; + + println!("Starting import process..."); + + for e in mbox.iter() { + let body = e.data; + let message = MessageParser::new().parse(body).unwrap(); + + let folder_name = match target_folder { + Some(ref folder_name) => folder_name.clone(), + None => { + let labels = message + .header("X-Gmail-Labels") + .and_then(|h| h.as_text()) + .unwrap_or("Inbox"); + determine_folder(labels) + } + }; + let b64_eml = base64_encode_url_safe!(&body); + let buffer = folder_buffers + .entry(folder_name.clone()) + .or_insert_with(|| Vec::new()); + buffer.push(b64_eml); + + if buffer.len() >= batch_limit { + let emls_to_send = folder_buffers.remove(&folder_name).unwrap(); + send_batch_request(&client, config, account_id, &folder_name, emls_to_send).await; + } + } + + for (folder_name, emls) in folder_buffers { + if !emls.is_empty() { + send_batch_request(&client, config, account_id, &folder_name, emls).await; + } + } + + println!("{}", style("Import completed successfully!").green().bold()); +} diff --git a/src/modules/cli/mbox/reader.rs b/src/modules/cli/mbox/reader.rs new file mode 100644 index 0000000..14ff3e9 --- /dev/null +++ b/src/modules/cli/mbox/reader.rs @@ -0,0 +1,210 @@ +use memmap2::Mmap; +use std::fs; +use std::io; +use std::path::Path; + +pub struct MboxFile { + map: Mmap, +} + +impl MboxFile { + pub fn from_file(name: &Path) -> io::Result { + let file = fs::File::open(name)?; + let metadata = file.metadata()?; + if metadata.len() == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Empty MBOX file", + )); + } + let map = unsafe { Mmap::map(&file)? }; + Ok(Self { map }) + } + + pub fn iter(&self) -> MboxReader<'_> { + MboxReader::new(&self.map) + } +} + +pub struct Entry<'a> { + pub offset: usize, + pub data: &'a [u8], +} + +pub struct MboxReader<'a> { + data: &'a [u8], + len: usize, + scan_pos: usize, + body_start: Option, +} + +impl<'a> MboxReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { + data, + len: data.len(), + scan_pos: 0, + body_start: None, + } + } + + fn is_from_line(&self, i: usize) -> bool { + if i + 5 > self.len { + return false; + } + + if i == 0 { + &self.data[0..5] == b"From " + } else { + self.data[i - 1] == b'\n' && &self.data[i..i + 5] == b"From " + } + } + + fn skip_from_line(&self, mut i: usize) -> usize { + while i < self.len && self.data[i] != b'\n' { + i += 1; + } + if i < self.len { + i += 1; + } + i + } +} + +impl<'a> Iterator for MboxReader<'a> { + type Item = Entry<'a>; + + fn next(&mut self) -> Option { + while self.scan_pos < self.len { + if self.is_from_line(self.scan_pos) { + let from_pos = self.scan_pos; + let body_pos = self.skip_from_line(from_pos); + + if let Some(start) = self.body_start { + let entry = Entry { + offset: start, + data: &self.data[start..from_pos], + }; + self.body_start = Some(body_pos); + self.scan_pos = body_pos; + return Some(entry); + } else { + self.body_start = Some(body_pos); + self.scan_pos = body_pos; + continue; + } + } + + self.scan_pos += 1; + } + if let Some(start) = self.body_start.take() { + return Some(Entry { + offset: start, + data: &self.data[start..self.len], + }); + } + None + } +} + +#[cfg(test)] +mod tests { + use mail_parser::MessageParser; + + use crate::modules::cli::mbox::gmail::determine_folder; + + use super::*; + + fn collect_entries(data: &[u8]) -> Vec<&[u8]> { + let reader = MboxReader::new(data); + reader.map(|e| e.data).collect() + } + + #[test] + fn two_mails() { + let data = b"From a\nmail1\nFrom b\nmail2\n"; + let e = collect_entries(data); + assert_eq!(e, vec![b"mail1\n", b"mail2\n"]); + } + #[test] + fn no_trailing_newline() { + let data = b"From a\nmail1"; + let e = collect_entries(data); + assert_eq!(e, vec![b"mail1"]); + } + + #[test] + fn from_inside_body() { + let data = b"From a\nhello\nFrom is here\nbye\n"; + let e = collect_entries(data); + assert_eq!(e.len(), 2); + } + + #[test] + fn inline_from_not_separator() { + let data = b"From a\nhello From world\n"; + let e = collect_entries(data); + assert_eq!(e.len(), 1); + } + + #[test] + fn realistic_mbox() { + let data = b"From a\nH:1\n\nbody1\nFrom b\nH:2\n\nbody2\n"; + let e = collect_entries(data); + assert_eq!(e.len(), 2); + } + + #[test] + fn empty_body() { + let data = b"From a\nFrom b\nbody\n"; + let e = collect_entries(data); + assert_eq!(e[0], b""); + assert_eq!(e[1], b"body\n"); + } + + #[test] + fn only_from_line() { + let data = b"From a\n"; + let e = collect_entries(data); + assert_eq!(e.len(), 1); + assert_eq!(e[0], b""); + } + + #[test] + fn windows_newlines() { + let data = b"From a\r\nbody\r\nFrom b\r\nbody2\r\n"; + let e = collect_entries(data); + assert_eq!(e.len(), 2); + } + + #[test] + fn many_small_mails() { + let mut data = Vec::new(); + for i in 0..1000 { + data.extend_from_slice(b"From a\nx\n"); + } + let e = collect_entries(&data); + assert_eq!(e.len(), 1000); + } + + #[test] + fn test11() { + let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap(); + + for e in mbox.iter() { + let body = e.data; + + let message = MessageParser::new().parse(body).unwrap(); + let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap(); + //println!("offset={} X-Gmail-Labels={:?}", e.offset, labels); + println!( + "X-Gmail-Labels={:?}, determine_folder={}", + labels, + determine_folder(labels) + ) + + + + } + } +} diff --git a/src/modules/cli/mod.rs b/src/modules/cli/mod.rs index e69de29..9c5da8f 100644 --- a/src/modules/cli/mod.rs +++ b/src/modules/cli/mod.rs @@ -0,0 +1,34 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +use crate::bichon_version; + +pub mod auth; +pub mod eml; +pub mod mbox; +pub mod sender; +pub mod thunderbird; +#[derive(Parser, Debug)] +#[command( + name = "bichonctl", + author = "rustmailer", + version = bichon_version!(), + about = "A CLI tool to import email data into Bichon service" +)] +pub struct BichonCli { + /// Path to the configuration file + #[arg( + short, + long, + default_value = "config.toml", + value_name = "FILE", + help = "Sets a custom config file" + )] + pub config: std::path::PathBuf, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BichonCtlConfig { + pub base_url: String, + pub api_token: String, +} diff --git a/src/modules/cli/sender.rs b/src/modules/cli/sender.rs new file mode 100644 index 0000000..9e9ce6a --- /dev/null +++ b/src/modules/cli/sender.rs @@ -0,0 +1,54 @@ +use console::style; +use reqwest::Client; + +use crate::modules::{cli::BichonCtlConfig, import::BatchEmlRequest}; + +pub async fn send_batch_request( + client: &Client, + config: &BichonCtlConfig, + account_id: u64, + folder: &str, + emls: Vec, +) { + let url = format!("{}/api/v1/import", config.base_url); + let payload = BatchEmlRequest { + account_id, + mail_folder: folder.to_string(), + emls, + }; + + let count = payload.emls.len(); + + match client + .post(&url) + .header("Authorization", format!("Bearer {}", config.api_token)) + .json(&payload) + .send() + .await + { + Ok(res) if res.status().is_success() => { + println!( + " {} Sent {} emails to [{}]", + style("✔").green(), + count, + folder + ); + } + Ok(res) => { + eprintln!( + " {} Failed to send to [{}]. Status: {}", + style("✘").red(), + folder, + res.status() + ); + } + Err(e) => { + eprintln!( + " {} Network error on [{}]: {}", + style("✘").red(), + folder, + e + ); + } + } +} diff --git a/src/modules/cli/thunderbird/mod.rs b/src/modules/cli/thunderbird/mod.rs new file mode 100644 index 0000000..a9bf6e1 --- /dev/null +++ b/src/modules/cli/thunderbird/mod.rs @@ -0,0 +1,114 @@ +use std::{collections::HashMap, path::PathBuf}; + +use crate::modules::cli::{mbox::run_import, BichonCtlConfig}; +use console::style; +use dialoguer::{theme::ColorfulTheme, Confirm, Input}; + +pub async fn handle_thunderbird_import( + config: &BichonCtlConfig, + account_id: u64, + theme: &ColorfulTheme, +) { + let root_str: String = Input::with_theme(theme) + .with_prompt("Enter your Thunderbird Mail/ImapMail directory") + .validate_with(|input: &String| { + let p = std::path::Path::new(input); + if p.exists() && p.is_dir() { + Ok(()) + } else { + Err("Directory not found.") + } + }) + .interact_text() + .unwrap(); + + let root_path = std::path::PathBuf::from(&root_str); + println!("{}", style("🔍 Scanning Thunderbird structure...").dim()); + + let mut mbox_tasks: HashMap = HashMap::new(); + + fn scan_thunderbird_dir( + root: &std::path::Path, + current: &std::path::Path, + tasks: &mut HashMap, + ) { + if let Ok(entries) = std::fs::read_dir(current) { + for entry in entries.flatten() { + let path = entry.path(); + let file_name = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + + if path.is_dir() { + scan_thunderbird_dir(root, &path, tasks); + } else { + let extension = path.extension().and_then(|s| s.to_str()).unwrap_or(""); + match extension { + "msf" | "dat" | "html" | "json" | "txt" | "sqlite" => continue, + _ => {} + } + + if file_name == "filterlog.html" || file_name == "msgFilterRules.dat" { + continue; + } + + if !extension.is_empty() { + continue; + } + + if let Ok(rel) = path.strip_prefix(root) { + let mailbox = rel.to_string_lossy().replace(".sbd", "").replace('\\', "/"); + tasks.insert(mailbox, path); + } + } + } + } + } + + scan_thunderbird_dir(&root_path, &root_path, &mut mbox_tasks); + if mbox_tasks.is_empty() { + println!( + "{}", + style("No mailboxes found in the specified directory.").yellow() + ); + return; + } + + println!("\n{}", style("🔍 Scanned Mailboxes:").bold().underlined()); + let mut sorted_keys: Vec<_> = mbox_tasks.keys().collect(); + sorted_keys.sort(); + + for name in &sorted_keys { + let path = &mbox_tasks[*name]; + let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + let size_mb = file_size as f64 / 1024.0 / 1024.0; + + println!( + " {} {} ({:.2} MB)", + style("•").dim(), + style(name).cyan(), + size_mb + ); + } + + println!(); + let prompt = format!("Ready to import {} mailboxes. Proceed?", mbox_tasks.len()); + if Confirm::with_theme(theme) + .with_prompt(prompt) + .default(true) + .interact() + .unwrap() + { + // 3. 执行导入 + for (mailbox_name, mbox_file) in mbox_tasks { + println!("\n🚀 Importing: {}", style(&mailbox_name).cyan().bold()); + run_import(account_id, &mbox_file, config, Some(mailbox_name)).await; + } + println!( + "\n{}", + style("✨ All mailboxes imported successfully!") + .green() + .bold() + ); + } else { + println!("{}", style("Import cancelled.").yellow()); + } +} diff --git a/src/modules/rest/api/account.rs b/src/modules/rest/api/account.rs index 1b14711..9583ccd 100644 --- a/src/modules/rest/api/account.rs +++ b/src/modules/rest/api/account.rs @@ -221,10 +221,13 @@ impl AccountApi { )] async fn minimal_accounts_list( &self, + only_nosync: Query>, context: ClientContext, ) -> ApiResult>> { let is_admin = context.user.is_admin().await; - let minimal_list = AccountModel::minimal_list().await?; + let only_nosync = only_nosync.0.unwrap_or_default(); + + let minimal_list = AccountModel::minimal_list(only_nosync).await?; if is_admin { return Ok(Json(minimal_list)); }