refactor(workspace): decompose project into multiple crates

This commit is contained in:
rustmailer
2026-04-23 21:45:34 +08:00
parent 5b884125f7
commit 0b866c81ff
171 changed files with 2203 additions and 2042 deletions
+188
View File
@@ -0,0 +1,188 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::process;
use console::style;
use dialoguer::{theme::ColorfulTheme, Select};
use reqwest::Client;
use bichon_core::{
account::payload::MinimalAccount,
users::{permissions::Permission, view::UserView},
};
use crate::BichonCtlConfig;
pub async fn verify_user_and_get_account(config: &BichonCtlConfig, theme: &ColorfulTheme) -> u64 {
let client = Client::new();
let url = format!("{}/api/v1/current-user", config.base_url);
let response = match client
.get(&url)
.header("Authorization", format!("Bearer {}", config.api_token))
.send()
.await
{
Ok(res) => res,
Err(e) => {
eprintln!(
"\n{} {}",
style("✘ Network Error:").red().bold(),
"Could not connect to Bichon service."
);
eprintln!("{} {}", style("Details:").dim(), e);
eprintln!(
"\n{} Please check if the Base URL is correct and the server is running.",
style("Tip:").cyan()
);
process::exit(1);
}
};
if !response.status().is_success() {
let status = response.status();
let error_body = response
.text()
.await
.unwrap_or_else(|_| "No error detail provided".to_string());
eprintln!(
"\n{} Server returned an error (Status: {})",
style("✘ API Error:").red().bold(),
style(status).yellow()
);
if status == 401 {
eprintln!(
"{} Your API Token seems to be invalid or expired.",
style("Context:").dim()
);
} else if status == 404 {
eprintln!(
"{} The endpoint was not found. Please check your Base URL.",
style("Context:").dim()
);
}
eprintln!("{} {}", style("Response:").dim(), error_body);
process::exit(1);
}
let user: UserView = response.json().await.expect("Failed to parse user data");
println!("Welcome, {}!", style(&user.username).cyan());
let account_list_url = format!(
"{}/api/v1/minimal-account-list?only_nosync=true",
config.base_url
);
let acc_response = client
.get(&account_list_url)
.header("Authorization", format!("Bearer {}", config.api_token))
.send()
.await
.expect("Failed to fetch account list");
if !acc_response.status().is_success() {
panic!(
"Failed to retrieve accounts. Status: {}",
acc_response.status()
);
}
let accounts: Vec<MinimalAccount> = acc_response
.json()
.await
.expect("Failed to parse minimal account list");
if accounts.is_empty() {
println!(
"\n{}",
style("Error: No 'nosync' accounts found.").red().bold()
);
println!(
"{}",
style("Mail import is only supported for 'nosync' type accounts.").dim()
);
println!(
"Please create a new {} account in the Bichon web interface first.",
style("Nosync").bold().yellow()
);
process::exit(1);
}
let required_permission = Permission::DATA_IMPORT_BATCH;
let mut selectable_accounts = Vec::new();
let mut options = Vec::new();
for acc in accounts {
let has_permission = if let Some(perms) = user.account_permissions.get(&acc.id) {
perms.iter().any(|p| p == required_permission)
} else {
user.global_permissions
.iter()
.any(|p| p == Permission::DATA_MANAGE_ALL || p == Permission::ROOT)
};
let status_prefix = if has_permission {
style(" [READY] ").green()
} else {
style(" [NO PERMISSION] ").red()
};
options.push(format!(
"{}{} - {}",
status_prefix,
style(&acc.email).bold(),
style(format!("ID: {}", acc.id)).dim()
));
selectable_accounts.push((acc, has_permission));
}
let selection = Select::with_theme(theme)
.with_prompt("Select the target account for import")
.items(&options)
.default(0)
.max_length(10)
.interact()
.unwrap();
let (selected_acc, can_import) = &selectable_accounts[selection];
if !*can_import {
eprintln!(
"\n{} You do not have '{}' permission for account {}.",
style("✘ Permission Denied:").red().bold(),
style(required_permission).yellow(),
style(&selected_acc.email).cyan()
);
eprintln!(
"{} Please contact your administrator to upgrade your role for this account.",
style("Tip:").dim()
);
process::exit(1);
}
println!(
"{} Targeting account: {}",
style("").green(),
style(&selected_acc.email).cyan().bold()
);
selected_acc.id
}
+150
View File
@@ -0,0 +1,150 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
collections::HashMap,
fs,
path::{Path, PathBuf},
};
use console::style;
use dialoguer::{theme::ColorfulTheme, Input};
use mail_parser::MessageParser;
use reqwest::Client;
use bichon_core::base64_encode_url_safe;
use crate::{sender::send_batch_request, BichonCtlConfig};
pub async fn handle_eml_directory_import(
config: &BichonCtlConfig,
account_id: u64,
theme: &ColorfulTheme,
) {
let root_str: String = Input::with_theme(theme)
.with_prompt("Enter the ROOT directory to scan for .eml files")
.validate_with(|input: &String| {
let p = std::path::Path::new(input);
if p.exists() && p.is_dir() {
Ok(())
} else {
Err("Directory not found.")
}
})
.interact_text()
.unwrap();
let root_path = std::path::PathBuf::from(root_str);
let mut tasks: HashMap<String, Vec<PathBuf>> = HashMap::new();
println!(
"{}",
style("🔍 Scanning recursively using std::fs...").dim()
);
if let Err(e) = scan_dir(&root_path, &root_path, &mut tasks) {
eprintln!("Error scanning directory: {}", e);
return;
}
if tasks.is_empty() {
println!("{}", style("No .eml files found.").yellow());
} else {
process_and_upload(config, account_id, tasks).await;
}
}
fn scan_dir(
root: &Path,
current: &Path,
tasks: &mut HashMap<String, Vec<PathBuf>>,
) -> std::io::Result<()> {
if current.is_dir() {
for entry in fs::read_dir(current)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
scan_dir(root, &path, tasks)?;
} else if path.is_file() {
if path.extension().and_then(|s| s.to_str()) == Some("eml") {
let rel_path = path.strip_prefix(root).unwrap_or(Path::new(""));
let mailbox_name = rel_path
.parent()
.map(|p| p.to_string_lossy().replace('\\', "/"))
.unwrap_or_default();
let folder = if mailbox_name.is_empty() {
"Inbox".to_string()
} else {
mailbox_name
};
tasks.entry(folder).or_insert_with(|| Vec::new()).push(path);
}
}
}
}
Ok(())
}
async fn process_and_upload(
config: &BichonCtlConfig,
account_id: u64,
tasks: HashMap<String, Vec<PathBuf>>,
) {
let client = Client::new();
let batch_size = 50;
for (mailbox, files) in tasks {
println!("\n🚀 Processing mailbox: {}", style(&mailbox).cyan().bold());
let mut current_batch = Vec::new();
for file_path in files {
let body = match fs::read(&file_path) {
Ok(b) => b,
Err(e) => {
eprintln!(
" {} Failed to read file {:?}: {}",
style("").red(),
file_path,
e
);
continue;
}
};
if MessageParser::new().parse(&body).is_some() {
let b64_content = base64_encode_url_safe!(&body);
current_batch.push(b64_content);
if current_batch.len() >= batch_size {
let to_send = current_batch;
current_batch = Vec::with_capacity(batch_size);
send_batch_request(&client, config, account_id, &mailbox, to_send).await;
}
} else {
eprintln!(
" {} Invalid format, skipping: {:?}",
style("").yellow(),
file_path
);
}
}
if !current_batch.is_empty() {
send_batch_request(&client, config, account_id, &mailbox, current_batch).await;
}
}
}
+150
View File
@@ -0,0 +1,150 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use bichon_core::bichon_version;
use clap::Parser;
use console::style;
use dialoguer::{theme::ColorfulTheme, Confirm, Input, Select};
use serde::{Deserialize, Serialize};
use std::fs;
use crate::{
auth::verify_user_and_get_account, eml::handle_eml_directory_import,
mbox::handle_mbox_single_file_import, pst::handle_pst_import,
thunderbird::handle_thunderbird_import,
};
pub mod auth;
pub mod eml;
pub mod mbox;
pub mod pst;
pub mod sender;
pub mod thunderbird;
#[derive(Parser, Debug)]
#[command(
name = "bichonctl",
author = "rustmailer",
version = bichon_version!(),
about = "A CLI tool to import email data into Bichon service"
)]
pub struct BichonCli {
/// Path to the configuration file
#[arg(
short,
long,
default_value = "config.toml",
value_name = "FILE",
help = "Sets a custom config file"
)]
pub config: std::path::PathBuf,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BichonCtlConfig {
pub base_url: String,
pub api_token: String,
}
#[tokio::main]
async fn main() {
let cli = BichonCli::parse();
let theme = ColorfulTheme::default();
let config_path = &cli.config;
let mut current_config: Option<BichonCtlConfig> = None;
if config_path.exists() {
if let Ok(content) = fs::read_to_string(config_path) {
if let Ok(config) = toml::from_str::<BichonCtlConfig>(&content) {
println!("{}", style("✔ Existing configuration found:").green());
println!(" Base URL: {}", style(&config.base_url).yellow());
println!(" API Token: {}", style(&config.api_token).yellow());
// Confirm with user
if Confirm::with_theme(&theme)
.with_prompt("Do you want to use this configuration?")
.default(true)
.interact()
.unwrap()
{
current_config = Some(config);
}
}
}
}
let final_config = match current_config {
Some(conf) => conf,
None => {
println!("\n{}", style("Please enter Bichon service details:").bold());
let url: String = Input::with_theme(&theme)
.with_prompt("Bichon Base URL")
.default("http://localhost:15630".into())
.interact_text()
.unwrap();
let token: String = Input::with_theme(&theme)
.with_prompt("API Token")
.interact_text()
.unwrap();
let conf = BichonCtlConfig {
base_url: url,
api_token: token,
};
// 3. Offer to save the new configuration
if Confirm::with_theme(&theme)
.with_prompt("Save this configuration for future use?")
.default(true)
.interact()
.unwrap()
{
let toml_str = toml::to_string(&conf).unwrap();
fs::write(config_path, toml_str).expect("Failed to save config file");
println!("{}", style("Configuration saved successfully!").green());
}
conf
}
};
let target_account_id = verify_user_and_get_account(&final_config, &theme).await;
let import_modes = &[
"1. EML: Scan directory recursively (Maintains folder structure)",
"2. MBOX: Single archive file (Stream from one file)",
"3. Thunderbird: Import from local profile directory",
"4. PST: Outlook Personal Storage (Single .pst file)",
];
let mode_idx = Select::with_theme(&theme)
.with_prompt("Select import method")
.items(import_modes)
.default(0)
.interact()
.unwrap();
match mode_idx {
0 => handle_eml_directory_import(&final_config, target_account_id, &theme).await,
1 => handle_mbox_single_file_import(&final_config, target_account_id, &theme).await,
2 => handle_thunderbird_import(&final_config, target_account_id, &theme).await,
3 => handle_pst_import(&final_config, target_account_id, &theme).await,
_ => unreachable!(),
}
}
+62
View File
@@ -0,0 +1,62 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::HashSet;
pub fn determine_folder(labels_raw: &str) -> String {
let mut status_blacklist = HashSet::new();
status_blacklist.insert("Opened");
status_blacklist.insert("Unread");
status_blacklist.insert("Archived");
let all_labels: Vec<&str> = labels_raw
.split(',')
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.collect();
if all_labels.is_empty() {
return "Unknown".to_string();
}
let filtered: Vec<&str> = all_labels
.iter()
.filter(|&&l| !status_blacklist.contains(l))
.cloned()
.collect();
match filtered.len() {
// Case A: If all labels were status labels, fallback to the first original label
0 => all_labels[0].to_string(),
// Case B: If only one label remains, that's our target destination
1 => filtered[0].to_string(),
// Case C: Multiple labels remain (e.g., ["Inbox", "medium"])
_ => {
// Prioritize custom business labels by excluding generic locations like "Inbox" or "Sent"
let business_label = filtered.iter().find(|&&l| l != "Inbox" && l != "Sent");
match business_label {
// Return the first non-generic label found
Some(label) => label.to_string(),
// If only generic labels remain (e.g., ["Sent", "Inbox"]), pick the first available
None => filtered[0].to_string(),
}
}
}
}
+186
View File
@@ -0,0 +1,186 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::HashMap;
use std::path::PathBuf;
use crate::mbox::gmail::determine_folder;
use crate::mbox::reader::MboxFile;
use crate::sender::send_batch_request;
use crate::BichonCtlConfig;
use bichon_core::base64_encode_url_safe;
use console::style;
use dialoguer::{theme::ColorfulTheme, Input};
use dialoguer::{Confirm, Select};
use mail_parser::parsers::MessageStream;
use mail_parser::MessageParser;
use reqwest::Client;
pub mod gmail;
pub mod reader;
pub async fn handle_mbox_single_file_import(
config: &BichonCtlConfig,
account_id: u64,
theme: &ColorfulTheme,
) {
let path_str: String = Input::with_theme(theme)
.with_prompt("Enter the path to your SINGLE .mbox file")
.validate_with(|input: &String| {
let p = std::path::Path::new(input);
if !p.exists() {
return Err("The specified path does not exist.");
}
if !p.is_file() {
return Err("MBOX mode requires a SINGLE file, not a directory.");
}
Ok(())
})
.interact_text()
.unwrap();
let mbox_path = PathBuf::from(path_str);
let options = vec![
"Use labels from mail headers (X-Gmail-Labels)",
"Specify a single target folder for all emails",
];
let selection = Select::with_theme(theme)
.with_prompt("How should we determine the target folder?")
.items(&options)
.default(0)
.interact()
.unwrap();
let target_folder: Option<String> = match selection {
0 => None,
1 => {
let folder: String = Input::with_theme(theme)
.with_prompt("Target folder name")
.default("INBOX".into())
.interact_text()
.unwrap();
Some(folder)
}
_ => unreachable!(),
};
if let Some(ref folder) = target_folder {
println!(
"{}",
style(format!("Mode: Fixed folder ({})", folder)).dim()
);
} else {
println!("{}", style("Mode: Dynamic (header-based)").dim());
}
println!(
"\n{} Ready to process MBOX file: {}",
style("").green(),
style(mbox_path.display()).cyan()
);
if let Ok(meta) = std::fs::metadata(&mbox_path) {
let size_mb = meta.len() as f64 / 1024.0 / 1024.0;
println!(
"{}",
style(format!("Processing file: {:.1} MB", size_mb)).dim()
);
}
if Confirm::with_theme(theme)
.with_prompt("Start importing?")
.default(true)
.interact()
.unwrap()
{
run_import(account_id, &mbox_path, config, target_folder).await
}
}
pub async fn run_import(
account_id: u64,
mbox_path: &PathBuf,
config: &BichonCtlConfig,
target_folder: Option<String>,
) {
let client = Client::new();
let mbox = match MboxFile::from_file(mbox_path) {
Ok(mbox) => mbox,
Err(err) => {
println!("Skipping invalid MBOX: {} ({})", mbox_path.display(), err);
return;
}
};
let mut folder_buffers: HashMap<String, Vec<String>> = HashMap::new();
let batch_limit = 50;
println!("Starting import process...");
for (index, e) in mbox.iter().enumerate() {
let msg_num = index + 1;
let body = e.data;
let message = match MessageParser::new().parse(body) {
Some(msg) => msg,
None => {
eprintln!(
"{} {}: {}",
style("Warning").yellow().bold(),
style(format!("at message #{}", msg_num)).dim(),
"Failed to parse email structure. Skipping..."
);
continue;
}
};
let folder_name = match target_folder {
Some(ref folder_name) => folder_name.clone(),
None => {
let gmail_labels = message.header_raw("X-Gmail-Labels").unwrap_or("INBOX");
let text_cow = MessageStream::new(gmail_labels.as_bytes())
.parse_unstructured()
.into_text();
let data: &str = match &text_cow {
Some(c) => c.as_ref(),
None => "INBOX",
};
determine_folder(data)
}
};
let b64_eml = base64_encode_url_safe!(&body);
let buffer = folder_buffers
.entry(folder_name.clone())
.or_insert_with(|| Vec::new());
buffer.push(b64_eml);
if buffer.len() >= batch_limit {
let emls_to_send = folder_buffers.remove(&folder_name).unwrap();
send_batch_request(&client, config, account_id, &folder_name, emls_to_send).await;
}
}
for (folder_name, emls) in folder_buffers {
if !emls.is_empty() {
send_batch_request(&client, config, account_id, &folder_name, emls).await;
}
}
println!("{}", style("Import completed successfully!").green().bold());
}
+225
View File
@@ -0,0 +1,225 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use memmap2::Mmap;
use std::fs;
use std::io;
use std::path::Path;
pub struct MboxFile {
map: Mmap,
}
impl MboxFile {
pub fn from_file(name: &Path) -> io::Result<Self> {
let file = fs::File::open(name)?;
let metadata = file.metadata()?;
if metadata.len() == 0 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"Empty MBOX file",
));
}
let map = unsafe { Mmap::map(&file)? };
Ok(Self { map })
}
pub fn iter(&self) -> MboxReader<'_> {
MboxReader::new(&self.map)
}
}
pub struct Entry<'a> {
pub offset: usize,
pub data: &'a [u8],
}
pub struct MboxReader<'a> {
data: &'a [u8],
len: usize,
scan_pos: usize,
body_start: Option<usize>,
}
impl<'a> MboxReader<'a> {
fn new(data: &'a [u8]) -> Self {
Self {
data,
len: data.len(),
scan_pos: 0,
body_start: None,
}
}
fn is_from_line(&self, i: usize) -> bool {
if i + 5 > self.len {
return false;
}
if i == 0 {
&self.data[0..5] == b"From "
} else {
self.data[i - 1] == b'\n' && &self.data[i..i + 5] == b"From "
}
}
fn skip_from_line(&self, mut i: usize) -> usize {
while i < self.len && self.data[i] != b'\n' {
i += 1;
}
if i < self.len {
i += 1;
}
i
}
}
impl<'a> Iterator for MboxReader<'a> {
type Item = Entry<'a>;
fn next(&mut self) -> Option<Self::Item> {
while self.scan_pos < self.len {
if self.is_from_line(self.scan_pos) {
let from_pos = self.scan_pos;
let body_pos = self.skip_from_line(from_pos);
if let Some(start) = self.body_start {
let entry = Entry {
offset: start,
data: &self.data[start..from_pos],
};
self.body_start = Some(body_pos);
self.scan_pos = body_pos;
return Some(entry);
} else {
self.body_start = Some(body_pos);
self.scan_pos = body_pos;
continue;
}
}
self.scan_pos += 1;
}
if let Some(start) = self.body_start.take() {
return Some(Entry {
offset: start,
data: &self.data[start..self.len],
});
}
None
}
}
#[cfg(test)]
mod tests {
use mail_parser::MessageParser;
use crate::mbox::gmail::determine_folder;
use super::*;
fn collect_entries(data: &[u8]) -> Vec<&[u8]> {
let reader = MboxReader::new(data);
reader.map(|e| e.data).collect()
}
#[test]
fn two_mails() {
let data = b"From a\nmail1\nFrom b\nmail2\n";
let e = collect_entries(data);
assert_eq!(e, vec![b"mail1\n", b"mail2\n"]);
}
#[test]
fn no_trailing_newline() {
let data = b"From a\nmail1";
let e = collect_entries(data);
assert_eq!(e, vec![b"mail1"]);
}
#[test]
fn from_inside_body() {
let data = b"From a\nhello\nFrom is here\nbye\n";
let e = collect_entries(data);
assert_eq!(e.len(), 2);
}
#[test]
fn inline_from_not_separator() {
let data = b"From a\nhello From world\n";
let e = collect_entries(data);
assert_eq!(e.len(), 1);
}
#[test]
fn realistic_mbox() {
let data = b"From a\nH:1\n\nbody1\nFrom b\nH:2\n\nbody2\n";
let e = collect_entries(data);
assert_eq!(e.len(), 2);
}
#[test]
fn empty_body() {
let data = b"From a\nFrom b\nbody\n";
let e = collect_entries(data);
assert_eq!(e[0], b"");
assert_eq!(e[1], b"body\n");
}
#[test]
fn only_from_line() {
let data = b"From a\n";
let e = collect_entries(data);
assert_eq!(e.len(), 1);
assert_eq!(e[0], b"");
}
#[test]
fn windows_newlines() {
let data = b"From a\r\nbody\r\nFrom b\r\nbody2\r\n";
let e = collect_entries(data);
assert_eq!(e.len(), 2);
}
#[test]
fn many_small_mails() {
let mut data = Vec::new();
for _ in 0..1000 {
data.extend_from_slice(b"From a\nx\n");
}
let e = collect_entries(&data);
assert_eq!(e.len(), 1000);
}
#[test]
fn test11() {
let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap();
for e in mbox.iter() {
let body = e.data;
let message = MessageParser::new().parse(body).unwrap();
let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap();
//println!("offset={} X-Gmail-Labels={:?}", e.offset, labels);
println!(
"X-Gmail-Labels={:?}, determine_folder={}",
labels,
determine_folder(labels)
)
}
}
}
+64
View File
@@ -0,0 +1,64 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use compressed_rtf::*;
use outlook_pst::ltp::prop_context::PropertyValue;
pub fn decode_subject(value: &PropertyValue) -> Option<String> {
match value {
PropertyValue::String8(value) => {
let offset = match value.buffer().first() {
Some(1) => 2,
_ => 0,
};
let buffer: Vec<_> = value
.buffer()
.iter()
.skip(offset)
.map(|&b| u16::from(b))
.collect();
Some(String::from_utf16_lossy(&buffer))
}
PropertyValue::Unicode(value) => {
let offset = match value.buffer().first() {
Some(1) => 2,
_ => 0,
};
Some(String::from_utf16_lossy(&value.buffer()[offset..]))
}
_ => None,
}
}
pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option<String> {
match code_page {
20127 => {
let buffer: Vec<_> = buffer.iter().map(|&b| u16::from(b)).collect();
Some(String::from_utf16_lossy(&buffer))
}
_ => {
let coding = codepage_strings::Coding::new(code_page).ok()?;
Some(coding.decode(buffer).ok()?.to_string())
}
}
}
pub fn decode_rtf_compressed(buffer: &[u8]) -> Option<String> {
decompress_rtf(buffer).ok()
}
+482
View File
@@ -0,0 +1,482 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use chrono::{DateTime, TimeZone, Utc};
use dialoguer::theme::ColorfulTheme;
use dialoguer::Input;
use mail_send::mail_builder::headers::text::Text;
use mail_send::mail_builder::MessageBuilder;
use outlook_pst::ltp::prop_context::PropertyValue;
use crate::BichonCtlConfig;
use crate::pst::encoding::decode_subject;
use crate::sender::send_batch_request;
use bichon_core::base64_encode_url_safe;
use dialoguer::Confirm;
use outlook_pst::messaging::attachment::AttachmentProperties;
use outlook_pst::messaging::folder::Folder;
use outlook_pst::messaging::message::{Message, MessageProperties};
use outlook_pst::ndb::node_id::NodeId;
use reqwest::Client;
use std::future::Future;
use std::path::PathBuf;
use std::pin::Pin;
use std::rc::Rc;
mod encoding;
#[derive(Debug, Default)]
pub struct EmailMetadata {
pub message_id: Option<String>,
pub subject: Option<String>,
pub from: Option<String>,
pub to: Option<Vec<String>>,
pub cc: Option<Vec<String>>,
pub bcc: Option<Vec<String>>,
pub html: Option<String>,
pub text: Option<String>,
pub in_reply_to: Option<String>,
}
#[derive(Debug, Default)]
pub struct EmailAttachment {
pub name: Option<String>,
pub mime_type: Option<String>,
pub data: Option<Vec<u8>>,
}
pub async fn handle_pst_import(config: &BichonCtlConfig, account_id: u64, theme: &ColorfulTheme) {
let path_str: String = Input::with_theme(theme)
.with_prompt("Enter the path to your SINGLE .pst file")
.validate_with(|input: &String| {
let p = std::path::Path::new(input);
if !p.exists() {
return Err("The specified path does not exist.");
}
if !p.is_file() {
return Err("PST mode requires a SINGLE file, not a directory.");
}
let is_pst = p
.extension()
.and_then(|ext| ext.to_str())
.map(|ext| ext.eq_ignore_ascii_case("pst"))
.unwrap_or(false);
if !is_pst {
return Err("The selected file must have a .pst extension.");
}
Ok(())
})
.interact_text()
.unwrap();
let pst_path = std::path::PathBuf::from(path_str);
println!(
"\n{} Ready to process PST file: {}",
console::style("").green(),
console::style(pst_path.display()).cyan()
);
if let Ok(meta) = std::fs::metadata(&pst_path) {
let size_mb = meta.len() as f64 / 1024.0 / 1024.0;
println!(
"{}",
console::style(format!("PST File Size: {:.1} MB", size_mb)).dim()
);
}
if Confirm::with_theme(theme)
.with_prompt("Start importing emails from this PST?")
.default(true)
.interact()
.unwrap()
{
parse_pst(pst_path, config, account_id).await;
} else {
println!("{}", console::style("Operation cancelled by user.").red());
}
}
async fn parse_pst(pst_path: PathBuf, config: &BichonCtlConfig, account_id: u64) {
let client = Client::new();
let pst_store = match outlook_pst::open_store(&pst_path) {
Ok(store) => store,
Err(e) => {
println!(
"{} Failed to open PST file: {}",
console::style("").red(),
console::style(format!("{:#?}", e)).dim()
);
return;
}
};
let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
Ok(id) => id,
Err(e) => {
println!(
"{} Could not find IPM_SUBTREE (Mailbox Root): {}",
console::style("").red(),
console::style(format!("{:#?}", e)).dim()
);
return;
}
};
let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
Ok(folder) => folder,
Err(e) => {
println!(
"{} Failed to open the root mailbox folder: {}",
console::style("").red(),
console::style(format!("{:#?}", e)).dim()
);
return;
}
};
process_folder_recursively(&client, &ipm_subtree_folder, "", config, account_id).await;
}
fn process_folder_recursively<'a>(
client: &'a Client,
folder: &'a Rc<dyn Folder>,
parent_path: &'a str,
config: &'a BichonCtlConfig,
account_id: u64,
) -> Pin<Box<dyn Future<Output = ()> + 'a>> {
Box::pin(async move {
let folder_name = folder
.properties()
.display_name()
.unwrap_or_else(|_| "Unknown".to_string());
let current_path = if parent_path.is_empty() {
folder_name
} else {
format!("{}/{}", parent_path, folder_name)
};
println!(
"{} {}",
console::style("📁 Folder:").dim(),
console::style(&current_path).cyan()
);
let mut emls_batch = Vec::new();
if let Some(contents_table) = folder.contents_table() {
for row in contents_table.rows_matrix() {
let store = folder.store().clone();
let entry_id = match store
.properties()
.make_entry_id(NodeId::from(u32::from(row.id())))
{
Ok(id) => id,
Err(e) => {
eprintln!(
" {} Skip row {}: {:?}",
console::style("").yellow(),
row.unique(),
e
);
continue;
}
};
match store.open_message(&entry_id, None) {
Ok(message) => match build_eml_base64(message) {
Some(base64_eml) => emls_batch.push(base64_eml),
None => {}
},
Err(e) => eprintln!(" {} Open error: {:?}", console::style("").yellow(), e),
}
if emls_batch.len() >= 50 {
let batch = emls_batch.clone();
emls_batch.clear();
send_to_bichon(client, config, account_id, &current_path, batch).await;
}
}
}
if !emls_batch.is_empty() {
send_to_bichon(client, config, account_id, &current_path, emls_batch).await;
}
if let Some(hierarchy_table) = folder.hierarchy_table() {
for row in hierarchy_table.rows_matrix() {
let node = NodeId::from(u32::from(row.id()));
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
process_folder_recursively(
client,
&sub_folder,
&current_path,
config,
account_id,
)
.await;
}
}
}
}
})
}
fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
let properties = message.properties();
let mut builder = MessageBuilder::new();
if let Some(sub) = extract_subject(properties) {
builder = builder.subject(sub);
}
if let Some(mid) = extract_string_property(properties, 0x1035) {
builder = builder.message_id(mid);
}
if let Some(irt) = extract_string_property(properties, 0x1042) {
builder = builder.in_reply_to(irt);
}
if let Some(refs) = extract_string_property(properties, 0x1039) {
builder = builder.header("References", Text::new(refs));
}
if let Some(cid_val) = properties.get(0x3013) {
if let PropertyValue::Binary(bin) = cid_val {
builder = builder.header(
"X-Bichon-Conversation-ID",
Text::new(hex::encode(bin.buffer())),
);
}
}
let from = extract_string_property(properties, 0x5D01)
.or_else(|| extract_string_property(properties, 0x5D02))
.or_else(|| extract_string_property(properties, 0x0C1F));
if let Some(f) = from {
builder = builder.from(f);
}
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
let dt = filetime_to_datetime(filetime).timestamp();
builder = builder.date(dt);
}
let (to, cc, bcc) = extract_recipients_list(&message);
if !to.is_empty() {
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
}
if !cc.is_empty() {
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
}
if !bcc.is_empty() {
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
}
if let Some(html) = extract_html(properties) {
builder = builder.html_body(html);
}
if let Some(text) = extract_text(properties) {
builder = builder.text_body(text);
}
if let Some(attachment_table) = message.attachment_table() {
for row in attachment_table.rows_matrix() {
let node_id = NodeId::from(u32::from(row.id()));
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
let att_props = attachment.properties();
let name = extract_attachment_string_property(att_props, 0x3707);
let mime = extract_attachment_string_property(att_props, 0x370E)
.unwrap_or_else(|| "application/octet-stream".into());
let cid = extract_attachment_string_property(att_props, 0x3712);
let is_inline = att_props
.get(0x3714)
.and_then(|val| {
if let PropertyValue::Integer32(f) = val {
Some(f)
} else {
None
}
})
.map(|flag| (flag & 0x4) != 0)
.unwrap_or(false);
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
let data = bin.buffer().to_vec();
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
if is_inline && cid.is_some() {
let content_id = cid.unwrap();
builder = builder.inline(mime, content_id, data);
} else {
builder = builder.attachment(mime, file_name, data);
}
}
}
}
}
match builder.write_to_vec() {
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
Err(e) => {
eprintln!("Failed to generate EML: {:?}", e);
None
}
}
}
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
let nsecs = (filetime % 10_000_000) * 100;
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
}
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
let mut to = Vec::new();
let mut cc = Vec::new();
let mut bcc = Vec::new();
let recipient_table = message.recipient_table();
if let Some(recipient_table) = recipient_table {
let context = recipient_table.context();
for row in recipient_table.rows_matrix() {
if let Ok(cols) = row.columns(context) {
let mut r_type = 0;
let mut email = String::new();
for (col, val) in context.columns().iter().zip(cols) {
let prop_val = val
.as_ref()
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
match col.prop_id() {
0x0C15 => {
if let Some(PropertyValue::Integer32(t)) = prop_val {
r_type = t;
}
}
0x39FE | 0x3003 => {
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
email = s;
}
}
_ => {}
}
}
if !email.is_empty() {
match r_type {
1 => to.push(email),
2 => cc.push(email),
3 => bcc.push(email),
_ => {}
}
}
}
}
} else {
let receiver = extract_string_property(message.properties(), 0x0076);
if let Some(receiver) = receiver {
to.push(receiver);
}
}
(to, cc, bcc)
}
async fn send_to_bichon(
client: &Client,
config: &BichonCtlConfig,
account_id: u64,
folder_path: &str,
emls: Vec<String>,
) {
send_batch_request(client, config, account_id, folder_path, emls).await;
}
fn extract_subject(props: &MessageProperties) -> Option<String> {
props.get(0x0037).and_then(|val| decode_subject(val))
}
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
properties
.get(prop_id)
.and_then(|value| extract_string(value))
}
fn extract_attachment_string_property(
properties: &AttachmentProperties,
prop_id: u16,
) -> Option<String> {
properties
.get(prop_id)
.and_then(|value| extract_string(value))
}
fn extract_string(value: &PropertyValue) -> Option<String> {
match value {
PropertyValue::String8(value) => Some(value.to_string()),
PropertyValue::Unicode(value) => Some(value.to_string()),
_ => None,
}
}
fn extract_text(properties: &MessageProperties) -> Option<String> {
properties.get(0x1000).and_then(extract_string).or_else(|| {
properties.get(0x1009).and_then(|value| match value {
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
_ => None,
})
})
}
fn extract_html(properties: &MessageProperties) -> Option<String> {
properties.get(0x1013).and_then(|value| match value {
PropertyValue::Binary(value) => {
let code_page = properties
.get(0x3FDE)
.and_then(|v| {
if let PropertyValue::Integer32(cpid) = v {
Some(*cpid as u16)
} else {
None
}
})
.unwrap_or(65001);
encoding::decode_html_body(value.buffer(), code_page)
}
PropertyValue::String8(value) => Some(value.to_string()),
PropertyValue::Unicode(value) => Some(value.to_string()),
_ => None,
})
}
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
for &prop_id in prop_ids {
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
return Some(*value);
}
}
None
}
+77
View File
@@ -0,0 +1,77 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use console::style;
use reqwest::Client;
use bichon_core::import::BatchEmlRequest;
use crate::BichonCtlConfig;
pub async fn send_batch_request(
client: &Client,
config: &BichonCtlConfig,
account_id: u64,
folder: &str,
emls: Vec<String>,
) {
let url = format!("{}/api/v1/import", config.base_url);
let payload = BatchEmlRequest {
account_id,
mail_folder: folder.to_string(),
emls,
};
let count = payload.emls.len();
match client
.post(&url)
.header("Authorization", format!("Bearer {}", config.api_token))
.json(&payload)
.send()
.await
{
Ok(res) if res.status().is_success() => {
println!(
" {} Sent {} emails to [{}]",
style("").green(),
count,
folder
);
}
Ok(res) => {
let status = res.status();
let error_body = res.text().await.unwrap_or_default();
eprintln!(
" {} Failed to send to [{}]. Status: {}\n Server error: {}",
style("").red(),
folder,
status,
error_body
);
}
Err(e) => {
eprintln!(
" {} Network error on [{}]: {}",
style("").red(),
folder,
e
);
}
}
}
+131
View File
@@ -0,0 +1,131 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{collections::HashMap, path::PathBuf};
use crate::{mbox::run_import, BichonCtlConfig};
use console::style;
use dialoguer::{theme::ColorfulTheme, Confirm, Input};
pub async fn handle_thunderbird_import(
config: &BichonCtlConfig,
account_id: u64,
theme: &ColorfulTheme,
) {
let root_str: String = Input::with_theme(theme)
.with_prompt("Enter your Thunderbird Mail/ImapMail directory")
.validate_with(|input: &String| {
let p = std::path::Path::new(input);
if p.exists() && p.is_dir() {
Ok(())
} else {
Err("Directory not found.")
}
})
.interact_text()
.unwrap();
let root_path = std::path::PathBuf::from(&root_str);
println!("{}", style("🔍 Scanning Thunderbird structure...").dim());
let mut mbox_tasks: HashMap<String, PathBuf> = HashMap::new();
fn scan_thunderbird_dir(
root: &std::path::Path,
current: &std::path::Path,
tasks: &mut HashMap<String, PathBuf>,
) {
if let Ok(entries) = std::fs::read_dir(current) {
for entry in entries.flatten() {
let path = entry.path();
let file_name = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
if path.is_dir() {
scan_thunderbird_dir(root, &path, tasks);
} else {
let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("");
match extension {
"msf" | "dat" | "html" | "json" | "txt" | "sqlite" => continue,
_ => {}
}
if file_name == "filterlog.html" || file_name == "msgFilterRules.dat" {
continue;
}
if !extension.is_empty() {
continue;
}
if let Ok(rel) = path.strip_prefix(root) {
let mailbox = rel.to_string_lossy().replace(".sbd", "").replace('\\', "/");
tasks.insert(mailbox, path);
}
}
}
}
}
scan_thunderbird_dir(&root_path, &root_path, &mut mbox_tasks);
if mbox_tasks.is_empty() {
println!(
"{}",
style("No mailboxes found in the specified directory.").yellow()
);
return;
}
println!("\n{}", style("🔍 Scanned Mailboxes:").bold().underlined());
let mut sorted_keys: Vec<_> = mbox_tasks.keys().collect();
sorted_keys.sort();
for name in &sorted_keys {
let path = &mbox_tasks[*name];
let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
let size_mb = file_size as f64 / 1024.0 / 1024.0;
println!(
" {} {} ({:.2} MB)",
style("").dim(),
style(name).cyan(),
size_mb
);
}
println!();
let prompt = format!("Ready to import {} mailboxes. Proceed?", mbox_tasks.len());
if Confirm::with_theme(theme)
.with_prompt(prompt)
.default(true)
.interact()
.unwrap()
{
for (mailbox_name, mbox_file) in mbox_tasks {
println!("\n🚀 Importing: {}", style(&mailbox_name).cyan().bold());
run_import(account_id, &mbox_file, config, Some(mailbox_name)).await;
}
println!(
"\n{}",
style("✨ All mailboxes imported successfully!")
.green()
.bold()
);
} else {
println!("{}", style("Import cancelled.").yellow());
}
}