External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(admin): add interactive data migration tool

Browse Source
This commit is contained in:
rustmailer
2026-05-12 01:56:24 +08:00
parent 123260b69d
commit 0abaa66a40
21 changed files with 1547 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4
View File
@@ -464,6 +464,7 @@ dependencies = [
"bichon-core",
"console",
"dialoguer",
"indicatif",
"tokio",
]
@@ -5022,6 +5023,8 @@ dependencies = [
"fastdivide",
"fnv",
"fs4",
"futures-channel",
"futures-util",
"htmlescape",
"itertools",
"levenshtein_automata",
@@ -5045,6 +5048,7 @@ dependencies = [
"tantivy-common",
"tantivy-fst",
"tantivy-query-grammar",
"tantivy-sstable",
"tantivy-stacker",
"tantivy-tokenizer-api",
"tempfile",
Cargo.toml
+1 -1
View File
@@ -84,7 +84,7 @@ uuid = { version = "1.23.1", features = ["v4", "serde"] }
fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] }
tracing-log = "0.2.0"
tokio-util = "0.7.18"
indicatif = "0.18.4"
[profile.release]
strip = true
crates/admin/Cargo.toml
+1
View File
@@ -9,3 +9,4 @@ bichon-core = { path = "../core" }
tokio.workspace = true
dialoguer.workspace = true
console.workspace = true
indicatif.workspace = true
crates/admin/src/main.rs
+16 -278
View File
@@ -16,19 +16,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
fs,
path::{Path, PathBuf},
};
use console::style;
use dialoguer::{theme::ColorfulTheme, Select};
use bichon_core::{
admin::meta::{find_admin, init_meta_database, update_admin_password},
error::BichonError,
utils::encrypt::internal_decrypt_string,
};
use console::{style, Emoji};
use dialoguer::Confirm;
use dialoguer::{theme::ColorfulTheme, Input, Password, Select};
use crate::{migrate::handle_migration, reset::handle_reset_password};
pub mod migrate;
pub mod reset;
#[tokio::main]
async fn main() {
@@ -38,7 +32,12 @@ async fn main() {
style("BICHON ADMINISTRATIVE TOOL").bold().bright().cyan()
);
let main_options = vec!["Reset Admin Password", "Exit"];
let main_options = vec![
"Reset Admin Password",
"Migrate Legacy v0.x Storage to v1.0",
"Exit",
];
let selection = Select::with_theme(&theme)
.with_prompt("Select an operation")
.default(0)
@@ -46,272 +45,11 @@ async fn main() {
.interact()
.unwrap();
if selection == 1 {
match selection {
0 => handle_reset_password(&theme),
1 => handle_migration(&theme),
_ => {
println!("{}", style("Exiting...").dim());
return;
}
let root_dir_str: String = Input::with_theme(&theme)
.with_prompt("Enter the absolute path for 'bichon_root_dir'")
.validate_with(|input: &String| -> Result<(), &str> {
let path = Path::new(input);
if !path.is_absolute() {
return Err("Path must be absolute.");
}
if !path.exists() {
return Err("Directory does not exist.");
}
let has_metadata = path.join("meta.db").exists();
if !has_metadata {
return Err("Invalid directory: 'meta.db' not found.");
}
Ok(())
})
.interact_text()
.unwrap();
let root_path = PathBuf::from(&root_dir_str);
let database = match init_meta_database(&root_path.join("meta.db")) {
Ok(database) => database,
Err(e) => match e {
BichonError::Generic {
message,
location,
code,
} => {
if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") {
println!("\n{}", style("ERROR: Database is locked.").red().bold());
println!(
"{}",
style("The Bichon service is likely still running.").yellow()
);
println!(
"Since the database cannot be shared between multiple instances, \n\
you must {} the Bichon service before proceeding.",
style("STOP").underlined().bold()
);
std::process::exit(1);
} else {
eprintln!(
"\n{} (Code: {:#?})\nLocation: {}\nMessage: {}",
style("A database error occurred:").red().bold(),
code,
location,
message
);
std::process::exit(1);
}
}
},
};
let admin = find_admin(&database);
match admin {
Ok(Some(user)) => {
println!("\n{}", style("Admin user found:").green().bold());
println!("----------------------------------------");
println!("{:<12} : {}", "Username", style(&user.username).cyan());
println!("{:<12} : {}", "Email", style(&user.email).cyan());
}
Ok(None) => {
println!(
"\n{}",
style("ERROR: No admin user found in the database.")
.red()
.bold()
);
println!("Please ensure the system has been initialized correctly.");
std::process::exit(1);
}
Err(e) => {
eprintln!(
"\n{} Failed to query admin user.",
style("ERROR:").red().bold()
);
eprintln!("Details: {:?}", e);
std::process::exit(1);
}
}
let encryption_key = loop {
let auth_methods = vec![
"Enter encryption password manually",
"Read from password file",
];
let method = Select::with_theme(&theme)
.with_prompt("How would you like to provide the database encryption key?")
.items(&auth_methods)
.interact()
.unwrap();
let raw_key = if method == 0 {
Password::with_theme(&theme)
.with_prompt("Enter Encryption Password")
.interact()
.unwrap()
} else {
let file_path: String = Input::with_theme(&theme)
.with_prompt("Enter path to encryption password file")
.interact_text()
.unwrap();
match fs::read_to_string(&file_path) {
Ok(content) => content.trim().to_string(),
Err(e) => {
println!("{}: {}", style("Failed to read file").red(), e);
continue;
}
}
};
if raw_key.is_empty() {
println!("{}", style("Key cannot be empty.").red());
continue;
}
let prompt_message = format!(
"Encryption key loaded: [ {} ]\n\n \
{}: This key must match the database encryption key used by the server.\n \
It corresponds to these settings in your service:\n \
- Arguments: {} or {}\n \
- Envs: {} or {}\n\n \
Do you want to continue?",
style(&raw_key).cyan().bold(),
style("IMPORTANT").yellow().bold(),
style("--bichon_encrypt_password").italic(),
style("--bichon_encrypt_password_file").italic(),
style("BICHON_ENCRYPT_PASSWORD").green(),
style("BICHON_ENCRYPT_PASSWORD_FILE").green()
);
if Confirm::with_theme(&theme)
.with_prompt(prompt_message)
.default(true)
.interact()
.unwrap()
{
break raw_key;
}
};
let admin = find_admin(&database);
match admin {
Ok(Some(user)) => {
println!("----------------------------------------");
println!("{:<12} : {}", "Username", style(&user.username).cyan());
println!("{:<12} : {}", "Email", style(&user.email).cyan());
let pwd_display = match &user.password {
Some(p) => {
let password = match internal_decrypt_string(&encryption_key, p) {
Ok(p) => p,
Err(e) => {
println!("\n{}", style("ERROR: Decryption Failed").red().bold());
println!(
"{}",
style("The provided encryption key is incorrect or invalid for this database.").yellow()
);
println!(
"{} Please verify your {} or the {} you provided.",
style("➔").cyan(),
style("encryption password").bold(),
style("key file").bold()
);
eprintln!("\nTechnical details: {:?}", e);
std::process::exit(1);
}
};
style(password).yellow().to_string()
}
None => style("None (No password set)").dim().italic().to_string(),
};
println!("{:<12} : {}", "Password", pwd_display);
println!("----------------------------------------");
if !dialoguer::Confirm::with_theme(&theme)
.with_prompt(format!(
"Do you want to reset the password for '{}'?",
user.username
))
.interact()
.unwrap()
{
println!("Operation cancelled.");
return;
}
}
Ok(None) => {
println!(
"\n{}",
style("ERROR: No admin user found in the database.")
.red()
.bold()
);
println!("Please ensure the system has been initialized correctly.");
std::process::exit(1);
}
Err(e) => {
eprintln!(
"\n{} Failed to query admin user.",
style("ERROR:").red().bold()
);
eprintln!("Details: {:?}", e);
std::process::exit(1);
}
}
println!(
"\n{}",
style("TARGET: Reset password for user 'admin'")
.yellow()
.bold()
);
let new_login_password = Password::with_theme(&theme)
.with_prompt("Enter new Admin Login Password")
.with_confirmation("Repeat password to confirm", "Passwords do not match!")
.interact()
.unwrap();
if !Confirm::with_theme(&theme)
.with_prompt("Proceed with database update?")
.interact()
.unwrap()
{
return;
}
println!("\n{} {}", style("⌛").yellow(), "Updating database...");
match update_admin_password(&database, new_login_password, &encryption_key) {
Ok(_) => {
println!(
"\n{} {}",
Emoji("✨", "*"),
style("Success! Admin password has been updated.")
.green()
.bold()
);
println!(
"{}",
style("You can now log in with the new password.").dim()
);
}
Err(e) => {
println!(
"\n{}",
style("ERROR: Failed to update database").red().bold()
);
eprintln!(
"{} Could not save the new password to the database.",
style("➔").cyan()
);
eprintln!("\nDetails: {:?}", e);
std::process::exit(1);
}
}
}
crates/admin/src/migrate.rs
+284
View File
@@ -0,0 +1,284 @@
use std::path::{Path, PathBuf};
use bichon_core::migrate::{
do_migrate, is_tantivy_index_dir,
store::{LegacyDirs, NewDirs},
};
use console::style;
use dialoguer::{theme::ColorfulTheme, Confirm, Input};
use indicatif::{ProgressBar, ProgressStyle};
pub fn handle_migration(theme: &ColorfulTheme) {
println!(
"\n{}",
style("MIGRATION: Bichon v0.x Storage Architecture → v1.0")
.bold()
.yellow()
);
println!(
"{}",
style(
"This tool migrates data from the legacy v0.x Tantivy-based storage \
architecture (used in versions 0.0.1 through 0.3.7) to the new v1.0 \
separated index and Fjall-backed storage format."
)
.dim()
);
println!(
"{}",
style(
"Legacy v0.x architecture:\n\
• envelope metadata stored in Tantivy\n\
• message data stored in Tantivy\n\n\
New v1.0 architecture:\n\
• mail indexes stored in Tantivy\n\
• attachment indexes stored in Tantivy\n\
• raw message data stored in Fjall\n\
• attachment blobs stored in Fjall"
)
.dim()
);
println!(
"\n{} {}",
style("IMPORTANT:").yellow().bold(),
style(
"The paths below must exactly match what your old bichon server was configured with."
)
.yellow()
);
// --- bichon-root-dir ---
let root_dir_str: String = Input::with_theme(theme)
.with_prompt("Enter --bichon-root-dir (same value used by the old server)")
.validate_with(|input: &String| -> Result<(), &str> {
let path = Path::new(input);
if !path.is_absolute() {
return Err("Path must be absolute.");
}
if !path.exists() {
return Err("Directory does not exist.");
}
Ok(())
})
.interact_text()
.unwrap();
let root_path = PathBuf::from(&root_dir_str);
// --- bichon-index-dir ---
let default_index = root_path.join("envelope");
let default_new_index = root_path.join("bichon-indices");
let index_dir_str: String = Input::with_theme(theme)
.with_prompt(format!(
"Enter --bichon-index-dir (leave blank to use default: {})",
style(default_index.display()).cyan()
))
.allow_empty(true)
.validate_with(|input: &String| -> Result<(), &str> {
if input.is_empty() {
return Ok(());
}
let path = Path::new(input);
if !path.is_absolute() {
return Err("Path must be absolute.");
}
if !path.exists() {
return Err("Directory does not exist.");
}
Ok(())
})
.interact_text()
.unwrap();
let index_path = if index_dir_str.is_empty() {
default_index
} else {
PathBuf::from(&index_dir_str)
};
let new_index_path = if index_dir_str.is_empty() {
default_new_index
} else {
PathBuf::from(&index_dir_str).join("bichon-indices")
};
// --- bichon-data-dir ---
let default_data = root_path.join("eml");
let default_new_data = root_path.join("bichon-storage");
let data_dir_str: String = Input::with_theme(theme)
.with_prompt(format!(
"Enter --bichon-data-dir (leave blank to use default: {})",
style(default_data.display()).cyan()
))
.allow_empty(true)
.validate_with(|input: &String| -> Result<(), &str> {
if input.is_empty() {
return Ok(());
}
let path = Path::new(input);
if !path.is_absolute() {
return Err("Path must be absolute.");
}
if !path.exists() {
return Err("Directory does not exist.");
}
Ok(())
})
.interact_text()
.unwrap();
let data_path = if data_dir_str.is_empty() {
default_data
} else {
PathBuf::from(&data_dir_str)
};
let new_data_path = if data_dir_str.is_empty() {
default_new_data
} else {
PathBuf::from(&data_dir_str).join("bichon-storage")
};
println!("\n{}", style("Paths to be migrated:").bold());
println!("----------------------------------------");
println!(
"{:<20} : {}",
"bichon-root-dir",
style(root_path.display()).cyan()
);
println!(
"{:<20} : {}",
"bichon-index-dir",
style(index_path.display()).cyan()
);
println!(
"{:<20} : {}",
"bichon-data-dir",
style(data_path.display()).cyan()
);
println!("----------------------------------------");
println!(
"\n{} Checking legacy v0.x storage layout...",
style("⌛").yellow()
);
match is_legacy_data_layout_with_paths(&index_path, &data_path) {
Ok(true) => {
println!(
"{} {}",
style("✔").green(),
style("Legacy v0.x Tantivy-based storage detected. Migration to v1.0 is required.")
.yellow()
);
}
Ok(false) => {
println!(
"{} {}",
style("✔").green(),
style("No legacy v0.x storage layout was detected at the specified paths.").green()
);
println!(
"{}",
style(
"The selected directories may already be using the v1.0 storage architecture."
)
.dim()
);
return;
}
Err(e) => {
eprintln!(
"{} Failed to verify legacy storage layout: {:?}",
style("ERROR:").red().bold(),
e
);
std::process::exit(1);
}
}
println!(
"\n{} {}",
style("⚠").yellow(),
style(
"This migration is non-destructive. Existing v0.x storage files will remain unchanged."
)
.yellow()
);
if !Confirm::with_theme(theme)
.with_prompt("Ready to migrate?")
.default(true)
.interact()
.unwrap()
{
println!("{}", style("Migration cancelled.").dim());
return;
}
println!("\n{} Migrating...", style("⌛").yellow());
let pb = ProgressBar::new(0);
pb.set_style(ProgressStyle::default_bar()
.template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}")
.unwrap()
.progress_chars("#>-"));
let legacy = LegacyDirs::new(index_path, data_path);
let new_dirs = NewDirs::new(new_index_path, new_data_path);
if let Err(e) = do_migrate(legacy, new_dirs, |msg| {
if let Some(data) = msg.strip_prefix("PROGRESS:") {
let parts: Vec<&str> = data.split(':').collect();
if parts.len() == 2 {
let migrated = parts[0].parse::<u64>().unwrap_or(0);
let skipped = parts[1].parse::<u64>().unwrap_or(0);
pb.set_position(migrated + skipped);
pb.set_message(format!(
"Migrated: {}, {} {}",
style(migrated).green(),
style(skipped).red(),
style("skipped").dim()
));
}
} else if let Some(total) = msg.strip_prefix("TOTAL:") {
pb.set_length(total.parse().unwrap_or(0));
} else if msg.starts_with("WARN:") {
pb.println(format!("{} {}", style("⚠").yellow(), &msg[5..]));
} else if let Some(done_data) = msg.strip_prefix("DONE:") {
let parts: Vec<&str> = done_data.split(':').collect();
pb.finish_with_message(format!(
"Migration finished. Total: {}, Skipped: {}",
parts.get(0).unwrap_or(&"0"),
parts.get(1).unwrap_or(&"0")
));
}
}) {
eprintln!(
"\n{} Migration failed:\n{:?}",
style("✘").red().bold(),
style(e).red()
);
return;
}
println!(
"{} {}",
style("✔").green(),
style("Migration completed successfully!").bold()
);
}
pub fn is_legacy_data_layout_with_paths(
envelope_dir: &PathBuf,
eml_dir: &PathBuf,
) -> std::io::Result<bool> {
let envelope_result = is_tantivy_index_dir(envelope_dir)?;
let eml_result = is_tantivy_index_dir(eml_dir)?;
Ok(envelope_result || eml_result)
}
crates/admin/src/reset.rs
+274
View File
@@ -0,0 +1,274 @@
use std::path::{Path, PathBuf};
use bichon_core::{
admin::meta::{find_admin, init_meta_database, update_admin_password},
error::BichonError,
utils::encrypt::internal_decrypt_string,
};
use console::{style, Emoji};
use dialoguer::{theme::ColorfulTheme, Confirm, Input, Password, Select};
pub fn handle_reset_password(theme: &ColorfulTheme) {
let root_dir_str: String = Input::with_theme(theme)
.with_prompt("Enter the absolute path for 'bichon_root_dir'")
.validate_with(|input: &String| -> Result<(), &str> {
let path = Path::new(input);
if !path.is_absolute() {
return Err("Path must be absolute.");
}
if !path.exists() {
return Err("Directory does not exist.");
}
let has_metadata = path.join("meta.db").exists();
if !has_metadata {
return Err("Invalid directory: 'meta.db' not found.");
}
Ok(())
})
.interact_text()
.unwrap();
let root_path = PathBuf::from(&root_dir_str);
let database = match init_meta_database(&root_path.join("meta.db")) {
Ok(database) => database,
Err(e) => match e {
BichonError::Generic {
message,
location,
code,
} => {
if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") {
println!("\n{}", style("ERROR: Database is locked.").red().bold());
println!(
"{}",
style("The Bichon service is likely still running.").yellow()
);
println!(
"Since the database cannot be shared between multiple instances, \n\
you must {} the Bichon service before proceeding.",
style("STOP").underlined().bold()
);
std::process::exit(1);
} else {
eprintln!(
"\n{} (Code: {:#?})\nLocation: {}\nMessage: {}",
style("A database error occurred:").red().bold(),
code,
location,
message
);
std::process::exit(1);
}
}
},
};
let admin = find_admin(&database);
match admin {
Ok(Some(user)) => {
println!("\n{}", style("Admin user found:").green().bold());
println!("----------------------------------------");
println!("{:<12} : {}", "Username", style(&user.username).cyan());
println!("{:<12} : {}", "Email", style(&user.email).cyan());
}
Ok(None) => {
println!(
"\n{}",
style("ERROR: No admin user found in the database.")
.red()
.bold()
);
println!("Please ensure the system has been initialized correctly.");
std::process::exit(1);
}
Err(e) => {
eprintln!(
"\n{} Failed to query admin user.",
style("ERROR:").red().bold()
);
eprintln!("Details: {:?}", e);
std::process::exit(1);
}
}
let encryption_key = loop {
let auth_methods = vec![
"Enter encryption password manually",
"Read from password file",
];
let method = Select::with_theme(theme)
.with_prompt("How would you like to provide the database encryption key?")
.items(&auth_methods)
.interact()
.unwrap();
let raw_key = if method == 0 {
Password::with_theme(theme)
.with_prompt("Enter Encryption Password")
.interact()
.unwrap()
} else {
let file_path: String = Input::with_theme(theme)
.with_prompt("Enter path to encryption password file")
.interact_text()
.unwrap();
match std::fs::read_to_string(&file_path) {
Ok(content) => content.trim().to_string(),
Err(e) => {
println!("{}: {}", style("Failed to read file").red(), e);
continue;
}
}
};
if raw_key.is_empty() {
println!("{}", style("Key cannot be empty.").red());
continue;
}
let prompt_message = format!(
"Encryption key loaded: [ {} ]\n\n \
{}: This key must match the database encryption key used by the server.\n \
It corresponds to these settings in your service:\n \
- Arguments: {} or {}\n \
- Envs: {} or {}\n\n \
Do you want to continue?",
style(&raw_key).cyan().bold(),
style("IMPORTANT").yellow().bold(),
style("--bichon_encrypt_password").italic(),
style("--bichon_encrypt_password_file").italic(),
style("BICHON_ENCRYPT_PASSWORD").green(),
style("BICHON_ENCRYPT_PASSWORD_FILE").green()
);
if Confirm::with_theme(theme)
.with_prompt(prompt_message)
.default(true)
.interact()
.unwrap()
{
break raw_key;
}
};
let admin = find_admin(&database);
match admin {
Ok(Some(user)) => {
println!("----------------------------------------");
println!("{:<12} : {}", "Username", style(&user.username).cyan());
println!("{:<12} : {}", "Email", style(&user.email).cyan());
let pwd_display = match &user.password {
Some(p) => {
let password = match internal_decrypt_string(&encryption_key, p) {
Ok(p) => p,
Err(e) => {
println!("\n{}", style("ERROR: Decryption Failed").red().bold());
println!(
"{}",
style("The provided encryption key is incorrect or invalid for this database.").yellow()
);
println!(
"{} Please verify your {} or the {} you provided.",
style("➔").cyan(),
style("encryption password").bold(),
style("key file").bold()
);
eprintln!("\nTechnical details: {:?}", e);
std::process::exit(1);
}
};
style(password).yellow().to_string()
}
None => style("None (No password set)").dim().italic().to_string(),
};
println!("{:<12} : {}", "Password", pwd_display);
println!("----------------------------------------");
if !dialoguer::Confirm::with_theme(theme)
.with_prompt(format!(
"Do you want to reset the password for '{}'?",
user.username
))
.interact()
.unwrap()
{
println!("Operation cancelled.");
return;
}
}
Ok(None) => {
println!(
"\n{}",
style("ERROR: No admin user found in the database.")
.red()
.bold()
);
println!("Please ensure the system has been initialized correctly.");
std::process::exit(1);
}
Err(e) => {
eprintln!(
"\n{} Failed to query admin user.",
style("ERROR:").red().bold()
);
eprintln!("Details: {:?}", e);
std::process::exit(1);
}
}
println!(
"\n{}",
style("TARGET: Reset password for user 'admin'")
.yellow()
.bold()
);
let new_login_password = Password::with_theme(theme)
.with_prompt("Enter new Admin Login Password")
.with_confirmation("Repeat password to confirm", "Passwords do not match!")
.interact()
.unwrap();
if !Confirm::with_theme(theme)
.with_prompt("Proceed with database update?")
.interact()
.unwrap()
{
return;
}
println!("\n{} {}", style("⌛").yellow(), "Updating database...");
match update_admin_password(&database, new_login_password, &encryption_key) {
Ok(_) => {
println!(
"\n{} {}",
Emoji("✨", "*"),
style("Success! Admin password has been updated.")
.green()
.bold()
);
println!(
"{}",
style("You can now log in with the new password.").dim()
);
}
Err(e) => {
println!(
"\n{}",
style("ERROR: Failed to update database").red().bold()
);
eprintln!(
"{} Could not save the new password to the database.",
style("➔").cyan()
);
eprintln!("\nDetails: {:?}", e);
std::process::exit(1);
}
}
}
crates/cli/Cargo.toml
+1 -1
View File
@@ -23,5 +23,5 @@ base64.workspace = true
codepage-strings = "1.0.2"
hex = "0.4.3"
sysinfo.workspace = true
indicatif = "0.18.4"
indicatif.workspace = true
serde_json.workspace = true
crates/core/Cargo.toml
+1 -1
View File
@@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m
"runtime-tokio",
"compress",
] }
tantivy = { version = "0.26.1", features = ["zstd-compression"] }
tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
webpki-roots.workspace = true
rustls.workspace = true
rustls-pki-types.workspace = true
crates/core/src/admin/meta.rs
+17 -5
View File
@@ -16,21 +16,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{path::Path, rc::Rc};
use native_db::{Builder, Database};
use crate::{
{
account::migration::AccountV3,
database::META_MODELS,
error::{code::ErrorCode, BichonResult},
raise_error,
token::{AccessTokenModel, AccessTokenModelKey, TokenType},
users::{UserModel, DEFAULT_ADMIN_USER_ID},
utils::encrypt::internal_encrypt_string,
},
raise_error,
};
use itertools::Itertools;
@@ -48,6 +45,21 @@ pub fn init_meta_database(path: impl AsRef<Path>) -> BichonResult<Rc<Database<'s
Ok(Rc::new(database))
}
pub fn list_all_accounts(database: &Rc<Database<'static>>) -> BichonResult<Vec<AccountV3>> {
let r_transaction = database
.r_transaction()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
let entities: Vec<AccountV3> = r_transaction
.scan()
.primary()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.all()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.try_collect()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(entities)
}
pub fn find_admin(database: &Rc<Database<'static>>) -> BichonResult<Option<UserModel>> {
let r_transaction = database
.r_transaction()
crates/core/src/envelope/extractor.rs
+2 -2
View File
@@ -22,7 +22,7 @@ use crate::envelope::utils::normalize_subject;
use crate::error::code::ErrorCode;
use crate::error::BichonResult;
use crate::message::content::AttachmentInfo;
use crate::store::storage::{DetachedEmail, BLOB_MANAGER};
use crate::store::blob::{DetachedEmail, BLOB_MANAGER};
use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments};
@@ -355,7 +355,7 @@ pub fn generate_message_id() -> String {
format!("<{:016x}.{}.{}@{}>", id!(128), ts, pid, "bichon")
}
fn extract_references(message: &Message<'_>) -> Option<Vec<String>> {
pub fn extract_references(message: &Message<'_>) -> Option<Vec<String>> {
match message.references() {
mail_parser::HeaderValue::Text(cow) => Some(vec![cow.to_string()]),
mail_parser::HeaderValue::TextList(vec) => {
crates/core/src/migrate/legacy/fields.rs
+69
View File
@@ -0,0 +1,69 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use tantivy::schema::Field;
pub const F_MESSAGE_ID: &str = "message_id";
pub const F_ACCOUNT_ID: &str = "account_id";
pub const F_MAILBOX_ID: &str = "mailbox_id";
pub const F_UID: &str = "uid";
pub const F_SUBJECT: &str = "subject";
pub const F_TEXT: &str = "text";
pub const F_FROM: &str = "from";
pub const F_TO: &str = "to";
pub const F_CC: &str = "cc";
pub const F_BCC: &str = "bcc";
pub const F_DATE: &str = "date";
pub const F_INTERNAL_DATE: &str = "internal_date";
pub const F_SIZE: &str = "size";
pub const F_THREAD_ID: &str = "thread_id";
pub const F_ATTACHMENTS: &str = "attachments";
pub const F_HAS_ATTACHMENT: &str = "has_attachment";
pub const F_TAGS: &str = "tags";
pub const F_ID: &str = "id";
pub struct EnvelopeFields {
pub f_id: Field,
pub f_message_id: Field,
pub f_account_id: Field,
pub f_mailbox_id: Field,
pub f_uid: Field,
pub f_subject: Field,
pub f_text: Field,
pub f_from: Field,
pub f_to: Field,
pub f_cc: Field,
pub f_bcc: Field,
pub f_date: Field,
pub f_internal_date: Field,
pub f_size: Field,
pub f_thread_id: Field,
pub f_attachments: Field,
pub f_has_attachment: Field,
pub f_tags: Field,
}
pub const F_EML: &str = "eml";
pub struct EmlFields {
pub f_id: Field,
pub f_account_id: Field,
pub f_mailbox_id: Field,
pub f_eml: Field,
}
crates/core/src/migrate/legacy/mod.rs
+2
View File
@@ -0,0 +1,2 @@
pub mod fields;
pub mod schema;
crates/core/src/migrate/legacy/schema.rs
+114
View File
@@ -0,0 +1,114 @@
use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
use crate::migrate::legacy::fields::{EmlFields, EnvelopeFields, *};
pub struct SchemaTools;
impl SchemaTools {
pub fn envelope_schema() -> Schema {
EnvelopeSchema::build().0
}
pub fn eml_schema() -> Schema {
EmlSchema::build().0
}
pub fn envelope_fields() -> EnvelopeFields {
EnvelopeSchema::fields()
}
pub fn eml_fields() -> EmlFields {
EmlSchema::fields()
}
pub fn envelope_default_fields() -> Vec<Field> {
let f = Self::envelope_fields();
vec![f.f_subject, f.f_text, f.f_attachments]
}
}
// ─── Schema builders ──────────────────────────────────────────────────────────
struct EnvelopeSchema;
impl EnvelopeSchema {
fn build() -> (Schema, EnvelopeFields) {
let mut b = Schema::builder();
let f_id = b.add_u64_field(F_ID, INDEXED | STORED | FAST);
let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
let f_thread_id = b.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST);
let f_subject = b.add_text_field(F_SUBJECT, TEXT | STORED);
let f_text = b.add_text_field(F_TEXT, TEXT | STORED);
let f_attachments = b.add_text_field(F_ATTACHMENTS, TEXT | STORED);
let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
let f_to = b.add_text_field(F_TO, STRING | STORED);
let f_cc = b.add_text_field(F_CC, STRING | STORED);
let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
let f_date = b.add_i64_field(F_DATE, STORED | FAST);
let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, STORED | FAST);
let f_size = b.add_u64_field(F_SIZE, STORED | FAST);
let f_has_attachment = b.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST);
let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let fields = EnvelopeFields {
f_id,
f_account_id,
f_mailbox_id,
f_uid,
f_thread_id,
f_subject,
f_text,
f_attachments,
f_from,
f_to,
f_cc,
f_bcc,
f_message_id,
f_date,
f_internal_date,
f_size,
f_has_attachment,
f_tags,
};
(b.build(), fields)
}
fn fields() -> EnvelopeFields {
Self::build().1
}
}
struct EmlSchema;
impl EmlSchema {
fn build() -> (Schema, EmlFields) {
let mut b = Schema::builder();
let f_id = b.add_u64_field(F_ID, INDEXED | FAST);
let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_eml = b.add_bytes_field(F_EML, STORED);
let fields = EmlFields {
f_id,
f_account_id,
f_mailbox_id,
f_eml,
};
(b.build(), fields)
}
fn fields() -> EmlFields {
Self::build().1
}
}
crates/core/src/migrate/mod.rs
+194 -13
View File
@@ -1,6 +1,20 @@
use std::path::PathBuf;
use crate::settings::cli::SETTINGS;
use crate::{
error::{code::ErrorCode, BichonResult},
migrate::{
legacy::schema::SchemaTools,
store::{LegacyDirs, NewDirs, NewIndexWriter},
},
raise_error,
settings::cli::SETTINGS,
};
use tantivy::{
collector::TopDocs, query::AllQuery, schema::Value, DocAddress, Index, TantivyDocument,
};
pub mod legacy;
pub mod store;
pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
if !dir.exists() || !dir.is_dir() {
@@ -29,23 +43,190 @@ pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
Ok(has_meta_json && match_count >= 3)
}
pub fn is_legacy_data_layout() -> std::io::Result<bool> {
pub fn check_data_status() -> std::io::Result<bool> {
let root_dir = PathBuf::from(&SETTINGS.bichon_root_dir);
let envelope_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir {
PathBuf::from(index_dir)
let new_indices_base = SETTINGS
.bichon_index_dir
.as_ref()
.map(PathBuf::from)
.unwrap_or_else(|| root_dir.clone());
let new_indices_path = new_indices_base.join("bichon-indices");
let new_data_base = SETTINGS
.bichon_data_dir
.as_ref()
.map(PathBuf::from)
.unwrap_or_else(|| root_dir.clone());
let new_storage_path = new_data_base.join("bichon-storage");
let has_new_indices = is_tantivy_index_dir(&new_indices_path.join("attachment_metadata"))?
&& is_tantivy_index_dir(&new_indices_path.join("mail_metadata"))?;
let has_new_storage = is_dir_not_empty(&new_storage_path)?;
if has_new_indices && has_new_storage {
return Ok(true);
}
let legacy_index_root = SETTINGS
.bichon_index_dir
.as_ref()
.map(PathBuf::from)
.unwrap_or_else(|| root_dir.join("envelope"));
let legacy_data_root = SETTINGS
.bichon_data_dir
.as_ref()
.map(PathBuf::from)
.unwrap_or_else(|| root_dir.join("eml"));
let has_legacy_index = is_tantivy_index_dir(&legacy_index_root)?;
let has_legacy_data = is_tantivy_index_dir(&legacy_data_root)?;
if has_legacy_index || has_legacy_data {
Ok(false)
} else {
root_dir.join("envelope")
Ok(true)
}
}
fn is_dir_not_empty(path: &PathBuf) -> std::io::Result<bool> {
if !path.exists() || !path.is_dir() {
return Ok(false);
}
let mut entries = std::fs::read_dir(path)?;
Ok(entries.next().is_some())
}
const PAGE_SIZE: usize = 100;
pub fn do_migrate<F>(legacy: LegacyDirs, new_dirs: NewDirs, mut on_progress: F) -> BichonResult<()>
where
F: FnMut(&str),
{
let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let eml_index = Index::open_in_dir(&legacy.eml_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let envelope_reader = envelope_index
.reader()
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let eml_reader = eml_index
.reader()
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let envelope_searcher = envelope_reader.searcher();
let eml_searcher = eml_reader.searcher();
let total_count = envelope_searcher.num_docs();
on_progress(&format!("TOTAL:{}", total_count));
let ef = SchemaTools::envelope_fields();
let mf = SchemaTools::eml_fields();
let mut writer = NewIndexWriter::open(new_dirs)?;
let mut offset = 0usize;
let mut total_migrated = 0usize;
let mut total_skipped = 0usize;
loop {
let page: Vec<(_, DocAddress)> = envelope_searcher
.search(
&AllQuery,
&TopDocs::with_limit(PAGE_SIZE)
.and_offset(offset)
.order_by_score(),
)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
if page.is_empty() {
break;
}
let fetched = page.len();
for (_, doc_address) in page {
let doc: TantivyDocument = envelope_searcher
.doc(doc_address)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let eid = match doc.get_first(ef.f_id).and_then(|v| v.as_u64()) {
Some(v) => v,
None => {
total_skipped += 1;
continue;
}
};
let account_id = match doc.get_first(ef.f_account_id).and_then(|v| v.as_u64()) {
Some(v) => v,
None => {
total_skipped += 1;
continue;
}
};
let mailbox_id = doc
.get_first(ef.f_mailbox_id)
.and_then(|v| v.as_u64())
.unwrap_or(0);
let uid = doc
.get_first(ef.f_uid)
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
let internal_date = doc
.get_first(ef.f_internal_date)
.and_then(|v| v.as_i64())
.unwrap_or(0);
let eml_term = tantivy::Term::from_field_u64(mf.f_id, eid);
let eml_query =
tantivy::query::TermQuery::new(eml_term, tantivy::schema::IndexRecordOption::Basic);
let eml_hits: Vec<(_, DocAddress)> = eml_searcher
.search(&eml_query, &TopDocs::with_limit(1).order_by_score())
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let eml_bytes = match eml_hits.first() {
Some((_, addr)) => {
let eml_doc: TantivyDocument = eml_searcher
.doc(*addr)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
Some(b) => b.to_vec(),
None => {
on_progress(&format!("WARN: Account {} ID {} eml field missing", account_id, eid));
total_skipped += 1;
continue;
}
}
}
None => {
on_progress(&format!("WARN:Account {} ID {} eml not found", account_id, eid));
total_skipped += 1;
continue;
}
};
let eml_dir = if let Some(ref data_dir) = SETTINGS.bichon_data_dir {
PathBuf::from(data_dir)
} else {
root_dir.join("eml")
};
if let Err(e) = writer.ingest(&eml_bytes, account_id, mailbox_id, uid, internal_date) {
on_progress(&format!("ERROR:Account {} ID {} ingest failed: {}", account_id, eid, e));
total_skipped += 1;
continue;
}
let envelope_result = is_tantivy_index_dir(&envelope_dir)?;
let eml_result = is_tantivy_index_dir(&eml_dir)?;
Ok(envelope_result || eml_result)
total_migrated += 1;
if total_migrated % 100 == 0 || total_migrated == total_count as usize {
on_progress(&format!("PROGRESS:{}:{}", total_migrated, total_skipped));
}
}
offset += fetched;
if fetched < PAGE_SIZE {
break;
}
}
writer.commit()?;
Ok(())
}
#[cfg(test)]
crates/core/src/migrate/store.rs
+424
View File
@@ -0,0 +1,424 @@
use std::path::PathBuf;
use bytes::Bytes;
use mail_parser::MimeHeaders;
use crate::{
envelope::extractor::extract_references, message::content::AttachmentInfo,
store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
};
use fjall::{
config::{BlockSizePolicy, CompressionPolicy},
CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
};
use mail_parser::MessageParser;
use tantivy::{Index, IndexWriter, TantivyDocument};
use uuid::Uuid;
use crate::{
common::AddrVec,
envelope::extractor::{compute_thread_id, generate_message_id},
error::{code::ErrorCode, BichonResult},
raise_error,
store::envelope::Envelope,
store::tantivy::{
model::{AttachmentModel, EnvelopeWithAttachments},
schema::SchemaTools,
},
utc_now,
};
pub struct LegacyDirs {
pub envelope_dir: PathBuf,
pub eml_dir: PathBuf,
}
pub struct NewDirs {
pub envelope_dir: PathBuf,
pub attachment_dir: PathBuf,
pub storage_dir: PathBuf,
}
impl LegacyDirs {
pub fn new(index: PathBuf, data: PathBuf) -> Self {
Self {
envelope_dir: index,
eml_dir: data,
}
}
}
impl NewDirs {
pub fn new(index: PathBuf, data: PathBuf) -> Self {
Self {
envelope_dir: index.join("mail_metadata"),
attachment_dir: index.join("attachment_metadata"),
storage_dir: data,
}
}
}
pub struct DetachOutput {
pub infos: Vec<AttachmentInfo>,
pub blobs: Vec<(String, Bytes)>,
}
pub fn detach_attachments_standalone(
original_body: &[u8],
message: &mail_parser::Message<'_>,
) -> (Vec<u8>, DetachOutput) {
let mut stripped_eml = original_body.to_vec();
let mut infos = Vec::new();
let mut blobs = Vec::new();
let mut ranges: Vec<_> = message
.attachments()
.map(|att| {
(
att.raw_body_offset() as usize,
att.raw_end_offset() as usize,
att,
)
})
.collect();
ranges.sort_by(|a, b| b.0.cmp(&a.0));
for (raw_start, raw_end, att) in ranges {
let content_hash = compute_content_hash(att.contents());
blobs.push((
content_hash.clone(),
Bytes::copy_from_slice(&original_body[raw_start..raw_end]),
));
let placeholder = format!("<<BICHON_DETACH_HASH:{}>>", &content_hash);
stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned());
infos.push(AttachmentInfo {
filename: att.attachment_name().map(|n| n.to_string()),
size: att.contents().len(),
inline: att
.content_disposition()
.map(|d| d.is_inline())
.unwrap_or(false),
file_type: att
.content_type()
.map(|ct| {
format!(
"{}/{}",
ct.c_type.as_ref(),
ct.c_subtype.as_deref().unwrap_or("")
)
})
.unwrap_or_else(|| "application/octet-stream".to_string()),
content_id: att.content_id().map(|id| id.to_string()),
content_hash,
is_message: att.is_message(),
});
}
(stripped_eml, DetachOutput { infos, blobs })
}
pub struct NewIndexWriter {
pub envelope_writer: IndexWriter,
pub attachment_writer: IndexWriter,
pub email_ks: Keyspace,
pub attachment_ks: Keyspace,
pending: usize,
}
const COMMIT_THRESHOLD: usize = 500;
impl NewIndexWriter {
pub fn open(dirs: NewDirs) -> BichonResult<Self> {
// ── envelope index ──────────────────────────────────────────────
std::fs::create_dir_all(&dirs.envelope_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let envelope_index = if dirs
.envelope_dir
.read_dir()
.map(|mut d| d.next().is_none())
.unwrap_or(true)
{
Index::create_in_dir(&dirs.envelope_dir, SchemaTools::email_schema())
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
} else {
Index::open_in_dir(&dirs.envelope_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
};
envelope_index
.tokenizers()
.register("euro", EuroTokenizer::new());
let envelope_writer = envelope_index
.writer_with_num_threads(2, 128 * 1024 * 1024)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
// ── attachment index ─────────────────────────────────────────────
std::fs::create_dir_all(&dirs.attachment_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let attachment_index = if dirs
.attachment_dir
.read_dir()
.map(|mut d| d.next().is_none())
.unwrap_or(true)
{
Index::create_in_dir(&dirs.attachment_dir, SchemaTools::attachment_schema())
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
} else {
Index::open_in_dir(&dirs.attachment_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
};
attachment_index
.tokenizers()
.register("euro", EuroTokenizer::new());
let attachment_writer = attachment_index
.writer_with_num_threads(2, 64 * 1024 * 1024)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
// ── blob store ───────────────────────────────────────────────────
std::fs::create_dir_all(&dirs.storage_dir)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let db = Database::builder(&dirs.storage_dir)
.cache_size(64 * 1024 * 1024)
.open()
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let email_ks = db
.keyspace("email", || {
KeyspaceCreateOptions::default()
.max_memtable_size(16 * 1024 * 1024)
.data_block_size_policy(BlockSizePolicy::all(4 * 1024))
.data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
.with_kv_separation(Some(
KvSeparationOptions::default()
.separation_threshold(1024)
.compression(CompressionType::Lz4)
.file_target_size(512 * 1024 * 1024),
))
})
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
let attachment_ks = db
.keyspace("attachments", || {
KeyspaceCreateOptions::default()
.max_memtable_size(16 * 1024 * 1024)
.data_block_size_policy(BlockSizePolicy::all(4 * 1024))
.data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
.with_kv_separation(Some(
KvSeparationOptions::default()
.separation_threshold(1024)
.compression(CompressionType::Lz4)
.file_target_size(512 * 1024 * 1024),
))
})
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
Ok(Self {
envelope_writer,
attachment_writer,
email_ks,
attachment_ks,
pending: 0,
})
}
pub fn ingest(
&mut self,
eml_bytes: &[u8],
account_id: u64,
mailbox_id: u64,
uid: u32,
internal_date: i64,
) -> BichonResult<()> {
let email_content_hash = compute_content_hash(eml_bytes);
let message = MessageParser::new()
.parse(eml_bytes)
.ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
// ── text / preview ────────────────────────────────────────────────
let text = message
.body_text(0)
.map(|c| c.into_owned())
.or_else(|| {
message
.body_html(0)
.map(|html| crate::utils::html::extract_text(html.into_owned()))
})
.unwrap_or_default();
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
let preview = if text.chars().count() > 100 {
text.chars().take(100).collect::<String>() + "..."
} else {
text.clone()
};
// ── headers ───────────────────────────────────────────────────────
let message_id = message
.message_id()
.map(String::from)
.unwrap_or_else(generate_message_id);
let in_reply_to = message.in_reply_to().as_text().map(String::from);
let references = extract_references(&message);
let thread_id = compute_thread_id(in_reply_to, references, &message_id);
let subject = message.subject().map(String::from).unwrap_or_default();
let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
let internal_date = if internal_date == 0 {
date
} else {
internal_date
};
let parse_addrs = |addrs: Option<&mail_parser::Address<'_>>| {
addrs
.map(|addr| {
AddrVec::from(addr)
.0
.into_iter()
.filter_map(|a| a.address)
.collect::<Vec<_>>()
})
.unwrap_or_default()
};
let from = message
.from()
.and_then(|addr| AddrVec::from(addr).0.into_iter().next())
.and_then(|a| a.address)
.unwrap_or_else(|| "unknown".to_string());
let to = parse_addrs(message.to());
let cc = parse_addrs(message.cc());
let bcc = parse_addrs(message.bcc());
// ── detach attachments → blob ──────────────────────────────────────
let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
if !self
.email_ks
.contains_key(&email_content_hash)
.unwrap_or(false)
{
self.email_ks
.insert(&email_content_hash, stripped_eml.as_slice())
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
}
// write attachment blobs
for (hash, data) in &attachment_output.blobs {
if !self.attachment_ks.contains_key(hash).unwrap_or(false) {
self.attachment_ks
.insert(hash, data.as_ref())
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
}
}
// ── build envelope doc ────────────────────────────────────────────
let envelope_id = Uuid::new_v4().to_string();
let now = utc_now!();
let attachment_docs: Vec<TantivyDocument> = attachment_output
.infos
.iter()
.filter(|a| !a.inline || a.content_id.is_none())
.map(|a| {
AttachmentModel {
id: Uuid::new_v4().to_string(),
envelope_id: envelope_id.clone(),
account_id,
account_email: None,
mailbox_id,
mailbox_name: None,
subject: subject.clone(),
content_hash: a.content_hash.clone(),
from: from.clone(),
date,
ingest_at: now,
size: a.size as u64,
ext: a.get_extension(),
category: a.get_category().to_string(),
content_type: a.file_type.clone(),
shard_id: 0,
text: None,
has_text: false,
is_ocr: false,
page_count: None,
is_indexed: false,
is_message: a.is_message,
name: a.filename.clone(),
tags: None,
auto_tags: None,
}
.into_document()
})
.collect();
let envelope = Envelope {
id: envelope_id,
message_id,
account_id,
mailbox_id,
uid,
subject,
preview,
from,
to,
cc,
bcc,
date,
internal_date,
ingest_at: now,
size: eml_bytes.len() as u32,
thread_id,
attachment_count: message.attachment_count(),
regular_attachment_count: attachment_docs.len(),
tags: None,
account_email: None,
mailbox_name: None,
content_hash: email_content_hash,
};
let ea = EnvelopeWithAttachments {
envelope,
attachments: Some(attachment_output.infos),
};
let envelope_doc = ea.to_document(&text, 0)?;
self.envelope_writer
.add_document(envelope_doc)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
for doc in attachment_docs {
self.attachment_writer
.add_document(doc)
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
}
self.pending += 1;
if self.pending >= COMMIT_THRESHOLD {
self.commit()?;
}
Ok(())
}
pub fn commit(&mut self) -> BichonResult<()> {
if self.pending == 0 {
return Ok(());
}
self.envelope_writer
.commit()
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
self.attachment_writer
.commit()
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
tracing::info!(count = self.pending, "committed batch");
self.pending = 0;
Ok(())
}
}
crates/core/src/store/storage.rs → crates/core/src/store/blob.rs
View File
crates/core/src/store/mod.rs
+1 -1
View File
@@ -17,5 +17,5 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
pub mod envelope;
pub mod storage;
pub mod blob;
pub mod tantivy;
crates/core/src/store/tantivy/envelope.rs
+1 -2
View File
@@ -37,7 +37,7 @@ use crate::{
settings::dir::DATA_DIR_MANAGER,
store::{
envelope::Envelope,
storage::BLOB_MANAGER,
blob::BLOB_MANAGER,
tantivy::{
fatal_commit,
fields::{
@@ -317,7 +317,6 @@ impl IndexManager {
let q = query_parser
.parse_query(subject_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
println!("{:#?}", &q);
subqueries.push((Occur::Must, q));
}
crates/core/src/store/tantivy/schema.rs
+118 -107
View File
@@ -32,84 +32,104 @@ use crate::store::tantivy::fields::{
F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID,
};
static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| {
let (_, fields) = SchemaTools::create_email_schema();
Arc::new(fields)
});
// ─── Lazy Globals ─────────────────────────────────────────────────────────────
static ATTACHMENT_FIELDS: LazyLock<Arc<AttachmentFields>> = LazyLock::new(|| {
let (_, fields) = SchemaTools::create_attachment_schema();
Arc::new(fields)
});
static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| Arc::new(EmailSchema::fields()));
static ATTACHMENT_FIELDS: LazyLock<Arc<AttachmentFields>> =
LazyLock::new(|| Arc::new(AttachmentSchema::fields()));
// ─── Public API ───────────────────────────────────────────────────────────────
pub struct SchemaTools;
impl SchemaTools {
pub fn email_schema() -> Schema {
let (schema, _) = Self::create_email_schema();
schema
EmailSchema::build().0
}
pub fn attachment_schema() -> Schema {
AttachmentSchema::build().0
}
pub fn email_fields() -> &'static EmailFields {
&EMAIL_FIELDS
}
pub fn attachment_fields() -> &'static AttachmentFields {
&ATTACHMENT_FIELDS
}
pub fn email_default_fields() -> Vec<Field> {
let fields = Self::email_fields();
let f = Self::email_fields();
vec![
fields.f_subject,
fields.f_body,
fields.f_attachment_name_text,
fields.f_from_text,
fields.f_to_text,
fields.f_cc_text,
fields.f_bcc_text,
f.f_subject,
f.f_body,
f.f_attachment_name_text,
f.f_from_text,
f.f_to_text,
f.f_cc_text,
f.f_bcc_text,
]
}
pub fn attachment_default_fields() -> Vec<Field> {
let f = Self::attachment_fields();
vec![f.f_subject, f.f_text, f.f_name_text, f.f_from_text]
}
pub fn create_email_schema() -> (Schema, EmailFields) {
let mut builder = Schema::builder();
let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
let f_message_id = builder.add_text_field(F_MESSAGE_ID, STRING | STORED);
let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro"));
let f_preview = builder.add_text_field(F_PREVIEW, STORED);
let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
EmailSchema::build()
}
pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
AttachmentSchema::build()
}
}
let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
let f_to = builder.add_text_field(F_TO, STRING | STORED);
let f_cc = builder.add_text_field(F_CC, STRING | STORED);
let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
// ─── Schema builders ──────────────────────────────────────────────────────────
let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro"));
let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro"));
let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro"));
struct EmailSchema;
let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
let f_thread_id = builder.add_text_field(F_THREAD_ID, STRING | STORED | FAST);
let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
impl EmailSchema {
fn build() -> (Schema, EmailFields) {
let mut b = Schema::builder();
let f_id = b.add_text_field(F_ID, STRING | STORED | FAST);
let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
let f_subject = b.add_text_field(F_SUBJECT, text_store("euro"));
let f_body = b.add_text_field(F_BODY, text_no_store("euro"));
let f_preview = b.add_text_field(F_PREVIEW, STORED);
let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
let f_to = b.add_text_field(F_TO, STRING | STORED);
let f_cc = b.add_text_field(F_CC, STRING | STORED);
let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro"));
let f_to_text = b.add_text_field(F_TO_TEXT, text_no_store("euro"));
let f_cc_text = b.add_text_field(F_CC_TEXT, text_no_store("euro"));
let f_bcc_text = b.add_text_field(F_BCC_TEXT, text_no_store("euro"));
let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
let f_thread_id = b.add_text_field(F_THREAD_ID, STRING | STORED | FAST);
let f_attachment_count = b.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_regular_attachment_count =
builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
b.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_attachment_name_text =
builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro"));
let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED);
b.add_text_field(F_ATTACHMENT_NAME_TEXT, text_no_store("euro"));
let f_attachment_name_exact = b.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
let f_attachments = b.add_text_field(F_ATTACHMENTS, STORED);
let f_attachment_content_hash =
builder.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST);
let f_attachment_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
let f_attachment_category =
builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
b.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST);
let f_attachment_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
let f_attachment_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
let f_attachment_content_type =
builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
let fields = EmailFields {
f_id,
f_message_id,
@@ -145,57 +165,47 @@ impl SchemaTools {
f_tags,
f_shard_id,
};
(builder.build(), fields)
(b.build(), fields)
}
pub fn attachment_schema() -> Schema {
let (schema, _) = Self::create_attachment_schema();
schema
fn fields() -> EmailFields {
Self::build().1
}
}
pub fn attachment_fields() -> &'static AttachmentFields {
&ATTACHMENT_FIELDS
}
struct AttachmentSchema;
pub fn attachment_default_fields() -> Vec<Field> {
let fields = Self::attachment_fields();
vec![
fields.f_subject,
fields.f_text,
fields.f_name_text,
fields.f_from_text,
]
}
impl AttachmentSchema {
fn build() -> (Schema, AttachmentFields) {
let mut b = Schema::builder();
let f_id = b.add_text_field(F_ID, STRING | STORED | FAST);
let f_envelope_id = b.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_subject = b.add_text_field(F_SUBJECT, text_store("euro"));
let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro"));
let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
let f_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
let f_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
let f_content_type = b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
let f_text = b.add_text_field(F_TEXT, text_no_store("euro"));
let f_has_text = b.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
let f_is_ocr = b.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
let f_page_count = b.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
let f_is_indexed = b.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
let f_is_message = b.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
let f_name_text = b.add_text_field(F_NAME_TEXT, text_no_store("euro"));
let f_name_exact = b.add_text_field(F_NAME_EXACT, STRING | STORED);
let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let f_auto_tags = b.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored());
pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
let mut builder = Schema::builder();
let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
let f_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
let f_category = builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
let f_content_type =
builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro"));
let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro"));
let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED);
let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let f_auto_tags =
builder.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored());
let fields = AttachmentFields {
f_id,
f_envelope_id,
@@ -223,9 +233,17 @@ impl SchemaTools {
f_tags,
f_auto_tags,
};
(builder.build(), fields)
(b.build(), fields)
}
fn fields() -> AttachmentFields {
Self::build().1
}
}
// ─── Helpers ──────────────────────────────────────────────────────────────────
fn text_no_store(tokenizer: &str) -> TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
@@ -235,12 +253,5 @@ impl SchemaTools {
}
fn text_store(tokenizer: &str) -> TextOptions {
TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored()
}
text_no_store(tokenizer).set_stored()
}
crates/server/src/main.rs
+6 -6
View File
@@ -25,11 +25,11 @@ use bichon_core::{
context::{executors::BichonContext, Initialize},
error::{code::ErrorCode, BichonResult},
logger,
migrate::is_legacy_data_layout,
migrate::check_data_status,
raise_error,
settings::cli::SETTINGS,
store::{
storage::BLOB_MANAGER,
blob::BLOB_MANAGER,
tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
},
tasks::PeriodicTasks,
@@ -69,11 +69,11 @@ async fn main() -> BichonResult<()> {
info!("Git: [{}]", env!("GIT_HASH"));
info!("GitHub: https://github.com/rustmailer/bichon");
match is_legacy_data_layout() {
Ok(true) => {
match check_data_status() {
Ok(false) => {
error!("Incompatible data format detected.");
error!("Your data was created by an older version of Bichon and must be migrated before use.");
error!("Please run: bichon-migrate");
error!("Please run: bichon-admin");
error!("Documentation: https://github.com/rustmailer/bichon/wiki/migration");
return Err(raise_error!(
"Legacy data layout detected".into(),
@@ -84,7 +84,7 @@ async fn main() -> BichonResult<()> {
error!("Failed to check data layout: {:#?}", e);
return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
}
Ok(false) => {}
Ok(true) => {}
}
if let Err(error) = initialize().await {
crates/server/src/rest/api/message.rs
+1 -1
View File
@@ -36,7 +36,7 @@ use bichon_core::message::tags::TagCount;
use bichon_core::message::tags::TagsRequest;
use bichon_core::raise_error;
use bichon_core::store::envelope::Envelope;
use bichon_core::store::storage::get_reader;
use bichon_core::store::blob::get_reader;
use bichon_core::store::tantivy::envelope::ENVELOPE_MANAGER;
use bichon_core::store::tantivy::validate_facet;
use bichon_core::users::permissions::Permission;