diff --git a/Cargo.lock b/Cargo.lock
index d1031f3..a5cddcd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -464,6 +464,7 @@ dependencies = [
"bichon-core",
"console",
"dialoguer",
+ "indicatif",
"tokio",
]
@@ -5022,6 +5023,8 @@ dependencies = [
"fastdivide",
"fnv",
"fs4",
+ "futures-channel",
+ "futures-util",
"htmlescape",
"itertools",
"levenshtein_automata",
@@ -5045,6 +5048,7 @@ dependencies = [
"tantivy-common",
"tantivy-fst",
"tantivy-query-grammar",
+ "tantivy-sstable",
"tantivy-stacker",
"tantivy-tokenizer-api",
"tempfile",
diff --git a/Cargo.toml b/Cargo.toml
index 44d763f..4c458e0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -84,7 +84,7 @@ uuid = { version = "1.23.1", features = ["v4", "serde"] }
fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] }
tracing-log = "0.2.0"
tokio-util = "0.7.18"
-
+indicatif = "0.18.4"
[profile.release]
strip = true
diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml
index f65744d..b1c91db 100644
--- a/crates/admin/Cargo.toml
+++ b/crates/admin/Cargo.toml
@@ -8,4 +8,5 @@ edition.workspace = true
bichon-core = { path = "../core" }
tokio.workspace = true
dialoguer.workspace = true
-console.workspace = true
\ No newline at end of file
+console.workspace = true
+indicatif.workspace = true
diff --git a/crates/admin/src/main.rs b/crates/admin/src/main.rs
index f94146b..2e15120 100644
--- a/crates/admin/src/main.rs
+++ b/crates/admin/src/main.rs
@@ -16,19 +16,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use std::{
- fs,
- path::{Path, PathBuf},
-};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Select};
-use bichon_core::{
- admin::meta::{find_admin, init_meta_database, update_admin_password},
- error::BichonError,
- utils::encrypt::internal_decrypt_string,
-};
-use console::{style, Emoji};
-use dialoguer::Confirm;
-use dialoguer::{theme::ColorfulTheme, Input, Password, Select};
+use crate::{migrate::handle_migration, reset::handle_reset_password};
+
+pub mod migrate;
+pub mod reset;
#[tokio::main]
async fn main() {
@@ -38,7 +32,12 @@ async fn main() {
style("BICHON ADMINISTRATIVE TOOL").bold().bright().cyan()
);
- let main_options = vec!["Reset Admin Password", "Exit"];
+ let main_options = vec![
+ "Reset Admin Password",
+ "Migrate Legacy v0.x Storage to v1.0",
+ "Exit",
+ ];
+
let selection = Select::with_theme(&theme)
.with_prompt("Select an operation")
.default(0)
@@ -46,272 +45,11 @@ async fn main() {
.interact()
.unwrap();
- if selection == 1 {
- println!("{}", style("Exiting...").dim());
- return;
- }
-
- let root_dir_str: String = Input::with_theme(&theme)
- .with_prompt("Enter the absolute path for 'bichon_root_dir'")
- .validate_with(|input: &String| -> Result<(), &str> {
- let path = Path::new(input);
- if !path.is_absolute() {
- return Err("Path must be absolute.");
- }
- if !path.exists() {
- return Err("Directory does not exist.");
- }
- let has_metadata = path.join("meta.db").exists();
- if !has_metadata {
- return Err("Invalid directory: 'meta.db' not found.");
- }
- Ok(())
- })
- .interact_text()
- .unwrap();
-
- let root_path = PathBuf::from(&root_dir_str);
-
- let database = match init_meta_database(&root_path.join("meta.db")) {
- Ok(database) => database,
- Err(e) => match e {
- BichonError::Generic {
- message,
- location,
- code,
- } => {
- if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") {
- println!("\n{}", style("ERROR: Database is locked.").red().bold());
- println!(
- "{}",
- style("The Bichon service is likely still running.").yellow()
- );
- println!(
- "Since the database cannot be shared between multiple instances, \n\
- you must {} the Bichon service before proceeding.",
- style("STOP").underlined().bold()
- );
- std::process::exit(1);
- } else {
- eprintln!(
- "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}",
- style("A database error occurred:").red().bold(),
- code,
- location,
- message
- );
- std::process::exit(1);
- }
- }
- },
- };
-
- let admin = find_admin(&database);
-
- match admin {
- Ok(Some(user)) => {
- println!("\n{}", style("Admin user found:").green().bold());
- println!("----------------------------------------");
- println!("{:<12} : {}", "Username", style(&user.username).cyan());
- println!("{:<12} : {}", "Email", style(&user.email).cyan());
- }
- Ok(None) => {
- println!(
- "\n{}",
- style("ERROR: No admin user found in the database.")
- .red()
- .bold()
- );
- println!("Please ensure the system has been initialized correctly.");
- std::process::exit(1);
- }
- Err(e) => {
- eprintln!(
- "\n{} Failed to query admin user.",
- style("ERROR:").red().bold()
- );
- eprintln!("Details: {:?}", e);
- std::process::exit(1);
- }
- }
-
- let encryption_key = loop {
- let auth_methods = vec![
- "Enter encryption password manually",
- "Read from password file",
- ];
- let method = Select::with_theme(&theme)
- .with_prompt("How would you like to provide the database encryption key?")
- .items(&auth_methods)
- .interact()
- .unwrap();
-
- let raw_key = if method == 0 {
- Password::with_theme(&theme)
- .with_prompt("Enter Encryption Password")
- .interact()
- .unwrap()
- } else {
- let file_path: String = Input::with_theme(&theme)
- .with_prompt("Enter path to encryption password file")
- .interact_text()
- .unwrap();
-
- match fs::read_to_string(&file_path) {
- Ok(content) => content.trim().to_string(),
- Err(e) => {
- println!("{}: {}", style("Failed to read file").red(), e);
- continue;
- }
- }
- };
-
- if raw_key.is_empty() {
- println!("{}", style("Key cannot be empty.").red());
- continue;
- }
-
- let prompt_message = format!(
- "Encryption key loaded: [ {} ]\n\n \
- {}: This key must match the database encryption key used by the server.\n \
- It corresponds to these settings in your service:\n \
- - Arguments: {} or {}\n \
- - Envs: {} or {}\n\n \
- Do you want to continue?",
- style(&raw_key).cyan().bold(),
- style("IMPORTANT").yellow().bold(),
- style("--bichon_encrypt_password").italic(),
- style("--bichon_encrypt_password_file").italic(),
- style("BICHON_ENCRYPT_PASSWORD").green(),
- style("BICHON_ENCRYPT_PASSWORD_FILE").green()
- );
-
- if Confirm::with_theme(&theme)
- .with_prompt(prompt_message)
- .default(true)
- .interact()
- .unwrap()
- {
- break raw_key;
- }
- };
-
- let admin = find_admin(&database);
-
- match admin {
- Ok(Some(user)) => {
- println!("----------------------------------------");
- println!("{:<12} : {}", "Username", style(&user.username).cyan());
- println!("{:<12} : {}", "Email", style(&user.email).cyan());
-
- let pwd_display = match &user.password {
- Some(p) => {
- let password = match internal_decrypt_string(&encryption_key, p) {
- Ok(p) => p,
- Err(e) => {
- println!("\n{}", style("ERROR: Decryption Failed").red().bold());
- println!(
- "{}",
- style("The provided encryption key is incorrect or invalid for this database.").yellow()
- );
- println!(
- "{} Please verify your {} or the {} you provided.",
- style("➔").cyan(),
- style("encryption password").bold(),
- style("key file").bold()
- );
- eprintln!("\nTechnical details: {:?}", e);
- std::process::exit(1);
- }
- };
- style(password).yellow().to_string()
- }
- None => style("None (No password set)").dim().italic().to_string(),
- };
-
- println!("{:<12} : {}", "Password", pwd_display);
- println!("----------------------------------------");
-
- if !dialoguer::Confirm::with_theme(&theme)
- .with_prompt(format!(
- "Do you want to reset the password for '{}'?",
- user.username
- ))
- .interact()
- .unwrap()
- {
- println!("Operation cancelled.");
- return;
- }
- }
- Ok(None) => {
- println!(
- "\n{}",
- style("ERROR: No admin user found in the database.")
- .red()
- .bold()
- );
- println!("Please ensure the system has been initialized correctly.");
- std::process::exit(1);
- }
- Err(e) => {
- eprintln!(
- "\n{} Failed to query admin user.",
- style("ERROR:").red().bold()
- );
- eprintln!("Details: {:?}", e);
- std::process::exit(1);
- }
- }
-
- println!(
- "\n{}",
- style("TARGET: Reset password for user 'admin'")
- .yellow()
- .bold()
- );
-
- let new_login_password = Password::with_theme(&theme)
- .with_prompt("Enter new Admin Login Password")
- .with_confirmation("Repeat password to confirm", "Passwords do not match!")
- .interact()
- .unwrap();
-
- if !Confirm::with_theme(&theme)
- .with_prompt("Proceed with database update?")
- .interact()
- .unwrap()
- {
- return;
- }
-
- println!("\n{} {}", style("⌛").yellow(), "Updating database...");
-
- match update_admin_password(&database, new_login_password, &encryption_key) {
- Ok(_) => {
- println!(
- "\n{} {}",
- Emoji("✨", "*"),
- style("Success! Admin password has been updated.")
- .green()
- .bold()
- );
- println!(
- "{}",
- style("You can now log in with the new password.").dim()
- );
- }
- Err(e) => {
- println!(
- "\n{}",
- style("ERROR: Failed to update database").red().bold()
- );
- eprintln!(
- "{} Could not save the new password to the database.",
- style("➔").cyan()
- );
- eprintln!("\nDetails: {:?}", e);
- std::process::exit(1);
+ match selection {
+ 0 => handle_reset_password(&theme),
+ 1 => handle_migration(&theme),
+ _ => {
+ println!("{}", style("Exiting...").dim());
}
}
}
diff --git a/crates/admin/src/migrate.rs b/crates/admin/src/migrate.rs
new file mode 100644
index 0000000..5e9da7c
--- /dev/null
+++ b/crates/admin/src/migrate.rs
@@ -0,0 +1,284 @@
+use std::path::{Path, PathBuf};
+
+use bichon_core::migrate::{
+ do_migrate, is_tantivy_index_dir,
+ store::{LegacyDirs, NewDirs},
+};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Confirm, Input};
+use indicatif::{ProgressBar, ProgressStyle};
+
+pub fn handle_migration(theme: &ColorfulTheme) {
+ println!(
+ "\n{}",
+ style("MIGRATION: Bichon v0.x Storage Architecture → v1.0")
+ .bold()
+ .yellow()
+ );
+
+ println!(
+ "{}",
+ style(
+ "This tool migrates data from the legacy v0.x Tantivy-based storage \
+ architecture (used in versions 0.0.1 through 0.3.7) to the new v1.0 \
+ separated index and Fjall-backed storage format."
+ )
+ .dim()
+ );
+
+ println!(
+ "{}",
+ style(
+ "Legacy v0.x architecture:\n\
+ • envelope metadata stored in Tantivy\n\
+ • message data stored in Tantivy\n\n\
+ New v1.0 architecture:\n\
+ • mail indexes stored in Tantivy\n\
+ • attachment indexes stored in Tantivy\n\
+ • raw message data stored in Fjall\n\
+ • attachment blobs stored in Fjall"
+ )
+ .dim()
+ );
+
+ println!(
+ "\n{} {}",
+ style("IMPORTANT:").yellow().bold(),
+ style(
+ "The paths below must exactly match what your old bichon server was configured with."
+ )
+ .yellow()
+ );
+
+ // --- bichon-root-dir ---
+ let root_dir_str: String = Input::with_theme(theme)
+ .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
+ .validate_with(|input: &String| -> Result<(), &str> {
+ let path = Path::new(input);
+ if !path.is_absolute() {
+ return Err("Path must be absolute.");
+ }
+ if !path.exists() {
+ return Err("Directory does not exist.");
+ }
+ Ok(())
+ })
+ .interact_text()
+ .unwrap();
+
+ let root_path = PathBuf::from(&root_dir_str);
+
+ // --- bichon-index-dir ---
+ let default_index = root_path.join("envelope");
+ let default_new_index = root_path.join("bichon-indices");
+ let index_dir_str: String = Input::with_theme(theme)
+ .with_prompt(format!(
+ "Enter --bichon-index-dir (leave blank to use default: {})",
+ style(default_index.display()).cyan()
+ ))
+ .allow_empty(true)
+ .validate_with(|input: &String| -> Result<(), &str> {
+ if input.is_empty() {
+ return Ok(());
+ }
+ let path = Path::new(input);
+ if !path.is_absolute() {
+ return Err("Path must be absolute.");
+ }
+
+ if !path.exists() {
+ return Err("Directory does not exist.");
+ }
+ Ok(())
+ })
+ .interact_text()
+ .unwrap();
+
+ let index_path = if index_dir_str.is_empty() {
+ default_index
+ } else {
+ PathBuf::from(&index_dir_str)
+ };
+
+ let new_index_path = if index_dir_str.is_empty() {
+ default_new_index
+ } else {
+ PathBuf::from(&index_dir_str).join("bichon-indices")
+ };
+
+ // --- bichon-data-dir ---
+ let default_data = root_path.join("eml");
+ let default_new_data = root_path.join("bichon-storage");
+ let data_dir_str: String = Input::with_theme(theme)
+ .with_prompt(format!(
+ "Enter --bichon-data-dir (leave blank to use default: {})",
+ style(default_data.display()).cyan()
+ ))
+ .allow_empty(true)
+ .validate_with(|input: &String| -> Result<(), &str> {
+ if input.is_empty() {
+ return Ok(());
+ }
+ let path = Path::new(input);
+ if !path.is_absolute() {
+ return Err("Path must be absolute.");
+ }
+ if !path.exists() {
+ return Err("Directory does not exist.");
+ }
+ Ok(())
+ })
+ .interact_text()
+ .unwrap();
+
+ let data_path = if data_dir_str.is_empty() {
+ default_data
+ } else {
+ PathBuf::from(&data_dir_str)
+ };
+
+ let new_data_path = if data_dir_str.is_empty() {
+ default_new_data
+ } else {
+ PathBuf::from(&data_dir_str).join("bichon-storage")
+ };
+
+ println!("\n{}", style("Paths to be migrated:").bold());
+ println!("----------------------------------------");
+ println!(
+ "{:<20} : {}",
+ "bichon-root-dir",
+ style(root_path.display()).cyan()
+ );
+ println!(
+ "{:<20} : {}",
+ "bichon-index-dir",
+ style(index_path.display()).cyan()
+ );
+ println!(
+ "{:<20} : {}",
+ "bichon-data-dir",
+ style(data_path.display()).cyan()
+ );
+ println!("----------------------------------------");
+
+ println!(
+ "\n{} Checking legacy v0.x storage layout...",
+ style("⌛").yellow()
+ );
+
+ match is_legacy_data_layout_with_paths(&index_path, &data_path) {
+ Ok(true) => {
+ println!(
+ "{} {}",
+ style("✔").green(),
+ style("Legacy v0.x Tantivy-based storage detected. Migration to v1.0 is required.")
+ .yellow()
+ );
+ }
+ Ok(false) => {
+ println!(
+ "{} {}",
+ style("✔").green(),
+ style("No legacy v0.x storage layout was detected at the specified paths.").green()
+ );
+
+ println!(
+ "{}",
+ style(
+ "The selected directories may already be using the v1.0 storage architecture."
+ )
+ .dim()
+ );
+
+ return;
+ }
+ Err(e) => {
+ eprintln!(
+ "{} Failed to verify legacy storage layout: {:?}",
+ style("ERROR:").red().bold(),
+ e
+ );
+
+ std::process::exit(1);
+ }
+ }
+
+ println!(
+ "\n{} {}",
+ style("⚠").yellow(),
+ style(
+ "This migration is non-destructive. Existing v0.x storage files will remain unchanged."
+ )
+ .yellow()
+ );
+
+ if !Confirm::with_theme(theme)
+ .with_prompt("Ready to migrate?")
+ .default(true)
+ .interact()
+ .unwrap()
+ {
+ println!("{}", style("Migration cancelled.").dim());
+ return;
+ }
+
+ println!("\n{} Migrating...", style("⌛").yellow());
+ let pb = ProgressBar::new(0);
+ pb.set_style(ProgressStyle::default_bar()
+ .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}")
+ .unwrap()
+ .progress_chars("#>-"));
+ let legacy = LegacyDirs::new(index_path, data_path);
+ let new_dirs = NewDirs::new(new_index_path, new_data_path);
+ if let Err(e) = do_migrate(legacy, new_dirs, |msg| {
+ if let Some(data) = msg.strip_prefix("PROGRESS:") {
+ let parts: Vec<&str> = data.split(':').collect();
+ if parts.len() == 2 {
+ let migrated = parts[0].parse::().unwrap_or(0);
+ let skipped = parts[1].parse::().unwrap_or(0);
+
+ pb.set_position(migrated + skipped);
+ pb.set_message(format!(
+ "Migrated: {}, {} {}",
+ style(migrated).green(),
+ style(skipped).red(),
+ style("skipped").dim()
+ ));
+ }
+ } else if let Some(total) = msg.strip_prefix("TOTAL:") {
+ pb.set_length(total.parse().unwrap_or(0));
+ } else if msg.starts_with("WARN:") {
+ pb.println(format!("{} {}", style("⚠").yellow(), &msg[5..]));
+ } else if let Some(done_data) = msg.strip_prefix("DONE:") {
+ let parts: Vec<&str> = done_data.split(':').collect();
+ pb.finish_with_message(format!(
+ "Migration finished. Total: {}, Skipped: {}",
+ parts.get(0).unwrap_or(&"0"),
+ parts.get(1).unwrap_or(&"0")
+ ));
+ }
+ }) {
+ eprintln!(
+ "\n{} Migration failed:\n{:?}",
+ style("✘").red().bold(),
+ style(e).red()
+ );
+ return;
+ }
+ println!(
+ "{} {}",
+ style("✔").green(),
+ style("Migration completed successfully!").bold()
+ );
+}
+
+pub fn is_legacy_data_layout_with_paths(
+ envelope_dir: &PathBuf,
+ eml_dir: &PathBuf,
+) -> std::io::Result {
+ let envelope_result = is_tantivy_index_dir(envelope_dir)?;
+ let eml_result = is_tantivy_index_dir(eml_dir)?;
+
+ Ok(envelope_result || eml_result)
+}
diff --git a/crates/admin/src/reset.rs b/crates/admin/src/reset.rs
new file mode 100644
index 0000000..050d5f5
--- /dev/null
+++ b/crates/admin/src/reset.rs
@@ -0,0 +1,274 @@
+use std::path::{Path, PathBuf};
+
+use bichon_core::{
+ admin::meta::{find_admin, init_meta_database, update_admin_password},
+ error::BichonError,
+ utils::encrypt::internal_decrypt_string,
+};
+use console::{style, Emoji};
+use dialoguer::{theme::ColorfulTheme, Confirm, Input, Password, Select};
+
+pub fn handle_reset_password(theme: &ColorfulTheme) {
+ let root_dir_str: String = Input::with_theme(theme)
+ .with_prompt("Enter the absolute path for 'bichon_root_dir'")
+ .validate_with(|input: &String| -> Result<(), &str> {
+ let path = Path::new(input);
+ if !path.is_absolute() {
+ return Err("Path must be absolute.");
+ }
+ if !path.exists() {
+ return Err("Directory does not exist.");
+ }
+ let has_metadata = path.join("meta.db").exists();
+ if !has_metadata {
+ return Err("Invalid directory: 'meta.db' not found.");
+ }
+ Ok(())
+ })
+ .interact_text()
+ .unwrap();
+
+ let root_path = PathBuf::from(&root_dir_str);
+ let database = match init_meta_database(&root_path.join("meta.db")) {
+ Ok(database) => database,
+ Err(e) => match e {
+ BichonError::Generic {
+ message,
+ location,
+ code,
+ } => {
+ if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") {
+ println!("\n{}", style("ERROR: Database is locked.").red().bold());
+ println!(
+ "{}",
+ style("The Bichon service is likely still running.").yellow()
+ );
+ println!(
+ "Since the database cannot be shared between multiple instances, \n\
+ you must {} the Bichon service before proceeding.",
+ style("STOP").underlined().bold()
+ );
+ std::process::exit(1);
+ } else {
+ eprintln!(
+ "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}",
+ style("A database error occurred:").red().bold(),
+ code,
+ location,
+ message
+ );
+ std::process::exit(1);
+ }
+ }
+ },
+ };
+
+ let admin = find_admin(&database);
+
+ match admin {
+ Ok(Some(user)) => {
+ println!("\n{}", style("Admin user found:").green().bold());
+ println!("----------------------------------------");
+ println!("{:<12} : {}", "Username", style(&user.username).cyan());
+ println!("{:<12} : {}", "Email", style(&user.email).cyan());
+ }
+ Ok(None) => {
+ println!(
+ "\n{}",
+ style("ERROR: No admin user found in the database.")
+ .red()
+ .bold()
+ );
+ println!("Please ensure the system has been initialized correctly.");
+ std::process::exit(1);
+ }
+ Err(e) => {
+ eprintln!(
+ "\n{} Failed to query admin user.",
+ style("ERROR:").red().bold()
+ );
+ eprintln!("Details: {:?}", e);
+ std::process::exit(1);
+ }
+ }
+
+ let encryption_key = loop {
+ let auth_methods = vec![
+ "Enter encryption password manually",
+ "Read from password file",
+ ];
+ let method = Select::with_theme(theme)
+ .with_prompt("How would you like to provide the database encryption key?")
+ .items(&auth_methods)
+ .interact()
+ .unwrap();
+
+ let raw_key = if method == 0 {
+ Password::with_theme(theme)
+ .with_prompt("Enter Encryption Password")
+ .interact()
+ .unwrap()
+ } else {
+ let file_path: String = Input::with_theme(theme)
+ .with_prompt("Enter path to encryption password file")
+ .interact_text()
+ .unwrap();
+
+ match std::fs::read_to_string(&file_path) {
+ Ok(content) => content.trim().to_string(),
+ Err(e) => {
+ println!("{}: {}", style("Failed to read file").red(), e);
+ continue;
+ }
+ }
+ };
+
+ if raw_key.is_empty() {
+ println!("{}", style("Key cannot be empty.").red());
+ continue;
+ }
+
+ let prompt_message = format!(
+ "Encryption key loaded: [ {} ]\n\n \
+ {}: This key must match the database encryption key used by the server.\n \
+ It corresponds to these settings in your service:\n \
+ - Arguments: {} or {}\n \
+ - Envs: {} or {}\n\n \
+ Do you want to continue?",
+ style(&raw_key).cyan().bold(),
+ style("IMPORTANT").yellow().bold(),
+ style("--bichon_encrypt_password").italic(),
+ style("--bichon_encrypt_password_file").italic(),
+ style("BICHON_ENCRYPT_PASSWORD").green(),
+ style("BICHON_ENCRYPT_PASSWORD_FILE").green()
+ );
+
+ if Confirm::with_theme(theme)
+ .with_prompt(prompt_message)
+ .default(true)
+ .interact()
+ .unwrap()
+ {
+ break raw_key;
+ }
+ };
+
+ let admin = find_admin(&database);
+
+ match admin {
+ Ok(Some(user)) => {
+ println!("----------------------------------------");
+ println!("{:<12} : {}", "Username", style(&user.username).cyan());
+ println!("{:<12} : {}", "Email", style(&user.email).cyan());
+
+ let pwd_display = match &user.password {
+ Some(p) => {
+ let password = match internal_decrypt_string(&encryption_key, p) {
+ Ok(p) => p,
+ Err(e) => {
+ println!("\n{}", style("ERROR: Decryption Failed").red().bold());
+ println!(
+ "{}",
+ style("The provided encryption key is incorrect or invalid for this database.").yellow()
+ );
+ println!(
+ "{} Please verify your {} or the {} you provided.",
+ style("➔").cyan(),
+ style("encryption password").bold(),
+ style("key file").bold()
+ );
+ eprintln!("\nTechnical details: {:?}", e);
+ std::process::exit(1);
+ }
+ };
+ style(password).yellow().to_string()
+ }
+ None => style("None (No password set)").dim().italic().to_string(),
+ };
+
+ println!("{:<12} : {}", "Password", pwd_display);
+ println!("----------------------------------------");
+
+ if !dialoguer::Confirm::with_theme(theme)
+ .with_prompt(format!(
+ "Do you want to reset the password for '{}'?",
+ user.username
+ ))
+ .interact()
+ .unwrap()
+ {
+ println!("Operation cancelled.");
+ return;
+ }
+ }
+ Ok(None) => {
+ println!(
+ "\n{}",
+ style("ERROR: No admin user found in the database.")
+ .red()
+ .bold()
+ );
+ println!("Please ensure the system has been initialized correctly.");
+ std::process::exit(1);
+ }
+ Err(e) => {
+ eprintln!(
+ "\n{} Failed to query admin user.",
+ style("ERROR:").red().bold()
+ );
+ eprintln!("Details: {:?}", e);
+ std::process::exit(1);
+ }
+ }
+
+ println!(
+ "\n{}",
+ style("TARGET: Reset password for user 'admin'")
+ .yellow()
+ .bold()
+ );
+
+ let new_login_password = Password::with_theme(theme)
+ .with_prompt("Enter new Admin Login Password")
+ .with_confirmation("Repeat password to confirm", "Passwords do not match!")
+ .interact()
+ .unwrap();
+
+ if !Confirm::with_theme(theme)
+ .with_prompt("Proceed with database update?")
+ .interact()
+ .unwrap()
+ {
+ return;
+ }
+
+ println!("\n{} {}", style("⌛").yellow(), "Updating database...");
+
+ match update_admin_password(&database, new_login_password, &encryption_key) {
+ Ok(_) => {
+ println!(
+ "\n{} {}",
+ Emoji("✨", "*"),
+ style("Success! Admin password has been updated.")
+ .green()
+ .bold()
+ );
+ println!(
+ "{}",
+ style("You can now log in with the new password.").dim()
+ );
+ }
+ Err(e) => {
+ println!(
+ "\n{}",
+ style("ERROR: Failed to update database").red().bold()
+ );
+ eprintln!(
+ "{} Could not save the new password to the database.",
+ style("➔").cyan()
+ );
+ eprintln!("\nDetails: {:?}", e);
+ std::process::exit(1);
+ }
+ }
+}
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
index 92e52de..08de615 100644
--- a/crates/cli/Cargo.toml
+++ b/crates/cli/Cargo.toml
@@ -23,5 +23,5 @@ base64.workspace = true
codepage-strings = "1.0.2"
hex = "0.4.3"
sysinfo.workspace = true
-indicatif = "0.18.4"
+indicatif.workspace = true
serde_json.workspace = true
\ No newline at end of file
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 65cbcab..220090c 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m
"runtime-tokio",
"compress",
] }
-tantivy = { version = "0.26.1", features = ["zstd-compression"] }
+tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
webpki-roots.workspace = true
rustls.workspace = true
rustls-pki-types.workspace = true
diff --git a/crates/core/src/admin/meta.rs b/crates/core/src/admin/meta.rs
index 7ec75e8..10c8d6b 100644
--- a/crates/core/src/admin/meta.rs
+++ b/crates/core/src/admin/meta.rs
@@ -16,21 +16,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-
-
use std::{path::Path, rc::Rc};
use native_db::{Builder, Database};
use crate::{
- {
- database::META_MODELS,
- error::{code::ErrorCode, BichonResult},
- token::{AccessTokenModel, AccessTokenModelKey, TokenType},
- users::{UserModel, DEFAULT_ADMIN_USER_ID},
- utils::encrypt::internal_encrypt_string,
- },
+ account::migration::AccountV3,
+ database::META_MODELS,
+ error::{code::ErrorCode, BichonResult},
raise_error,
+ token::{AccessTokenModel, AccessTokenModelKey, TokenType},
+ users::{UserModel, DEFAULT_ADMIN_USER_ID},
+ utils::encrypt::internal_encrypt_string,
};
use itertools::Itertools;
@@ -48,6 +45,21 @@ pub fn init_meta_database(path: impl AsRef) -> BichonResult>) -> BichonResult> {
+ let r_transaction = database
+ .r_transaction()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let entities: Vec = r_transaction
+ .scan()
+ .primary()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+ .all()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+ .try_collect()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ Ok(entities)
+}
+
pub fn find_admin(database: &Rc>) -> BichonResult> {
let r_transaction = database
.r_transaction()
diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs
index 845f895..1ee5416 100644
--- a/crates/core/src/envelope/extractor.rs
+++ b/crates/core/src/envelope/extractor.rs
@@ -22,7 +22,7 @@ use crate::envelope::utils::normalize_subject;
use crate::error::code::ErrorCode;
use crate::error::BichonResult;
use crate::message::content::AttachmentInfo;
-use crate::store::storage::{DetachedEmail, BLOB_MANAGER};
+use crate::store::blob::{DetachedEmail, BLOB_MANAGER};
use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments};
@@ -355,7 +355,7 @@ pub fn generate_message_id() -> String {
format!("<{:016x}.{}.{}@{}>", id!(128), ts, pid, "bichon")
}
-fn extract_references(message: &Message<'_>) -> Option> {
+pub fn extract_references(message: &Message<'_>) -> Option> {
match message.references() {
mail_parser::HeaderValue::Text(cow) => Some(vec![cow.to_string()]),
mail_parser::HeaderValue::TextList(vec) => {
diff --git a/crates/core/src/migrate/legacy/fields.rs b/crates/core/src/migrate/legacy/fields.rs
new file mode 100644
index 0000000..1ff9afa
--- /dev/null
+++ b/crates/core/src/migrate/legacy/fields.rs
@@ -0,0 +1,69 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+
+use tantivy::schema::Field;
+
+pub const F_MESSAGE_ID: &str = "message_id";
+pub const F_ACCOUNT_ID: &str = "account_id";
+pub const F_MAILBOX_ID: &str = "mailbox_id";
+pub const F_UID: &str = "uid";
+pub const F_SUBJECT: &str = "subject";
+pub const F_TEXT: &str = "text";
+pub const F_FROM: &str = "from";
+pub const F_TO: &str = "to";
+pub const F_CC: &str = "cc";
+pub const F_BCC: &str = "bcc";
+pub const F_DATE: &str = "date";
+pub const F_INTERNAL_DATE: &str = "internal_date";
+pub const F_SIZE: &str = "size";
+pub const F_THREAD_ID: &str = "thread_id";
+pub const F_ATTACHMENTS: &str = "attachments";
+pub const F_HAS_ATTACHMENT: &str = "has_attachment";
+pub const F_TAGS: &str = "tags";
+
+pub const F_ID: &str = "id";
+pub struct EnvelopeFields {
+ pub f_id: Field,
+ pub f_message_id: Field,
+ pub f_account_id: Field,
+ pub f_mailbox_id: Field,
+ pub f_uid: Field,
+ pub f_subject: Field,
+ pub f_text: Field,
+ pub f_from: Field,
+ pub f_to: Field,
+ pub f_cc: Field,
+ pub f_bcc: Field,
+ pub f_date: Field,
+ pub f_internal_date: Field,
+ pub f_size: Field,
+ pub f_thread_id: Field,
+ pub f_attachments: Field,
+ pub f_has_attachment: Field,
+ pub f_tags: Field,
+}
+
+pub const F_EML: &str = "eml";
+
+pub struct EmlFields {
+ pub f_id: Field,
+ pub f_account_id: Field,
+ pub f_mailbox_id: Field,
+ pub f_eml: Field,
+}
\ No newline at end of file
diff --git a/crates/core/src/migrate/legacy/mod.rs b/crates/core/src/migrate/legacy/mod.rs
new file mode 100644
index 0000000..ce95d06
--- /dev/null
+++ b/crates/core/src/migrate/legacy/mod.rs
@@ -0,0 +1,2 @@
+pub mod fields;
+pub mod schema;
diff --git a/crates/core/src/migrate/legacy/schema.rs b/crates/core/src/migrate/legacy/schema.rs
new file mode 100644
index 0000000..9a0d762
--- /dev/null
+++ b/crates/core/src/migrate/legacy/schema.rs
@@ -0,0 +1,114 @@
+use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
+
+use crate::migrate::legacy::fields::{EmlFields, EnvelopeFields, *};
+
+pub struct SchemaTools;
+
+impl SchemaTools {
+ pub fn envelope_schema() -> Schema {
+ EnvelopeSchema::build().0
+ }
+ pub fn eml_schema() -> Schema {
+ EmlSchema::build().0
+ }
+
+ pub fn envelope_fields() -> EnvelopeFields {
+ EnvelopeSchema::fields()
+ }
+ pub fn eml_fields() -> EmlFields {
+ EmlSchema::fields()
+ }
+
+ pub fn envelope_default_fields() -> Vec {
+ let f = Self::envelope_fields();
+ vec![f.f_subject, f.f_text, f.f_attachments]
+ }
+}
+
+// ─── Schema builders ──────────────────────────────────────────────────────────
+
+struct EnvelopeSchema;
+
+impl EnvelopeSchema {
+ fn build() -> (Schema, EnvelopeFields) {
+ let mut b = Schema::builder();
+
+ let f_id = b.add_u64_field(F_ID, INDEXED | STORED | FAST);
+ let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+ let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+ let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
+ let f_thread_id = b.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST);
+
+ let f_subject = b.add_text_field(F_SUBJECT, TEXT | STORED);
+ let f_text = b.add_text_field(F_TEXT, TEXT | STORED);
+ let f_attachments = b.add_text_field(F_ATTACHMENTS, TEXT | STORED);
+
+ let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+ let f_to = b.add_text_field(F_TO, STRING | STORED);
+ let f_cc = b.add_text_field(F_CC, STRING | STORED);
+ let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
+
+ let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
+
+ let f_date = b.add_i64_field(F_DATE, STORED | FAST);
+ let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, STORED | FAST);
+
+ let f_size = b.add_u64_field(F_SIZE, STORED | FAST);
+ let f_has_attachment = b.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST);
+
+ let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+
+ let fields = EnvelopeFields {
+ f_id,
+ f_account_id,
+ f_mailbox_id,
+ f_uid,
+ f_thread_id,
+ f_subject,
+ f_text,
+ f_attachments,
+ f_from,
+ f_to,
+ f_cc,
+ f_bcc,
+ f_message_id,
+ f_date,
+ f_internal_date,
+ f_size,
+ f_has_attachment,
+ f_tags,
+ };
+
+ (b.build(), fields)
+ }
+
+ fn fields() -> EnvelopeFields {
+ Self::build().1
+ }
+}
+
+struct EmlSchema;
+
+impl EmlSchema {
+ fn build() -> (Schema, EmlFields) {
+ let mut b = Schema::builder();
+
+ let f_id = b.add_u64_field(F_ID, INDEXED | FAST);
+ let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+ let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+ let f_eml = b.add_bytes_field(F_EML, STORED);
+
+ let fields = EmlFields {
+ f_id,
+ f_account_id,
+ f_mailbox_id,
+ f_eml,
+ };
+
+ (b.build(), fields)
+ }
+
+ fn fields() -> EmlFields {
+ Self::build().1
+ }
+}
diff --git a/crates/core/src/migrate/mod.rs b/crates/core/src/migrate/mod.rs
index 7d2feae..444cb3b 100644
--- a/crates/core/src/migrate/mod.rs
+++ b/crates/core/src/migrate/mod.rs
@@ -1,6 +1,20 @@
use std::path::PathBuf;
-use crate::settings::cli::SETTINGS;
+use crate::{
+ error::{code::ErrorCode, BichonResult},
+ migrate::{
+ legacy::schema::SchemaTools,
+ store::{LegacyDirs, NewDirs, NewIndexWriter},
+ },
+ raise_error,
+ settings::cli::SETTINGS,
+};
+use tantivy::{
+ collector::TopDocs, query::AllQuery, schema::Value, DocAddress, Index, TantivyDocument,
+};
+
+pub mod legacy;
+pub mod store;
pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result {
if !dir.exists() || !dir.is_dir() {
@@ -29,23 +43,190 @@ pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result {
Ok(has_meta_json && match_count >= 3)
}
-pub fn is_legacy_data_layout() -> std::io::Result {
+pub fn check_data_status() -> std::io::Result {
let root_dir = PathBuf::from(&SETTINGS.bichon_root_dir);
- let envelope_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir {
- PathBuf::from(index_dir)
- } else {
- root_dir.join("envelope")
- };
- let eml_dir = if let Some(ref data_dir) = SETTINGS.bichon_data_dir {
- PathBuf::from(data_dir)
- } else {
- root_dir.join("eml")
- };
+ let new_indices_base = SETTINGS
+ .bichon_index_dir
+ .as_ref()
+ .map(PathBuf::from)
+ .unwrap_or_else(|| root_dir.clone());
+ let new_indices_path = new_indices_base.join("bichon-indices");
- let envelope_result = is_tantivy_index_dir(&envelope_dir)?;
- let eml_result = is_tantivy_index_dir(&eml_dir)?;
- Ok(envelope_result || eml_result)
+ let new_data_base = SETTINGS
+ .bichon_data_dir
+ .as_ref()
+ .map(PathBuf::from)
+ .unwrap_or_else(|| root_dir.clone());
+ let new_storage_path = new_data_base.join("bichon-storage");
+
+ let has_new_indices = is_tantivy_index_dir(&new_indices_path.join("attachment_metadata"))?
+ && is_tantivy_index_dir(&new_indices_path.join("mail_metadata"))?;
+ let has_new_storage = is_dir_not_empty(&new_storage_path)?;
+
+ if has_new_indices && has_new_storage {
+ return Ok(true);
+ }
+
+ let legacy_index_root = SETTINGS
+ .bichon_index_dir
+ .as_ref()
+ .map(PathBuf::from)
+ .unwrap_or_else(|| root_dir.join("envelope"));
+ let legacy_data_root = SETTINGS
+ .bichon_data_dir
+ .as_ref()
+ .map(PathBuf::from)
+ .unwrap_or_else(|| root_dir.join("eml"));
+
+ let has_legacy_index = is_tantivy_index_dir(&legacy_index_root)?;
+ let has_legacy_data = is_tantivy_index_dir(&legacy_data_root)?;
+
+ if has_legacy_index || has_legacy_data {
+ Ok(false)
+ } else {
+ Ok(true)
+ }
+}
+
+fn is_dir_not_empty(path: &PathBuf) -> std::io::Result {
+ if !path.exists() || !path.is_dir() {
+ return Ok(false);
+ }
+ let mut entries = std::fs::read_dir(path)?;
+ Ok(entries.next().is_some())
+}
+
+const PAGE_SIZE: usize = 100;
+
+pub fn do_migrate(legacy: LegacyDirs, new_dirs: NewDirs, mut on_progress: F) -> BichonResult<()>
+where
+ F: FnMut(&str),
+{
+ let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ let eml_index = Index::open_in_dir(&legacy.eml_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ let envelope_reader = envelope_index
+ .reader()
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ let eml_reader = eml_index
+ .reader()
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ let envelope_searcher = envelope_reader.searcher();
+ let eml_searcher = eml_reader.searcher();
+
+ let total_count = envelope_searcher.num_docs();
+ on_progress(&format!("TOTAL:{}", total_count));
+
+ let ef = SchemaTools::envelope_fields();
+ let mf = SchemaTools::eml_fields();
+
+ let mut writer = NewIndexWriter::open(new_dirs)?;
+
+ let mut offset = 0usize;
+ let mut total_migrated = 0usize;
+ let mut total_skipped = 0usize;
+
+ loop {
+ let page: Vec<(_, DocAddress)> = envelope_searcher
+ .search(
+ &AllQuery,
+ &TopDocs::with_limit(PAGE_SIZE)
+ .and_offset(offset)
+ .order_by_score(),
+ )
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ if page.is_empty() {
+ break;
+ }
+ let fetched = page.len();
+
+ for (_, doc_address) in page {
+ let doc: TantivyDocument = envelope_searcher
+ .doc(doc_address)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ let eid = match doc.get_first(ef.f_id).and_then(|v| v.as_u64()) {
+ Some(v) => v,
+ None => {
+ total_skipped += 1;
+ continue;
+ }
+ };
+ let account_id = match doc.get_first(ef.f_account_id).and_then(|v| v.as_u64()) {
+ Some(v) => v,
+ None => {
+ total_skipped += 1;
+ continue;
+ }
+ };
+ let mailbox_id = doc
+ .get_first(ef.f_mailbox_id)
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0);
+ let uid = doc
+ .get_first(ef.f_uid)
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0) as u32;
+ let internal_date = doc
+ .get_first(ef.f_internal_date)
+ .and_then(|v| v.as_i64())
+ .unwrap_or(0);
+
+ let eml_term = tantivy::Term::from_field_u64(mf.f_id, eid);
+ let eml_query =
+ tantivy::query::TermQuery::new(eml_term, tantivy::schema::IndexRecordOption::Basic);
+ let eml_hits: Vec<(_, DocAddress)> = eml_searcher
+ .search(&eml_query, &TopDocs::with_limit(1).order_by_score())
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ let eml_bytes = match eml_hits.first() {
+ Some((_, addr)) => {
+ let eml_doc: TantivyDocument = eml_searcher
+ .doc(*addr)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
+ Some(b) => b.to_vec(),
+ None => {
+ on_progress(&format!("WARN: Account {} ID {} eml field missing", account_id, eid));
+ total_skipped += 1;
+ continue;
+ }
+ }
+ }
+ None => {
+ on_progress(&format!("WARN:Account {} ID {} eml not found", account_id, eid));
+ total_skipped += 1;
+ continue;
+ }
+ };
+
+ if let Err(e) = writer.ingest(&eml_bytes, account_id, mailbox_id, uid, internal_date) {
+ on_progress(&format!("ERROR:Account {} ID {} ingest failed: {}", account_id, eid, e));
+ total_skipped += 1;
+ continue;
+ }
+
+ total_migrated += 1;
+
+ if total_migrated % 100 == 0 || total_migrated == total_count as usize {
+ on_progress(&format!("PROGRESS:{}:{}", total_migrated, total_skipped));
+ }
+ }
+
+ offset += fetched;
+ if fetched < PAGE_SIZE {
+ break;
+ }
+ }
+
+ writer.commit()?;
+
+ Ok(())
}
#[cfg(test)]
diff --git a/crates/core/src/migrate/store.rs b/crates/core/src/migrate/store.rs
new file mode 100644
index 0000000..691b204
--- /dev/null
+++ b/crates/core/src/migrate/store.rs
@@ -0,0 +1,424 @@
+use std::path::PathBuf;
+
+use bytes::Bytes;
+use mail_parser::MimeHeaders;
+
+use crate::{
+ envelope::extractor::extract_references, message::content::AttachmentInfo,
+ store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
+};
+
+use fjall::{
+ config::{BlockSizePolicy, CompressionPolicy},
+ CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
+};
+use mail_parser::MessageParser;
+use tantivy::{Index, IndexWriter, TantivyDocument};
+use uuid::Uuid;
+
+use crate::{
+ common::AddrVec,
+ envelope::extractor::{compute_thread_id, generate_message_id},
+ error::{code::ErrorCode, BichonResult},
+ raise_error,
+ store::envelope::Envelope,
+ store::tantivy::{
+ model::{AttachmentModel, EnvelopeWithAttachments},
+ schema::SchemaTools,
+ },
+ utc_now,
+};
+
+pub struct LegacyDirs {
+ pub envelope_dir: PathBuf,
+ pub eml_dir: PathBuf,
+}
+
+pub struct NewDirs {
+ pub envelope_dir: PathBuf,
+ pub attachment_dir: PathBuf,
+ pub storage_dir: PathBuf,
+}
+
+impl LegacyDirs {
+ pub fn new(index: PathBuf, data: PathBuf) -> Self {
+ Self {
+ envelope_dir: index,
+ eml_dir: data,
+ }
+ }
+}
+
+impl NewDirs {
+ pub fn new(index: PathBuf, data: PathBuf) -> Self {
+ Self {
+ envelope_dir: index.join("mail_metadata"),
+ attachment_dir: index.join("attachment_metadata"),
+ storage_dir: data,
+ }
+ }
+}
+
+pub struct DetachOutput {
+ pub infos: Vec,
+ pub blobs: Vec<(String, Bytes)>,
+}
+
+pub fn detach_attachments_standalone(
+ original_body: &[u8],
+ message: &mail_parser::Message<'_>,
+) -> (Vec, DetachOutput) {
+ let mut stripped_eml = original_body.to_vec();
+ let mut infos = Vec::new();
+ let mut blobs = Vec::new();
+
+ let mut ranges: Vec<_> = message
+ .attachments()
+ .map(|att| {
+ (
+ att.raw_body_offset() as usize,
+ att.raw_end_offset() as usize,
+ att,
+ )
+ })
+ .collect();
+ ranges.sort_by(|a, b| b.0.cmp(&a.0));
+
+ for (raw_start, raw_end, att) in ranges {
+ let content_hash = compute_content_hash(att.contents());
+ blobs.push((
+ content_hash.clone(),
+ Bytes::copy_from_slice(&original_body[raw_start..raw_end]),
+ ));
+
+ let placeholder = format!("<>", &content_hash);
+ stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned());
+
+ infos.push(AttachmentInfo {
+ filename: att.attachment_name().map(|n| n.to_string()),
+ size: att.contents().len(),
+ inline: att
+ .content_disposition()
+ .map(|d| d.is_inline())
+ .unwrap_or(false),
+ file_type: att
+ .content_type()
+ .map(|ct| {
+ format!(
+ "{}/{}",
+ ct.c_type.as_ref(),
+ ct.c_subtype.as_deref().unwrap_or("")
+ )
+ })
+ .unwrap_or_else(|| "application/octet-stream".to_string()),
+ content_id: att.content_id().map(|id| id.to_string()),
+ content_hash,
+ is_message: att.is_message(),
+ });
+ }
+
+ (stripped_eml, DetachOutput { infos, blobs })
+}
+
+pub struct NewIndexWriter {
+ pub envelope_writer: IndexWriter,
+ pub attachment_writer: IndexWriter,
+ pub email_ks: Keyspace,
+ pub attachment_ks: Keyspace,
+ pending: usize,
+}
+
+const COMMIT_THRESHOLD: usize = 500;
+
+impl NewIndexWriter {
+ pub fn open(dirs: NewDirs) -> BichonResult {
+ // ── envelope index ──────────────────────────────────────────────
+ std::fs::create_dir_all(&dirs.envelope_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ let envelope_index = if dirs
+ .envelope_dir
+ .read_dir()
+ .map(|mut d| d.next().is_none())
+ .unwrap_or(true)
+ {
+ Index::create_in_dir(&dirs.envelope_dir, SchemaTools::email_schema())
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+ } else {
+ Index::open_in_dir(&dirs.envelope_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+ };
+
+ envelope_index
+ .tokenizers()
+ .register("euro", EuroTokenizer::new());
+
+ let envelope_writer = envelope_index
+ .writer_with_num_threads(2, 128 * 1024 * 1024)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ // ── attachment index ─────────────────────────────────────────────
+ std::fs::create_dir_all(&dirs.attachment_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ let attachment_index = if dirs
+ .attachment_dir
+ .read_dir()
+ .map(|mut d| d.next().is_none())
+ .unwrap_or(true)
+ {
+ Index::create_in_dir(&dirs.attachment_dir, SchemaTools::attachment_schema())
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+ } else {
+ Index::open_in_dir(&dirs.attachment_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+ };
+
+ attachment_index
+ .tokenizers()
+ .register("euro", EuroTokenizer::new());
+ let attachment_writer = attachment_index
+ .writer_with_num_threads(2, 64 * 1024 * 1024)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ // ── blob store ───────────────────────────────────────────────────
+ std::fs::create_dir_all(&dirs.storage_dir)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ let db = Database::builder(&dirs.storage_dir)
+ .cache_size(64 * 1024 * 1024)
+ .open()
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ let email_ks = db
+ .keyspace("email", || {
+ KeyspaceCreateOptions::default()
+ .max_memtable_size(16 * 1024 * 1024)
+ .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+ .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
+ .with_kv_separation(Some(
+ KvSeparationOptions::default()
+ .separation_threshold(1024)
+ .compression(CompressionType::Lz4)
+ .file_target_size(512 * 1024 * 1024),
+ ))
+ })
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ let attachment_ks = db
+ .keyspace("attachments", || {
+ KeyspaceCreateOptions::default()
+ .max_memtable_size(16 * 1024 * 1024)
+ .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+ .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
+ .with_kv_separation(Some(
+ KvSeparationOptions::default()
+ .separation_threshold(1024)
+ .compression(CompressionType::Lz4)
+ .file_target_size(512 * 1024 * 1024),
+ ))
+ })
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ Ok(Self {
+ envelope_writer,
+ attachment_writer,
+ email_ks,
+ attachment_ks,
+ pending: 0,
+ })
+ }
+
+ pub fn ingest(
+ &mut self,
+ eml_bytes: &[u8],
+ account_id: u64,
+ mailbox_id: u64,
+ uid: u32,
+ internal_date: i64,
+ ) -> BichonResult<()> {
+ let email_content_hash = compute_content_hash(eml_bytes);
+
+ let message = MessageParser::new()
+ .parse(eml_bytes)
+ .ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
+
+ // ── text / preview ────────────────────────────────────────────────
+ let text = message
+ .body_text(0)
+ .map(|c| c.into_owned())
+ .or_else(|| {
+ message
+ .body_html(0)
+ .map(|html| crate::utils::html::extract_text(html.into_owned()))
+ })
+ .unwrap_or_default();
+ let text = text.split_whitespace().collect::>().join(" ");
+ let preview = if text.chars().count() > 100 {
+ text.chars().take(100).collect::() + "..."
+ } else {
+ text.clone()
+ };
+
+ // ── headers ───────────────────────────────────────────────────────
+ let message_id = message
+ .message_id()
+ .map(String::from)
+ .unwrap_or_else(generate_message_id);
+
+ let in_reply_to = message.in_reply_to().as_text().map(String::from);
+ let references = extract_references(&message);
+ let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+
+ let subject = message.subject().map(String::from).unwrap_or_default();
+ let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+ let internal_date = if internal_date == 0 {
+ date
+ } else {
+ internal_date
+ };
+
+ let parse_addrs = |addrs: Option<&mail_parser::Address<'_>>| {
+ addrs
+ .map(|addr| {
+ AddrVec::from(addr)
+ .0
+ .into_iter()
+ .filter_map(|a| a.address)
+ .collect::>()
+ })
+ .unwrap_or_default()
+ };
+
+ let from = message
+ .from()
+ .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+ .and_then(|a| a.address)
+ .unwrap_or_else(|| "unknown".to_string());
+ let to = parse_addrs(message.to());
+ let cc = parse_addrs(message.cc());
+ let bcc = parse_addrs(message.bcc());
+
+ // ── detach attachments → blob ──────────────────────────────────────
+ let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
+
+ if !self
+ .email_ks
+ .contains_key(&email_content_hash)
+ .unwrap_or(false)
+ {
+ self.email_ks
+ .insert(&email_content_hash, stripped_eml.as_slice())
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ }
+
+ // write attachment blobs
+ for (hash, data) in &attachment_output.blobs {
+ if !self.attachment_ks.contains_key(hash).unwrap_or(false) {
+ self.attachment_ks
+ .insert(hash, data.as_ref())
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ }
+ }
+
+ // ── build envelope doc ────────────────────────────────────────────
+ let envelope_id = Uuid::new_v4().to_string();
+ let now = utc_now!();
+
+ let attachment_docs: Vec = attachment_output
+ .infos
+ .iter()
+ .filter(|a| !a.inline || a.content_id.is_none())
+ .map(|a| {
+ AttachmentModel {
+ id: Uuid::new_v4().to_string(),
+ envelope_id: envelope_id.clone(),
+ account_id,
+ account_email: None,
+ mailbox_id,
+ mailbox_name: None,
+ subject: subject.clone(),
+ content_hash: a.content_hash.clone(),
+ from: from.clone(),
+ date,
+ ingest_at: now,
+ size: a.size as u64,
+ ext: a.get_extension(),
+ category: a.get_category().to_string(),
+ content_type: a.file_type.clone(),
+ shard_id: 0,
+ text: None,
+ has_text: false,
+ is_ocr: false,
+ page_count: None,
+ is_indexed: false,
+ is_message: a.is_message,
+ name: a.filename.clone(),
+ tags: None,
+ auto_tags: None,
+ }
+ .into_document()
+ })
+ .collect();
+
+ let envelope = Envelope {
+ id: envelope_id,
+ message_id,
+ account_id,
+ mailbox_id,
+ uid,
+ subject,
+ preview,
+ from,
+ to,
+ cc,
+ bcc,
+ date,
+ internal_date,
+ ingest_at: now,
+ size: eml_bytes.len() as u32,
+ thread_id,
+ attachment_count: message.attachment_count(),
+ regular_attachment_count: attachment_docs.len(),
+ tags: None,
+ account_email: None,
+ mailbox_name: None,
+ content_hash: email_content_hash,
+ };
+
+ let ea = EnvelopeWithAttachments {
+ envelope,
+ attachments: Some(attachment_output.infos),
+ };
+ let envelope_doc = ea.to_document(&text, 0)?;
+
+ self.envelope_writer
+ .add_document(envelope_doc)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+ for doc in attachment_docs {
+ self.attachment_writer
+ .add_document(doc)
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ }
+
+ self.pending += 1;
+ if self.pending >= COMMIT_THRESHOLD {
+ self.commit()?;
+ }
+
+ Ok(())
+ }
+
+ pub fn commit(&mut self) -> BichonResult<()> {
+ if self.pending == 0 {
+ return Ok(());
+ }
+ self.envelope_writer
+ .commit()
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ self.attachment_writer
+ .commit()
+ .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+ tracing::info!(count = self.pending, "committed batch");
+ self.pending = 0;
+ Ok(())
+ }
+}
diff --git a/crates/core/src/store/storage.rs b/crates/core/src/store/blob.rs
similarity index 100%
rename from crates/core/src/store/storage.rs
rename to crates/core/src/store/blob.rs
diff --git a/crates/core/src/store/mod.rs b/crates/core/src/store/mod.rs
index 89220a3..6ae1b0c 100644
--- a/crates/core/src/store/mod.rs
+++ b/crates/core/src/store/mod.rs
@@ -17,5 +17,5 @@
// along with this program. If not, see .
pub mod envelope;
-pub mod storage;
+pub mod blob;
pub mod tantivy;
diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs
index f77c810..38eead0 100644
--- a/crates/core/src/store/tantivy/envelope.rs
+++ b/crates/core/src/store/tantivy/envelope.rs
@@ -37,7 +37,7 @@ use crate::{
settings::dir::DATA_DIR_MANAGER,
store::{
envelope::Envelope,
- storage::BLOB_MANAGER,
+ blob::BLOB_MANAGER,
tantivy::{
fatal_commit,
fields::{
@@ -317,7 +317,6 @@ impl IndexManager {
let q = query_parser
.parse_query(subject_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
- println!("{:#?}", &q);
subqueries.push((Occur::Must, q));
}
diff --git a/crates/core/src/store/tantivy/schema.rs b/crates/core/src/store/tantivy/schema.rs
index 430b969..29c8c01 100644
--- a/crates/core/src/store/tantivy/schema.rs
+++ b/crates/core/src/store/tantivy/schema.rs
@@ -32,84 +32,104 @@ use crate::store::tantivy::fields::{
F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID,
};
-static EMAIL_FIELDS: LazyLock> = LazyLock::new(|| {
- let (_, fields) = SchemaTools::create_email_schema();
- Arc::new(fields)
-});
+// ─── Lazy Globals ─────────────────────────────────────────────────────────────
-static ATTACHMENT_FIELDS: LazyLock> = LazyLock::new(|| {
- let (_, fields) = SchemaTools::create_attachment_schema();
- Arc::new(fields)
-});
+static EMAIL_FIELDS: LazyLock> = LazyLock::new(|| Arc::new(EmailSchema::fields()));
+
+static ATTACHMENT_FIELDS: LazyLock> =
+ LazyLock::new(|| Arc::new(AttachmentSchema::fields()));
+
+// ─── Public API ───────────────────────────────────────────────────────────────
pub struct SchemaTools;
impl SchemaTools {
pub fn email_schema() -> Schema {
- let (schema, _) = Self::create_email_schema();
- schema
+ EmailSchema::build().0
+ }
+ pub fn attachment_schema() -> Schema {
+ AttachmentSchema::build().0
}
pub fn email_fields() -> &'static EmailFields {
&EMAIL_FIELDS
}
+ pub fn attachment_fields() -> &'static AttachmentFields {
+ &ATTACHMENT_FIELDS
+ }
pub fn email_default_fields() -> Vec {
- let fields = Self::email_fields();
+ let f = Self::email_fields();
vec![
- fields.f_subject,
- fields.f_body,
- fields.f_attachment_name_text,
- fields.f_from_text,
- fields.f_to_text,
- fields.f_cc_text,
- fields.f_bcc_text,
+ f.f_subject,
+ f.f_body,
+ f.f_attachment_name_text,
+ f.f_from_text,
+ f.f_to_text,
+ f.f_cc_text,
+ f.f_bcc_text,
]
}
+ pub fn attachment_default_fields() -> Vec {
+ let f = Self::attachment_fields();
+ vec![f.f_subject, f.f_text, f.f_name_text, f.f_from_text]
+ }
+
pub fn create_email_schema() -> (Schema, EmailFields) {
- let mut builder = Schema::builder();
- let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
- let f_message_id = builder.add_text_field(F_MESSAGE_ID, STRING | STORED);
- let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
- let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
- let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
- let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
- let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro"));
- let f_preview = builder.add_text_field(F_PREVIEW, STORED);
- let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+ EmailSchema::build()
+ }
+ pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
+ AttachmentSchema::build()
+ }
+}
- let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
- let f_to = builder.add_text_field(F_TO, STRING | STORED);
- let f_cc = builder.add_text_field(F_CC, STRING | STORED);
- let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
+// ─── Schema builders ──────────────────────────────────────────────────────────
- let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
- let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro"));
- let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro"));
- let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro"));
+struct EmailSchema;
- let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
- let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
- let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
- let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
- let f_thread_id = builder.add_text_field(F_THREAD_ID, STRING | STORED | FAST);
- let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
+impl EmailSchema {
+ fn build() -> (Schema, EmailFields) {
+ let mut b = Schema::builder();
+
+ let f_id = b.add_text_field(F_ID, STRING | STORED | FAST);
+ let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
+ let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+ let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+ let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
+ let f_subject = b.add_text_field(F_SUBJECT, text_store("euro"));
+ let f_body = b.add_text_field(F_BODY, text_no_store("euro"));
+ let f_preview = b.add_text_field(F_PREVIEW, STORED);
+ let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+ let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+ let f_to = b.add_text_field(F_TO, STRING | STORED);
+ let f_cc = b.add_text_field(F_CC, STRING | STORED);
+ let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
+ let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro"));
+ let f_to_text = b.add_text_field(F_TO_TEXT, text_no_store("euro"));
+ let f_cc_text = b.add_text_field(F_CC_TEXT, text_no_store("euro"));
+ let f_bcc_text = b.add_text_field(F_BCC_TEXT, text_no_store("euro"));
+ let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST);
+ let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
+ let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
+ let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
+ let f_thread_id = b.add_text_field(F_THREAD_ID, STRING | STORED | FAST);
+ let f_attachment_count = b.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_regular_attachment_count =
- builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
+ b.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_attachment_name_text =
- builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro"));
- let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
- let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED);
+ b.add_text_field(F_ATTACHMENT_NAME_TEXT, text_no_store("euro"));
+ let f_attachment_name_exact = b.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
+ let f_attachments = b.add_text_field(F_ATTACHMENTS, STORED);
let f_attachment_content_hash =
- builder.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST);
- let f_attachment_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
- let f_attachment_category =
- builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
+ b.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST);
+ let f_attachment_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
+ let f_attachment_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
let f_attachment_content_type =
- builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
- let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
- let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
+ b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
+ let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+ let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
+
let fields = EmailFields {
f_id,
f_message_id,
@@ -145,57 +165,47 @@ impl SchemaTools {
f_tags,
f_shard_id,
};
- (builder.build(), fields)
+
+ (b.build(), fields)
}
- pub fn attachment_schema() -> Schema {
- let (schema, _) = Self::create_attachment_schema();
- schema
+ fn fields() -> EmailFields {
+ Self::build().1
}
+}
- pub fn attachment_fields() -> &'static AttachmentFields {
- &ATTACHMENT_FIELDS
- }
+struct AttachmentSchema;
- pub fn attachment_default_fields() -> Vec {
- let fields = Self::attachment_fields();
- vec![
- fields.f_subject,
- fields.f_text,
- fields.f_name_text,
- fields.f_from_text,
- ]
- }
+impl AttachmentSchema {
+ fn build() -> (Schema, AttachmentFields) {
+ let mut b = Schema::builder();
+
+ let f_id = b.add_text_field(F_ID, STRING | STORED | FAST);
+ let f_envelope_id = b.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
+ let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+ let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+ let f_subject = b.add_text_field(F_SUBJECT, text_store("euro"));
+ let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+ let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+ let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro"));
+ let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST);
+ let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
+ let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
+ let f_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
+ let f_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
+ let f_content_type = b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
+ let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
+ let f_text = b.add_text_field(F_TEXT, text_no_store("euro"));
+ let f_has_text = b.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
+ let f_is_ocr = b.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
+ let f_page_count = b.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
+ let f_is_indexed = b.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
+ let f_is_message = b.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
+ let f_name_text = b.add_text_field(F_NAME_TEXT, text_no_store("euro"));
+ let f_name_exact = b.add_text_field(F_NAME_EXACT, STRING | STORED);
+ let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+ let f_auto_tags = b.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored());
- pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
- let mut builder = Schema::builder();
- let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
- let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
- let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
- let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
- let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
- let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
- let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
- let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
- let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
- let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
- let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
- let f_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
- let f_category = builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
- let f_content_type =
- builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
- let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
- let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro"));
- let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
- let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
- let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
- let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
- let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
- let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro"));
- let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED);
- let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
- let f_auto_tags =
- builder.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored());
let fields = AttachmentFields {
f_id,
f_envelope_id,
@@ -223,24 +233,25 @@ impl SchemaTools {
f_tags,
f_auto_tags,
};
- (builder.build(), fields)
+
+ (b.build(), fields)
}
- fn text_no_store(tokenizer: &str) -> TextOptions {
- TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(tokenizer)
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- )
- }
-
- fn text_store(tokenizer: &str) -> TextOptions {
- TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(tokenizer)
- .set_index_option(IndexRecordOption::WithFreqsAndPositions),
- )
- .set_stored()
+ fn fields() -> AttachmentFields {
+ Self::build().1
}
}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+fn text_no_store(tokenizer: &str) -> TextOptions {
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(tokenizer)
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ )
+}
+
+fn text_store(tokenizer: &str) -> TextOptions {
+ text_no_store(tokenizer).set_stored()
+}
diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs
index 1512153..cb1de30 100644
--- a/crates/server/src/main.rs
+++ b/crates/server/src/main.rs
@@ -25,11 +25,11 @@ use bichon_core::{
context::{executors::BichonContext, Initialize},
error::{code::ErrorCode, BichonResult},
logger,
- migrate::is_legacy_data_layout,
+ migrate::check_data_status,
raise_error,
settings::cli::SETTINGS,
store::{
- storage::BLOB_MANAGER,
+ blob::BLOB_MANAGER,
tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
},
tasks::PeriodicTasks,
@@ -69,11 +69,11 @@ async fn main() -> BichonResult<()> {
info!("Git: [{}]", env!("GIT_HASH"));
info!("GitHub: https://github.com/rustmailer/bichon");
- match is_legacy_data_layout() {
- Ok(true) => {
+ match check_data_status() {
+ Ok(false) => {
error!("Incompatible data format detected.");
error!("Your data was created by an older version of Bichon and must be migrated before use.");
- error!("Please run: bichon-migrate");
+ error!("Please run: bichon-admin");
error!("Documentation: https://github.com/rustmailer/bichon/wiki/migration");
return Err(raise_error!(
"Legacy data layout detected".into(),
@@ -84,7 +84,7 @@ async fn main() -> BichonResult<()> {
error!("Failed to check data layout: {:#?}", e);
return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
}
- Ok(false) => {}
+ Ok(true) => {}
}
if let Err(error) = initialize().await {
diff --git a/crates/server/src/rest/api/message.rs b/crates/server/src/rest/api/message.rs
index 3df383a..e52b6cd 100644
--- a/crates/server/src/rest/api/message.rs
+++ b/crates/server/src/rest/api/message.rs
@@ -36,7 +36,7 @@ use bichon_core::message::tags::TagCount;
use bichon_core::message::tags::TagsRequest;
use bichon_core::raise_error;
use bichon_core::store::envelope::Envelope;
-use bichon_core::store::storage::get_reader;
+use bichon_core::store::blob::get_reader;
use bichon_core::store::tantivy::envelope::ENVELOPE_MANAGER;
use bichon_core::store::tantivy::validate_facet;
use bichon_core::users::permissions::Permission;