From 0abaa66a408a04316ecf9c97a5ccbf00d0f56da4 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 12 May 2026 01:56:24 +0800 Subject: [PATCH] feat(admin): add interactive data migration tool --- Cargo.lock | 4 + Cargo.toml | 2 +- crates/admin/Cargo.toml | 3 +- crates/admin/src/main.rs | 296 +----------- crates/admin/src/migrate.rs | 284 ++++++++++++ crates/admin/src/reset.rs | 274 +++++++++++ crates/cli/Cargo.toml | 2 +- crates/core/Cargo.toml | 2 +- crates/core/src/admin/meta.rs | 30 +- crates/core/src/envelope/extractor.rs | 4 +- crates/core/src/migrate/legacy/fields.rs | 69 +++ crates/core/src/migrate/legacy/mod.rs | 2 + crates/core/src/migrate/legacy/schema.rs | 114 +++++ crates/core/src/migrate/mod.rs | 211 ++++++++- crates/core/src/migrate/store.rs | 424 ++++++++++++++++++ crates/core/src/store/{storage.rs => blob.rs} | 0 crates/core/src/store/mod.rs | 2 +- crates/core/src/store/tantivy/envelope.rs | 3 +- crates/core/src/store/tantivy/schema.rs | 241 +++++----- crates/server/src/main.rs | 12 +- crates/server/src/rest/api/message.rs | 2 +- 21 files changed, 1547 insertions(+), 434 deletions(-) create mode 100644 crates/admin/src/migrate.rs create mode 100644 crates/admin/src/reset.rs create mode 100644 crates/core/src/migrate/legacy/fields.rs create mode 100644 crates/core/src/migrate/legacy/mod.rs create mode 100644 crates/core/src/migrate/legacy/schema.rs create mode 100644 crates/core/src/migrate/store.rs rename crates/core/src/store/{storage.rs => blob.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index d1031f3..a5cddcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -464,6 +464,7 @@ dependencies = [ "bichon-core", "console", "dialoguer", + "indicatif", "tokio", ] @@ -5022,6 +5023,8 @@ dependencies = [ "fastdivide", "fnv", "fs4", + "futures-channel", + "futures-util", "htmlescape", "itertools", "levenshtein_automata", @@ -5045,6 +5048,7 @@ dependencies = [ "tantivy-common", "tantivy-fst", "tantivy-query-grammar", + "tantivy-sstable", "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 44d763f..4c458e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,7 +84,7 @@ uuid = { version = "1.23.1", features = ["v4", "serde"] } fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] } tracing-log = "0.2.0" tokio-util = "0.7.18" - +indicatif = "0.18.4" [profile.release] strip = true diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index f65744d..b1c91db 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -8,4 +8,5 @@ edition.workspace = true bichon-core = { path = "../core" } tokio.workspace = true dialoguer.workspace = true -console.workspace = true \ No newline at end of file +console.workspace = true +indicatif.workspace = true diff --git a/crates/admin/src/main.rs b/crates/admin/src/main.rs index f94146b..2e15120 100644 --- a/crates/admin/src/main.rs +++ b/crates/admin/src/main.rs @@ -16,19 +16,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - fs, - path::{Path, PathBuf}, -}; +use console::style; +use dialoguer::{theme::ColorfulTheme, Select}; -use bichon_core::{ - admin::meta::{find_admin, init_meta_database, update_admin_password}, - error::BichonError, - utils::encrypt::internal_decrypt_string, -}; -use console::{style, Emoji}; -use dialoguer::Confirm; -use dialoguer::{theme::ColorfulTheme, Input, Password, Select}; +use crate::{migrate::handle_migration, reset::handle_reset_password}; + +pub mod migrate; +pub mod reset; #[tokio::main] async fn main() { @@ -38,7 +32,12 @@ async fn main() { style("BICHON ADMINISTRATIVE TOOL").bold().bright().cyan() ); - let main_options = vec!["Reset Admin Password", "Exit"]; + let main_options = vec![ + "Reset Admin Password", + "Migrate Legacy v0.x Storage to v1.0", + "Exit", + ]; + let selection = Select::with_theme(&theme) .with_prompt("Select an operation") .default(0) @@ -46,272 +45,11 @@ async fn main() { .interact() .unwrap(); - if selection == 1 { - println!("{}", style("Exiting...").dim()); - return; - } - - let root_dir_str: String = Input::with_theme(&theme) - .with_prompt("Enter the absolute path for 'bichon_root_dir'") - .validate_with(|input: &String| -> Result<(), &str> { - let path = Path::new(input); - if !path.is_absolute() { - return Err("Path must be absolute."); - } - if !path.exists() { - return Err("Directory does not exist."); - } - let has_metadata = path.join("meta.db").exists(); - if !has_metadata { - return Err("Invalid directory: 'meta.db' not found."); - } - Ok(()) - }) - .interact_text() - .unwrap(); - - let root_path = PathBuf::from(&root_dir_str); - - let database = match init_meta_database(&root_path.join("meta.db")) { - Ok(database) => database, - Err(e) => match e { - BichonError::Generic { - message, - location, - code, - } => { - if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") { - println!("\n{}", style("ERROR: Database is locked.").red().bold()); - println!( - "{}", - style("The Bichon service is likely still running.").yellow() - ); - println!( - "Since the database cannot be shared between multiple instances, \n\ - you must {} the Bichon service before proceeding.", - style("STOP").underlined().bold() - ); - std::process::exit(1); - } else { - eprintln!( - "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}", - style("A database error occurred:").red().bold(), - code, - location, - message - ); - std::process::exit(1); - } - } - }, - }; - - let admin = find_admin(&database); - - match admin { - Ok(Some(user)) => { - println!("\n{}", style("Admin user found:").green().bold()); - println!("----------------------------------------"); - println!("{:<12} : {}", "Username", style(&user.username).cyan()); - println!("{:<12} : {}", "Email", style(&user.email).cyan()); - } - Ok(None) => { - println!( - "\n{}", - style("ERROR: No admin user found in the database.") - .red() - .bold() - ); - println!("Please ensure the system has been initialized correctly."); - std::process::exit(1); - } - Err(e) => { - eprintln!( - "\n{} Failed to query admin user.", - style("ERROR:").red().bold() - ); - eprintln!("Details: {:?}", e); - std::process::exit(1); - } - } - - let encryption_key = loop { - let auth_methods = vec![ - "Enter encryption password manually", - "Read from password file", - ]; - let method = Select::with_theme(&theme) - .with_prompt("How would you like to provide the database encryption key?") - .items(&auth_methods) - .interact() - .unwrap(); - - let raw_key = if method == 0 { - Password::with_theme(&theme) - .with_prompt("Enter Encryption Password") - .interact() - .unwrap() - } else { - let file_path: String = Input::with_theme(&theme) - .with_prompt("Enter path to encryption password file") - .interact_text() - .unwrap(); - - match fs::read_to_string(&file_path) { - Ok(content) => content.trim().to_string(), - Err(e) => { - println!("{}: {}", style("Failed to read file").red(), e); - continue; - } - } - }; - - if raw_key.is_empty() { - println!("{}", style("Key cannot be empty.").red()); - continue; - } - - let prompt_message = format!( - "Encryption key loaded: [ {} ]\n\n \ - {}: This key must match the database encryption key used by the server.\n \ - It corresponds to these settings in your service:\n \ - - Arguments: {} or {}\n \ - - Envs: {} or {}\n\n \ - Do you want to continue?", - style(&raw_key).cyan().bold(), - style("IMPORTANT").yellow().bold(), - style("--bichon_encrypt_password").italic(), - style("--bichon_encrypt_password_file").italic(), - style("BICHON_ENCRYPT_PASSWORD").green(), - style("BICHON_ENCRYPT_PASSWORD_FILE").green() - ); - - if Confirm::with_theme(&theme) - .with_prompt(prompt_message) - .default(true) - .interact() - .unwrap() - { - break raw_key; - } - }; - - let admin = find_admin(&database); - - match admin { - Ok(Some(user)) => { - println!("----------------------------------------"); - println!("{:<12} : {}", "Username", style(&user.username).cyan()); - println!("{:<12} : {}", "Email", style(&user.email).cyan()); - - let pwd_display = match &user.password { - Some(p) => { - let password = match internal_decrypt_string(&encryption_key, p) { - Ok(p) => p, - Err(e) => { - println!("\n{}", style("ERROR: Decryption Failed").red().bold()); - println!( - "{}", - style("The provided encryption key is incorrect or invalid for this database.").yellow() - ); - println!( - "{} Please verify your {} or the {} you provided.", - style("➔").cyan(), - style("encryption password").bold(), - style("key file").bold() - ); - eprintln!("\nTechnical details: {:?}", e); - std::process::exit(1); - } - }; - style(password).yellow().to_string() - } - None => style("None (No password set)").dim().italic().to_string(), - }; - - println!("{:<12} : {}", "Password", pwd_display); - println!("----------------------------------------"); - - if !dialoguer::Confirm::with_theme(&theme) - .with_prompt(format!( - "Do you want to reset the password for '{}'?", - user.username - )) - .interact() - .unwrap() - { - println!("Operation cancelled."); - return; - } - } - Ok(None) => { - println!( - "\n{}", - style("ERROR: No admin user found in the database.") - .red() - .bold() - ); - println!("Please ensure the system has been initialized correctly."); - std::process::exit(1); - } - Err(e) => { - eprintln!( - "\n{} Failed to query admin user.", - style("ERROR:").red().bold() - ); - eprintln!("Details: {:?}", e); - std::process::exit(1); - } - } - - println!( - "\n{}", - style("TARGET: Reset password for user 'admin'") - .yellow() - .bold() - ); - - let new_login_password = Password::with_theme(&theme) - .with_prompt("Enter new Admin Login Password") - .with_confirmation("Repeat password to confirm", "Passwords do not match!") - .interact() - .unwrap(); - - if !Confirm::with_theme(&theme) - .with_prompt("Proceed with database update?") - .interact() - .unwrap() - { - return; - } - - println!("\n{} {}", style("⌛").yellow(), "Updating database..."); - - match update_admin_password(&database, new_login_password, &encryption_key) { - Ok(_) => { - println!( - "\n{} {}", - Emoji("✨", "*"), - style("Success! Admin password has been updated.") - .green() - .bold() - ); - println!( - "{}", - style("You can now log in with the new password.").dim() - ); - } - Err(e) => { - println!( - "\n{}", - style("ERROR: Failed to update database").red().bold() - ); - eprintln!( - "{} Could not save the new password to the database.", - style("➔").cyan() - ); - eprintln!("\nDetails: {:?}", e); - std::process::exit(1); + match selection { + 0 => handle_reset_password(&theme), + 1 => handle_migration(&theme), + _ => { + println!("{}", style("Exiting...").dim()); } } } diff --git a/crates/admin/src/migrate.rs b/crates/admin/src/migrate.rs new file mode 100644 index 0000000..5e9da7c --- /dev/null +++ b/crates/admin/src/migrate.rs @@ -0,0 +1,284 @@ +use std::path::{Path, PathBuf}; + +use bichon_core::migrate::{ + do_migrate, is_tantivy_index_dir, + store::{LegacyDirs, NewDirs}, +}; +use console::style; +use dialoguer::{theme::ColorfulTheme, Confirm, Input}; +use indicatif::{ProgressBar, ProgressStyle}; + +pub fn handle_migration(theme: &ColorfulTheme) { + println!( + "\n{}", + style("MIGRATION: Bichon v0.x Storage Architecture → v1.0") + .bold() + .yellow() + ); + + println!( + "{}", + style( + "This tool migrates data from the legacy v0.x Tantivy-based storage \ + architecture (used in versions 0.0.1 through 0.3.7) to the new v1.0 \ + separated index and Fjall-backed storage format." + ) + .dim() + ); + + println!( + "{}", + style( + "Legacy v0.x architecture:\n\ + • envelope metadata stored in Tantivy\n\ + • message data stored in Tantivy\n\n\ + New v1.0 architecture:\n\ + • mail indexes stored in Tantivy\n\ + • attachment indexes stored in Tantivy\n\ + • raw message data stored in Fjall\n\ + • attachment blobs stored in Fjall" + ) + .dim() + ); + + println!( + "\n{} {}", + style("IMPORTANT:").yellow().bold(), + style( + "The paths below must exactly match what your old bichon server was configured with." + ) + .yellow() + ); + + // --- bichon-root-dir --- + let root_dir_str: String = Input::with_theme(theme) + .with_prompt("Enter --bichon-root-dir (same value used by the old server)") + .validate_with(|input: &String| -> Result<(), &str> { + let path = Path::new(input); + if !path.is_absolute() { + return Err("Path must be absolute."); + } + if !path.exists() { + return Err("Directory does not exist."); + } + Ok(()) + }) + .interact_text() + .unwrap(); + + let root_path = PathBuf::from(&root_dir_str); + + // --- bichon-index-dir --- + let default_index = root_path.join("envelope"); + let default_new_index = root_path.join("bichon-indices"); + let index_dir_str: String = Input::with_theme(theme) + .with_prompt(format!( + "Enter --bichon-index-dir (leave blank to use default: {})", + style(default_index.display()).cyan() + )) + .allow_empty(true) + .validate_with(|input: &String| -> Result<(), &str> { + if input.is_empty() { + return Ok(()); + } + let path = Path::new(input); + if !path.is_absolute() { + return Err("Path must be absolute."); + } + + if !path.exists() { + return Err("Directory does not exist."); + } + Ok(()) + }) + .interact_text() + .unwrap(); + + let index_path = if index_dir_str.is_empty() { + default_index + } else { + PathBuf::from(&index_dir_str) + }; + + let new_index_path = if index_dir_str.is_empty() { + default_new_index + } else { + PathBuf::from(&index_dir_str).join("bichon-indices") + }; + + // --- bichon-data-dir --- + let default_data = root_path.join("eml"); + let default_new_data = root_path.join("bichon-storage"); + let data_dir_str: String = Input::with_theme(theme) + .with_prompt(format!( + "Enter --bichon-data-dir (leave blank to use default: {})", + style(default_data.display()).cyan() + )) + .allow_empty(true) + .validate_with(|input: &String| -> Result<(), &str> { + if input.is_empty() { + return Ok(()); + } + let path = Path::new(input); + if !path.is_absolute() { + return Err("Path must be absolute."); + } + if !path.exists() { + return Err("Directory does not exist."); + } + Ok(()) + }) + .interact_text() + .unwrap(); + + let data_path = if data_dir_str.is_empty() { + default_data + } else { + PathBuf::from(&data_dir_str) + }; + + let new_data_path = if data_dir_str.is_empty() { + default_new_data + } else { + PathBuf::from(&data_dir_str).join("bichon-storage") + }; + + println!("\n{}", style("Paths to be migrated:").bold()); + println!("----------------------------------------"); + println!( + "{:<20} : {}", + "bichon-root-dir", + style(root_path.display()).cyan() + ); + println!( + "{:<20} : {}", + "bichon-index-dir", + style(index_path.display()).cyan() + ); + println!( + "{:<20} : {}", + "bichon-data-dir", + style(data_path.display()).cyan() + ); + println!("----------------------------------------"); + + println!( + "\n{} Checking legacy v0.x storage layout...", + style("⌛").yellow() + ); + + match is_legacy_data_layout_with_paths(&index_path, &data_path) { + Ok(true) => { + println!( + "{} {}", + style("✔").green(), + style("Legacy v0.x Tantivy-based storage detected. Migration to v1.0 is required.") + .yellow() + ); + } + Ok(false) => { + println!( + "{} {}", + style("✔").green(), + style("No legacy v0.x storage layout was detected at the specified paths.").green() + ); + + println!( + "{}", + style( + "The selected directories may already be using the v1.0 storage architecture." + ) + .dim() + ); + + return; + } + Err(e) => { + eprintln!( + "{} Failed to verify legacy storage layout: {:?}", + style("ERROR:").red().bold(), + e + ); + + std::process::exit(1); + } + } + + println!( + "\n{} {}", + style("⚠").yellow(), + style( + "This migration is non-destructive. Existing v0.x storage files will remain unchanged." + ) + .yellow() + ); + + if !Confirm::with_theme(theme) + .with_prompt("Ready to migrate?") + .default(true) + .interact() + .unwrap() + { + println!("{}", style("Migration cancelled.").dim()); + return; + } + + println!("\n{} Migrating...", style("⌛").yellow()); + let pb = ProgressBar::new(0); + pb.set_style(ProgressStyle::default_bar() + .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}") + .unwrap() + .progress_chars("#>-")); + let legacy = LegacyDirs::new(index_path, data_path); + let new_dirs = NewDirs::new(new_index_path, new_data_path); + if let Err(e) = do_migrate(legacy, new_dirs, |msg| { + if let Some(data) = msg.strip_prefix("PROGRESS:") { + let parts: Vec<&str> = data.split(':').collect(); + if parts.len() == 2 { + let migrated = parts[0].parse::().unwrap_or(0); + let skipped = parts[1].parse::().unwrap_or(0); + + pb.set_position(migrated + skipped); + pb.set_message(format!( + "Migrated: {}, {} {}", + style(migrated).green(), + style(skipped).red(), + style("skipped").dim() + )); + } + } else if let Some(total) = msg.strip_prefix("TOTAL:") { + pb.set_length(total.parse().unwrap_or(0)); + } else if msg.starts_with("WARN:") { + pb.println(format!("{} {}", style("⚠").yellow(), &msg[5..])); + } else if let Some(done_data) = msg.strip_prefix("DONE:") { + let parts: Vec<&str> = done_data.split(':').collect(); + pb.finish_with_message(format!( + "Migration finished. Total: {}, Skipped: {}", + parts.get(0).unwrap_or(&"0"), + parts.get(1).unwrap_or(&"0") + )); + } + }) { + eprintln!( + "\n{} Migration failed:\n{:?}", + style("✘").red().bold(), + style(e).red() + ); + return; + } + println!( + "{} {}", + style("✔").green(), + style("Migration completed successfully!").bold() + ); +} + +pub fn is_legacy_data_layout_with_paths( + envelope_dir: &PathBuf, + eml_dir: &PathBuf, +) -> std::io::Result { + let envelope_result = is_tantivy_index_dir(envelope_dir)?; + let eml_result = is_tantivy_index_dir(eml_dir)?; + + Ok(envelope_result || eml_result) +} diff --git a/crates/admin/src/reset.rs b/crates/admin/src/reset.rs new file mode 100644 index 0000000..050d5f5 --- /dev/null +++ b/crates/admin/src/reset.rs @@ -0,0 +1,274 @@ +use std::path::{Path, PathBuf}; + +use bichon_core::{ + admin::meta::{find_admin, init_meta_database, update_admin_password}, + error::BichonError, + utils::encrypt::internal_decrypt_string, +}; +use console::{style, Emoji}; +use dialoguer::{theme::ColorfulTheme, Confirm, Input, Password, Select}; + +pub fn handle_reset_password(theme: &ColorfulTheme) { + let root_dir_str: String = Input::with_theme(theme) + .with_prompt("Enter the absolute path for 'bichon_root_dir'") + .validate_with(|input: &String| -> Result<(), &str> { + let path = Path::new(input); + if !path.is_absolute() { + return Err("Path must be absolute."); + } + if !path.exists() { + return Err("Directory does not exist."); + } + let has_metadata = path.join("meta.db").exists(); + if !has_metadata { + return Err("Invalid directory: 'meta.db' not found."); + } + Ok(()) + }) + .interact_text() + .unwrap(); + + let root_path = PathBuf::from(&root_dir_str); + let database = match init_meta_database(&root_path.join("meta.db")) { + Ok(database) => database, + Err(e) => match e { + BichonError::Generic { + message, + location, + code, + } => { + if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") { + println!("\n{}", style("ERROR: Database is locked.").red().bold()); + println!( + "{}", + style("The Bichon service is likely still running.").yellow() + ); + println!( + "Since the database cannot be shared between multiple instances, \n\ + you must {} the Bichon service before proceeding.", + style("STOP").underlined().bold() + ); + std::process::exit(1); + } else { + eprintln!( + "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}", + style("A database error occurred:").red().bold(), + code, + location, + message + ); + std::process::exit(1); + } + } + }, + }; + + let admin = find_admin(&database); + + match admin { + Ok(Some(user)) => { + println!("\n{}", style("Admin user found:").green().bold()); + println!("----------------------------------------"); + println!("{:<12} : {}", "Username", style(&user.username).cyan()); + println!("{:<12} : {}", "Email", style(&user.email).cyan()); + } + Ok(None) => { + println!( + "\n{}", + style("ERROR: No admin user found in the database.") + .red() + .bold() + ); + println!("Please ensure the system has been initialized correctly."); + std::process::exit(1); + } + Err(e) => { + eprintln!( + "\n{} Failed to query admin user.", + style("ERROR:").red().bold() + ); + eprintln!("Details: {:?}", e); + std::process::exit(1); + } + } + + let encryption_key = loop { + let auth_methods = vec![ + "Enter encryption password manually", + "Read from password file", + ]; + let method = Select::with_theme(theme) + .with_prompt("How would you like to provide the database encryption key?") + .items(&auth_methods) + .interact() + .unwrap(); + + let raw_key = if method == 0 { + Password::with_theme(theme) + .with_prompt("Enter Encryption Password") + .interact() + .unwrap() + } else { + let file_path: String = Input::with_theme(theme) + .with_prompt("Enter path to encryption password file") + .interact_text() + .unwrap(); + + match std::fs::read_to_string(&file_path) { + Ok(content) => content.trim().to_string(), + Err(e) => { + println!("{}: {}", style("Failed to read file").red(), e); + continue; + } + } + }; + + if raw_key.is_empty() { + println!("{}", style("Key cannot be empty.").red()); + continue; + } + + let prompt_message = format!( + "Encryption key loaded: [ {} ]\n\n \ + {}: This key must match the database encryption key used by the server.\n \ + It corresponds to these settings in your service:\n \ + - Arguments: {} or {}\n \ + - Envs: {} or {}\n\n \ + Do you want to continue?", + style(&raw_key).cyan().bold(), + style("IMPORTANT").yellow().bold(), + style("--bichon_encrypt_password").italic(), + style("--bichon_encrypt_password_file").italic(), + style("BICHON_ENCRYPT_PASSWORD").green(), + style("BICHON_ENCRYPT_PASSWORD_FILE").green() + ); + + if Confirm::with_theme(theme) + .with_prompt(prompt_message) + .default(true) + .interact() + .unwrap() + { + break raw_key; + } + }; + + let admin = find_admin(&database); + + match admin { + Ok(Some(user)) => { + println!("----------------------------------------"); + println!("{:<12} : {}", "Username", style(&user.username).cyan()); + println!("{:<12} : {}", "Email", style(&user.email).cyan()); + + let pwd_display = match &user.password { + Some(p) => { + let password = match internal_decrypt_string(&encryption_key, p) { + Ok(p) => p, + Err(e) => { + println!("\n{}", style("ERROR: Decryption Failed").red().bold()); + println!( + "{}", + style("The provided encryption key is incorrect or invalid for this database.").yellow() + ); + println!( + "{} Please verify your {} or the {} you provided.", + style("➔").cyan(), + style("encryption password").bold(), + style("key file").bold() + ); + eprintln!("\nTechnical details: {:?}", e); + std::process::exit(1); + } + }; + style(password).yellow().to_string() + } + None => style("None (No password set)").dim().italic().to_string(), + }; + + println!("{:<12} : {}", "Password", pwd_display); + println!("----------------------------------------"); + + if !dialoguer::Confirm::with_theme(theme) + .with_prompt(format!( + "Do you want to reset the password for '{}'?", + user.username + )) + .interact() + .unwrap() + { + println!("Operation cancelled."); + return; + } + } + Ok(None) => { + println!( + "\n{}", + style("ERROR: No admin user found in the database.") + .red() + .bold() + ); + println!("Please ensure the system has been initialized correctly."); + std::process::exit(1); + } + Err(e) => { + eprintln!( + "\n{} Failed to query admin user.", + style("ERROR:").red().bold() + ); + eprintln!("Details: {:?}", e); + std::process::exit(1); + } + } + + println!( + "\n{}", + style("TARGET: Reset password for user 'admin'") + .yellow() + .bold() + ); + + let new_login_password = Password::with_theme(theme) + .with_prompt("Enter new Admin Login Password") + .with_confirmation("Repeat password to confirm", "Passwords do not match!") + .interact() + .unwrap(); + + if !Confirm::with_theme(theme) + .with_prompt("Proceed with database update?") + .interact() + .unwrap() + { + return; + } + + println!("\n{} {}", style("⌛").yellow(), "Updating database..."); + + match update_admin_password(&database, new_login_password, &encryption_key) { + Ok(_) => { + println!( + "\n{} {}", + Emoji("✨", "*"), + style("Success! Admin password has been updated.") + .green() + .bold() + ); + println!( + "{}", + style("You can now log in with the new password.").dim() + ); + } + Err(e) => { + println!( + "\n{}", + style("ERROR: Failed to update database").red().bold() + ); + eprintln!( + "{} Could not save the new password to the database.", + style("➔").cyan() + ); + eprintln!("\nDetails: {:?}", e); + std::process::exit(1); + } + } +} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 92e52de..08de615 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,5 +23,5 @@ base64.workspace = true codepage-strings = "1.0.2" hex = "0.4.3" sysinfo.workspace = true -indicatif = "0.18.4" +indicatif.workspace = true serde_json.workspace = true \ No newline at end of file diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 65cbcab..220090c 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m "runtime-tokio", "compress", ] } -tantivy = { version = "0.26.1", features = ["zstd-compression"] } +tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] } webpki-roots.workspace = true rustls.workspace = true rustls-pki-types.workspace = true diff --git a/crates/core/src/admin/meta.rs b/crates/core/src/admin/meta.rs index 7ec75e8..10c8d6b 100644 --- a/crates/core/src/admin/meta.rs +++ b/crates/core/src/admin/meta.rs @@ -16,21 +16,18 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - - use std::{path::Path, rc::Rc}; use native_db::{Builder, Database}; use crate::{ - { - database::META_MODELS, - error::{code::ErrorCode, BichonResult}, - token::{AccessTokenModel, AccessTokenModelKey, TokenType}, - users::{UserModel, DEFAULT_ADMIN_USER_ID}, - utils::encrypt::internal_encrypt_string, - }, + account::migration::AccountV3, + database::META_MODELS, + error::{code::ErrorCode, BichonResult}, raise_error, + token::{AccessTokenModel, AccessTokenModelKey, TokenType}, + users::{UserModel, DEFAULT_ADMIN_USER_ID}, + utils::encrypt::internal_encrypt_string, }; use itertools::Itertools; @@ -48,6 +45,21 @@ pub fn init_meta_database(path: impl AsRef) -> BichonResult>) -> BichonResult> { + let r_transaction = database + .r_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let entities: Vec = r_transaction + .scan() + .primary() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .all() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .try_collect() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(entities) +} + pub fn find_admin(database: &Rc>) -> BichonResult> { let r_transaction = database .r_transaction() diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 845f895..1ee5416 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -22,7 +22,7 @@ use crate::envelope::utils::normalize_subject; use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::message::content::AttachmentInfo; -use crate::store::storage::{DetachedEmail, BLOB_MANAGER}; +use crate::store::blob::{DetachedEmail, BLOB_MANAGER}; use crate::store::tantivy::attachment::ATTACHMENT_MANAGER; use crate::store::tantivy::envelope::ENVELOPE_MANAGER; use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments}; @@ -355,7 +355,7 @@ pub fn generate_message_id() -> String { format!("<{:016x}.{}.{}@{}>", id!(128), ts, pid, "bichon") } -fn extract_references(message: &Message<'_>) -> Option> { +pub fn extract_references(message: &Message<'_>) -> Option> { match message.references() { mail_parser::HeaderValue::Text(cow) => Some(vec![cow.to_string()]), mail_parser::HeaderValue::TextList(vec) => { diff --git a/crates/core/src/migrate/legacy/fields.rs b/crates/core/src/migrate/legacy/fields.rs new file mode 100644 index 0000000..1ff9afa --- /dev/null +++ b/crates/core/src/migrate/legacy/fields.rs @@ -0,0 +1,69 @@ +// +// Copyright (c) 2025 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + + +use tantivy::schema::Field; + +pub const F_MESSAGE_ID: &str = "message_id"; +pub const F_ACCOUNT_ID: &str = "account_id"; +pub const F_MAILBOX_ID: &str = "mailbox_id"; +pub const F_UID: &str = "uid"; +pub const F_SUBJECT: &str = "subject"; +pub const F_TEXT: &str = "text"; +pub const F_FROM: &str = "from"; +pub const F_TO: &str = "to"; +pub const F_CC: &str = "cc"; +pub const F_BCC: &str = "bcc"; +pub const F_DATE: &str = "date"; +pub const F_INTERNAL_DATE: &str = "internal_date"; +pub const F_SIZE: &str = "size"; +pub const F_THREAD_ID: &str = "thread_id"; +pub const F_ATTACHMENTS: &str = "attachments"; +pub const F_HAS_ATTACHMENT: &str = "has_attachment"; +pub const F_TAGS: &str = "tags"; + +pub const F_ID: &str = "id"; +pub struct EnvelopeFields { + pub f_id: Field, + pub f_message_id: Field, + pub f_account_id: Field, + pub f_mailbox_id: Field, + pub f_uid: Field, + pub f_subject: Field, + pub f_text: Field, + pub f_from: Field, + pub f_to: Field, + pub f_cc: Field, + pub f_bcc: Field, + pub f_date: Field, + pub f_internal_date: Field, + pub f_size: Field, + pub f_thread_id: Field, + pub f_attachments: Field, + pub f_has_attachment: Field, + pub f_tags: Field, +} + +pub const F_EML: &str = "eml"; + +pub struct EmlFields { + pub f_id: Field, + pub f_account_id: Field, + pub f_mailbox_id: Field, + pub f_eml: Field, +} \ No newline at end of file diff --git a/crates/core/src/migrate/legacy/mod.rs b/crates/core/src/migrate/legacy/mod.rs new file mode 100644 index 0000000..ce95d06 --- /dev/null +++ b/crates/core/src/migrate/legacy/mod.rs @@ -0,0 +1,2 @@ +pub mod fields; +pub mod schema; diff --git a/crates/core/src/migrate/legacy/schema.rs b/crates/core/src/migrate/legacy/schema.rs new file mode 100644 index 0000000..9a0d762 --- /dev/null +++ b/crates/core/src/migrate/legacy/schema.rs @@ -0,0 +1,114 @@ +use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT}; + +use crate::migrate::legacy::fields::{EmlFields, EnvelopeFields, *}; + +pub struct SchemaTools; + +impl SchemaTools { + pub fn envelope_schema() -> Schema { + EnvelopeSchema::build().0 + } + pub fn eml_schema() -> Schema { + EmlSchema::build().0 + } + + pub fn envelope_fields() -> EnvelopeFields { + EnvelopeSchema::fields() + } + pub fn eml_fields() -> EmlFields { + EmlSchema::fields() + } + + pub fn envelope_default_fields() -> Vec { + let f = Self::envelope_fields(); + vec![f.f_subject, f.f_text, f.f_attachments] + } +} + +// ─── Schema builders ────────────────────────────────────────────────────────── + +struct EnvelopeSchema; + +impl EnvelopeSchema { + fn build() -> (Schema, EnvelopeFields) { + let mut b = Schema::builder(); + + let f_id = b.add_u64_field(F_ID, INDEXED | STORED | FAST); + let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); + let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); + let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST); + let f_thread_id = b.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST); + + let f_subject = b.add_text_field(F_SUBJECT, TEXT | STORED); + let f_text = b.add_text_field(F_TEXT, TEXT | STORED); + let f_attachments = b.add_text_field(F_ATTACHMENTS, TEXT | STORED); + + let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST); + let f_to = b.add_text_field(F_TO, STRING | STORED); + let f_cc = b.add_text_field(F_CC, STRING | STORED); + let f_bcc = b.add_text_field(F_BCC, STRING | STORED); + + let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED); + + let f_date = b.add_i64_field(F_DATE, STORED | FAST); + let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, STORED | FAST); + + let f_size = b.add_u64_field(F_SIZE, STORED | FAST); + let f_has_attachment = b.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST); + + let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); + + let fields = EnvelopeFields { + f_id, + f_account_id, + f_mailbox_id, + f_uid, + f_thread_id, + f_subject, + f_text, + f_attachments, + f_from, + f_to, + f_cc, + f_bcc, + f_message_id, + f_date, + f_internal_date, + f_size, + f_has_attachment, + f_tags, + }; + + (b.build(), fields) + } + + fn fields() -> EnvelopeFields { + Self::build().1 + } +} + +struct EmlSchema; + +impl EmlSchema { + fn build() -> (Schema, EmlFields) { + let mut b = Schema::builder(); + + let f_id = b.add_u64_field(F_ID, INDEXED | FAST); + let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); + let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); + let f_eml = b.add_bytes_field(F_EML, STORED); + + let fields = EmlFields { + f_id, + f_account_id, + f_mailbox_id, + f_eml, + }; + + (b.build(), fields) + } + + fn fields() -> EmlFields { + Self::build().1 + } +} diff --git a/crates/core/src/migrate/mod.rs b/crates/core/src/migrate/mod.rs index 7d2feae..444cb3b 100644 --- a/crates/core/src/migrate/mod.rs +++ b/crates/core/src/migrate/mod.rs @@ -1,6 +1,20 @@ use std::path::PathBuf; -use crate::settings::cli::SETTINGS; +use crate::{ + error::{code::ErrorCode, BichonResult}, + migrate::{ + legacy::schema::SchemaTools, + store::{LegacyDirs, NewDirs, NewIndexWriter}, + }, + raise_error, + settings::cli::SETTINGS, +}; +use tantivy::{ + collector::TopDocs, query::AllQuery, schema::Value, DocAddress, Index, TantivyDocument, +}; + +pub mod legacy; +pub mod store; pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result { if !dir.exists() || !dir.is_dir() { @@ -29,23 +43,190 @@ pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result { Ok(has_meta_json && match_count >= 3) } -pub fn is_legacy_data_layout() -> std::io::Result { +pub fn check_data_status() -> std::io::Result { let root_dir = PathBuf::from(&SETTINGS.bichon_root_dir); - let envelope_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir { - PathBuf::from(index_dir) - } else { - root_dir.join("envelope") - }; - let eml_dir = if let Some(ref data_dir) = SETTINGS.bichon_data_dir { - PathBuf::from(data_dir) - } else { - root_dir.join("eml") - }; + let new_indices_base = SETTINGS + .bichon_index_dir + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| root_dir.clone()); + let new_indices_path = new_indices_base.join("bichon-indices"); - let envelope_result = is_tantivy_index_dir(&envelope_dir)?; - let eml_result = is_tantivy_index_dir(&eml_dir)?; - Ok(envelope_result || eml_result) + let new_data_base = SETTINGS + .bichon_data_dir + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| root_dir.clone()); + let new_storage_path = new_data_base.join("bichon-storage"); + + let has_new_indices = is_tantivy_index_dir(&new_indices_path.join("attachment_metadata"))? + && is_tantivy_index_dir(&new_indices_path.join("mail_metadata"))?; + let has_new_storage = is_dir_not_empty(&new_storage_path)?; + + if has_new_indices && has_new_storage { + return Ok(true); + } + + let legacy_index_root = SETTINGS + .bichon_index_dir + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| root_dir.join("envelope")); + let legacy_data_root = SETTINGS + .bichon_data_dir + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| root_dir.join("eml")); + + let has_legacy_index = is_tantivy_index_dir(&legacy_index_root)?; + let has_legacy_data = is_tantivy_index_dir(&legacy_data_root)?; + + if has_legacy_index || has_legacy_data { + Ok(false) + } else { + Ok(true) + } +} + +fn is_dir_not_empty(path: &PathBuf) -> std::io::Result { + if !path.exists() || !path.is_dir() { + return Ok(false); + } + let mut entries = std::fs::read_dir(path)?; + Ok(entries.next().is_some()) +} + +const PAGE_SIZE: usize = 100; + +pub fn do_migrate(legacy: LegacyDirs, new_dirs: NewDirs, mut on_progress: F) -> BichonResult<()> +where + F: FnMut(&str), +{ + let envelope_index = Index::open_in_dir(&legacy.envelope_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let eml_index = Index::open_in_dir(&legacy.eml_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + let envelope_reader = envelope_index + .reader() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let eml_reader = eml_index + .reader() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + let envelope_searcher = envelope_reader.searcher(); + let eml_searcher = eml_reader.searcher(); + + let total_count = envelope_searcher.num_docs(); + on_progress(&format!("TOTAL:{}", total_count)); + + let ef = SchemaTools::envelope_fields(); + let mf = SchemaTools::eml_fields(); + + let mut writer = NewIndexWriter::open(new_dirs)?; + + let mut offset = 0usize; + let mut total_migrated = 0usize; + let mut total_skipped = 0usize; + + loop { + let page: Vec<(_, DocAddress)> = envelope_searcher + .search( + &AllQuery, + &TopDocs::with_limit(PAGE_SIZE) + .and_offset(offset) + .order_by_score(), + ) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + if page.is_empty() { + break; + } + let fetched = page.len(); + + for (_, doc_address) in page { + let doc: TantivyDocument = envelope_searcher + .doc(doc_address) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + let eid = match doc.get_first(ef.f_id).and_then(|v| v.as_u64()) { + Some(v) => v, + None => { + total_skipped += 1; + continue; + } + }; + let account_id = match doc.get_first(ef.f_account_id).and_then(|v| v.as_u64()) { + Some(v) => v, + None => { + total_skipped += 1; + continue; + } + }; + let mailbox_id = doc + .get_first(ef.f_mailbox_id) + .and_then(|v| v.as_u64()) + .unwrap_or(0); + let uid = doc + .get_first(ef.f_uid) + .and_then(|v| v.as_u64()) + .unwrap_or(0) as u32; + let internal_date = doc + .get_first(ef.f_internal_date) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + + let eml_term = tantivy::Term::from_field_u64(mf.f_id, eid); + let eml_query = + tantivy::query::TermQuery::new(eml_term, tantivy::schema::IndexRecordOption::Basic); + let eml_hits: Vec<(_, DocAddress)> = eml_searcher + .search(&eml_query, &TopDocs::with_limit(1).order_by_score()) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + let eml_bytes = match eml_hits.first() { + Some((_, addr)) => { + let eml_doc: TantivyDocument = eml_searcher + .doc(*addr) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) { + Some(b) => b.to_vec(), + None => { + on_progress(&format!("WARN: Account {} ID {} eml field missing", account_id, eid)); + total_skipped += 1; + continue; + } + } + } + None => { + on_progress(&format!("WARN:Account {} ID {} eml not found", account_id, eid)); + total_skipped += 1; + continue; + } + }; + + if let Err(e) = writer.ingest(&eml_bytes, account_id, mailbox_id, uid, internal_date) { + on_progress(&format!("ERROR:Account {} ID {} ingest failed: {}", account_id, eid, e)); + total_skipped += 1; + continue; + } + + total_migrated += 1; + + if total_migrated % 100 == 0 || total_migrated == total_count as usize { + on_progress(&format!("PROGRESS:{}:{}", total_migrated, total_skipped)); + } + } + + offset += fetched; + if fetched < PAGE_SIZE { + break; + } + } + + writer.commit()?; + + Ok(()) } #[cfg(test)] diff --git a/crates/core/src/migrate/store.rs b/crates/core/src/migrate/store.rs new file mode 100644 index 0000000..691b204 --- /dev/null +++ b/crates/core/src/migrate/store.rs @@ -0,0 +1,424 @@ +use std::path::PathBuf; + +use bytes::Bytes; +use mail_parser::MimeHeaders; + +use crate::{ + envelope::extractor::extract_references, message::content::AttachmentInfo, + store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash, +}; + +use fjall::{ + config::{BlockSizePolicy, CompressionPolicy}, + CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions, +}; +use mail_parser::MessageParser; +use tantivy::{Index, IndexWriter, TantivyDocument}; +use uuid::Uuid; + +use crate::{ + common::AddrVec, + envelope::extractor::{compute_thread_id, generate_message_id}, + error::{code::ErrorCode, BichonResult}, + raise_error, + store::envelope::Envelope, + store::tantivy::{ + model::{AttachmentModel, EnvelopeWithAttachments}, + schema::SchemaTools, + }, + utc_now, +}; + +pub struct LegacyDirs { + pub envelope_dir: PathBuf, + pub eml_dir: PathBuf, +} + +pub struct NewDirs { + pub envelope_dir: PathBuf, + pub attachment_dir: PathBuf, + pub storage_dir: PathBuf, +} + +impl LegacyDirs { + pub fn new(index: PathBuf, data: PathBuf) -> Self { + Self { + envelope_dir: index, + eml_dir: data, + } + } +} + +impl NewDirs { + pub fn new(index: PathBuf, data: PathBuf) -> Self { + Self { + envelope_dir: index.join("mail_metadata"), + attachment_dir: index.join("attachment_metadata"), + storage_dir: data, + } + } +} + +pub struct DetachOutput { + pub infos: Vec, + pub blobs: Vec<(String, Bytes)>, +} + +pub fn detach_attachments_standalone( + original_body: &[u8], + message: &mail_parser::Message<'_>, +) -> (Vec, DetachOutput) { + let mut stripped_eml = original_body.to_vec(); + let mut infos = Vec::new(); + let mut blobs = Vec::new(); + + let mut ranges: Vec<_> = message + .attachments() + .map(|att| { + ( + att.raw_body_offset() as usize, + att.raw_end_offset() as usize, + att, + ) + }) + .collect(); + ranges.sort_by(|a, b| b.0.cmp(&a.0)); + + for (raw_start, raw_end, att) in ranges { + let content_hash = compute_content_hash(att.contents()); + blobs.push(( + content_hash.clone(), + Bytes::copy_from_slice(&original_body[raw_start..raw_end]), + )); + + let placeholder = format!("<>", &content_hash); + stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + + infos.push(AttachmentInfo { + filename: att.attachment_name().map(|n| n.to_string()), + size: att.contents().len(), + inline: att + .content_disposition() + .map(|d| d.is_inline()) + .unwrap_or(false), + file_type: att + .content_type() + .map(|ct| { + format!( + "{}/{}", + ct.c_type.as_ref(), + ct.c_subtype.as_deref().unwrap_or("") + ) + }) + .unwrap_or_else(|| "application/octet-stream".to_string()), + content_id: att.content_id().map(|id| id.to_string()), + content_hash, + is_message: att.is_message(), + }); + } + + (stripped_eml, DetachOutput { infos, blobs }) +} + +pub struct NewIndexWriter { + pub envelope_writer: IndexWriter, + pub attachment_writer: IndexWriter, + pub email_ks: Keyspace, + pub attachment_ks: Keyspace, + pending: usize, +} + +const COMMIT_THRESHOLD: usize = 500; + +impl NewIndexWriter { + pub fn open(dirs: NewDirs) -> BichonResult { + // ── envelope index ────────────────────────────────────────────── + std::fs::create_dir_all(&dirs.envelope_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let envelope_index = if dirs + .envelope_dir + .read_dir() + .map(|mut d| d.next().is_none()) + .unwrap_or(true) + { + Index::create_in_dir(&dirs.envelope_dir, SchemaTools::email_schema()) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + } else { + Index::open_in_dir(&dirs.envelope_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + }; + + envelope_index + .tokenizers() + .register("euro", EuroTokenizer::new()); + + let envelope_writer = envelope_index + .writer_with_num_threads(2, 128 * 1024 * 1024) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + // ── attachment index ───────────────────────────────────────────── + std::fs::create_dir_all(&dirs.attachment_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let attachment_index = if dirs + .attachment_dir + .read_dir() + .map(|mut d| d.next().is_none()) + .unwrap_or(true) + { + Index::create_in_dir(&dirs.attachment_dir, SchemaTools::attachment_schema()) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + } else { + Index::open_in_dir(&dirs.attachment_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))? + }; + + attachment_index + .tokenizers() + .register("euro", EuroTokenizer::new()); + let attachment_writer = attachment_index + .writer_with_num_threads(2, 64 * 1024 * 1024) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + // ── blob store ─────────────────────────────────────────────────── + std::fs::create_dir_all(&dirs.storage_dir) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + let db = Database::builder(&dirs.storage_dir) + .cache_size(64 * 1024 * 1024) + .open() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + let email_ks = db + .keyspace("email", || { + KeyspaceCreateOptions::default() + .max_memtable_size(16 * 1024 * 1024) + .data_block_size_policy(BlockSizePolicy::all(4 * 1024)) + .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4)) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(1024) + .compression(CompressionType::Lz4) + .file_target_size(512 * 1024 * 1024), + )) + }) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + let attachment_ks = db + .keyspace("attachments", || { + KeyspaceCreateOptions::default() + .max_memtable_size(16 * 1024 * 1024) + .data_block_size_policy(BlockSizePolicy::all(4 * 1024)) + .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4)) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(1024) + .compression(CompressionType::Lz4) + .file_target_size(512 * 1024 * 1024), + )) + }) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + Ok(Self { + envelope_writer, + attachment_writer, + email_ks, + attachment_ks, + pending: 0, + }) + } + + pub fn ingest( + &mut self, + eml_bytes: &[u8], + account_id: u64, + mailbox_id: u64, + uid: u32, + internal_date: i64, + ) -> BichonResult<()> { + let email_content_hash = compute_content_hash(eml_bytes); + + let message = MessageParser::new() + .parse(eml_bytes) + .ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?; + + // ── text / preview ──────────────────────────────────────────────── + let text = message + .body_text(0) + .map(|c| c.into_owned()) + .or_else(|| { + message + .body_html(0) + .map(|html| crate::utils::html::extract_text(html.into_owned())) + }) + .unwrap_or_default(); + let text = text.split_whitespace().collect::>().join(" "); + let preview = if text.chars().count() > 100 { + text.chars().take(100).collect::() + "..." + } else { + text.clone() + }; + + // ── headers ─────────────────────────────────────────────────────── + let message_id = message + .message_id() + .map(String::from) + .unwrap_or_else(generate_message_id); + + let in_reply_to = message.in_reply_to().as_text().map(String::from); + let references = extract_references(&message); + let thread_id = compute_thread_id(in_reply_to, references, &message_id); + + let subject = message.subject().map(String::from).unwrap_or_default(); + let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0); + let internal_date = if internal_date == 0 { + date + } else { + internal_date + }; + + let parse_addrs = |addrs: Option<&mail_parser::Address<'_>>| { + addrs + .map(|addr| { + AddrVec::from(addr) + .0 + .into_iter() + .filter_map(|a| a.address) + .collect::>() + }) + .unwrap_or_default() + }; + + let from = message + .from() + .and_then(|addr| AddrVec::from(addr).0.into_iter().next()) + .and_then(|a| a.address) + .unwrap_or_else(|| "unknown".to_string()); + let to = parse_addrs(message.to()); + let cc = parse_addrs(message.cc()); + let bcc = parse_addrs(message.bcc()); + + // ── detach attachments → blob ────────────────────────────────────── + let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message); + + if !self + .email_ks + .contains_key(&email_content_hash) + .unwrap_or(false) + { + self.email_ks + .insert(&email_content_hash, stripped_eml.as_slice()) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + } + + // write attachment blobs + for (hash, data) in &attachment_output.blobs { + if !self.attachment_ks.contains_key(hash).unwrap_or(false) { + self.attachment_ks + .insert(hash, data.as_ref()) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + } + } + + // ── build envelope doc ──────────────────────────────────────────── + let envelope_id = Uuid::new_v4().to_string(); + let now = utc_now!(); + + let attachment_docs: Vec = attachment_output + .infos + .iter() + .filter(|a| !a.inline || a.content_id.is_none()) + .map(|a| { + AttachmentModel { + id: Uuid::new_v4().to_string(), + envelope_id: envelope_id.clone(), + account_id, + account_email: None, + mailbox_id, + mailbox_name: None, + subject: subject.clone(), + content_hash: a.content_hash.clone(), + from: from.clone(), + date, + ingest_at: now, + size: a.size as u64, + ext: a.get_extension(), + category: a.get_category().to_string(), + content_type: a.file_type.clone(), + shard_id: 0, + text: None, + has_text: false, + is_ocr: false, + page_count: None, + is_indexed: false, + is_message: a.is_message, + name: a.filename.clone(), + tags: None, + auto_tags: None, + } + .into_document() + }) + .collect(); + + let envelope = Envelope { + id: envelope_id, + message_id, + account_id, + mailbox_id, + uid, + subject, + preview, + from, + to, + cc, + bcc, + date, + internal_date, + ingest_at: now, + size: eml_bytes.len() as u32, + thread_id, + attachment_count: message.attachment_count(), + regular_attachment_count: attachment_docs.len(), + tags: None, + account_email: None, + mailbox_name: None, + content_hash: email_content_hash, + }; + + let ea = EnvelopeWithAttachments { + envelope, + attachments: Some(attachment_output.infos), + }; + let envelope_doc = ea.to_document(&text, 0)?; + + self.envelope_writer + .add_document(envelope_doc) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + + for doc in attachment_docs { + self.attachment_writer + .add_document(doc) + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + } + + self.pending += 1; + if self.pending >= COMMIT_THRESHOLD { + self.commit()?; + } + + Ok(()) + } + + pub fn commit(&mut self) -> BichonResult<()> { + if self.pending == 0 { + return Ok(()); + } + self.envelope_writer + .commit() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + self.attachment_writer + .commit() + .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?; + tracing::info!(count = self.pending, "committed batch"); + self.pending = 0; + Ok(()) + } +} diff --git a/crates/core/src/store/storage.rs b/crates/core/src/store/blob.rs similarity index 100% rename from crates/core/src/store/storage.rs rename to crates/core/src/store/blob.rs diff --git a/crates/core/src/store/mod.rs b/crates/core/src/store/mod.rs index 89220a3..6ae1b0c 100644 --- a/crates/core/src/store/mod.rs +++ b/crates/core/src/store/mod.rs @@ -17,5 +17,5 @@ // along with this program. If not, see . pub mod envelope; -pub mod storage; +pub mod blob; pub mod tantivy; diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index f77c810..38eead0 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -37,7 +37,7 @@ use crate::{ settings::dir::DATA_DIR_MANAGER, store::{ envelope::Envelope, - storage::BLOB_MANAGER, + blob::BLOB_MANAGER, tantivy::{ fatal_commit, fields::{ @@ -317,7 +317,6 @@ impl IndexManager { let q = query_parser .parse_query(subject_val) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; - println!("{:#?}", &q); subqueries.push((Occur::Must, q)); } diff --git a/crates/core/src/store/tantivy/schema.rs b/crates/core/src/store/tantivy/schema.rs index 430b969..29c8c01 100644 --- a/crates/core/src/store/tantivy/schema.rs +++ b/crates/core/src/store/tantivy/schema.rs @@ -32,84 +32,104 @@ use crate::store::tantivy::fields::{ F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID, }; -static EMAIL_FIELDS: LazyLock> = LazyLock::new(|| { - let (_, fields) = SchemaTools::create_email_schema(); - Arc::new(fields) -}); +// ─── Lazy Globals ───────────────────────────────────────────────────────────── -static ATTACHMENT_FIELDS: LazyLock> = LazyLock::new(|| { - let (_, fields) = SchemaTools::create_attachment_schema(); - Arc::new(fields) -}); +static EMAIL_FIELDS: LazyLock> = LazyLock::new(|| Arc::new(EmailSchema::fields())); + +static ATTACHMENT_FIELDS: LazyLock> = + LazyLock::new(|| Arc::new(AttachmentSchema::fields())); + +// ─── Public API ─────────────────────────────────────────────────────────────── pub struct SchemaTools; impl SchemaTools { pub fn email_schema() -> Schema { - let (schema, _) = Self::create_email_schema(); - schema + EmailSchema::build().0 + } + pub fn attachment_schema() -> Schema { + AttachmentSchema::build().0 } pub fn email_fields() -> &'static EmailFields { &EMAIL_FIELDS } + pub fn attachment_fields() -> &'static AttachmentFields { + &ATTACHMENT_FIELDS + } pub fn email_default_fields() -> Vec { - let fields = Self::email_fields(); + let f = Self::email_fields(); vec![ - fields.f_subject, - fields.f_body, - fields.f_attachment_name_text, - fields.f_from_text, - fields.f_to_text, - fields.f_cc_text, - fields.f_bcc_text, + f.f_subject, + f.f_body, + f.f_attachment_name_text, + f.f_from_text, + f.f_to_text, + f.f_cc_text, + f.f_bcc_text, ] } + pub fn attachment_default_fields() -> Vec { + let f = Self::attachment_fields(); + vec![f.f_subject, f.f_text, f.f_name_text, f.f_from_text] + } + pub fn create_email_schema() -> (Schema, EmailFields) { - let mut builder = Schema::builder(); - let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST); - let f_message_id = builder.add_text_field(F_MESSAGE_ID, STRING | STORED); - let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); - let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); - let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST); - let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro")); - let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro")); - let f_preview = builder.add_text_field(F_PREVIEW, STORED); - let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST); + EmailSchema::build() + } + pub fn create_attachment_schema() -> (Schema, AttachmentFields) { + AttachmentSchema::build() + } +} - let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST); - let f_to = builder.add_text_field(F_TO, STRING | STORED); - let f_cc = builder.add_text_field(F_CC, STRING | STORED); - let f_bcc = builder.add_text_field(F_BCC, STRING | STORED); +// ─── Schema builders ────────────────────────────────────────────────────────── - let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro")); - let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro")); - let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro")); - let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro")); +struct EmailSchema; - let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST); - let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST); - let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); - let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST); - let f_thread_id = builder.add_text_field(F_THREAD_ID, STRING | STORED | FAST); - let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST); +impl EmailSchema { + fn build() -> (Schema, EmailFields) { + let mut b = Schema::builder(); + + let f_id = b.add_text_field(F_ID, STRING | STORED | FAST); + let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED); + let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); + let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); + let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST); + let f_subject = b.add_text_field(F_SUBJECT, text_store("euro")); + let f_body = b.add_text_field(F_BODY, text_no_store("euro")); + let f_preview = b.add_text_field(F_PREVIEW, STORED); + let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST); + let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST); + let f_to = b.add_text_field(F_TO, STRING | STORED); + let f_cc = b.add_text_field(F_CC, STRING | STORED); + let f_bcc = b.add_text_field(F_BCC, STRING | STORED); + let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro")); + let f_to_text = b.add_text_field(F_TO_TEXT, text_no_store("euro")); + let f_cc_text = b.add_text_field(F_CC_TEXT, text_no_store("euro")); + let f_bcc_text = b.add_text_field(F_BCC_TEXT, text_no_store("euro")); + let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST); + let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST); + let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); + let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST); + let f_thread_id = b.add_text_field(F_THREAD_ID, STRING | STORED | FAST); + let f_attachment_count = b.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST); let f_regular_attachment_count = - builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST); + b.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST); let f_attachment_name_text = - builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro")); - let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING); - let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED); + b.add_text_field(F_ATTACHMENT_NAME_TEXT, text_no_store("euro")); + let f_attachment_name_exact = b.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING); + let f_attachments = b.add_text_field(F_ATTACHMENTS, STORED); let f_attachment_content_hash = - builder.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST); - let f_attachment_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST); - let f_attachment_category = - builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST); + b.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST); + let f_attachment_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST); + let f_attachment_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST); let f_attachment_content_type = - builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST); - let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); - let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST); + b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST); + let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); + let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST); + let fields = EmailFields { f_id, f_message_id, @@ -145,57 +165,47 @@ impl SchemaTools { f_tags, f_shard_id, }; - (builder.build(), fields) + + (b.build(), fields) } - pub fn attachment_schema() -> Schema { - let (schema, _) = Self::create_attachment_schema(); - schema + fn fields() -> EmailFields { + Self::build().1 } +} - pub fn attachment_fields() -> &'static AttachmentFields { - &ATTACHMENT_FIELDS - } +struct AttachmentSchema; - pub fn attachment_default_fields() -> Vec { - let fields = Self::attachment_fields(); - vec![ - fields.f_subject, - fields.f_text, - fields.f_name_text, - fields.f_from_text, - ] - } +impl AttachmentSchema { + fn build() -> (Schema, AttachmentFields) { + let mut b = Schema::builder(); + + let f_id = b.add_text_field(F_ID, STRING | STORED | FAST); + let f_envelope_id = b.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST); + let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); + let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); + let f_subject = b.add_text_field(F_SUBJECT, text_store("euro")); + let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST); + let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST); + let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro")); + let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST); + let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); + let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST); + let f_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST); + let f_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST); + let f_content_type = b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST); + let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST); + let f_text = b.add_text_field(F_TEXT, text_no_store("euro")); + let f_has_text = b.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST); + let f_is_ocr = b.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST); + let f_page_count = b.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST); + let f_is_indexed = b.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST); + let f_is_message = b.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST); + let f_name_text = b.add_text_field(F_NAME_TEXT, text_no_store("euro")); + let f_name_exact = b.add_text_field(F_NAME_EXACT, STRING | STORED); + let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); + let f_auto_tags = b.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored()); - pub fn create_attachment_schema() -> (Schema, AttachmentFields) { - let mut builder = Schema::builder(); - let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST); - let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST); - let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); - let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); - let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro")); - let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST); - let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST); - let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro")); - let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST); - let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); - let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST); - let f_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST); - let f_category = builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST); - let f_content_type = - builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST); - let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST); - let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro")); - let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST); - let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST); - let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST); - let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST); - let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST); - let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro")); - let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED); - let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); - let f_auto_tags = - builder.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored()); let fields = AttachmentFields { f_id, f_envelope_id, @@ -223,24 +233,25 @@ impl SchemaTools { f_tags, f_auto_tags, }; - (builder.build(), fields) + + (b.build(), fields) } - fn text_no_store(tokenizer: &str) -> TextOptions { - TextOptions::default().set_indexing_options( - TextFieldIndexing::default() - .set_tokenizer(tokenizer) - .set_index_option(IndexRecordOption::WithFreqsAndPositions), - ) - } - - fn text_store(tokenizer: &str) -> TextOptions { - TextOptions::default() - .set_indexing_options( - TextFieldIndexing::default() - .set_tokenizer(tokenizer) - .set_index_option(IndexRecordOption::WithFreqsAndPositions), - ) - .set_stored() + fn fields() -> AttachmentFields { + Self::build().1 } } + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +fn text_no_store(tokenizer: &str) -> TextOptions { + TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(tokenizer) + .set_index_option(IndexRecordOption::WithFreqsAndPositions), + ) +} + +fn text_store(tokenizer: &str) -> TextOptions { + text_no_store(tokenizer).set_stored() +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 1512153..cb1de30 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -25,11 +25,11 @@ use bichon_core::{ context::{executors::BichonContext, Initialize}, error::{code::ErrorCode, BichonResult}, logger, - migrate::is_legacy_data_layout, + migrate::check_data_status, raise_error, settings::cli::SETTINGS, store::{ - storage::BLOB_MANAGER, + blob::BLOB_MANAGER, tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER}, }, tasks::PeriodicTasks, @@ -69,11 +69,11 @@ async fn main() -> BichonResult<()> { info!("Git: [{}]", env!("GIT_HASH")); info!("GitHub: https://github.com/rustmailer/bichon"); - match is_legacy_data_layout() { - Ok(true) => { + match check_data_status() { + Ok(false) => { error!("Incompatible data format detected."); error!("Your data was created by an older version of Bichon and must be migrated before use."); - error!("Please run: bichon-migrate"); + error!("Please run: bichon-admin"); error!("Documentation: https://github.com/rustmailer/bichon/wiki/migration"); return Err(raise_error!( "Legacy data layout detected".into(), @@ -84,7 +84,7 @@ async fn main() -> BichonResult<()> { error!("Failed to check data layout: {:#?}", e); return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)); } - Ok(false) => {} + Ok(true) => {} } if let Err(error) = initialize().await { diff --git a/crates/server/src/rest/api/message.rs b/crates/server/src/rest/api/message.rs index 3df383a..e52b6cd 100644 --- a/crates/server/src/rest/api/message.rs +++ b/crates/server/src/rest/api/message.rs @@ -36,7 +36,7 @@ use bichon_core::message::tags::TagCount; use bichon_core::message::tags::TagsRequest; use bichon_core::raise_error; use bichon_core::store::envelope::Envelope; -use bichon_core::store::storage::get_reader; +use bichon_core::store::blob::get_reader; use bichon_core::store::tantivy::envelope::ENVELOPE_MANAGER; use bichon_core::store::tantivy::validate_facet; use bichon_core::users::permissions::Permission;