feat(admin): add interactive data migration tool

Cargo.lock

@@ -464,6 +464,7 @@ dependencies = [
  "bichon-core",
  "console",
  "dialoguer",
+ "indicatif",
  "tokio",
 ]
 
@@ -5022,6 +5023,8 @@ dependencies = [
  "fastdivide",
  "fnv",
  "fs4",
+ "futures-channel",
+ "futures-util",
  "htmlescape",
  "itertools",
  "levenshtein_automata",
@@ -5045,6 +5048,7 @@ dependencies = [
  "tantivy-common",
  "tantivy-fst",
  "tantivy-query-grammar",
+ "tantivy-sstable",
  "tantivy-stacker",
  "tantivy-tokenizer-api",
  "tempfile",

Cargo.toml

@@ -84,7 +84,7 @@ uuid = { version = "1.23.1", features = ["v4", "serde"] }
 fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] }
 tracing-log = "0.2.0"
 tokio-util = "0.7.18"
-
+indicatif = "0.18.4"
 
 [profile.release]
 strip = true

crates/admin/Cargo.toml

@@ -9,3 +9,4 @@ bichon-core = { path = "../core" }
 tokio.workspace = true
 dialoguer.workspace = true
 console.workspace = true
+indicatif.workspace = true

crates/admin/src/main.rs

@@ -16,19 +16,13 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::{
-    fs,
-    path::{Path, PathBuf},
-};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Select};
 
-use bichon_core::{
-    admin::meta::{find_admin, init_meta_database, update_admin_password},
-    error::BichonError,
-    utils::encrypt::internal_decrypt_string,
-};
-use console::{style, Emoji};
-use dialoguer::Confirm;
-use dialoguer::{theme::ColorfulTheme, Input, Password, Select};
+use crate::{migrate::handle_migration, reset::handle_reset_password};
+
+pub mod migrate;
+pub mod reset;
 
 #[tokio::main]
 async fn main() {
@@ -38,7 +32,12 @@ async fn main() {
         style("BICHON ADMINISTRATIVE TOOL").bold().bright().cyan()
     );
 
-    let main_options = vec!["Reset Admin Password", "Exit"];
+    let main_options = vec![
+        "Reset Admin Password",
+        "Migrate Legacy v0.x Storage to v1.0",
+        "Exit",
+    ];
+
     let selection = Select::with_theme(&theme)
         .with_prompt("Select an operation")
         .default(0)
@@ -46,272 +45,11 @@ async fn main() {
         .interact()
         .unwrap();
 
-    if selection == 1 {
+    match selection {
+        0 => handle_reset_password(&theme),
+        1 => handle_migration(&theme),
+        _ => {
             println!("{}", style("Exiting...").dim());
-        return;
-    }
-
-    let root_dir_str: String = Input::with_theme(&theme)
-        .with_prompt("Enter the absolute path for 'bichon_root_dir'")
-        .validate_with(|input: &String| -> Result<(), &str> {
-            let path = Path::new(input);
-            if !path.is_absolute() {
-                return Err("Path must be absolute.");
-            }
-            if !path.exists() {
-                return Err("Directory does not exist.");
-            }
-            let has_metadata = path.join("meta.db").exists();
-            if !has_metadata {
-                return Err("Invalid directory: 'meta.db' not found.");
-            }
-            Ok(())
-        })
-        .interact_text()
-        .unwrap();
-
-    let root_path = PathBuf::from(&root_dir_str);
-
-    let database = match init_meta_database(&root_path.join("meta.db")) {
-        Ok(database) => database,
-        Err(e) => match e {
-            BichonError::Generic {
-                message,
-                location,
-                code,
-            } => {
-                if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") {
-                    println!("\n{}", style("ERROR: Database is locked.").red().bold());
-                    println!(
-                        "{}",
-                        style("The Bichon service is likely still running.").yellow()
-                    );
-                    println!(
-                        "Since the database cannot be shared between multiple instances, \n\
-                     you must {} the Bichon service before proceeding.",
-                        style("STOP").underlined().bold()
-                    );
-                    std::process::exit(1);
-                } else {
-                    eprintln!(
-                        "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}",
-                        style("A database error occurred:").red().bold(),
-                        code,
-                        location,
-                        message
-                    );
-                    std::process::exit(1);
-                }
-            }
-        },
-    };
-
-    let admin = find_admin(&database);
-
-    match admin {
-        Ok(Some(user)) => {
-            println!("\n{}", style("Admin user found:").green().bold());
-            println!("----------------------------------------");
-            println!("{:<12} : {}", "Username", style(&user.username).cyan());
-            println!("{:<12} : {}", "Email", style(&user.email).cyan());
-        }
-        Ok(None) => {
-            println!(
-                "\n{}",
-                style("ERROR: No admin user found in the database.")
-                    .red()
-                    .bold()
-            );
-            println!("Please ensure the system has been initialized correctly.");
-            std::process::exit(1);
-        }
-        Err(e) => {
-            eprintln!(
-                "\n{} Failed to query admin user.",
-                style("ERROR:").red().bold()
-            );
-            eprintln!("Details: {:?}", e);
-            std::process::exit(1);
-        }
-    }
-
-    let encryption_key = loop {
-        let auth_methods = vec![
-            "Enter encryption password manually",
-            "Read from password file",
-        ];
-        let method = Select::with_theme(&theme)
-            .with_prompt("How would you like to provide the database encryption key?")
-            .items(&auth_methods)
-            .interact()
-            .unwrap();
-
-        let raw_key = if method == 0 {
-            Password::with_theme(&theme)
-                .with_prompt("Enter Encryption Password")
-                .interact()
-                .unwrap()
-        } else {
-            let file_path: String = Input::with_theme(&theme)
-                .with_prompt("Enter path to encryption password file")
-                .interact_text()
-                .unwrap();
-
-            match fs::read_to_string(&file_path) {
-                Ok(content) => content.trim().to_string(),
-                Err(e) => {
-                    println!("{}: {}", style("Failed to read file").red(), e);
-                    continue;
-                }
-            }
-        };
-
-        if raw_key.is_empty() {
-            println!("{}", style("Key cannot be empty.").red());
-            continue;
-        }
-
-        let prompt_message = format!(
-            "Encryption key loaded: [ {} ]\n\n  \
-                {}: This key must match the database encryption key used by the server.\n  \
-                It corresponds to these settings in your service:\n    \
-                - Arguments: {} or {}\n    \
-                - Envs:      {} or {}\n\n  \
-                Do you want to continue?",
-            style(&raw_key).cyan().bold(),
-            style("IMPORTANT").yellow().bold(),
-            style("--bichon_encrypt_password").italic(),
-            style("--bichon_encrypt_password_file").italic(),
-            style("BICHON_ENCRYPT_PASSWORD").green(),
-            style("BICHON_ENCRYPT_PASSWORD_FILE").green()
-        );
-
-        if Confirm::with_theme(&theme)
-            .with_prompt(prompt_message)
-            .default(true)
-            .interact()
-            .unwrap()
-        {
-            break raw_key;
-        }
-    };
-
-    let admin = find_admin(&database);
-
-    match admin {
-        Ok(Some(user)) => {
-            println!("----------------------------------------");
-            println!("{:<12} : {}", "Username", style(&user.username).cyan());
-            println!("{:<12} : {}", "Email", style(&user.email).cyan());
-
-            let pwd_display = match &user.password {
-                Some(p) => {
-                    let password = match internal_decrypt_string(&encryption_key, p) {
-                        Ok(p) => p,
-                        Err(e) => {
-                            println!("\n{}", style("ERROR: Decryption Failed").red().bold());
-                            println!(
-                                "{}",
-                                style("The provided encryption key is incorrect or invalid for this database.").yellow()
-                            );
-                            println!(
-                                "{} Please verify your {} or the {} you provided.",
-                                style("➔").cyan(),
-                                style("encryption password").bold(),
-                                style("key file").bold()
-                            );
-                            eprintln!("\nTechnical details: {:?}", e);
-                            std::process::exit(1);
-                        }
-                    };
-                    style(password).yellow().to_string()
-                }
-                None => style("None (No password set)").dim().italic().to_string(),
-            };
-
-            println!("{:<12} : {}", "Password", pwd_display);
-            println!("----------------------------------------");
-
-            if !dialoguer::Confirm::with_theme(&theme)
-                .with_prompt(format!(
-                    "Do you want to reset the password for '{}'?",
-                    user.username
-                ))
-                .interact()
-                .unwrap()
-            {
-                println!("Operation cancelled.");
-                return;
-            }
-        }
-        Ok(None) => {
-            println!(
-                "\n{}",
-                style("ERROR: No admin user found in the database.")
-                    .red()
-                    .bold()
-            );
-            println!("Please ensure the system has been initialized correctly.");
-            std::process::exit(1);
-        }
-        Err(e) => {
-            eprintln!(
-                "\n{} Failed to query admin user.",
-                style("ERROR:").red().bold()
-            );
-            eprintln!("Details: {:?}", e);
-            std::process::exit(1);
-        }
-    }
-
-    println!(
-        "\n{}",
-        style("TARGET: Reset password for user 'admin'")
-            .yellow()
-            .bold()
-    );
-
-    let new_login_password = Password::with_theme(&theme)
-        .with_prompt("Enter new Admin Login Password")
-        .with_confirmation("Repeat password to confirm", "Passwords do not match!")
-        .interact()
-        .unwrap();
-
-    if !Confirm::with_theme(&theme)
-        .with_prompt("Proceed with database update?")
-        .interact()
-        .unwrap()
-    {
-        return;
-    }
-
-    println!("\n{} {}", style("⌛").yellow(), "Updating database...");
-
-    match update_admin_password(&database, new_login_password, &encryption_key) {
-        Ok(_) => {
-            println!(
-                "\n{} {}",
-                Emoji("✨", "*"),
-                style("Success! Admin password has been updated.")
-                    .green()
-                    .bold()
-            );
-            println!(
-                "{}",
-                style("You can now log in with the new password.").dim()
-            );
-        }
-        Err(e) => {
-            println!(
-                "\n{}",
-                style("ERROR: Failed to update database").red().bold()
-            );
-            eprintln!(
-                "{} Could not save the new password to the database.",
-                style("➔").cyan()
-            );
-            eprintln!("\nDetails: {:?}", e);
-            std::process::exit(1);
         }
     }
 }

crates/admin/src/migrate.rs

@@ -0,0 +1,284 @@
+use std::path::{Path, PathBuf};
+
+use bichon_core::migrate::{
+    do_migrate, is_tantivy_index_dir,
+    store::{LegacyDirs, NewDirs},
+};
+use console::style;
+use dialoguer::{theme::ColorfulTheme, Confirm, Input};
+use indicatif::{ProgressBar, ProgressStyle};
+
+pub fn handle_migration(theme: &ColorfulTheme) {
+    println!(
+        "\n{}",
+        style("MIGRATION: Bichon v0.x Storage Architecture → v1.0")
+            .bold()
+            .yellow()
+    );
+
+    println!(
+        "{}",
+        style(
+            "This tool migrates data from the legacy v0.x Tantivy-based storage \
+            architecture (used in versions 0.0.1 through 0.3.7) to the new v1.0 \
+            separated index and Fjall-backed storage format."
+        )
+        .dim()
+    );
+
+    println!(
+        "{}",
+        style(
+            "Legacy v0.x architecture:\n\
+            • envelope metadata stored in Tantivy\n\
+            • message data stored in Tantivy\n\n\
+                New v1.0 architecture:\n\
+            • mail indexes stored in Tantivy\n\
+            • attachment indexes stored in Tantivy\n\
+            • raw message data stored in Fjall\n\
+            • attachment blobs stored in Fjall"
+        )
+        .dim()
+    );
+
+    println!(
+        "\n{} {}",
+        style("IMPORTANT:").yellow().bold(),
+        style(
+            "The paths below must exactly match what your old bichon server was configured with."
+        )
+        .yellow()
+    );
+
+    // --- bichon-root-dir ---
+    let root_dir_str: String = Input::with_theme(theme)
+        .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
+        .validate_with(|input: &String| -> Result<(), &str> {
+            let path = Path::new(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let root_path = PathBuf::from(&root_dir_str);
+
+    // --- bichon-index-dir ---
+    let default_index = root_path.join("envelope");
+    let default_new_index = root_path.join("bichon-indices");
+    let index_dir_str: String = Input::with_theme(theme)
+        .with_prompt(format!(
+            "Enter --bichon-index-dir (leave blank to use default: {})",
+            style(default_index.display()).cyan()
+        ))
+        .allow_empty(true)
+        .validate_with(|input: &String| -> Result<(), &str> {
+            if input.is_empty() {
+                return Ok(());
+            }
+            let path = Path::new(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let index_path = if index_dir_str.is_empty() {
+        default_index
+    } else {
+        PathBuf::from(&index_dir_str)
+    };
+
+    let new_index_path = if index_dir_str.is_empty() {
+        default_new_index
+    } else {
+        PathBuf::from(&index_dir_str).join("bichon-indices")
+    };
+
+    // --- bichon-data-dir ---
+    let default_data = root_path.join("eml");
+    let default_new_data = root_path.join("bichon-storage");
+    let data_dir_str: String = Input::with_theme(theme)
+        .with_prompt(format!(
+            "Enter --bichon-data-dir (leave blank to use default: {})",
+            style(default_data.display()).cyan()
+        ))
+        .allow_empty(true)
+        .validate_with(|input: &String| -> Result<(), &str> {
+            if input.is_empty() {
+                return Ok(());
+            }
+            let path = Path::new(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let data_path = if data_dir_str.is_empty() {
+        default_data
+    } else {
+        PathBuf::from(&data_dir_str)
+    };
+
+    let new_data_path = if data_dir_str.is_empty() {
+        default_new_data
+    } else {
+        PathBuf::from(&data_dir_str).join("bichon-storage")
+    };
+
+    println!("\n{}", style("Paths to be migrated:").bold());
+    println!("----------------------------------------");
+    println!(
+        "{:<20} : {}",
+        "bichon-root-dir",
+        style(root_path.display()).cyan()
+    );
+    println!(
+        "{:<20} : {}",
+        "bichon-index-dir",
+        style(index_path.display()).cyan()
+    );
+    println!(
+        "{:<20} : {}",
+        "bichon-data-dir",
+        style(data_path.display()).cyan()
+    );
+    println!("----------------------------------------");
+
+    println!(
+        "\n{} Checking legacy v0.x storage layout...",
+        style("⌛").yellow()
+    );
+
+    match is_legacy_data_layout_with_paths(&index_path, &data_path) {
+        Ok(true) => {
+            println!(
+                "{} {}",
+                style("✔").green(),
+                style("Legacy v0.x Tantivy-based storage detected. Migration to v1.0 is required.")
+                    .yellow()
+            );
+        }
+        Ok(false) => {
+            println!(
+                "{} {}",
+                style("✔").green(),
+                style("No legacy v0.x storage layout was detected at the specified paths.").green()
+            );
+
+            println!(
+                "{}",
+                style(
+                    "The selected directories may already be using the v1.0 storage architecture."
+                )
+                .dim()
+            );
+
+            return;
+        }
+        Err(e) => {
+            eprintln!(
+                "{} Failed to verify legacy storage layout: {:?}",
+                style("ERROR:").red().bold(),
+                e
+            );
+
+            std::process::exit(1);
+        }
+    }
+
+    println!(
+        "\n{} {}",
+        style("⚠").yellow(),
+        style(
+            "This migration is non-destructive. Existing v0.x storage files will remain unchanged."
+        )
+        .yellow()
+    );
+
+    if !Confirm::with_theme(theme)
+        .with_prompt("Ready to migrate?")
+        .default(true)
+        .interact()
+        .unwrap()
+    {
+        println!("{}", style("Migration cancelled.").dim());
+        return;
+    }
+
+    println!("\n{} Migrating...", style("⌛").yellow());
+    let pb = ProgressBar::new(0);
+    pb.set_style(ProgressStyle::default_bar()
+    .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}")
+    .unwrap()
+    .progress_chars("#>-"));
+    let legacy = LegacyDirs::new(index_path, data_path);
+    let new_dirs = NewDirs::new(new_index_path, new_data_path);
+    if let Err(e) = do_migrate(legacy, new_dirs, |msg| {
+        if let Some(data) = msg.strip_prefix("PROGRESS:") {
+            let parts: Vec<&str> = data.split(':').collect();
+            if parts.len() == 2 {
+                let migrated = parts[0].parse::<u64>().unwrap_or(0);
+                let skipped = parts[1].parse::<u64>().unwrap_or(0);
+
+                pb.set_position(migrated + skipped);
+                pb.set_message(format!(
+                    "Migrated: {}, {} {}",
+                    style(migrated).green(),
+                    style(skipped).red(),
+                    style("skipped").dim()
+                ));
+            }
+        } else if let Some(total) = msg.strip_prefix("TOTAL:") {
+            pb.set_length(total.parse().unwrap_or(0));
+        } else if msg.starts_with("WARN:") {
+            pb.println(format!("{} {}", style("⚠").yellow(), &msg[5..]));
+        } else if let Some(done_data) = msg.strip_prefix("DONE:") {
+            let parts: Vec<&str> = done_data.split(':').collect();
+            pb.finish_with_message(format!(
+                "Migration finished. Total: {}, Skipped: {}",
+                parts.get(0).unwrap_or(&"0"),
+                parts.get(1).unwrap_or(&"0")
+            ));
+        }
+    }) {
+        eprintln!(
+            "\n{} Migration failed:\n{:?}",
+            style("✘").red().bold(),
+            style(e).red()
+        );
+        return;
+    }
+    println!(
+        "{} {}",
+        style("✔").green(),
+        style("Migration completed successfully!").bold()
+    );
+}
+
+pub fn is_legacy_data_layout_with_paths(
+    envelope_dir: &PathBuf,
+    eml_dir: &PathBuf,
+) -> std::io::Result<bool> {
+    let envelope_result = is_tantivy_index_dir(envelope_dir)?;
+    let eml_result = is_tantivy_index_dir(eml_dir)?;
+
+    Ok(envelope_result || eml_result)
+}

crates/admin/src/reset.rs

@@ -0,0 +1,274 @@
+use std::path::{Path, PathBuf};
+
+use bichon_core::{
+    admin::meta::{find_admin, init_meta_database, update_admin_password},
+    error::BichonError,
+    utils::encrypt::internal_decrypt_string,
+};
+use console::{style, Emoji};
+use dialoguer::{theme::ColorfulTheme, Confirm, Input, Password, Select};
+
+pub fn handle_reset_password(theme: &ColorfulTheme) {
+    let root_dir_str: String = Input::with_theme(theme)
+        .with_prompt("Enter the absolute path for 'bichon_root_dir'")
+        .validate_with(|input: &String| -> Result<(), &str> {
+            let path = Path::new(input);
+            if !path.is_absolute() {
+                return Err("Path must be absolute.");
+            }
+            if !path.exists() {
+                return Err("Directory does not exist.");
+            }
+            let has_metadata = path.join("meta.db").exists();
+            if !has_metadata {
+                return Err("Invalid directory: 'meta.db' not found.");
+            }
+            Ok(())
+        })
+        .interact_text()
+        .unwrap();
+
+    let root_path = PathBuf::from(&root_dir_str);
+    let database = match init_meta_database(&root_path.join("meta.db")) {
+        Ok(database) => database,
+        Err(e) => match e {
+            BichonError::Generic {
+                message,
+                location,
+                code,
+            } => {
+                if message.contains("RedbDatabaseError(DatabaseAlreadyOpen") {
+                    println!("\n{}", style("ERROR: Database is locked.").red().bold());
+                    println!(
+                        "{}",
+                        style("The Bichon service is likely still running.").yellow()
+                    );
+                    println!(
+                        "Since the database cannot be shared between multiple instances, \n\
+                     you must {} the Bichon service before proceeding.",
+                        style("STOP").underlined().bold()
+                    );
+                    std::process::exit(1);
+                } else {
+                    eprintln!(
+                        "\n{} (Code: {:#?})\nLocation: {}\nMessage: {}",
+                        style("A database error occurred:").red().bold(),
+                        code,
+                        location,
+                        message
+                    );
+                    std::process::exit(1);
+                }
+            }
+        },
+    };
+
+    let admin = find_admin(&database);
+
+    match admin {
+        Ok(Some(user)) => {
+            println!("\n{}", style("Admin user found:").green().bold());
+            println!("----------------------------------------");
+            println!("{:<12} : {}", "Username", style(&user.username).cyan());
+            println!("{:<12} : {}", "Email", style(&user.email).cyan());
+        }
+        Ok(None) => {
+            println!(
+                "\n{}",
+                style("ERROR: No admin user found in the database.")
+                    .red()
+                    .bold()
+            );
+            println!("Please ensure the system has been initialized correctly.");
+            std::process::exit(1);
+        }
+        Err(e) => {
+            eprintln!(
+                "\n{} Failed to query admin user.",
+                style("ERROR:").red().bold()
+            );
+            eprintln!("Details: {:?}", e);
+            std::process::exit(1);
+        }
+    }
+
+    let encryption_key = loop {
+        let auth_methods = vec![
+            "Enter encryption password manually",
+            "Read from password file",
+        ];
+        let method = Select::with_theme(theme)
+            .with_prompt("How would you like to provide the database encryption key?")
+            .items(&auth_methods)
+            .interact()
+            .unwrap();
+
+        let raw_key = if method == 0 {
+            Password::with_theme(theme)
+                .with_prompt("Enter Encryption Password")
+                .interact()
+                .unwrap()
+        } else {
+            let file_path: String = Input::with_theme(theme)
+                .with_prompt("Enter path to encryption password file")
+                .interact_text()
+                .unwrap();
+
+            match std::fs::read_to_string(&file_path) {
+                Ok(content) => content.trim().to_string(),
+                Err(e) => {
+                    println!("{}: {}", style("Failed to read file").red(), e);
+                    continue;
+                }
+            }
+        };
+
+        if raw_key.is_empty() {
+            println!("{}", style("Key cannot be empty.").red());
+            continue;
+        }
+
+        let prompt_message = format!(
+            "Encryption key loaded: [ {} ]\n\n  \
+                {}: This key must match the database encryption key used by the server.\n  \
+                It corresponds to these settings in your service:\n    \
+                - Arguments: {} or {}\n    \
+                - Envs:      {} or {}\n\n  \
+                Do you want to continue?",
+            style(&raw_key).cyan().bold(),
+            style("IMPORTANT").yellow().bold(),
+            style("--bichon_encrypt_password").italic(),
+            style("--bichon_encrypt_password_file").italic(),
+            style("BICHON_ENCRYPT_PASSWORD").green(),
+            style("BICHON_ENCRYPT_PASSWORD_FILE").green()
+        );
+
+        if Confirm::with_theme(theme)
+            .with_prompt(prompt_message)
+            .default(true)
+            .interact()
+            .unwrap()
+        {
+            break raw_key;
+        }
+    };
+
+    let admin = find_admin(&database);
+
+    match admin {
+        Ok(Some(user)) => {
+            println!("----------------------------------------");
+            println!("{:<12} : {}", "Username", style(&user.username).cyan());
+            println!("{:<12} : {}", "Email", style(&user.email).cyan());
+
+            let pwd_display = match &user.password {
+                Some(p) => {
+                    let password = match internal_decrypt_string(&encryption_key, p) {
+                        Ok(p) => p,
+                        Err(e) => {
+                            println!("\n{}", style("ERROR: Decryption Failed").red().bold());
+                            println!(
+                                "{}",
+                                style("The provided encryption key is incorrect or invalid for this database.").yellow()
+                            );
+                            println!(
+                                "{} Please verify your {} or the {} you provided.",
+                                style("➔").cyan(),
+                                style("encryption password").bold(),
+                                style("key file").bold()
+                            );
+                            eprintln!("\nTechnical details: {:?}", e);
+                            std::process::exit(1);
+                        }
+                    };
+                    style(password).yellow().to_string()
+                }
+                None => style("None (No password set)").dim().italic().to_string(),
+            };
+
+            println!("{:<12} : {}", "Password", pwd_display);
+            println!("----------------------------------------");
+
+            if !dialoguer::Confirm::with_theme(theme)
+                .with_prompt(format!(
+                    "Do you want to reset the password for '{}'?",
+                    user.username
+                ))
+                .interact()
+                .unwrap()
+            {
+                println!("Operation cancelled.");
+                return;
+            }
+        }
+        Ok(None) => {
+            println!(
+                "\n{}",
+                style("ERROR: No admin user found in the database.")
+                    .red()
+                    .bold()
+            );
+            println!("Please ensure the system has been initialized correctly.");
+            std::process::exit(1);
+        }
+        Err(e) => {
+            eprintln!(
+                "\n{} Failed to query admin user.",
+                style("ERROR:").red().bold()
+            );
+            eprintln!("Details: {:?}", e);
+            std::process::exit(1);
+        }
+    }
+
+    println!(
+        "\n{}",
+        style("TARGET: Reset password for user 'admin'")
+            .yellow()
+            .bold()
+    );
+
+    let new_login_password = Password::with_theme(theme)
+        .with_prompt("Enter new Admin Login Password")
+        .with_confirmation("Repeat password to confirm", "Passwords do not match!")
+        .interact()
+        .unwrap();
+
+    if !Confirm::with_theme(theme)
+        .with_prompt("Proceed with database update?")
+        .interact()
+        .unwrap()
+    {
+        return;
+    }
+
+    println!("\n{} {}", style("⌛").yellow(), "Updating database...");
+
+    match update_admin_password(&database, new_login_password, &encryption_key) {
+        Ok(_) => {
+            println!(
+                "\n{} {}",
+                Emoji("✨", "*"),
+                style("Success! Admin password has been updated.")
+                    .green()
+                    .bold()
+            );
+            println!(
+                "{}",
+                style("You can now log in with the new password.").dim()
+            );
+        }
+        Err(e) => {
+            println!(
+                "\n{}",
+                style("ERROR: Failed to update database").red().bold()
+            );
+            eprintln!(
+                "{} Could not save the new password to the database.",
+                style("➔").cyan()
+            );
+            eprintln!("\nDetails: {:?}", e);
+            std::process::exit(1);
+        }
+    }
+}

crates/cli/Cargo.toml

@@ -23,5 +23,5 @@ base64.workspace = true
 codepage-strings = "1.0.2"
 hex = "0.4.3"
 sysinfo.workspace = true
-indicatif = "0.18.4"
+indicatif.workspace = true
 serde_json.workspace = true

crates/core/Cargo.toml

@@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m
     "runtime-tokio",
     "compress",
 ] }
-tantivy = { version = "0.26.1", features = ["zstd-compression"] }
+tantivy = { version = "0.26.1", features = ["zstd-compression", "quickwit"] }
 webpki-roots.workspace = true
 rustls.workspace = true
 rustls-pki-types.workspace = true

crates/core/src/admin/meta.rs

@@ -16,21 +16,18 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
-
 use std::{path::Path, rc::Rc};
 
 use native_db::{Builder, Database};
 
 use crate::{
-    {
+    account::migration::AccountV3,
     database::META_MODELS,
     error::{code::ErrorCode, BichonResult},
+    raise_error,
     token::{AccessTokenModel, AccessTokenModelKey, TokenType},
     users::{UserModel, DEFAULT_ADMIN_USER_ID},
     utils::encrypt::internal_encrypt_string,
-    },
-    raise_error,
 };
 use itertools::Itertools;
 
@@ -48,6 +45,21 @@ pub fn init_meta_database(path: impl AsRef<Path>) -> BichonResult<Rc<Database<'s
     Ok(Rc::new(database))
 }
 
+pub fn list_all_accounts(database: &Rc<Database<'static>>) -> BichonResult<Vec<AccountV3>> {
+    let r_transaction = database
+        .r_transaction()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    let entities: Vec<AccountV3> = r_transaction
+        .scan()
+        .primary()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+        .all()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+        .try_collect()
+        .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+    Ok(entities)
+}
+
 pub fn find_admin(database: &Rc<Database<'static>>) -> BichonResult<Option<UserModel>> {
     let r_transaction = database
         .r_transaction()

crates/core/src/envelope/extractor.rs

@@ -22,7 +22,7 @@ use crate::envelope::utils::normalize_subject;
 use crate::error::code::ErrorCode;
 use crate::error::BichonResult;
 use crate::message::content::AttachmentInfo;
-use crate::store::storage::{DetachedEmail, BLOB_MANAGER};
+use crate::store::blob::{DetachedEmail, BLOB_MANAGER};
 use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
 use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
 use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments};
@@ -355,7 +355,7 @@ pub fn generate_message_id() -> String {
     format!("<{:016x}.{}.{}@{}>", id!(128), ts, pid, "bichon")
 }
 
-fn extract_references(message: &Message<'_>) -> Option<Vec<String>> {
+pub fn extract_references(message: &Message<'_>) -> Option<Vec<String>> {
     match message.references() {
         mail_parser::HeaderValue::Text(cow) => Some(vec![cow.to_string()]),
         mail_parser::HeaderValue::TextList(vec) => {

crates/core/src/migrate/legacy/fields.rs

@@ -0,0 +1,69 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use tantivy::schema::Field;
+
+pub const F_MESSAGE_ID: &str = "message_id";
+pub const F_ACCOUNT_ID: &str = "account_id";
+pub const F_MAILBOX_ID: &str = "mailbox_id";
+pub const F_UID: &str = "uid";
+pub const F_SUBJECT: &str = "subject";
+pub const F_TEXT: &str = "text";
+pub const F_FROM: &str = "from";
+pub const F_TO: &str = "to";
+pub const F_CC: &str = "cc";
+pub const F_BCC: &str = "bcc";
+pub const F_DATE: &str = "date";
+pub const F_INTERNAL_DATE: &str = "internal_date";
+pub const F_SIZE: &str = "size";
+pub const F_THREAD_ID: &str = "thread_id";
+pub const F_ATTACHMENTS: &str = "attachments";
+pub const F_HAS_ATTACHMENT: &str = "has_attachment";
+pub const F_TAGS: &str = "tags";
+
+pub const F_ID: &str = "id";
+pub struct EnvelopeFields {
+    pub f_id: Field,
+    pub f_message_id: Field,
+    pub f_account_id: Field,
+    pub f_mailbox_id: Field,
+    pub f_uid: Field,
+    pub f_subject: Field,
+    pub f_text: Field,
+    pub f_from: Field,
+    pub f_to: Field,
+    pub f_cc: Field,
+    pub f_bcc: Field,
+    pub f_date: Field,
+    pub f_internal_date: Field,
+    pub f_size: Field,
+    pub f_thread_id: Field,
+    pub f_attachments: Field,
+    pub f_has_attachment: Field,
+    pub f_tags: Field,
+}
+
+pub const F_EML: &str = "eml";
+
+pub struct EmlFields {
+    pub f_id: Field,
+    pub f_account_id: Field,
+    pub f_mailbox_id: Field,
+    pub f_eml: Field,
+}

crates/core/src/migrate/legacy/mod.rs

@@ -0,0 +1,2 @@
+pub mod fields;
+pub mod schema;

crates/core/src/migrate/legacy/schema.rs

@@ -0,0 +1,114 @@
+use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
+
+use crate::migrate::legacy::fields::{EmlFields, EnvelopeFields, *};
+
+pub struct SchemaTools;
+
+impl SchemaTools {
+    pub fn envelope_schema() -> Schema {
+        EnvelopeSchema::build().0
+    }
+    pub fn eml_schema() -> Schema {
+        EmlSchema::build().0
+    }
+
+    pub fn envelope_fields() -> EnvelopeFields {
+        EnvelopeSchema::fields()
+    }
+    pub fn eml_fields() -> EmlFields {
+        EmlSchema::fields()
+    }
+
+    pub fn envelope_default_fields() -> Vec<Field> {
+        let f = Self::envelope_fields();
+        vec![f.f_subject, f.f_text, f.f_attachments]
+    }
+}
+
+// ─── Schema builders ──────────────────────────────────────────────────────────
+
+struct EnvelopeSchema;
+
+impl EnvelopeSchema {
+    fn build() -> (Schema, EnvelopeFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_u64_field(F_ID, INDEXED | STORED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
+        let f_thread_id = b.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST);
+
+        let f_subject = b.add_text_field(F_SUBJECT, TEXT | STORED);
+        let f_text = b.add_text_field(F_TEXT, TEXT | STORED);
+        let f_attachments = b.add_text_field(F_ATTACHMENTS, TEXT | STORED);
+
+        let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_to = b.add_text_field(F_TO, STRING | STORED);
+        let f_cc = b.add_text_field(F_CC, STRING | STORED);
+        let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
+
+        let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
+
+        let f_date = b.add_i64_field(F_DATE, STORED | FAST);
+        let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, STORED | FAST);
+
+        let f_size = b.add_u64_field(F_SIZE, STORED | FAST);
+        let f_has_attachment = b.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST);
+
+        let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+
+        let fields = EnvelopeFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_uid,
+            f_thread_id,
+            f_subject,
+            f_text,
+            f_attachments,
+            f_from,
+            f_to,
+            f_cc,
+            f_bcc,
+            f_message_id,
+            f_date,
+            f_internal_date,
+            f_size,
+            f_has_attachment,
+            f_tags,
+        };
+
+        (b.build(), fields)
+    }
+
+    fn fields() -> EnvelopeFields {
+        Self::build().1
+    }
+}
+
+struct EmlSchema;
+
+impl EmlSchema {
+    fn build() -> (Schema, EmlFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_u64_field(F_ID, INDEXED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_eml = b.add_bytes_field(F_EML, STORED);
+
+        let fields = EmlFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_eml,
+        };
+
+        (b.build(), fields)
+    }
+
+    fn fields() -> EmlFields {
+        Self::build().1
+    }
+}

crates/core/src/migrate/mod.rs

@@ -1,6 +1,20 @@
 use std::path::PathBuf;
 
-use crate::settings::cli::SETTINGS;
+use crate::{
+    error::{code::ErrorCode, BichonResult},
+    migrate::{
+        legacy::schema::SchemaTools,
+        store::{LegacyDirs, NewDirs, NewIndexWriter},
+    },
+    raise_error,
+    settings::cli::SETTINGS,
+};
+use tantivy::{
+    collector::TopDocs, query::AllQuery, schema::Value, DocAddress, Index, TantivyDocument,
+};
+
+pub mod legacy;
+pub mod store;
 
 pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
     if !dir.exists() || !dir.is_dir() {
@@ -29,23 +43,190 @@ pub fn is_tantivy_index_dir(dir: &PathBuf) -> std::io::Result<bool> {
     Ok(has_meta_json && match_count >= 3)
 }
 
-pub fn is_legacy_data_layout() -> std::io::Result<bool> {
+pub fn check_data_status() -> std::io::Result<bool> {
     let root_dir = PathBuf::from(&SETTINGS.bichon_root_dir);
-    let envelope_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir {
-        PathBuf::from(index_dir)
+
+    let new_indices_base = SETTINGS
+        .bichon_index_dir
+        .as_ref()
+        .map(PathBuf::from)
+        .unwrap_or_else(|| root_dir.clone());
+    let new_indices_path = new_indices_base.join("bichon-indices");
+
+    let new_data_base = SETTINGS
+        .bichon_data_dir
+        .as_ref()
+        .map(PathBuf::from)
+        .unwrap_or_else(|| root_dir.clone());
+    let new_storage_path = new_data_base.join("bichon-storage");
+
+    let has_new_indices = is_tantivy_index_dir(&new_indices_path.join("attachment_metadata"))?
+        && is_tantivy_index_dir(&new_indices_path.join("mail_metadata"))?;
+    let has_new_storage = is_dir_not_empty(&new_storage_path)?;
+
+    if has_new_indices && has_new_storage {
+        return Ok(true);
+    }
+
+    let legacy_index_root = SETTINGS
+        .bichon_index_dir
+        .as_ref()
+        .map(PathBuf::from)
+        .unwrap_or_else(|| root_dir.join("envelope"));
+    let legacy_data_root = SETTINGS
+        .bichon_data_dir
+        .as_ref()
+        .map(PathBuf::from)
+        .unwrap_or_else(|| root_dir.join("eml"));
+
+    let has_legacy_index = is_tantivy_index_dir(&legacy_index_root)?;
+    let has_legacy_data = is_tantivy_index_dir(&legacy_data_root)?;
+
+    if has_legacy_index || has_legacy_data {
+        Ok(false)
     } else {
-        root_dir.join("envelope")
+        Ok(true)
+    }
+}
+
+fn is_dir_not_empty(path: &PathBuf) -> std::io::Result<bool> {
+    if !path.exists() || !path.is_dir() {
+        return Ok(false);
+    }
+    let mut entries = std::fs::read_dir(path)?;
+    Ok(entries.next().is_some())
+}
+
+const PAGE_SIZE: usize = 100;
+
+pub fn do_migrate<F>(legacy: LegacyDirs, new_dirs: NewDirs, mut on_progress: F) -> BichonResult<()>
+where
+    F: FnMut(&str),
+{
+    let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_reader = envelope_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_searcher = envelope_reader.searcher();
+    let eml_searcher = eml_reader.searcher();
+
+    let total_count = envelope_searcher.num_docs();
+    on_progress(&format!("TOTAL:{}", total_count));
+
+    let ef = SchemaTools::envelope_fields();
+    let mf = SchemaTools::eml_fields();
+
+    let mut writer = NewIndexWriter::open(new_dirs)?;
+
+    let mut offset = 0usize;
+    let mut total_migrated = 0usize;
+    let mut total_skipped = 0usize;
+
+    loop {
+        let page: Vec<(_, DocAddress)> = envelope_searcher
+            .search(
+                &AllQuery,
+                &TopDocs::with_limit(PAGE_SIZE)
+                    .and_offset(offset)
+                    .order_by_score(),
+            )
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        if page.is_empty() {
+            break;
+        }
+        let fetched = page.len();
+
+        for (_, doc_address) in page {
+            let doc: TantivyDocument = envelope_searcher
+                .doc(doc_address)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+            let eid = match doc.get_first(ef.f_id).and_then(|v| v.as_u64()) {
+                Some(v) => v,
+                None => {
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+            let account_id = match doc.get_first(ef.f_account_id).and_then(|v| v.as_u64()) {
+                Some(v) => v,
+                None => {
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+            let mailbox_id = doc
+                .get_first(ef.f_mailbox_id)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0);
+            let uid = doc
+                .get_first(ef.f_uid)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0) as u32;
+            let internal_date = doc
+                .get_first(ef.f_internal_date)
+                .and_then(|v| v.as_i64())
+                .unwrap_or(0);
+
+            let eml_term = tantivy::Term::from_field_u64(mf.f_id, eid);
+            let eml_query =
+                tantivy::query::TermQuery::new(eml_term, tantivy::schema::IndexRecordOption::Basic);
+            let eml_hits: Vec<(_, DocAddress)> = eml_searcher
+                .search(&eml_query, &TopDocs::with_limit(1).order_by_score())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+            let eml_bytes = match eml_hits.first() {
+                Some((_, addr)) => {
+                    let eml_doc: TantivyDocument = eml_searcher
+                        .doc(*addr)
+                        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+                    match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
+                        Some(b) => b.to_vec(),
+                        None => {
+                            on_progress(&format!("WARN: Account {} ID {} eml field missing", account_id, eid));
+                            total_skipped += 1;
+                            continue;
+                        }
+                    }
+                }
+                None => {
+                    on_progress(&format!("WARN:Account {} ID {} eml not found", account_id, eid));
+                    total_skipped += 1;
+                    continue;
+                }
             };
 
-    let eml_dir = if let Some(ref data_dir) = SETTINGS.bichon_data_dir {
-        PathBuf::from(data_dir)
-    } else {
-        root_dir.join("eml")
-    };
+            if let Err(e) = writer.ingest(&eml_bytes, account_id, mailbox_id, uid, internal_date) {
+                on_progress(&format!("ERROR:Account {} ID {} ingest failed: {}", account_id, eid, e));
+                total_skipped += 1;
+                continue;
+            }
 
-    let envelope_result = is_tantivy_index_dir(&envelope_dir)?;
-    let eml_result = is_tantivy_index_dir(&eml_dir)?;
-    Ok(envelope_result || eml_result)
+            total_migrated += 1;
+
+            if total_migrated % 100 == 0 || total_migrated == total_count as usize {
+                on_progress(&format!("PROGRESS:{}:{}", total_migrated, total_skipped));
+            }
+        }
+
+        offset += fetched;
+        if fetched < PAGE_SIZE {
+            break;
+        }
+    }
+
+    writer.commit()?;
+
+    Ok(())
 }
 
 #[cfg(test)]

crates/core/src/migrate/store.rs

@@ -0,0 +1,424 @@
+use std::path::PathBuf;
+
+use bytes::Bytes;
+use mail_parser::MimeHeaders;
+
+use crate::{
+    envelope::extractor::extract_references, message::content::AttachmentInfo,
+    store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
+};
+
+use fjall::{
+    config::{BlockSizePolicy, CompressionPolicy},
+    CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
+};
+use mail_parser::MessageParser;
+use tantivy::{Index, IndexWriter, TantivyDocument};
+use uuid::Uuid;
+
+use crate::{
+    common::AddrVec,
+    envelope::extractor::{compute_thread_id, generate_message_id},
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+    store::envelope::Envelope,
+    store::tantivy::{
+        model::{AttachmentModel, EnvelopeWithAttachments},
+        schema::SchemaTools,
+    },
+    utc_now,
+};
+
+pub struct LegacyDirs {
+    pub envelope_dir: PathBuf,
+    pub eml_dir: PathBuf,
+}
+
+pub struct NewDirs {
+    pub envelope_dir: PathBuf,
+    pub attachment_dir: PathBuf,
+    pub storage_dir: PathBuf,
+}
+
+impl LegacyDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index,
+            eml_dir: data,
+        }
+    }
+}
+
+impl NewDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index.join("mail_metadata"),
+            attachment_dir: index.join("attachment_metadata"),
+            storage_dir: data,
+        }
+    }
+}
+
+pub struct DetachOutput {
+    pub infos: Vec<AttachmentInfo>,
+    pub blobs: Vec<(String, Bytes)>,
+}
+
+pub fn detach_attachments_standalone(
+    original_body: &[u8],
+    message: &mail_parser::Message<'_>,
+) -> (Vec<u8>, DetachOutput) {
+    let mut stripped_eml = original_body.to_vec();
+    let mut infos = Vec::new();
+    let mut blobs = Vec::new();
+
+    let mut ranges: Vec<_> = message
+        .attachments()
+        .map(|att| {
+            (
+                att.raw_body_offset() as usize,
+                att.raw_end_offset() as usize,
+                att,
+            )
+        })
+        .collect();
+    ranges.sort_by(|a, b| b.0.cmp(&a.0));
+
+    for (raw_start, raw_end, att) in ranges {
+        let content_hash = compute_content_hash(att.contents());
+        blobs.push((
+            content_hash.clone(),
+            Bytes::copy_from_slice(&original_body[raw_start..raw_end]),
+        ));
+
+        let placeholder = format!("<<BICHON_DETACH_HASH:{}>>", &content_hash);
+        stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned());
+
+        infos.push(AttachmentInfo {
+            filename: att.attachment_name().map(|n| n.to_string()),
+            size: att.contents().len(),
+            inline: att
+                .content_disposition()
+                .map(|d| d.is_inline())
+                .unwrap_or(false),
+            file_type: att
+                .content_type()
+                .map(|ct| {
+                    format!(
+                        "{}/{}",
+                        ct.c_type.as_ref(),
+                        ct.c_subtype.as_deref().unwrap_or("")
+                    )
+                })
+                .unwrap_or_else(|| "application/octet-stream".to_string()),
+            content_id: att.content_id().map(|id| id.to_string()),
+            content_hash,
+            is_message: att.is_message(),
+        });
+    }
+
+    (stripped_eml, DetachOutput { infos, blobs })
+}
+
+pub struct NewIndexWriter {
+    pub envelope_writer: IndexWriter,
+    pub attachment_writer: IndexWriter,
+    pub email_ks: Keyspace,
+    pub attachment_ks: Keyspace,
+    pending: usize,
+}
+
+const COMMIT_THRESHOLD: usize = 500;
+
+impl NewIndexWriter {
+    pub fn open(dirs: NewDirs) -> BichonResult<Self> {
+        // ── envelope index ──────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.envelope_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let envelope_index = if dirs
+            .envelope_dir
+            .read_dir()
+            .map(|mut d| d.next().is_none())
+            .unwrap_or(true)
+        {
+            Index::create_in_dir(&dirs.envelope_dir, SchemaTools::email_schema())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        } else {
+            Index::open_in_dir(&dirs.envelope_dir)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        };
+
+        envelope_index
+            .tokenizers()
+            .register("euro", EuroTokenizer::new());
+
+        let envelope_writer = envelope_index
+            .writer_with_num_threads(2, 128 * 1024 * 1024)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        // ── attachment index ─────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.attachment_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let attachment_index = if dirs
+            .attachment_dir
+            .read_dir()
+            .map(|mut d| d.next().is_none())
+            .unwrap_or(true)
+        {
+            Index::create_in_dir(&dirs.attachment_dir, SchemaTools::attachment_schema())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        } else {
+            Index::open_in_dir(&dirs.attachment_dir)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        };
+
+        attachment_index
+            .tokenizers()
+            .register("euro", EuroTokenizer::new());
+        let attachment_writer = attachment_index
+            .writer_with_num_threads(2, 64 * 1024 * 1024)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        // ── blob store ───────────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.storage_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let db = Database::builder(&dirs.storage_dir)
+            .cache_size(64 * 1024 * 1024)
+            .open()
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        let email_ks = db
+            .keyspace("email", || {
+                KeyspaceCreateOptions::default()
+                    .max_memtable_size(16 * 1024 * 1024)
+                    .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+                    .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
+                    .with_kv_separation(Some(
+                        KvSeparationOptions::default()
+                            .separation_threshold(1024)
+                            .compression(CompressionType::Lz4)
+                            .file_target_size(512 * 1024 * 1024),
+                    ))
+            })
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        let attachment_ks = db
+            .keyspace("attachments", || {
+                KeyspaceCreateOptions::default()
+                    .max_memtable_size(16 * 1024 * 1024)
+                    .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+                    .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
+                    .with_kv_separation(Some(
+                        KvSeparationOptions::default()
+                            .separation_threshold(1024)
+                            .compression(CompressionType::Lz4)
+                            .file_target_size(512 * 1024 * 1024),
+                    ))
+            })
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        Ok(Self {
+            envelope_writer,
+            attachment_writer,
+            email_ks,
+            attachment_ks,
+            pending: 0,
+        })
+    }
+
+    pub fn ingest(
+        &mut self,
+        eml_bytes: &[u8],
+        account_id: u64,
+        mailbox_id: u64,
+        uid: u32,
+        internal_date: i64,
+    ) -> BichonResult<()> {
+        let email_content_hash = compute_content_hash(eml_bytes);
+
+        let message = MessageParser::new()
+            .parse(eml_bytes)
+            .ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
+
+        // ── text / preview ────────────────────────────────────────────────
+        let text = message
+            .body_text(0)
+            .map(|c| c.into_owned())
+            .or_else(|| {
+                message
+                    .body_html(0)
+                    .map(|html| crate::utils::html::extract_text(html.into_owned()))
+            })
+            .unwrap_or_default();
+        let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
+        let preview = if text.chars().count() > 100 {
+            text.chars().take(100).collect::<String>() + "..."
+        } else {
+            text.clone()
+        };
+
+        // ── headers ───────────────────────────────────────────────────────
+        let message_id = message
+            .message_id()
+            .map(String::from)
+            .unwrap_or_else(generate_message_id);
+
+        let in_reply_to = message.in_reply_to().as_text().map(String::from);
+        let references = extract_references(&message);
+        let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+
+        let subject = message.subject().map(String::from).unwrap_or_default();
+        let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+        let internal_date = if internal_date == 0 {
+            date
+        } else {
+            internal_date
+        };
+
+        let parse_addrs = |addrs: Option<&mail_parser::Address<'_>>| {
+            addrs
+                .map(|addr| {
+                    AddrVec::from(addr)
+                        .0
+                        .into_iter()
+                        .filter_map(|a| a.address)
+                        .collect::<Vec<_>>()
+                })
+                .unwrap_or_default()
+        };
+
+        let from = message
+            .from()
+            .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+            .and_then(|a| a.address)
+            .unwrap_or_else(|| "unknown".to_string());
+        let to = parse_addrs(message.to());
+        let cc = parse_addrs(message.cc());
+        let bcc = parse_addrs(message.bcc());
+
+        // ── detach attachments → blob ──────────────────────────────────────
+        let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
+
+        if !self
+            .email_ks
+            .contains_key(&email_content_hash)
+            .unwrap_or(false)
+        {
+            self.email_ks
+                .insert(&email_content_hash, stripped_eml.as_slice())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+
+        // write attachment blobs
+        for (hash, data) in &attachment_output.blobs {
+            if !self.attachment_ks.contains_key(hash).unwrap_or(false) {
+                self.attachment_ks
+                    .insert(hash, data.as_ref())
+                    .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+            }
+        }
+
+        // ── build envelope doc ────────────────────────────────────────────
+        let envelope_id = Uuid::new_v4().to_string();
+        let now = utc_now!();
+
+        let attachment_docs: Vec<TantivyDocument> = attachment_output
+            .infos
+            .iter()
+            .filter(|a| !a.inline || a.content_id.is_none())
+            .map(|a| {
+                AttachmentModel {
+                    id: Uuid::new_v4().to_string(),
+                    envelope_id: envelope_id.clone(),
+                    account_id,
+                    account_email: None,
+                    mailbox_id,
+                    mailbox_name: None,
+                    subject: subject.clone(),
+                    content_hash: a.content_hash.clone(),
+                    from: from.clone(),
+                    date,
+                    ingest_at: now,
+                    size: a.size as u64,
+                    ext: a.get_extension(),
+                    category: a.get_category().to_string(),
+                    content_type: a.file_type.clone(),
+                    shard_id: 0,
+                    text: None,
+                    has_text: false,
+                    is_ocr: false,
+                    page_count: None,
+                    is_indexed: false,
+                    is_message: a.is_message,
+                    name: a.filename.clone(),
+                    tags: None,
+                    auto_tags: None,
+                }
+                .into_document()
+            })
+            .collect();
+
+        let envelope = Envelope {
+            id: envelope_id,
+            message_id,
+            account_id,
+            mailbox_id,
+            uid,
+            subject,
+            preview,
+            from,
+            to,
+            cc,
+            bcc,
+            date,
+            internal_date,
+            ingest_at: now,
+            size: eml_bytes.len() as u32,
+            thread_id,
+            attachment_count: message.attachment_count(),
+            regular_attachment_count: attachment_docs.len(),
+            tags: None,
+            account_email: None,
+            mailbox_name: None,
+            content_hash: email_content_hash,
+        };
+
+        let ea = EnvelopeWithAttachments {
+            envelope,
+            attachments: Some(attachment_output.infos),
+        };
+        let envelope_doc = ea.to_document(&text, 0)?;
+
+        self.envelope_writer
+            .add_document(envelope_doc)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        for doc in attachment_docs {
+            self.attachment_writer
+                .add_document(doc)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+
+        self.pending += 1;
+        if self.pending >= COMMIT_THRESHOLD {
+            self.commit()?;
+        }
+
+        Ok(())
+    }
+
+    pub fn commit(&mut self) -> BichonResult<()> {
+        if self.pending == 0 {
+            return Ok(());
+        }
+        self.envelope_writer
+            .commit()
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        self.attachment_writer
+            .commit()
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        tracing::info!(count = self.pending, "committed batch");
+        self.pending = 0;
+        Ok(())
+    }
+}

crates/core/src/store/mod.rs

@@ -17,5 +17,5 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 pub mod envelope;
-pub mod storage;
+pub mod blob;
 pub mod tantivy;

crates/core/src/store/tantivy/envelope.rs

@@ -37,7 +37,7 @@ use crate::{
     settings::dir::DATA_DIR_MANAGER,
     store::{
         envelope::Envelope,
-        storage::BLOB_MANAGER,
+        blob::BLOB_MANAGER,
         tantivy::{
             fatal_commit,
             fields::{
@@ -317,7 +317,6 @@ impl IndexManager {
             let q = query_parser
                 .parse_query(subject_val)
                 .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
-            println!("{:#?}", &q);
             subqueries.push((Occur::Must, q));
         }
 

crates/core/src/store/tantivy/schema.rs

@@ -32,84 +32,104 @@ use crate::store::tantivy::fields::{
     F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID,
 };
 
-static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| {
-    let (_, fields) = SchemaTools::create_email_schema();
-    Arc::new(fields)
-});
+// ─── Lazy Globals ─────────────────────────────────────────────────────────────
 
-static ATTACHMENT_FIELDS: LazyLock<Arc<AttachmentFields>> = LazyLock::new(|| {
-    let (_, fields) = SchemaTools::create_attachment_schema();
-    Arc::new(fields)
-});
+static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| Arc::new(EmailSchema::fields()));
+
+static ATTACHMENT_FIELDS: LazyLock<Arc<AttachmentFields>> =
+    LazyLock::new(|| Arc::new(AttachmentSchema::fields()));
+
+// ─── Public API ───────────────────────────────────────────────────────────────
 
 pub struct SchemaTools;
 
 impl SchemaTools {
     pub fn email_schema() -> Schema {
-        let (schema, _) = Self::create_email_schema();
-        schema
+        EmailSchema::build().0
+    }
+    pub fn attachment_schema() -> Schema {
+        AttachmentSchema::build().0
     }
 
     pub fn email_fields() -> &'static EmailFields {
         &EMAIL_FIELDS
     }
+    pub fn attachment_fields() -> &'static AttachmentFields {
+        &ATTACHMENT_FIELDS
+    }
 
     pub fn email_default_fields() -> Vec<Field> {
-        let fields = Self::email_fields();
+        let f = Self::email_fields();
         vec![
-            fields.f_subject,
-            fields.f_body,
-            fields.f_attachment_name_text,
-            fields.f_from_text,
-            fields.f_to_text,
-            fields.f_cc_text,
-            fields.f_bcc_text,
+            f.f_subject,
+            f.f_body,
+            f.f_attachment_name_text,
+            f.f_from_text,
+            f.f_to_text,
+            f.f_cc_text,
+            f.f_bcc_text,
         ]
     }
 
+    pub fn attachment_default_fields() -> Vec<Field> {
+        let f = Self::attachment_fields();
+        vec![f.f_subject, f.f_text, f.f_name_text, f.f_from_text]
+    }
+
     pub fn create_email_schema() -> (Schema, EmailFields) {
-        let mut builder = Schema::builder();
-        let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
-        let f_message_id = builder.add_text_field(F_MESSAGE_ID, STRING | STORED);
-        let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
-        let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
-        let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
-        let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
-        let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro"));
-        let f_preview = builder.add_text_field(F_PREVIEW, STORED);
-        let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+        EmailSchema::build()
+    }
+    pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
+        AttachmentSchema::build()
+    }
+}
 
-        let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
-        let f_to = builder.add_text_field(F_TO, STRING | STORED);
-        let f_cc = builder.add_text_field(F_CC, STRING | STORED);
-        let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
+// ─── Schema builders ──────────────────────────────────────────────────────────
 
-        let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
-        let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro"));
-        let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro"));
-        let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro"));
+struct EmailSchema;
 
-        let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
-        let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
-        let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
-        let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
-        let f_thread_id = builder.add_text_field(F_THREAD_ID, STRING | STORED | FAST);
-        let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
+impl EmailSchema {
+    fn build() -> (Schema, EmailFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_text_field(F_ID, STRING | STORED | FAST);
+        let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
+        let f_subject = b.add_text_field(F_SUBJECT, text_store("euro"));
+        let f_body = b.add_text_field(F_BODY, text_no_store("euro"));
+        let f_preview = b.add_text_field(F_PREVIEW, STORED);
+        let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+        let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_to = b.add_text_field(F_TO, STRING | STORED);
+        let f_cc = b.add_text_field(F_CC, STRING | STORED);
+        let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
+        let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro"));
+        let f_to_text = b.add_text_field(F_TO_TEXT, text_no_store("euro"));
+        let f_cc_text = b.add_text_field(F_CC_TEXT, text_no_store("euro"));
+        let f_bcc_text = b.add_text_field(F_BCC_TEXT, text_no_store("euro"));
+        let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST);
+        let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
+        let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
+        let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
+        let f_thread_id = b.add_text_field(F_THREAD_ID, STRING | STORED | FAST);
+        let f_attachment_count = b.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
         let f_regular_attachment_count =
-            builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
+            b.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
         let f_attachment_name_text =
-            builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro"));
-        let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
-        let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED);
+            b.add_text_field(F_ATTACHMENT_NAME_TEXT, text_no_store("euro"));
+        let f_attachment_name_exact = b.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
+        let f_attachments = b.add_text_field(F_ATTACHMENTS, STORED);
         let f_attachment_content_hash =
-            builder.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST);
-        let f_attachment_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
-        let f_attachment_category =
-            builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
+            b.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | STORED | FAST);
+        let f_attachment_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
+        let f_attachment_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
         let f_attachment_content_type =
-            builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
-        let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
-        let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
+            b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
+        let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+        let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
+
         let fields = EmailFields {
             f_id,
             f_message_id,
@@ -145,57 +165,47 @@ impl SchemaTools {
             f_tags,
             f_shard_id,
         };
-        (builder.build(), fields)
+
+        (b.build(), fields)
     }
 
-    pub fn attachment_schema() -> Schema {
-        let (schema, _) = Self::create_attachment_schema();
-        schema
+    fn fields() -> EmailFields {
+        Self::build().1
     }
+}
 
-    pub fn attachment_fields() -> &'static AttachmentFields {
-        &ATTACHMENT_FIELDS
-    }
+struct AttachmentSchema;
 
-    pub fn attachment_default_fields() -> Vec<Field> {
-        let fields = Self::attachment_fields();
-        vec![
-            fields.f_subject,
-            fields.f_text,
-            fields.f_name_text,
-            fields.f_from_text,
-        ]
-    }
+impl AttachmentSchema {
+    fn build() -> (Schema, AttachmentFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_text_field(F_ID, STRING | STORED | FAST);
+        let f_envelope_id = b.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_subject = b.add_text_field(F_SUBJECT, text_store("euro"));
+        let f_content_hash = b.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+        let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_from_text = b.add_text_field(F_FROM_TEXT, text_no_store("euro"));
+        let f_date = b.add_i64_field(F_DATE, INDEXED | STORED | FAST);
+        let f_ingest_at = b.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
+        let f_size = b.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
+        let f_ext = b.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
+        let f_category = b.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
+        let f_content_type = b.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
+        let f_shard_id = b.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
+        let f_text = b.add_text_field(F_TEXT, text_no_store("euro"));
+        let f_has_text = b.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
+        let f_is_ocr = b.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
+        let f_page_count = b.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
+        let f_is_indexed = b.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
+        let f_is_message = b.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
+        let f_name_text = b.add_text_field(F_NAME_TEXT, text_no_store("euro"));
+        let f_name_exact = b.add_text_field(F_NAME_EXACT, STRING | STORED);
+        let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+        let f_auto_tags = b.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored());
 
-    pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
-        let mut builder = Schema::builder();
-        let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
-        let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
-        let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
-        let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
-        let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
-        let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
-        let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
-        let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
-        let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
-        let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
-        let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
-        let f_ext = builder.add_text_field(F_ATTACHMENT_EXT, STRING | STORED | FAST);
-        let f_category = builder.add_text_field(F_ATTACHMENT_CATEGORY, STRING | STORED | FAST);
-        let f_content_type =
-            builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
-        let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
-        let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro"));
-        let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
-        let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
-        let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
-        let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
-        let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
-        let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro"));
-        let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED);
-        let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
-        let f_auto_tags =
-            builder.add_facet_field(F_AUTO_TAGS, FacetOptions::default().set_stored());
         let fields = AttachmentFields {
             f_id,
             f_envelope_id,
@@ -223,24 +233,25 @@ impl SchemaTools {
             f_tags,
             f_auto_tags,
         };
-        (builder.build(), fields)
+
+        (b.build(), fields)
     }
 
-    fn text_no_store(tokenizer: &str) -> TextOptions {
+    fn fields() -> AttachmentFields {
+        Self::build().1
+    }
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+fn text_no_store(tokenizer: &str) -> TextOptions {
     TextOptions::default().set_indexing_options(
         TextFieldIndexing::default()
             .set_tokenizer(tokenizer)
             .set_index_option(IndexRecordOption::WithFreqsAndPositions),
     )
-    }
+}
 
-    fn text_store(tokenizer: &str) -> TextOptions {
-        TextOptions::default()
-            .set_indexing_options(
-                TextFieldIndexing::default()
-                    .set_tokenizer(tokenizer)
-                    .set_index_option(IndexRecordOption::WithFreqsAndPositions),
-            )
-            .set_stored()
-    }
+fn text_store(tokenizer: &str) -> TextOptions {
+    text_no_store(tokenizer).set_stored()
 }

crates/server/src/main.rs

@@ -25,11 +25,11 @@ use bichon_core::{
     context::{executors::BichonContext, Initialize},
     error::{code::ErrorCode, BichonResult},
     logger,
-    migrate::is_legacy_data_layout,
+    migrate::check_data_status,
     raise_error,
     settings::cli::SETTINGS,
     store::{
-        storage::BLOB_MANAGER,
+        blob::BLOB_MANAGER,
         tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
     },
     tasks::PeriodicTasks,
@@ -69,11 +69,11 @@ async fn main() -> BichonResult<()> {
     info!("Git:      [{}]", env!("GIT_HASH"));
     info!("GitHub:   https://github.com/rustmailer/bichon");
 
-    match is_legacy_data_layout() {
-        Ok(true) => {
+    match check_data_status() {
+        Ok(false) => {
             error!("Incompatible data format detected.");
             error!("Your data was created by an older version of Bichon and must be migrated before use.");
-            error!("Please run: bichon-migrate");
+            error!("Please run: bichon-admin");
             error!("Documentation: https://github.com/rustmailer/bichon/wiki/migration");
             return Err(raise_error!(
                 "Legacy data layout detected".into(),
@@ -84,7 +84,7 @@ async fn main() -> BichonResult<()> {
             error!("Failed to check data layout: {:#?}", e);
             return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
         }
-        Ok(false) => {}
+        Ok(true) => {}
     }
 
     if let Err(error) = initialize().await {

crates/server/src/rest/api/message.rs

@@ -36,7 +36,7 @@ use bichon_core::message::tags::TagCount;
 use bichon_core::message::tags::TagsRequest;
 use bichon_core::raise_error;
 use bichon_core::store::envelope::Envelope;
-use bichon_core::store::storage::get_reader;
+use bichon_core::store::blob::get_reader;
 use bichon_core::store::tantivy::envelope::ENVELOPE_MANAGER;
 use bichon_core::store::tantivy::validate_facet;
 use bichon_core::users::permissions::Permission;