feat: replace fjall with bichon-blob for blob storage

- use a single Engine instance for email + attachment blobs
  - add delete_batch, gc_if_needed, background flush to blob crate
  - fix Entry.raw_size storing compressed length instead of original
  - move fjall-dependent migration code from core to admin crate
  - add STORAGE_VERSION file for layout version detection

crates/admin/src/legacy/fields.rs

@@ -0,0 +1,69 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use tantivy::schema::Field;
+
+pub const F_MESSAGE_ID: &str = "message_id";
+pub const F_ACCOUNT_ID: &str = "account_id";
+pub const F_MAILBOX_ID: &str = "mailbox_id";
+pub const F_UID: &str = "uid";
+pub const F_SUBJECT: &str = "subject";
+pub const F_TEXT: &str = "text";
+pub const F_FROM: &str = "from";
+pub const F_TO: &str = "to";
+pub const F_CC: &str = "cc";
+pub const F_BCC: &str = "bcc";
+pub const F_DATE: &str = "date";
+pub const F_INTERNAL_DATE: &str = "internal_date";
+pub const F_SIZE: &str = "size";
+pub const F_THREAD_ID: &str = "thread_id";
+pub const F_ATTACHMENTS: &str = "attachments";
+pub const F_HAS_ATTACHMENT: &str = "has_attachment";
+pub const F_TAGS: &str = "tags";
+
+pub const F_ID: &str = "id";
+pub struct EnvelopeFields {
+    pub f_id: Field,
+    pub f_message_id: Field,
+    pub f_account_id: Field,
+    pub f_mailbox_id: Field,
+    pub f_uid: Field,
+    pub f_subject: Field,
+    pub f_text: Field,
+    pub f_from: Field,
+    pub f_to: Field,
+    pub f_cc: Field,
+    pub f_bcc: Field,
+    pub f_date: Field,
+    pub f_internal_date: Field,
+    pub f_size: Field,
+    pub f_thread_id: Field,
+    pub f_attachments: Field,
+    pub f_has_attachment: Field,
+    pub f_tags: Field,
+}
+
+pub const F_EML: &str = "eml";
+
+pub struct EmlFields {
+    pub f_id: Field,
+    pub f_account_id: Field,
+    pub f_mailbox_id: Field,
+    pub f_eml: Field,
+}

crates/admin/src/legacy/mod.rs

@@ -0,0 +1,2 @@
+pub mod fields;
+pub mod schema;

crates/admin/src/legacy/schema.rs

@@ -0,0 +1,114 @@
+use tantivy::schema::{FacetOptions, Field, Schema, FAST, INDEXED, STORED, STRING, TEXT};
+
+use crate::legacy::fields::{EmlFields, EnvelopeFields, *};
+
+pub struct SchemaTools;
+
+impl SchemaTools {
+    pub fn envelope_schema() -> Schema {
+        EnvelopeSchema::build().0
+    }
+    pub fn eml_schema() -> Schema {
+        EmlSchema::build().0
+    }
+
+    pub fn envelope_fields() -> EnvelopeFields {
+        EnvelopeSchema::fields()
+    }
+    pub fn eml_fields() -> EmlFields {
+        EmlSchema::fields()
+    }
+
+    pub fn envelope_default_fields() -> Vec<Field> {
+        let f = Self::envelope_fields();
+        vec![f.f_subject, f.f_text, f.f_attachments]
+    }
+}
+
+// ─── Schema builders ──────────────────────────────────────────────────────────
+
+struct EnvelopeSchema;
+
+impl EnvelopeSchema {
+    fn build() -> (Schema, EnvelopeFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_u64_field(F_ID, INDEXED | STORED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_uid = b.add_u64_field(F_UID, INDEXED | STORED | FAST);
+        let f_thread_id = b.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST);
+
+        let f_subject = b.add_text_field(F_SUBJECT, TEXT | STORED);
+        let f_text = b.add_text_field(F_TEXT, TEXT | STORED);
+        let f_attachments = b.add_text_field(F_ATTACHMENTS, TEXT | STORED);
+
+        let f_from = b.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_to = b.add_text_field(F_TO, STRING | STORED);
+        let f_cc = b.add_text_field(F_CC, STRING | STORED);
+        let f_bcc = b.add_text_field(F_BCC, STRING | STORED);
+
+        let f_message_id = b.add_text_field(F_MESSAGE_ID, STRING | STORED);
+
+        let f_date = b.add_i64_field(F_DATE, STORED | FAST);
+        let f_internal_date = b.add_i64_field(F_INTERNAL_DATE, STORED | FAST);
+
+        let f_size = b.add_u64_field(F_SIZE, STORED | FAST);
+        let f_has_attachment = b.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST);
+
+        let f_tags = b.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+
+        let fields = EnvelopeFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_uid,
+            f_thread_id,
+            f_subject,
+            f_text,
+            f_attachments,
+            f_from,
+            f_to,
+            f_cc,
+            f_bcc,
+            f_message_id,
+            f_date,
+            f_internal_date,
+            f_size,
+            f_has_attachment,
+            f_tags,
+        };
+
+        (b.build(), fields)
+    }
+
+    fn fields() -> EnvelopeFields {
+        Self::build().1
+    }
+}
+
+struct EmlSchema;
+
+impl EmlSchema {
+    fn build() -> (Schema, EmlFields) {
+        let mut b = Schema::builder();
+
+        let f_id = b.add_u64_field(F_ID, INDEXED | FAST);
+        let f_account_id = b.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = b.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_eml = b.add_bytes_field(F_EML, STORED);
+
+        let fields = EmlFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_eml,
+        };
+
+        (b.build(), fields)
+    }
+
+    fn fields() -> EmlFields {
+        Self::build().1
+    }
+}

crates/admin/src/main.rs

@@ -21,8 +21,10 @@ use dialoguer::{theme::ColorfulTheme, Select};
 
 use crate::{migrate::handle_migration, reset::handle_reset_password};
 
+pub mod legacy;
 pub mod meta;
 pub mod migrate;
+pub mod migrate_store;
 pub mod reset;
 
 

crates/admin/src/migrate.rs

@@ -1,12 +1,23 @@
-use std::path::{Path, PathBuf};
+use std::{collections::HashMap, path::PathBuf};
 
-use bichon_core::migrate::{
-    count_eml_segments, do_migrate_segment, is_tantivy_index_dir,
-    store::{LegacyDirs, NewDirs, NewIndexWriter},
+use bichon_core::{
+    error::{code::ErrorCode, BichonResult},
+    migrate::is_tantivy_index_dir,
+    raise_error,
 };
 use console::style;
 use dialoguer::{theme::ColorfulTheme, Confirm, Input};
 use indicatif::{ProgressBar, ProgressStyle};
+use tantivy::{
+    collector::TopDocs,
+    columnar::Column,
+    query::TermQuery,
+    schema::{IndexRecordOption, Value},
+    DocAddress, Index, TantivyDocument, Term,
+};
+
+use crate::legacy::schema::SchemaTools;
+use crate::migrate_store::{LegacyDirs, NewDirs, NewIndexWriter};
 
 pub fn handle_migration(theme: &ColorfulTheme) {
     println!(
@@ -21,7 +32,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
         style(
             "This tool migrates data from the legacy v0.3.7 Tantivy-based storage \
             architecture to the new v1.x \
-            separated index and Fjall-backed storage format."
+            separated index and blob-backed storage format."
         )
         .dim()
     );
@@ -35,8 +46,8 @@ pub fn handle_migration(theme: &ColorfulTheme) {
                 New v1.x architecture:\n\
             • mail indexes stored in Tantivy\n\
             • attachment indexes stored in Tantivy\n\
-            • raw message data stored in Fjall\n\
-            • attachment blobs stored in Fjall"
+            • raw message data stored in blob engine\n\
+            • attachment blobs stored in blob engine"
         )
         .dim()
     );
@@ -54,7 +65,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
     let root_dir_str: String = Input::with_theme(theme)
         .with_prompt("Enter --bichon-root-dir (same value used by the old server)")
         .validate_with(|input: &String| -> Result<(), &str> {
-            let path = Path::new(input);
+            let path = PathBuf::from(input);
             if !path.is_absolute() {
                 return Err("Path must be absolute.");
             }
@@ -81,7 +92,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             if input.is_empty() {
                 return Ok(());
             }
-            let path = Path::new(input);
+            let path = PathBuf::from(input);
             if !path.is_absolute() {
                 return Err("Path must be absolute.");
             }
@@ -119,7 +130,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
             if input.is_empty() {
                 return Ok(());
             }
-            let path = Path::new(input);
+            let path = PathBuf::from(input);
             if !path.is_absolute() {
                 return Err("Path must be absolute.");
             }
@@ -447,3 +458,210 @@ pub fn is_legacy_data_layout_with_paths(
 
     Ok(envelope_result || eml_result)
 }
+
+/// Return the number of segments in the legacy EML Tantivy index.
+/// Each segment can be passed to `do_migrate_segment` for bounded-memory batch migration.
+pub fn count_eml_segments(legacy: &LegacyDirs) -> BichonResult<usize> {
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let searcher = reader.searcher();
+    Ok(searcher.segment_readers().len())
+}
+
+/// Migrate all documents from a single EML segment to the new storage layout.
+///
+/// This is the core of the batch migration strategy: each Process B invocation
+/// handles exactly one EML segment, so peak memory is bounded by that segment's
+/// size regardless of the total archive size.
+pub fn do_migrate_segment<F>(
+    batch_size: u32,
+    legacy: LegacyDirs,
+    writer: &mut NewIndexWriter,
+    segment_index: usize,
+    mut on_progress: F,
+) -> BichonResult<()>
+where
+    F: FnMut(&str),
+{
+    // ── open legacy indices ────────────────────────────────────────────
+    let envelope_index = Index::open_in_dir(&legacy.envelope_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_index = Index::open_in_dir(&legacy.eml_dir)
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_reader = envelope_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+    let eml_reader = eml_index
+        .reader()
+        .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+    let envelope_searcher = envelope_reader.searcher();
+    let eml_searcher = eml_reader.searcher();
+
+    let ef = SchemaTools::envelope_fields();
+    let mf = SchemaTools::eml_fields();
+
+    let eml_segments = eml_searcher.segment_readers();
+    let eml_segment = eml_segments.get(segment_index).ok_or_else(|| {
+        raise_error!(
+            format!(
+                "segment index {} out of range ({} segments)",
+                segment_index,
+                eml_segments.len()
+            ),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    let num_docs = eml_segment.num_docs();
+    if num_docs == 0 {
+        on_progress("TOTAL:0");
+        on_progress("DONE:0:0");
+        return Ok(());
+    }
+
+    on_progress(&format!("TOTAL:{}", num_docs));
+
+    let max_doc = eml_segment.max_doc();
+    let ff = eml_segment.fast_fields();
+    let f_id_col: Column<u64> = ff.u64("id").map_err(|e| {
+        raise_error!(
+            format!("failed to open f_id fast field: {e:#?}"),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    // ── Phase 1: build eid → (uid, internal_date) from envelope, then drop it ──
+    let mut envelope_map: HashMap<u64, (u32, i64)> = HashMap::with_capacity(num_docs as usize);
+
+    let mut env_scanned = 0u32;
+    let mut env_skipped = 0u32;
+    for doc_id in 0..max_doc {
+        if eml_segment.is_deleted(doc_id) {
+            continue;
+        }
+        let eid = f_id_col.values.get_val(doc_id);
+
+        let term = Term::from_field_u64(ef.f_id, eid);
+        let query = TermQuery::new(term, IndexRecordOption::Basic);
+        let hits: Vec<(_, DocAddress)> = envelope_searcher
+            .search(&query, &TopDocs::with_limit(1).order_by_score())
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        if let Some((_, addr)) = hits.first() {
+            let env_doc: TantivyDocument = envelope_searcher
+                .doc(*addr)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+            let uid = env_doc
+                .get_first(ef.f_uid)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0) as u32;
+            let internal_date = env_doc
+                .get_first(ef.f_internal_date)
+                .and_then(|v| v.as_i64())
+                .unwrap_or(0);
+            envelope_map.insert(eid, (uid, internal_date));
+            env_scanned += 1;
+        } else {
+            env_skipped += 1;
+        }
+
+        if env_scanned % 10 == 0 {
+            on_progress(&format!(
+                "PHASE1:{}/{} skipped:{}",
+                env_scanned, max_doc, env_skipped
+            ));
+        }
+    }
+
+    // Free the envelope index before the heavy EML processing.
+    drop(envelope_searcher);
+    drop(envelope_reader);
+    drop(envelope_index);
+
+    // ── Phase 2: process EML docs, streaming one at a time ─────────────
+    let mut total_migrated = 0usize;
+    let mut total_skipped = 0usize;
+
+    let mut chunk_start = 0u32;
+
+    while chunk_start < max_doc {
+        let chunk_end = (chunk_start + batch_size).min(max_doc);
+        let store_reader = eml_segment
+            .get_store_reader(2)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        for doc_id in chunk_start..chunk_end {
+            if eml_segment.is_deleted(doc_id) {
+                continue;
+            }
+
+            let eid = f_id_col.values.get_val(doc_id);
+
+            let (uid, internal_date) = match envelope_map.get(&eid) {
+                Some(v) => *v,
+                None => {
+                    on_progress(&format!("WARN: eid {} envelope not found", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+
+            let eml_doc: TantivyDocument = store_reader
+                .get(doc_id)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+            let account_id = match eml_doc.get_first(mf.f_account_id).and_then(|v| v.as_u64()) {
+                Some(v) => v,
+                None => {
+                    on_progress(&format!("WARN: eid {} account_id missing", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+            let mailbox_id = eml_doc
+                .get_first(mf.f_mailbox_id)
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0);
+
+            // Borrow directly from eml_doc — no .to_vec() clone.
+            let eml_bytes = match eml_doc.get_first(mf.f_eml).and_then(|v| v.as_bytes()) {
+                Some(b) => b,
+                None => {
+                    on_progress(&format!("WARN: eid {} eml bytes missing", eid));
+                    total_skipped += 1;
+                    continue;
+                }
+            };
+
+            if let Err(e) = writer.ingest(eml_bytes, account_id, mailbox_id, uid, internal_date) {
+                on_progress(&format!(
+                    "ERROR: Account {} eid {} ingest failed: {}",
+                    account_id, eid, e
+                ));
+                total_skipped += 1;
+                continue;
+            }
+
+            total_migrated += 1;
+
+            if total_migrated % 10 == 0 || total_migrated as u32 == num_docs {
+                on_progress(&format!("PROGRESS:{}:{}", total_migrated, num_docs));
+            }
+        }
+
+        drop(store_reader);
+
+        // Flush blob buffers via ingestion API — bypasses memtable/WAL.
+        writer.flush_fjall_buffers()?;
+
+        chunk_start = chunk_end;
+    }
+
+    on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
+    Ok(())
+}

crates/admin/src/migrate_store.rs

@@ -0,0 +1,560 @@
+use std::{path::PathBuf, time::Instant};
+
+use bytes::Bytes;
+use mail_parser::MimeHeaders;
+
+use bichon_core::{
+    envelope::extractor::extract_references, message::content::AttachmentInfo,
+    store::tantivy::tokenizers::EuroTokenizer, utils::compute_content_hash,
+};
+
+use fjall::{
+    config::{BlockSizePolicy, CompressionPolicy},
+    CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
+};
+use mail_parser::MessageParser;
+use tantivy::{indexer::NoMergePolicy, Index, IndexWriter, TantivyDocument};
+use uuid::Uuid;
+
+use bichon_core::{
+    common::AddrVec,
+    envelope::extractor::{compute_thread_id, generate_message_id},
+    error::{code::ErrorCode, BichonResult},
+    raise_error,
+    store::envelope::Envelope,
+    store::tantivy::{
+        model::{AttachmentModel, EnvelopeWithAttachments},
+        schema::SchemaTools,
+    },
+    utc_now,
+};
+
+pub struct LegacyDirs {
+    pub envelope_dir: PathBuf,
+    pub eml_dir: PathBuf,
+}
+
+pub struct NewDirs {
+    pub envelope_dir: PathBuf,
+    pub attachment_dir: PathBuf,
+    pub storage_dir: PathBuf,
+}
+
+impl LegacyDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index,
+            eml_dir: data,
+        }
+    }
+}
+
+impl NewDirs {
+    pub fn new(index: PathBuf, data: PathBuf) -> Self {
+        Self {
+            envelope_dir: index.join("mail_metadata"),
+            attachment_dir: index.join("attachment_metadata"),
+            storage_dir: data,
+        }
+    }
+}
+
+pub struct DetachOutput {
+    pub infos: Vec<AttachmentInfo>,
+    pub blobs: Vec<(String, Bytes)>,
+}
+
+pub fn detach_attachments_standalone(
+    original_body: &[u8],
+    message: &mail_parser::Message<'_>,
+) -> (Vec<u8>, DetachOutput) {
+    let mut stripped_eml = original_body.to_vec();
+    let mut infos = Vec::new();
+    let mut blobs = Vec::new();
+
+    let mut ranges: Vec<_> = message
+        .attachments()
+        .map(|att| {
+            (
+                att.raw_body_offset() as usize,
+                att.raw_end_offset() as usize,
+                att,
+            )
+        })
+        .collect();
+    ranges.sort_by(|a, b| b.0.cmp(&a.0));
+
+    for (raw_start, raw_end, att) in ranges {
+        let content_hash = compute_content_hash(att.contents());
+        let body_len = original_body.len();
+        let raw_start = raw_start.min(body_len);
+        let raw_end = raw_end.min(body_len);
+        let range_valid = raw_start < raw_end;
+
+        if range_valid {
+            blobs.push((
+                content_hash.clone(),
+                Bytes::copy_from_slice(&original_body[raw_start..raw_end]),
+            ));
+        }
+
+        if range_valid {
+            let placeholder = format!("<<BICHON_DETACH_HASH:{}>>", &content_hash);
+            stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned());
+        }
+
+        infos.push(AttachmentInfo {
+            filename: att.attachment_name().map(|n| n.to_string()),
+            size: att.contents().len(),
+            inline: att
+                .content_disposition()
+                .map(|d| d.is_inline())
+                .unwrap_or_else(|| att.content_id().is_some()),
+            file_type: att
+                .content_type()
+                .map(|ct| {
+                    format!(
+                        "{}/{}",
+                        ct.c_type.as_ref(),
+                        ct.c_subtype.as_deref().unwrap_or("")
+                    )
+                })
+                .unwrap_or_else(|| "application/octet-stream".to_string()),
+            content_id: att.content_id().map(|id| id.to_string()),
+            content_hash,
+            is_message: att.is_message(),
+            extracted_text: None,
+            extracted_page_count: None,
+            extracted_is_ocr: false,
+        });
+    }
+
+    (stripped_eml, DetachOutput { infos, blobs })
+}
+
+pub struct NewIndexWriter {
+    pub envelope_writer: Option<IndexWriter>,
+    pub attachment_writer: Option<IndexWriter>,
+    pub email_ks: Keyspace,
+    pub attachment_ks: Keyspace,
+    pending: usize,
+    email_buf: Vec<(String, Vec<u8>)>,
+    attachment_buf: Vec<(String, Vec<u8>)>,
+}
+
+//const COMMIT_THRESHOLD: usize = 500;
+
+impl NewIndexWriter {
+    pub fn open(dirs: NewDirs) -> BichonResult<Self> {
+        // ── envelope index ──────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.envelope_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let envelope_index = if dirs
+            .envelope_dir
+            .read_dir()
+            .map(|mut d| d.next().is_none())
+            .unwrap_or(true)
+        {
+            Index::create_in_dir(&dirs.envelope_dir, SchemaTools::email_schema())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        } else {
+            Index::open_in_dir(&dirs.envelope_dir)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        };
+
+        envelope_index
+            .tokenizers()
+            .register("euro", EuroTokenizer::new());
+
+        let envelope_writer = envelope_index
+            .writer_with_num_threads(3, 256 * 1024 * 1024)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        // let mut merge_policy = LogMergePolicy::default();
+        // merge_policy.set_min_num_segments(25);
+        // merge_policy.set_min_layer_size(10_000);
+        // merge_policy.set_max_docs_before_merge(100_000);
+
+        envelope_writer.set_merge_policy(Box::new(NoMergePolicy));
+        // ── attachment index ─────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.attachment_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let attachment_index = if dirs
+            .attachment_dir
+            .read_dir()
+            .map(|mut d| d.next().is_none())
+            .unwrap_or(true)
+        {
+            Index::create_in_dir(&dirs.attachment_dir, SchemaTools::attachment_schema())
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        } else {
+            Index::open_in_dir(&dirs.attachment_dir)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?
+        };
+
+        attachment_index
+            .tokenizers()
+            .register("euro", EuroTokenizer::new());
+        let attachment_writer = attachment_index
+            .writer_with_num_threads(3, 256 * 1024 * 1024)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        // let mut merge_policy = LogMergePolicy::default();
+        // merge_policy.set_min_num_segments(25);
+        // merge_policy.set_min_layer_size(10_000);
+        // merge_policy.set_max_docs_before_merge(100_000);
+
+        attachment_writer.set_merge_policy(Box::new(NoMergePolicy));
+
+        // ── blob store ───────────────────────────────────────────────────
+        std::fs::create_dir_all(&dirs.storage_dir)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        let db = Database::builder(&dirs.storage_dir)
+            .cache_size(8 * 1024 * 1024)
+            .journal_compression(CompressionType::None)
+            .max_journaling_size(64 * 1024 * 1024)
+            .open()
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        let email_ks = db
+            .keyspace("email", || {
+                KeyspaceCreateOptions::default()
+                    .max_memtable_size(4 * 1024 * 1024)
+                    .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+                    .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
+                    .with_kv_separation(Some(
+                        KvSeparationOptions::default()
+                            .separation_threshold(1024)
+                            .compression(CompressionType::Lz4)
+                            .file_target_size(512 * 1024 * 1024),
+                    ))
+            })
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        let attachment_ks = db
+            .keyspace("attachments", || {
+                KeyspaceCreateOptions::default()
+                    .max_memtable_size(4 * 1024 * 1024)
+                    .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+                    .data_block_compression_policy(CompressionPolicy::all(CompressionType::Lz4))
+                    .with_kv_separation(Some(
+                        KvSeparationOptions::default()
+                            .separation_threshold(1024)
+                            .compression(CompressionType::Lz4)
+                            .file_target_size(512 * 1024 * 1024),
+                    ))
+            })
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        Ok(Self {
+            envelope_writer: Some(envelope_writer),
+            attachment_writer: Some(attachment_writer),
+            email_ks,
+            attachment_ks,
+            pending: 0,
+            email_buf: Vec::new(),
+            attachment_buf: Vec::new(),
+        })
+    }
+
+    pub fn ingest(
+        &mut self,
+        eml_bytes: &[u8],
+        account_id: u64,
+        mailbox_id: u64,
+        uid: u32,
+        internal_date: i64,
+    ) -> BichonResult<()> {
+        let email_content_hash = compute_content_hash(eml_bytes);
+
+        let message = MessageParser::new()
+            .parse(eml_bytes)
+            .ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
+
+        if message.parts.is_empty() {
+            return Err(raise_error!(
+                "Malformed or completely empty EML (no parts found)".into(),
+                ErrorCode::InternalError
+            ));
+        }
+        // ── text / preview ────────────────────────────────────────────────
+        let text = message
+            .body_text(0)
+            .map(|c| c.into_owned())
+            .or_else(|| {
+                message
+                    .body_html(0)
+                    .map(|html| bichon_core::utils::html::extract_text(html.into_owned()))
+            })
+            .unwrap_or_default();
+        let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
+        let preview = if text.chars().count() > 100 {
+            text.chars().take(100).collect::<String>() + "..."
+        } else {
+            text.clone()
+        };
+
+        // ── headers ───────────────────────────────────────────────────────
+        let message_id = message
+            .message_id()
+            .map(String::from)
+            .unwrap_or_else(generate_message_id);
+
+        let in_reply_to = message.in_reply_to().as_text().map(String::from);
+        let references = extract_references(&message);
+        let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+
+        let subject = message.subject().map(String::from).unwrap_or_default();
+        let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+        let internal_date = if internal_date == 0 {
+            date
+        } else {
+            internal_date
+        };
+
+        let parse_addrs = |addrs: Option<&mail_parser::Address<'_>>| {
+            addrs
+                .map(|addr| {
+                    AddrVec::from(addr)
+                        .0
+                        .into_iter()
+                        .filter_map(|a| a.address)
+                        .collect::<Vec<_>>()
+                })
+                .unwrap_or_default()
+        };
+
+        let from = message
+            .from()
+            .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+            .and_then(|a| a.address)
+            .unwrap_or_else(|| "unknown".to_string());
+        let to = parse_addrs(message.to());
+        let cc = parse_addrs(message.cc());
+        let bcc = parse_addrs(message.bcc());
+
+        // ── detach attachments → blob ──────────────────────────────────────
+        let (stripped_eml, attachment_output) = detach_attachments_standalone(eml_bytes, &message);
+
+        // Buffer for bulk ingestion — sorted + flushed later.
+        self.email_buf
+            .push((email_content_hash.clone(), stripped_eml));
+        for (hash, data) in &attachment_output.blobs {
+            self.attachment_buf.push((hash.clone(), data.to_vec()));
+        }
+
+        // ── build envelope doc ────────────────────────────────────────────
+        let envelope_id = Uuid::new_v4().to_string();
+        let now = utc_now!();
+
+        let attachment_docs: Vec<TantivyDocument> = attachment_output
+            .infos
+            .iter()
+            .filter(|a| !a.inline || a.content_id.is_none())
+            .map(|a| {
+                AttachmentModel {
+                    id: Uuid::new_v4().to_string(),
+                    envelope_id: envelope_id.clone(),
+                    account_id,
+                    account_email: None,
+                    mailbox_id,
+                    mailbox_name: None,
+                    subject: subject.clone(),
+                    content_hash: a.content_hash.clone(),
+                    from: from.clone(),
+                    date,
+                    ingest_at: now,
+                    size: a.size as u64,
+                    ext: a.get_extension(),
+                    category: a.get_category().to_string(),
+                    content_type: a.file_type.clone(),
+                    shard_id: 0,
+                    text: None,
+                    has_text: false,
+                    is_ocr: false,
+                    page_count: None,
+                    is_indexed: false,
+                    is_message: a.is_message,
+                    name: a.filename.clone(),
+                    tags: None,
+                    auto_tags: None,
+                }
+                .into_document()
+            })
+            .collect();
+
+        let envelope = Envelope {
+            id: envelope_id,
+            message_id,
+            account_id,
+            mailbox_id,
+            uid,
+            subject,
+            preview,
+            from,
+            to,
+            cc,
+            bcc,
+            date,
+            internal_date,
+            ingest_at: now,
+            size: eml_bytes.len() as u32,
+            thread_id,
+            attachment_count: message.attachment_count(),
+            regular_attachment_count: attachment_docs.len(),
+            tags: None,
+            account_email: None,
+            account_name: None,
+            mailbox_name: None,
+            content_hash: email_content_hash,
+        };
+
+        let ea = EnvelopeWithAttachments {
+            envelope,
+            attachments: Some(attachment_output.infos),
+        };
+        let envelope_doc = ea.to_document(&text, 0)?;
+
+        self.envelope_writer
+            .as_mut()
+            .unwrap()
+            .add_document(envelope_doc)
+            .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+
+        for doc in attachment_docs {
+            self.attachment_writer
+                .as_mut()
+                .unwrap()
+                .add_document(doc)
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+
+        self.pending += 1;
+        // if self.pending >= COMMIT_THRESHOLD {
+        //     self.commit()?;
+        // }
+
+        Ok(())
+    }
+
+    /// Commit pending Tantivy documents (mid-stream) — frees the in-memory
+    /// term dictionary / postings that accumulate in the IndexWriter.
+    fn commit_tantivy(&mut self) -> BichonResult<()> {
+        if self.pending == 0 {
+            return Ok(());
+        }
+        println!("Tantivy committing... this may take 2-3 minutes, please wait.");
+        let start = Instant::now();
+        if let Some(writer) = self.envelope_writer.as_mut() {
+            writer
+                .commit()
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+        if let Some(writer) = self.attachment_writer.as_mut() {
+            writer
+                .commit()
+                .map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
+        }
+        println!("tantivy commit elapsed: {:#?}", start.elapsed());
+        tracing::info!(count = self.pending, "committed tantivy batch");
+        self.pending = 0;
+        Ok(())
+    }
+
+    /// Final commit + segment merge for Tantivy writers (called once at end).
+    pub fn finish_writers(&mut self) -> BichonResult<()> {
+        self.commit_tantivy()?;
+
+        for (name, writer_opt) in [
+            ("envelope", &mut self.envelope_writer),
+            ("attachment", &mut self.attachment_writer),
+        ] {
+            if let Some(writer) = writer_opt.as_mut() {
+                let seg_ids = writer
+                    .index()
+                    .searchable_segment_ids()
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+                println!("merging {} {} segments...", seg_ids.len(), name);
+                if seg_ids.len() > 1 {
+                    let _ = writer.merge(&seg_ids);
+                }
+            }
+
+            if let Some(writer) = writer_opt.take() {
+                println!("waiting for {} merge to finish...", name);
+                let start = std::time::Instant::now();
+                let _ = writer.wait_merging_threads();
+                println!("{} merge done: {:#?}", name, start.elapsed());
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Sort buffered (hash, data) pairs, dedup, and write via Fjall's
+    /// ingestion API — writes SSTables directly, bypassing memtable and WAL.
+    /// Also commits the Tantivy writers to bound their in-memory state.
+    pub fn flush_fjall_buffers(&mut self) -> BichonResult<()> {
+        self.commit_tantivy()?;
+
+        if !self.email_buf.is_empty() {
+            self.email_buf.sort_by(|a, b| a.0.cmp(&b.0));
+            self.email_buf.dedup_by(|a, b| a.0 == b.0);
+
+            let mut ingestion = self.email_ks.start_ingestion().map_err(|e| {
+                raise_error!(
+                    format!("email ingestion start: {e:#?}"),
+                    ErrorCode::InternalError
+                )
+            })?;
+            for (hash, data) in &self.email_buf {
+                ingestion
+                    .write(hash.as_bytes(), data.as_slice())
+                    .map_err(|e| {
+                        raise_error!(
+                            format!("email ingestion write: {e:#?}"),
+                            ErrorCode::InternalError
+                        )
+                    })?;
+            }
+            ingestion.finish().map_err(|e| {
+                raise_error!(
+                    format!("email ingestion finish: {e:#?}"),
+                    ErrorCode::InternalError
+                )
+            })?;
+            self.email_buf.clear();
+        }
+
+        if !self.attachment_buf.is_empty() {
+            self.attachment_buf.sort_by(|a, b| a.0.cmp(&b.0));
+            self.attachment_buf.dedup_by(|a, b| a.0 == b.0);
+
+            let mut ingestion = self.attachment_ks.start_ingestion().map_err(|e| {
+                raise_error!(
+                    format!("attachment ingestion start: {e:#?}"),
+                    ErrorCode::InternalError
+                )
+            })?;
+            for (hash, data) in &self.attachment_buf {
+                ingestion
+                    .write(hash.as_bytes(), data.as_slice())
+                    .map_err(|e| {
+                        raise_error!(
+                            format!("attachment ingestion write: {e:#?}"),
+                            ErrorCode::InternalError
+                        )
+                    })?;
+            }
+            ingestion.finish().map_err(|e| {
+                raise_error!(
+                    format!("attachment ingestion finish: {e:#?}"),
+                    ErrorCode::InternalError
+                )
+            })?;
+            self.attachment_buf.clear();
+        }
+
+        Ok(())
+    }
+}