feat: use stemmer for multilingual token matching

Cargo.lock

@@ -505,6 +505,7 @@ dependencies = [
  "chrono",
  "clap",
  "dashmap",
+ "deunicode",
  "email_address",
  "encoding_rs",
  "fjall",
@@ -513,7 +514,7 @@ dependencies = [
  "html2text",
  "itertools",
  "itoa",
- "lru 0.17.0",
+ "lru 0.18.0",
  "mail-parser",
  "mail-send",
  "murmur3",
@@ -546,6 +547,7 @@ dependencies = [
  "utf7-imap",
  "uuid",
  "webpki-roots",
+ "whichlang",
 ]
 
 [[package]]
@@ -1331,6 +1333,12 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "deunicode"
+version = "1.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
+
 [[package]]
 name = "dialoguer"
 version = "0.12.0"
@@ -2719,9 +2727,9 @@ checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
 
 [[package]]
 name = "libmimalloc-sys"
-version = "0.1.46"
+version = "0.1.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc89deee4af0429081d2a518c0431ae068222a5a262a3bc6ff4d8535ec2e02fe"
+checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
 dependencies = [
  "cc",
 ]
@@ -2795,9 +2803,9 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.17.0"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e0b564323a0fb6d54b864f625ae139de9612e27edb944dda37c109f05aac531"
+checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
 dependencies = [
  "hashbrown 0.17.0",
 ]
@@ -2879,9 +2887,9 @@ dependencies = [
 
 [[package]]
 name = "mail-parser"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f82a3d6522697593ba4c683e0a6ee5a40fee93bc1a525e3cc6eeb3da11fd8897"
+checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7"
 dependencies = [
  "encoding_rs",
  "hashify",
@@ -2966,9 +2974,9 @@ dependencies = [
 
 [[package]]
 name = "mimalloc"
-version = "0.1.49"
+version = "0.1.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aca3c01a711f395b4257b81674c0e90e8dd1f1e62c4b7db45f684cc7a4fcb18a"
+checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
 dependencies = [
  "libmimalloc-sys",
 ]
@@ -4274,9 +4282,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.39"
+version = "0.23.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e"
+checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
 dependencies = [
  "aws-lc-rs",
  "log",
@@ -6097,6 +6105,12 @@ dependencies = [
  "rustls-pki-types",
 ]
 
+[[package]]
+name = "whichlang"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b9aa3ad29c3d08283ac6b769e3ec15ad1ddb88af7d2e9bc402c574973b937e7"
+
 [[package]]
 name = "widestring"
 version = "1.2.1"

Cargo.toml

@@ -11,7 +11,7 @@ edition = "2021"
 [workspace.dependencies]
 chrono = "0.4.44"
 clap = { version = "4.6.1", features = ["derive", "env"] }
-mimalloc = "0.1.49"
+mimalloc = "0.1.50"
 native_db = "0.8.2"
 itertools = "0.14.0"
 native_model = "0.4.20"
@@ -38,7 +38,7 @@ regex = "1.12.3"
 email_address = "0.2.9"
 futures = "0.3.32"
 utf7-imap = "0.3.2"
-mail-parser = { version = '0.11.2', features = ["serde"] }
+mail-parser = { version = '0.11.3', features = ["serde"] }
 # mail-send = "0.5.2"
 tokio-rustls = { version = "0.26.4", default-features = false, features = [
     "ring",
@@ -52,12 +52,12 @@ num_cpus = "1.17.0"
 rand = "0.10.1"
 encoding_rs = "0.8.35"
 webpki-roots = "1.0.7"
-rustls = { version = "0.23.39", default-features = false, features = ["ring"] }
+rustls = { version = "0.23.40", default-features = false, features = ["ring"] }
 rustls-pki-types = "1.14.1"
 tokio-io-timeout = "1.2.1"
 semver = "1.0.28"
 governor = "0.10.4"
-lru = "0.17.0"
+lru = "0.18.0"
 mime_guess = "2.0.5"
 hex = "0.4.3"
 time = { version = "0.3.47", features = [

crates/core/Cargo.toml

@@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m
     "runtime-tokio",
     "compress",
 ] }
-tantivy = { version = "0.26.0", features = ["zstd-compression"] }
+tantivy = { version = "0.26.1", features = ["zstd-compression"] }
 webpki-roots.workspace = true
 rustls.workspace = true
 rustls-pki-types.workspace = true
@@ -68,3 +68,5 @@ uuid.workspace = true
 fjall.workspace = true
 tracing-log.workspace = true
 tokio-util.workspace = true
+whichlang = "0.1.1"
+deunicode = "1.6.2"

crates/core/src/store/tantivy/attachment.rs

@@ -25,19 +25,26 @@ use std::{
 };
 
 use crate::{
-    common::{paginated::DataPage, signal::SIGNAL_MANAGER}, dashboard::{Group, LargestAttachment}, error::{BichonResult, code::ErrorCode}, message::{
+    common::{paginated::DataPage, signal::SIGNAL_MANAGER},
-            attachment::AttachmentMetadata,
+    dashboard::{Group, LargestAttachment},
-            search::{AttachmentSearchFilter, SortBy},
+    error::{code::ErrorCode, BichonResult},
-            tags::{TagAction, TagCount, TagsRequest},
+    message::{
-        }, raise_error, settings::dir::DATA_DIR_MANAGER, store::tantivy::{
+        attachment::AttachmentMetadata,
-            fatal_commit,
+        search::{AttachmentSearchFilter, SortBy},
-            fields::{
+        tags::{TagAction, TagCount, TagsRequest},
-                F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
+    },
-                F_TAGS,
+    raise_error,
-            },
+    settings::dir::DATA_DIR_MANAGER,
-            model::{AttachmentModel, extract_senders},
+    store::tantivy::{
-            schema::SchemaTools,
+        fatal_commit,
-        }
+        fields::{
+            F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
+            F_TAGS,
+        },
+        model::{extract_senders, AttachmentModel},
+        schema::SchemaTools,
+        tokenizers::EuroTokenizer,
+    },
 };
 
 use serde_json::json;
@@ -67,7 +74,6 @@ pub struct IndexManager {
     index_writer: Arc<Mutex<IndexWriter>>,
     sender: mpsc::Sender<TantivyDocument>,
     reader: IndexReader,
-    query_parser: QueryParser,
     handle: Mutex<Option<JoinHandle<()>>>,
 }
 
@@ -80,6 +86,7 @@ impl IndexManager {
     }
     pub fn new() -> Self {
         let index = Self::open_or_create_index(&DATA_DIR_MANAGER.attachment_dir);
+        index.tokenizers().register("euro", EuroTokenizer::new());
         let mut merge_policy = LogMergePolicy::default();
         merge_policy.set_min_num_segments(25);
         merge_policy.set_min_layer_size(10_000);
@@ -101,9 +108,6 @@ impl IndexManager {
                 &DATA_DIR_MANAGER.envelope_dir, e
             )
         });
-        let mut query_parser =
-            QueryParser::for_index(&index, SchemaTools::attachment_default_fields());
-        query_parser.set_conjunction_by_default();
 
         let (sender, mut receiver) = mpsc::channel::<TantivyDocument>(100);
 
@@ -186,7 +190,6 @@ impl IndexManager {
             index_writer,
             sender,
             reader,
-            query_parser,
             handle: Mutex::new(Some(handler)),
         }
     }
@@ -267,7 +270,6 @@ impl IndexManager {
         &self,
         accounts: Option<HashSet<u64>>,
         filter: AttachmentSearchFilter,
-        parser: QueryParser,
     ) -> BichonResult<Box<dyn Query>> {
         let f = SchemaTools::attachment_fields();
         let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
@@ -296,16 +298,21 @@ impl IndexManager {
         }
 
         if let Some(ref text) = filter.text {
-            let query = parser
+            let query_parser =
+                QueryParser::for_index(&self.index, SchemaTools::attachment_default_fields());
+
+            let query = query_parser
                 .parse_query(text)
                 .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
             subqueries.push((Occur::Must, Box::new(query)));
         }
 
         if let Some(ref subject_val) = filter.subject {
-            let term = Term::from_field_text(f.f_subject, subject_val);
+            let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]);
-            let query = TermQuery::new(term, IndexRecordOption::Basic);
+            let q = query_parser
-            subqueries.push((Occur::Must, Box::new(query)));
+                .parse_query(subject_val)
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+            subqueries.push((Occur::Must, q));
         }
 
         if let Some(ref tags) = filter.tags {
@@ -329,9 +336,11 @@ impl IndexManager {
         }
 
         if let Some(from_query) = &filter.from {
-            let term = Term::from_field_text(f.f_from, from_query);
+            let query_parser = QueryParser::for_index(&self.index, vec![f.f_from_text]);
-            let query = TermQuery::new(term, IndexRecordOption::Basic);
+            let q = query_parser
-            subqueries.push((Occur::Must, Box::new(query)));
+                .parse_query(from_query)
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+            subqueries.push((Occur::Must, q))
         }
 
         if let Some(content_hash) = &filter.content_hash {
@@ -349,9 +358,11 @@ impl IndexManager {
         if let Some(ref name) = filter.attachment_name {
             let query_parser =
                 QueryParser::for_index(&self.index, vec![f.f_name_text, f.f_name_exact]);
-            if let Ok(q) = query_parser.parse_query(name) {
+            
-                subqueries.push((Occur::Must, q));
+            let q = query_parser
-            }
+                .parse_query(name)
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+            subqueries.push((Occur::Must, q));
         }
 
         if let Some(ref extension) = filter.attachment_extension {
@@ -774,7 +785,7 @@ impl IndexManager {
     ) -> BichonResult<DataPage<AttachmentModel>> {
         assert!(page > 0, "Page number must be greater than 0");
         assert!(page_size > 0, "Page size must be greater than 0");
-        let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
+        let query = self.filter_query(accounts, filter)?;
         let searcher = self.create_searcher()?;
         let total = searcher
             .search(&query, &Count)

crates/core/src/store/tantivy/envelope.rs

@@ -46,6 +46,7 @@ use crate::{
             },
             model::{extract_contacts, EnvelopeWithAttachments},
             schema::SchemaTools,
+            tokenizers::EuroTokenizer,
         },
     },
     utc_now,
@@ -67,7 +68,7 @@ use tantivy::{
     schema::{IndexRecordOption, Value},
     DocAddress, Index, IndexReader, IndexWriter, Order, TantivyDocument, Term,
 };
-use tantivy::{query::RegexQuery, schema::Facet, Searcher};
+use tantivy::{schema::Facet, Searcher};
 use tokio::{
     sync::{mpsc, Mutex},
     task::{self, JoinHandle},
@@ -81,7 +82,6 @@ pub struct IndexManager {
     index_writer: Arc<Mutex<IndexWriter>>,
     sender: mpsc::Sender<TantivyDocument>,
     reader: IndexReader,
-    query_parser: QueryParser,
     handle: Mutex<Option<JoinHandle<()>>>,
 }
 
@@ -94,6 +94,7 @@ impl IndexManager {
     }
     pub fn new() -> Self {
         let index = Self::open_or_create_index(&DATA_DIR_MANAGER.envelope_dir);
+        index.tokenizers().register("euro", EuroTokenizer::new());
         let mut merge_policy = LogMergePolicy::default();
         merge_policy.set_min_num_segments(25);
         merge_policy.set_min_layer_size(10_000);
@@ -115,8 +116,6 @@ impl IndexManager {
                 &DATA_DIR_MANAGER.envelope_dir, e
             )
         });
-        let mut query_parser = QueryParser::for_index(&index, SchemaTools::email_default_fields());
-        query_parser.set_conjunction_by_default();
 
         let (sender, mut receiver) = mpsc::channel::<TantivyDocument>(100);
 
@@ -199,7 +198,6 @@ impl IndexManager {
             index_writer,
             sender,
             reader,
-            query_parser,
             handle: Mutex::new(Some(handler)),
         }
     }
@@ -277,7 +275,6 @@ impl IndexManager {
         &self,
         accounts: Option<HashSet<u64>>,
         filter: EmailSearchFilter,
-        parser: QueryParser,
     ) -> BichonResult<Box<dyn Query>> {
         let f = SchemaTools::email_fields();
         let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
@@ -306,7 +303,10 @@ impl IndexManager {
         }
 
         if let Some(ref text) = filter.text {
-            let query = parser
+            let query_parser =
+                QueryParser::for_index(&self.index, SchemaTools::email_default_fields());
+
+            let query = query_parser
                 .parse_query(text)
                 .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
             subqueries.push((Occur::Must, Box::new(query)));
@@ -314,16 +314,20 @@ impl IndexManager {
 
         if let Some(ref subject_val) = filter.subject {
             let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]);
-            if let Ok(q) = query_parser.parse_query(subject_val) {
+            let q = query_parser
-                subqueries.push((Occur::Must, q));
+                .parse_query(subject_val)
-            }
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+            println!("{:#?}", &q);
+            subqueries.push((Occur::Must, q));
         }
 
         if let Some(ref body_val) = filter.body {
             let query_parser = QueryParser::for_index(&self.index, vec![f.f_body]);
-            if let Ok(q) = query_parser.parse_query(body_val) {
+            
-                subqueries.push((Occur::Must, q));
+            let q = query_parser
-            }
+                .parse_query(body_val)
+                .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+            subqueries.push((Occur::Must, q));
         }
 
         if let Some(ref tags) = filter.tags {
@@ -347,15 +351,17 @@ impl IndexManager {
         }
 
         for (field, opt_value) in [
-            (f.f_from, &filter.from),
+            (f.f_from_text, &filter.from),
-            (f.f_to, &filter.to),
+            (f.f_to_text, &filter.to),
-            (f.f_cc, &filter.cc),
+            (f.f_cc_text, &filter.cc),
-            (f.f_bcc, &filter.bcc),
+            (f.f_bcc_text, &filter.bcc),
         ] {
             if let Some(ref v) = opt_value {
-                if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) {
+                let query_parser = QueryParser::for_index(&self.index, vec![field]);
-                    subqueries.push((Occur::Must, Box::new(query)));
+                let q = query_parser
-                }
+                    .parse_query(v)
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+                subqueries.push((Occur::Must, q));
             }
         }
 
@@ -378,12 +384,24 @@ impl IndexManager {
         if let Some(ref name) = filter.attachment_name {
             if name.contains('.') {
                 let term = Term::from_field_text(f.f_attachment_name_exact, name);
-                let query = TermQuery::new(term, IndexRecordOption::Basic);
+                let exact_query = TermQuery::new(term, IndexRecordOption::Basic);
-                subqueries.push((Occur::Should, Box::new(query)));
-            }
 
-            let query_parser = QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
+                let query_parser =
-            if let Ok(q) = query_parser.parse_query(name) {
+                    QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
+                let q: Box<dyn Query> = query_parser
+                    .parse_query(name)
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+                let query = BooleanQuery::new(vec![
+                    (Occur::Should, Box::new(exact_query)),
+                    (Occur::Should, Box::new(q)),
+                ]);
+                subqueries.push((Occur::Must, Box::new(query)));
+            } else {
+                let query_parser =
+                    QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
+                let q = query_parser
+                    .parse_query(name)
+                    .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
                 subqueries.push((Occur::Must, q));
             }
         }
@@ -1061,7 +1079,7 @@ impl IndexManager {
     ) -> BichonResult<DataPage<Envelope>> {
         assert!(page > 0, "Page number must be greater than 0");
         assert!(page_size > 0, "Page size must be greater than 0");
-        let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
+        let query = self.filter_query(accounts, filter)?;
         let searcher = self.create_searcher()?;
         let total = searcher
             .search(&query, &Count)

crates/core/src/store/tantivy/fields.rs

@@ -27,10 +27,17 @@ pub const F_SUBJECT: &str = "subject";
 pub const F_BODY: &str = "body";
 pub const F_PREVIEW: &str = "preview";
 pub const F_CONTENT_HASH: &str = "content_hash";
+
 pub const F_FROM: &str = "from";
 pub const F_TO: &str = "to";
 pub const F_CC: &str = "cc";
 pub const F_BCC: &str = "bcc";
+
+pub const F_FROM_TEXT: &str = "from_text";
+pub const F_TO_TEXT: &str = "to_text";
+pub const F_CC_TEXT: &str = "cc_text";
+pub const F_BCC_TEXT: &str = "bcc_text";
+
 pub const F_DATE: &str = "date";
 pub const F_INTERNAL_DATE: &str = "internal_date";
 pub const F_INGEST_AT: &str = "ingest_at";
@@ -73,6 +80,10 @@ pub struct EmailFields {
     pub f_to: Field,
     pub f_cc: Field,
     pub f_bcc: Field,
+    pub f_from_text: Field,
+    pub f_to_text: Field,
+    pub f_cc_text: Field,
+    pub f_bcc_text: Field,
     pub f_date: Field,
     pub f_internal_date: Field,
     pub f_ingest_at: Field,
@@ -97,6 +108,7 @@ pub struct AttachmentFields {
     pub f_account_id: Field,
     pub f_mailbox_id: Field,
     pub f_from: Field,
+    pub f_from_text: Field,
     pub f_subject: Field,
     pub f_content_hash: Field,
     pub f_text: Field,

crates/core/src/store/tantivy/filter.rs

@@ -0,0 +1,73 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use deunicode::deunicode;
+use tantivy::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
+
+#[derive(Clone, Debug, Default)]
+pub struct DeunicodeFilter;
+
+impl TokenFilter for DeunicodeFilter {
+    type Tokenizer<T: Tokenizer> = DeunicodeFilterWrapper<T>;
+
+    fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
+        DeunicodeFilterWrapper { inner: tokenizer }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct DeunicodeFilterWrapper<T> {
+    inner: T,
+}
+
+impl<T: Tokenizer> Tokenizer for DeunicodeFilterWrapper<T> {
+    type TokenStream<'a> = DeunicodeTokenStream<T::TokenStream<'a>>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        DeunicodeTokenStream {
+            inner: self.inner.token_stream(text),
+        }
+    }
+}
+
+pub struct DeunicodeTokenStream<S> {
+    inner: S,
+}
+
+impl<S: TokenStream> TokenStream for DeunicodeTokenStream<S> {
+    fn advance(&mut self) -> bool {
+        if !self.inner.advance() {
+            return false;
+        }
+        let token: &mut Token = self.inner.token_mut();
+        // Only allocate a new String when the text actually contains
+        // non-ASCII characters to avoid unnecessary overhead.
+        if !token.text.is_ascii() {
+            token.text = deunicode(&token.text);
+        }
+        true
+    }
+
+    fn token(&self) -> &Token {
+        self.inner.token()
+    }
+
+    fn token_mut(&mut self) -> &mut Token {
+        self.inner.token_mut()
+    }
+}

crates/core/src/store/tantivy/mod.rs

@@ -26,8 +26,10 @@ use crate::{
 pub mod attachment;
 pub mod envelope;
 pub mod fields;
+pub mod filter;
 pub mod model;
 pub mod schema;
+pub mod tokenizers;
 
 pub fn fatal_commit(writer: &mut IndexWriter) {
     const MAX_RETRIES: usize = 3;

crates/core/src/store/tantivy/model.rs

@@ -67,16 +67,20 @@ impl EnvelopeWithAttachments {
         doc.add_text(fields.f_preview, &self.envelope.preview);
         doc.add_text(fields.f_content_hash, &self.envelope.content_hash);
         doc.add_text(fields.f_from, &self.envelope.from);
+        doc.add_text(fields.f_from_text, &self.envelope.from);
         doc.add_text(fields.f_body, body_text);
 
         for to in &self.envelope.to {
             doc.add_text(fields.f_to, to);
+            doc.add_text(fields.f_to_text, to);
         }
         for cc in &self.envelope.cc {
             doc.add_text(fields.f_cc, cc);
+            doc.add_text(fields.f_cc_text, cc);
         }
         for bcc in &self.envelope.bcc {
             doc.add_text(fields.f_bcc, bcc);
+            doc.add_text(fields.f_bcc_text, bcc);
         }
 
         doc.add_i64(fields.f_date, self.envelope.date);
@@ -353,9 +357,10 @@ impl AttachmentModel {
         doc.add_text(f.f_envelope_id, self.envelope_id);
         doc.add_u64(f.f_account_id, self.account_id);
         doc.add_u64(f.f_mailbox_id, self.mailbox_id);
-        doc.add_text(f.f_subject, self.subject);
+        doc.add_text(f.f_subject, &self.subject);
         doc.add_text(f.f_content_hash, self.content_hash);
-        doc.add_text(f.f_from, self.from);
+        doc.add_text(f.f_from, &self.from);
+        doc.add_text(f.f_from_text, &self.from);
         doc.add_i64(f.f_date, self.date);
         doc.add_i64(f.f_ingest_at, self.ingest_at);
         doc.add_u64(f.f_size, self.size);
@@ -385,6 +390,18 @@ impl AttachmentModel {
             doc.add_text(f.f_name_exact, name);
         }
 
+        if let Some(tags) = &self.tags {
+            for tag in tags {
+                doc.add_facet(f.f_tags, tag);
+            }
+        }
+
+        if let Some(tags) = &self.auto_tags {
+            for tag in tags {
+                doc.add_facet(f.f_auto_tags, tag);
+            }
+        }
+
         doc
     }
 

crates/core/src/store/tantivy/schema.rs

@@ -17,17 +17,19 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::sync::{Arc, LazyLock};
-use tantivy::schema::{FacetOptions, Field, INDEXED};
+use tantivy::schema::{
-use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
+    FacetOptions, Field, IndexRecordOption, TextFieldIndexing, TextOptions, INDEXED,
+};
+use tantivy::schema::{Schema, FAST, STORED, STRING};
 
 use crate::store::tantivy::fields::{
     AttachmentFields, EmailFields, F_ACCOUNT_ID, F_ATTACHMENTS, F_ATTACHMENT_CATEGORY,
     F_ATTACHMENT_CONTENT_HASH, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_COUNT, F_ATTACHMENT_EXT,
-    F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BODY, F_CC,
+    F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BCC_TEXT, F_BODY, F_CC,
-    F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_HAS_TEXT, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
+    F_CC_TEXT, F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_FROM_TEXT, F_HAS_TEXT, F_ID,
-    F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID, F_NAME_EXACT, F_NAME_TEXT,
+    F_INGEST_AT, F_INTERNAL_DATE, F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID,
-    F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID, F_SIZE, F_SUBJECT, F_TAGS,
+    F_NAME_EXACT, F_NAME_TEXT, F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID,
-    F_TEXT, F_THREAD_ID, F_TO, F_UID,
+    F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID,
 };
 
 static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| {
@@ -58,29 +60,10 @@ impl SchemaTools {
             fields.f_subject,
             fields.f_body,
             fields.f_attachment_name_text,
-            fields.f_attachment_name_exact,
+            fields.f_from_text,
-            fields.f_from,
+            fields.f_to_text,
-            fields.f_to,
+            fields.f_cc_text,
-        ]
+            fields.f_bcc_text,
-    }
-
-    pub fn attachment_schema() -> Schema {
-        let (schema, _) = Self::create_attachment_schema();
-        schema
-    }
-
-    pub fn attachment_fields() -> &'static AttachmentFields {
-        &ATTACHMENT_FIELDS
-    }
-
-    pub fn attachment_default_fields() -> Vec<Field> {
-        let fields = Self::attachment_fields();
-        vec![
-            fields.f_subject,
-            fields.f_text,
-            fields.f_name_exact,
-            fields.f_name_text,
-            fields.f_from,
         ]
     }
 
@@ -91,14 +74,21 @@ impl SchemaTools {
         let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
         let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
         let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
-        let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
+        let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
-        let f_body = builder.add_text_field(F_BODY, TEXT);
+        let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro"));
         let f_preview = builder.add_text_field(F_PREVIEW, STORED);
         let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
+
         let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
         let f_to = builder.add_text_field(F_TO, STRING | STORED);
         let f_cc = builder.add_text_field(F_CC, STRING | STORED);
         let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
+
+        let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
+        let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro"));
+        let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro"));
+        let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro"));
+
         let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
         let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
         let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
@@ -107,7 +97,8 @@ impl SchemaTools {
         let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
         let f_regular_attachment_count =
             builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
-        let f_attachment_name_text = builder.add_text_field(F_ATTACHMENT_NAME_TEXT, TEXT);
+        let f_attachment_name_text =
+            builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro"));
         let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
         let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED);
         let f_attachment_content_hash =
@@ -133,6 +124,10 @@ impl SchemaTools {
             f_to,
             f_cc,
             f_bcc,
+            f_from_text,
+            f_to_text,
+            f_cc_text,
+            f_bcc_text,
             f_date,
             f_internal_date,
             f_ingest_at,
@@ -153,15 +148,35 @@ impl SchemaTools {
         (builder.build(), fields)
     }
 
+    pub fn attachment_schema() -> Schema {
+        let (schema, _) = Self::create_attachment_schema();
+        schema
+    }
+
+    pub fn attachment_fields() -> &'static AttachmentFields {
+        &ATTACHMENT_FIELDS
+    }
+
+    pub fn attachment_default_fields() -> Vec<Field> {
+        let fields = Self::attachment_fields();
+        vec![
+            fields.f_subject,
+            fields.f_text,
+            fields.f_name_text,
+            fields.f_from_text,
+        ]
+    }
+
     pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
         let mut builder = Schema::builder();
         let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
         let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
         let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
         let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
-        let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
+        let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
         let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
         let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
         let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
         let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
         let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
@@ -170,13 +185,13 @@ impl SchemaTools {
         let f_content_type =
             builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
         let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
-        let f_text = builder.add_text_field(F_TEXT, TEXT);
+        let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro"));
         let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
         let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
         let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
         let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
         let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
-        let f_name_text = builder.add_text_field(F_NAME_TEXT, TEXT);
+        let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro"));
         let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED);
         let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
         let f_auto_tags =
@@ -189,6 +204,7 @@ impl SchemaTools {
             f_subject,
             f_content_hash,
             f_from,
+            f_from_text,
             f_date,
             f_ingest_at,
             f_size,
@@ -209,4 +225,22 @@ impl SchemaTools {
         };
         (builder.build(), fields)
     }
+
+    fn text_no_store(tokenizer: &str) -> TextOptions {
+        TextOptions::default().set_indexing_options(
+            TextFieldIndexing::default()
+                .set_tokenizer(tokenizer)
+                .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+        )
+    }
+
+    fn text_store(tokenizer: &str) -> TextOptions {
+        TextOptions::default()
+            .set_indexing_options(
+                TextFieldIndexing::default()
+                    .set_tokenizer(tokenizer)
+                    .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+            )
+            .set_stored()
+    }
 }

crates/core/src/store/tantivy/tokenizers.rs

@@ -0,0 +1,72 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use tantivy::tokenizer::*;
+use whichlang::{detect_language, Lang};
+
+use crate::store::tantivy::filter::DeunicodeFilter;
+
+#[derive(Clone)]
+pub struct EuroTokenizer {
+    en: TextAnalyzer,
+    fr: TextAnalyzer,
+    de: TextAnalyzer,
+    es: TextAnalyzer,
+    nl: TextAnalyzer,
+    default: TextAnalyzer,
+}
+
+impl EuroTokenizer {
+    pub fn new() -> Self {
+        fn build(lang: Language) -> TextAnalyzer {
+            TextAnalyzer::builder(SimpleTokenizer::default())
+                .filter(RemoveLongFilter::limit(40))
+                .filter(LowerCaser)
+                .filter(DeunicodeFilter)
+                .filter(Stemmer::new(lang))
+                .build()
+        }
+
+        Self {
+            en: build(Language::English),
+            fr: build(Language::French),
+            de: build(Language::German),
+            es: build(Language::Spanish),
+            nl: build(Language::Dutch),
+            default: TextAnalyzer::builder(SimpleTokenizer::default())
+                .filter(RemoveLongFilter::limit(40))
+                .filter(LowerCaser)
+                .build(),
+        }
+    }
+}
+
+impl Tokenizer for EuroTokenizer {
+    type TokenStream<'a> = BoxTokenStream<'a>;
+
+    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
+        match detect_language(text) {
+            Lang::Eng => self.en.token_stream(text),
+            Lang::Fra => self.fr.token_stream(text),
+            Lang::Deu => self.de.token_stream(text),
+            Lang::Spa => self.es.token_stream(text),
+            Lang::Nld => self.nl.token_stream(text),
+            _ => self.default.token_stream(text), // fallback
+        }
+    }
+}

crates/server/src/main.rs

@@ -117,5 +117,7 @@ async fn initialize() -> BichonResult<()> {
     BichonTls::initialize().await?;
     BichonContext::initialize().await?;
     LazyLock::force(&BLOB_MANAGER);
+    LazyLock::force(&ENVELOPE_MANAGER);
+    LazyLock::force(&ATTACHMENT_MANAGER);
     Ok(())
 }