External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: use stemmer for multilingual token matching

Browse Source
This commit is contained in:
rustmailer
2026-05-05 03:49:20 +08:00
parent 9bacf3fb7a
commit f583c3413c
12 changed files with 367 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+25 -11
View File
@@ -505,6 +505,7 @@ dependencies = [
"chrono",
"clap",
"dashmap",
"deunicode",
"email_address",
"encoding_rs",
"fjall",
@@ -513,7 +514,7 @@ dependencies = [
"html2text",
"itertools",
"itoa",
"lru 0.17.0",
"lru 0.18.0",
"mail-parser",
"mail-send",
"murmur3",
@@ -546,6 +547,7 @@ dependencies = [
"utf7-imap",
"uuid",
"webpki-roots",
"whichlang",
]
[[package]]
@@ -1331,6 +1333,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "deunicode"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
[[package]]
name = "dialoguer"
version = "0.12.0"
@@ -2719,9 +2727,9 @@ checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
[[package]]
name = "libmimalloc-sys"
version = "0.1.46"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc89deee4af0429081d2a518c0431ae068222a5a262a3bc6ff4d8535ec2e02fe"
checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
dependencies = [
"cc",
]
@@ -2795,9 +2803,9 @@ dependencies = [
[[package]]
name = "lru"
version = "0.17.0"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e0b564323a0fb6d54b864f625ae139de9612e27edb944dda37c109f05aac531"
checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
dependencies = [
"hashbrown 0.17.0",
]
@@ -2879,9 +2887,9 @@ dependencies = [
[[package]]
name = "mail-parser"
version = "0.11.2"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f82a3d6522697593ba4c683e0a6ee5a40fee93bc1a525e3cc6eeb3da11fd8897"
checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7"
dependencies = [
"encoding_rs",
"hashify",
@@ -2966,9 +2974,9 @@ dependencies = [
[[package]]
name = "mimalloc"
version = "0.1.49"
version = "0.1.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aca3c01a711f395b4257b81674c0e90e8dd1f1e62c4b7db45f684cc7a4fcb18a"
checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
dependencies = [
"libmimalloc-sys",
]
@@ -4274,9 +4282,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.39"
version = "0.23.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
dependencies = [
"aws-lc-rs",
"log",
@@ -6097,6 +6105,12 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "whichlang"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b9aa3ad29c3d08283ac6b769e3ec15ad1ddb88af7d2e9bc402c574973b937e7"
[[package]]
name = "widestring"
version = "1.2.1"
Cargo.toml
+4 -4
View File
@@ -11,7 +11,7 @@ edition = "2021"
[workspace.dependencies]
chrono = "0.4.44"
clap = { version = "4.6.1", features = ["derive", "env"] }
mimalloc = "0.1.49"
mimalloc = "0.1.50"
native_db = "0.8.2"
itertools = "0.14.0"
native_model = "0.4.20"
@@ -38,7 +38,7 @@ regex = "1.12.3"
email_address = "0.2.9"
futures = "0.3.32"
utf7-imap = "0.3.2"
mail-parser = { version = '0.11.2', features = ["serde"] }
mail-parser = { version = '0.11.3', features = ["serde"] }
# mail-send = "0.5.2"
tokio-rustls = { version = "0.26.4", default-features = false, features = [
"ring",
@@ -52,12 +52,12 @@ num_cpus = "1.17.0"
rand = "0.10.1"
encoding_rs = "0.8.35"
webpki-roots = "1.0.7"
rustls = { version = "0.23.39", default-features = false, features = ["ring"] }
rustls = { version = "0.23.40", default-features = false, features = ["ring"] }
rustls-pki-types = "1.14.1"
tokio-io-timeout = "1.2.1"
semver = "1.0.28"
governor = "0.10.4"
lru = "0.17.0"
lru = "0.18.0"
mime_guess = "2.0.5"
hex = "0.4.3"
time = { version = "0.3.47", features = [
crates/core/Cargo.toml
+3 -1
View File
@@ -48,7 +48,7 @@ async-imap = { git = "https://github.com/rustmailer/async-imap.git", branch = "m
"runtime-tokio",
"compress",
] }
tantivy = { version = "0.26.0", features = ["zstd-compression"] }
tantivy = { version = "0.26.1", features = ["zstd-compression"] }
webpki-roots.workspace = true
rustls.workspace = true
rustls-pki-types.workspace = true
@@ -68,3 +68,5 @@ uuid.workspace = true
fjall.workspace = true
tracing-log.workspace = true
tokio-util.workspace = true
whichlang = "0.1.1"
deunicode = "1.6.2"
crates/core/src/store/tantivy/attachment.rs
+41 -30
View File
@@ -25,19 +25,26 @@ use std::{
};
use crate::{
common::{paginated::DataPage, signal::SIGNAL_MANAGER}, dashboard::{Group, LargestAttachment}, error::{BichonResult, code::ErrorCode}, message::{
attachment::AttachmentMetadata,
search::{AttachmentSearchFilter, SortBy},
tags::{TagAction, TagCount, TagsRequest},
}, raise_error, settings::dir::DATA_DIR_MANAGER, store::tantivy::{
fatal_commit,
fields::{
F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
F_TAGS,
},
model::{AttachmentModel, extract_senders},
schema::SchemaTools,
}
common::{paginated::DataPage, signal::SIGNAL_MANAGER},
dashboard::{Group, LargestAttachment},
error::{code::ErrorCode, BichonResult},
message::{
attachment::AttachmentMetadata,
search::{AttachmentSearchFilter, SortBy},
tags::{TagAction, TagCount, TagsRequest},
},
raise_error,
settings::dir::DATA_DIR_MANAGER,
store::tantivy::{
fatal_commit,
fields::{
F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
F_TAGS,
},
model::{extract_senders, AttachmentModel},
schema::SchemaTools,
tokenizers::EuroTokenizer,
},
};
use serde_json::json;
@@ -67,7 +74,6 @@ pub struct IndexManager {
index_writer: Arc<Mutex<IndexWriter>>,
sender: mpsc::Sender<TantivyDocument>,
reader: IndexReader,
query_parser: QueryParser,
handle: Mutex<Option<JoinHandle<()>>>,
}
@@ -80,6 +86,7 @@ impl IndexManager {
}
pub fn new() -> Self {
let index = Self::open_or_create_index(&DATA_DIR_MANAGER.attachment_dir);
index.tokenizers().register("euro", EuroTokenizer::new());
let mut merge_policy = LogMergePolicy::default();
merge_policy.set_min_num_segments(25);
merge_policy.set_min_layer_size(10_000);
@@ -101,9 +108,6 @@ impl IndexManager {
&DATA_DIR_MANAGER.envelope_dir, e
)
});
let mut query_parser =
QueryParser::for_index(&index, SchemaTools::attachment_default_fields());
query_parser.set_conjunction_by_default();
let (sender, mut receiver) = mpsc::channel::<TantivyDocument>(100);
@@ -186,7 +190,6 @@ impl IndexManager {
index_writer,
sender,
reader,
query_parser,
handle: Mutex::new(Some(handler)),
}
}
@@ -267,7 +270,6 @@ impl IndexManager {
&self,
accounts: Option<HashSet<u64>>,
filter: AttachmentSearchFilter,
parser: QueryParser,
) -> BichonResult<Box<dyn Query>> {
let f = SchemaTools::attachment_fields();
let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
@@ -296,16 +298,21 @@ impl IndexManager {
}
if let Some(ref text) = filter.text {
let query = parser
let query_parser =
QueryParser::for_index(&self.index, SchemaTools::attachment_default_fields());
let query = query_parser
.parse_query(text)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, Box::new(query)));
}
if let Some(ref subject_val) = filter.subject {
let term = Term::from_field_text(f.f_subject, subject_val);
let query = TermQuery::new(term, IndexRecordOption::Basic);
subqueries.push((Occur::Must, Box::new(query)));
let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]);
let q = query_parser
.parse_query(subject_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
if let Some(ref tags) = filter.tags {
@@ -329,9 +336,11 @@ impl IndexManager {
}
if let Some(from_query) = &filter.from {
let term = Term::from_field_text(f.f_from, from_query);
let query = TermQuery::new(term, IndexRecordOption::Basic);
subqueries.push((Occur::Must, Box::new(query)));
let query_parser = QueryParser::for_index(&self.index, vec![f.f_from_text]);
let q = query_parser
.parse_query(from_query)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q))
}
if let Some(content_hash) = &filter.content_hash {
@@ -349,9 +358,11 @@ impl IndexManager {
if let Some(ref name) = filter.attachment_name {
let query_parser =
QueryParser::for_index(&self.index, vec![f.f_name_text, f.f_name_exact]);
if let Ok(q) = query_parser.parse_query(name) {
subqueries.push((Occur::Must, q));
}
let q = query_parser
.parse_query(name)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
if let Some(ref extension) = filter.attachment_extension {
@@ -774,7 +785,7 @@ impl IndexManager {
) -> BichonResult<DataPage<AttachmentModel>> {
assert!(page > 0, "Page number must be greater than 0");
assert!(page_size > 0, "Page size must be greater than 0");
let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
let query = self.filter_query(accounts, filter)?;
let searcher = self.create_searcher()?;
let total = searcher
.search(&query, &Count)
crates/core/src/store/tantivy/envelope.rs
+44 -26
View File
@@ -46,6 +46,7 @@ use crate::{
},
model::{extract_contacts, EnvelopeWithAttachments},
schema::SchemaTools,
tokenizers::EuroTokenizer,
},
},
utc_now,
@@ -67,7 +68,7 @@ use tantivy::{
schema::{IndexRecordOption, Value},
DocAddress, Index, IndexReader, IndexWriter, Order, TantivyDocument, Term,
};
use tantivy::{query::RegexQuery, schema::Facet, Searcher};
use tantivy::{schema::Facet, Searcher};
use tokio::{
sync::{mpsc, Mutex},
task::{self, JoinHandle},
@@ -81,7 +82,6 @@ pub struct IndexManager {
index_writer: Arc<Mutex<IndexWriter>>,
sender: mpsc::Sender<TantivyDocument>,
reader: IndexReader,
query_parser: QueryParser,
handle: Mutex<Option<JoinHandle<()>>>,
}
@@ -94,6 +94,7 @@ impl IndexManager {
}
pub fn new() -> Self {
let index = Self::open_or_create_index(&DATA_DIR_MANAGER.envelope_dir);
index.tokenizers().register("euro", EuroTokenizer::new());
let mut merge_policy = LogMergePolicy::default();
merge_policy.set_min_num_segments(25);
merge_policy.set_min_layer_size(10_000);
@@ -115,8 +116,6 @@ impl IndexManager {
&DATA_DIR_MANAGER.envelope_dir, e
)
});
let mut query_parser = QueryParser::for_index(&index, SchemaTools::email_default_fields());
query_parser.set_conjunction_by_default();
let (sender, mut receiver) = mpsc::channel::<TantivyDocument>(100);
@@ -199,7 +198,6 @@ impl IndexManager {
index_writer,
sender,
reader,
query_parser,
handle: Mutex::new(Some(handler)),
}
}
@@ -277,7 +275,6 @@ impl IndexManager {
&self,
accounts: Option<HashSet<u64>>,
filter: EmailSearchFilter,
parser: QueryParser,
) -> BichonResult<Box<dyn Query>> {
let f = SchemaTools::email_fields();
let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
@@ -306,7 +303,10 @@ impl IndexManager {
}
if let Some(ref text) = filter.text {
let query = parser
let query_parser =
QueryParser::for_index(&self.index, SchemaTools::email_default_fields());
let query = query_parser
.parse_query(text)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, Box::new(query)));
@@ -314,16 +314,20 @@ impl IndexManager {
if let Some(ref subject_val) = filter.subject {
let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]);
if let Ok(q) = query_parser.parse_query(subject_val) {
subqueries.push((Occur::Must, q));
}
let q = query_parser
.parse_query(subject_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
println!("{:#?}", &q);
subqueries.push((Occur::Must, q));
}
if let Some(ref body_val) = filter.body {
let query_parser = QueryParser::for_index(&self.index, vec![f.f_body]);
if let Ok(q) = query_parser.parse_query(body_val) {
subqueries.push((Occur::Must, q));
}
let q = query_parser
.parse_query(body_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
if let Some(ref tags) = filter.tags {
@@ -347,15 +351,17 @@ impl IndexManager {
}
for (field, opt_value) in [
(f.f_from, &filter.from),
(f.f_to, &filter.to),
(f.f_cc, &filter.cc),
(f.f_bcc, &filter.bcc),
(f.f_from_text, &filter.from),
(f.f_to_text, &filter.to),
(f.f_cc_text, &filter.cc),
(f.f_bcc_text, &filter.bcc),
] {
if let Some(ref v) = opt_value {
if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) {
subqueries.push((Occur::Must, Box::new(query)));
}
let query_parser = QueryParser::for_index(&self.index, vec![field]);
let q = query_parser
.parse_query(v)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
}
@@ -378,12 +384,24 @@ impl IndexManager {
if let Some(ref name) = filter.attachment_name {
if name.contains('.') {
let term = Term::from_field_text(f.f_attachment_name_exact, name);
let query = TermQuery::new(term, IndexRecordOption::Basic);
subqueries.push((Occur::Should, Box::new(query)));
}
let exact_query = TermQuery::new(term, IndexRecordOption::Basic);
let query_parser = QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
if let Ok(q) = query_parser.parse_query(name) {
let query_parser =
QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
let q: Box<dyn Query> = query_parser
.parse_query(name)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
let query = BooleanQuery::new(vec![
(Occur::Should, Box::new(exact_query)),
(Occur::Should, Box::new(q)),
]);
subqueries.push((Occur::Must, Box::new(query)));
} else {
let query_parser =
QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
let q = query_parser
.parse_query(name)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
}
@@ -1061,7 +1079,7 @@ impl IndexManager {
) -> BichonResult<DataPage<Envelope>> {
assert!(page > 0, "Page number must be greater than 0");
assert!(page_size > 0, "Page size must be greater than 0");
let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
let query = self.filter_query(accounts, filter)?;
let searcher = self.create_searcher()?;
let total = searcher
.search(&query, &Count)
crates/core/src/store/tantivy/fields.rs
+12
View File
@@ -27,10 +27,17 @@ pub const F_SUBJECT: &str = "subject";
pub const F_BODY: &str = "body";
pub const F_PREVIEW: &str = "preview";
pub const F_CONTENT_HASH: &str = "content_hash";
pub const F_FROM: &str = "from";
pub const F_TO: &str = "to";
pub const F_CC: &str = "cc";
pub const F_BCC: &str = "bcc";
pub const F_FROM_TEXT: &str = "from_text";
pub const F_TO_TEXT: &str = "to_text";
pub const F_CC_TEXT: &str = "cc_text";
pub const F_BCC_TEXT: &str = "bcc_text";
pub const F_DATE: &str = "date";
pub const F_INTERNAL_DATE: &str = "internal_date";
pub const F_INGEST_AT: &str = "ingest_at";
@@ -73,6 +80,10 @@ pub struct EmailFields {
pub f_to: Field,
pub f_cc: Field,
pub f_bcc: Field,
pub f_from_text: Field,
pub f_to_text: Field,
pub f_cc_text: Field,
pub f_bcc_text: Field,
pub f_date: Field,
pub f_internal_date: Field,
pub f_ingest_at: Field,
@@ -97,6 +108,7 @@ pub struct AttachmentFields {
pub f_account_id: Field,
pub f_mailbox_id: Field,
pub f_from: Field,
pub f_from_text: Field,
pub f_subject: Field,
pub f_content_hash: Field,
pub f_text: Field,
crates/core/src/store/tantivy/filter.rs
+73
View File
@@ -0,0 +1,73 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use deunicode::deunicode;
use tantivy::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
#[derive(Clone, Debug, Default)]
pub struct DeunicodeFilter;
impl TokenFilter for DeunicodeFilter {
type Tokenizer<T: Tokenizer> = DeunicodeFilterWrapper<T>;
fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
DeunicodeFilterWrapper { inner: tokenizer }
}
}
#[derive(Clone, Debug)]
pub struct DeunicodeFilterWrapper<T> {
inner: T,
}
impl<T: Tokenizer> Tokenizer for DeunicodeFilterWrapper<T> {
type TokenStream<'a> = DeunicodeTokenStream<T::TokenStream<'a>>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
DeunicodeTokenStream {
inner: self.inner.token_stream(text),
}
}
}
pub struct DeunicodeTokenStream<S> {
inner: S,
}
impl<S: TokenStream> TokenStream for DeunicodeTokenStream<S> {
fn advance(&mut self) -> bool {
if !self.inner.advance() {
return false;
}
let token: &mut Token = self.inner.token_mut();
// Only allocate a new String when the text actually contains
// non-ASCII characters to avoid unnecessary overhead.
if !token.text.is_ascii() {
token.text = deunicode(&token.text);
}
true
}
fn token(&self) -> &Token {
self.inner.token()
}
fn token_mut(&mut self) -> &mut Token {
self.inner.token_mut()
}
}
crates/core/src/store/tantivy/mod.rs
+2
View File
@@ -26,8 +26,10 @@ use crate::{
pub mod attachment;
pub mod envelope;
pub mod fields;
pub mod filter;
pub mod model;
pub mod schema;
pub mod tokenizers;
pub fn fatal_commit(writer: &mut IndexWriter) {
const MAX_RETRIES: usize = 3;
crates/core/src/store/tantivy/model.rs
+19 -2
View File
@@ -67,16 +67,20 @@ impl EnvelopeWithAttachments {
doc.add_text(fields.f_preview, &self.envelope.preview);
doc.add_text(fields.f_content_hash, &self.envelope.content_hash);
doc.add_text(fields.f_from, &self.envelope.from);
doc.add_text(fields.f_from_text, &self.envelope.from);
doc.add_text(fields.f_body, body_text);
for to in &self.envelope.to {
doc.add_text(fields.f_to, to);
doc.add_text(fields.f_to_text, to);
}
for cc in &self.envelope.cc {
doc.add_text(fields.f_cc, cc);
doc.add_text(fields.f_cc_text, cc);
}
for bcc in &self.envelope.bcc {
doc.add_text(fields.f_bcc, bcc);
doc.add_text(fields.f_bcc_text, bcc);
}
doc.add_i64(fields.f_date, self.envelope.date);
@@ -353,9 +357,10 @@ impl AttachmentModel {
doc.add_text(f.f_envelope_id, self.envelope_id);
doc.add_u64(f.f_account_id, self.account_id);
doc.add_u64(f.f_mailbox_id, self.mailbox_id);
doc.add_text(f.f_subject, self.subject);
doc.add_text(f.f_subject, &self.subject);
doc.add_text(f.f_content_hash, self.content_hash);
doc.add_text(f.f_from, self.from);
doc.add_text(f.f_from, &self.from);
doc.add_text(f.f_from_text, &self.from);
doc.add_i64(f.f_date, self.date);
doc.add_i64(f.f_ingest_at, self.ingest_at);
doc.add_u64(f.f_size, self.size);
@@ -385,6 +390,18 @@ impl AttachmentModel {
doc.add_text(f.f_name_exact, name);
}
if let Some(tags) = &self.tags {
for tag in tags {
doc.add_facet(f.f_tags, tag);
}
}
if let Some(tags) = &self.auto_tags {
for tag in tags {
doc.add_facet(f.f_auto_tags, tag);
}
}
doc
}
crates/core/src/store/tantivy/schema.rs
+70 -36
View File
@@ -17,17 +17,19 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::sync::{Arc, LazyLock};
use tantivy::schema::{FacetOptions, Field, INDEXED};
use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
use tantivy::schema::{
FacetOptions, Field, IndexRecordOption, TextFieldIndexing, TextOptions, INDEXED,
};
use tantivy::schema::{Schema, FAST, STORED, STRING};
use crate::store::tantivy::fields::{
AttachmentFields, EmailFields, F_ACCOUNT_ID, F_ATTACHMENTS, F_ATTACHMENT_CATEGORY,
F_ATTACHMENT_CONTENT_HASH, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_COUNT, F_ATTACHMENT_EXT,
F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BODY, F_CC,
F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_HAS_TEXT, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID, F_NAME_EXACT, F_NAME_TEXT,
F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID, F_SIZE, F_SUBJECT, F_TAGS,
F_TEXT, F_THREAD_ID, F_TO, F_UID,
F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BCC_TEXT, F_BODY, F_CC,
F_CC_TEXT, F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_FROM_TEXT, F_HAS_TEXT, F_ID,
F_INGEST_AT, F_INTERNAL_DATE, F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID,
F_NAME_EXACT, F_NAME_TEXT, F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID,
F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID,
};
static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| {
@@ -58,29 +60,10 @@ impl SchemaTools {
fields.f_subject,
fields.f_body,
fields.f_attachment_name_text,
fields.f_attachment_name_exact,
fields.f_from,
fields.f_to,
]
}
pub fn attachment_schema() -> Schema {
let (schema, _) = Self::create_attachment_schema();
schema
}
pub fn attachment_fields() -> &'static AttachmentFields {
&ATTACHMENT_FIELDS
}
pub fn attachment_default_fields() -> Vec<Field> {
let fields = Self::attachment_fields();
vec![
fields.f_subject,
fields.f_text,
fields.f_name_exact,
fields.f_name_text,
fields.f_from,
fields.f_from_text,
fields.f_to_text,
fields.f_cc_text,
fields.f_bcc_text,
]
}
@@ -91,14 +74,21 @@ impl SchemaTools {
let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
let f_body = builder.add_text_field(F_BODY, TEXT);
let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro"));
let f_preview = builder.add_text_field(F_PREVIEW, STORED);
let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
let f_to = builder.add_text_field(F_TO, STRING | STORED);
let f_cc = builder.add_text_field(F_CC, STRING | STORED);
let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro"));
let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro"));
let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro"));
let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
@@ -107,7 +97,8 @@ impl SchemaTools {
let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_regular_attachment_count =
builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_attachment_name_text = builder.add_text_field(F_ATTACHMENT_NAME_TEXT, TEXT);
let f_attachment_name_text =
builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro"));
let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED);
let f_attachment_content_hash =
@@ -133,6 +124,10 @@ impl SchemaTools {
f_to,
f_cc,
f_bcc,
f_from_text,
f_to_text,
f_cc_text,
f_bcc_text,
f_date,
f_internal_date,
f_ingest_at,
@@ -153,15 +148,35 @@ impl SchemaTools {
(builder.build(), fields)
}
pub fn attachment_schema() -> Schema {
let (schema, _) = Self::create_attachment_schema();
schema
}
pub fn attachment_fields() -> &'static AttachmentFields {
&ATTACHMENT_FIELDS
}
pub fn attachment_default_fields() -> Vec<Field> {
let fields = Self::attachment_fields();
vec![
fields.f_subject,
fields.f_text,
fields.f_name_text,
fields.f_from_text,
]
}
pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
let mut builder = Schema::builder();
let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
@@ -170,13 +185,13 @@ impl SchemaTools {
let f_content_type =
builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
let f_text = builder.add_text_field(F_TEXT, TEXT);
let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro"));
let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
let f_name_text = builder.add_text_field(F_NAME_TEXT, TEXT);
let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro"));
let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED);
let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let f_auto_tags =
@@ -189,6 +204,7 @@ impl SchemaTools {
f_subject,
f_content_hash,
f_from,
f_from_text,
f_date,
f_ingest_at,
f_size,
@@ -209,4 +225,22 @@ impl SchemaTools {
};
(builder.build(), fields)
}
fn text_no_store(tokenizer: &str) -> TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
}
fn text_store(tokenizer: &str) -> TextOptions {
TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored()
}
}
crates/core/src/store/tantivy/tokenizers.rs
+72
View File
@@ -0,0 +1,72 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use tantivy::tokenizer::*;
use whichlang::{detect_language, Lang};
use crate::store::tantivy::filter::DeunicodeFilter;
#[derive(Clone)]
pub struct EuroTokenizer {
en: TextAnalyzer,
fr: TextAnalyzer,
de: TextAnalyzer,
es: TextAnalyzer,
nl: TextAnalyzer,
default: TextAnalyzer,
}
impl EuroTokenizer {
pub fn new() -> Self {
fn build(lang: Language) -> TextAnalyzer {
TextAnalyzer::builder(SimpleTokenizer::default())
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser)
.filter(DeunicodeFilter)
.filter(Stemmer::new(lang))
.build()
}
Self {
en: build(Language::English),
fr: build(Language::French),
de: build(Language::German),
es: build(Language::Spanish),
nl: build(Language::Dutch),
default: TextAnalyzer::builder(SimpleTokenizer::default())
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser)
.build(),
}
}
}
impl Tokenizer for EuroTokenizer {
type TokenStream<'a> = BoxTokenStream<'a>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
match detect_language(text) {
Lang::Eng => self.en.token_stream(text),
Lang::Fra => self.fr.token_stream(text),
Lang::Deu => self.de.token_stream(text),
Lang::Spa => self.es.token_stream(text),
Lang::Nld => self.nl.token_stream(text),
_ => self.default.token_stream(text), // fallback
}
}
}
crates/server/src/main.rs
+2
View File
@@ -117,5 +117,7 @@ async fn initialize() -> BichonResult<()> {
BichonTls::initialize().await?;
BichonContext::initialize().await?;
LazyLock::force(&BLOB_MANAGER);
LazyLock::force(&ENVELOPE_MANAGER);
LazyLock::force(&ATTACHMENT_MANAGER);
Ok(())
}