External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: use stemmer for multilingual token matching

Browse Source
This commit is contained in:
rustmailer
2026-05-05 03:49:20 +08:00
parent 9bacf3fb7a
commit f583c3413c
12 changed files with 367 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/core/src/store/tantivy/attachment.rs
+41 -30
View File
@@ -25,19 +25,26 @@ use std::{
};
use crate::{
common::{paginated::DataPage, signal::SIGNAL_MANAGER}, dashboard::{Group, LargestAttachment}, error::{BichonResult, code::ErrorCode}, message::{
attachment::AttachmentMetadata,
search::{AttachmentSearchFilter, SortBy},
tags::{TagAction, TagCount, TagsRequest},
}, raise_error, settings::dir::DATA_DIR_MANAGER, store::tantivy::{
fatal_commit,
fields::{
F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
F_TAGS,
},
model::{AttachmentModel, extract_senders},
schema::SchemaTools,
}
common::{paginated::DataPage, signal::SIGNAL_MANAGER},
dashboard::{Group, LargestAttachment},
error::{code::ErrorCode, BichonResult},
message::{
attachment::AttachmentMetadata,
search::{AttachmentSearchFilter, SortBy},
tags::{TagAction, TagCount, TagsRequest},
},
raise_error,
settings::dir::DATA_DIR_MANAGER,
store::tantivy::{
fatal_commit,
fields::{
F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
F_TAGS,
},
model::{extract_senders, AttachmentModel},
schema::SchemaTools,
tokenizers::EuroTokenizer,
},
};
use serde_json::json;
@@ -67,7 +74,6 @@ pub struct IndexManager {
index_writer: Arc<Mutex<IndexWriter>>,
sender: mpsc::Sender<TantivyDocument>,
reader: IndexReader,
query_parser: QueryParser,
handle: Mutex<Option<JoinHandle<()>>>,
}
@@ -80,6 +86,7 @@ impl IndexManager {
}
pub fn new() -> Self {
let index = Self::open_or_create_index(&DATA_DIR_MANAGER.attachment_dir);
index.tokenizers().register("euro", EuroTokenizer::new());
let mut merge_policy = LogMergePolicy::default();
merge_policy.set_min_num_segments(25);
merge_policy.set_min_layer_size(10_000);
@@ -101,9 +108,6 @@ impl IndexManager {
&DATA_DIR_MANAGER.envelope_dir, e
)
});
let mut query_parser =
QueryParser::for_index(&index, SchemaTools::attachment_default_fields());
query_parser.set_conjunction_by_default();
let (sender, mut receiver) = mpsc::channel::<TantivyDocument>(100);
@@ -186,7 +190,6 @@ impl IndexManager {
index_writer,
sender,
reader,
query_parser,
handle: Mutex::new(Some(handler)),
}
}
@@ -267,7 +270,6 @@ impl IndexManager {
&self,
accounts: Option<HashSet<u64>>,
filter: AttachmentSearchFilter,
parser: QueryParser,
) -> BichonResult<Box<dyn Query>> {
let f = SchemaTools::attachment_fields();
let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
@@ -296,16 +298,21 @@ impl IndexManager {
}
if let Some(ref text) = filter.text {
let query = parser
let query_parser =
QueryParser::for_index(&self.index, SchemaTools::attachment_default_fields());
let query = query_parser
.parse_query(text)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, Box::new(query)));
}
if let Some(ref subject_val) = filter.subject {
let term = Term::from_field_text(f.f_subject, subject_val);
let query = TermQuery::new(term, IndexRecordOption::Basic);
subqueries.push((Occur::Must, Box::new(query)));
let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]);
let q = query_parser
.parse_query(subject_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
if let Some(ref tags) = filter.tags {
@@ -329,9 +336,11 @@ impl IndexManager {
}
if let Some(from_query) = &filter.from {
let term = Term::from_field_text(f.f_from, from_query);
let query = TermQuery::new(term, IndexRecordOption::Basic);
subqueries.push((Occur::Must, Box::new(query)));
let query_parser = QueryParser::for_index(&self.index, vec![f.f_from_text]);
let q = query_parser
.parse_query(from_query)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q))
}
if let Some(content_hash) = &filter.content_hash {
@@ -349,9 +358,11 @@ impl IndexManager {
if let Some(ref name) = filter.attachment_name {
let query_parser =
QueryParser::for_index(&self.index, vec![f.f_name_text, f.f_name_exact]);
if let Ok(q) = query_parser.parse_query(name) {
subqueries.push((Occur::Must, q));
}
let q = query_parser
.parse_query(name)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
if let Some(ref extension) = filter.attachment_extension {
@@ -774,7 +785,7 @@ impl IndexManager {
) -> BichonResult<DataPage<AttachmentModel>> {
assert!(page > 0, "Page number must be greater than 0");
assert!(page_size > 0, "Page size must be greater than 0");
let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
let query = self.filter_query(accounts, filter)?;
let searcher = self.create_searcher()?;
let total = searcher
.search(&query, &Count)
crates/core/src/store/tantivy/envelope.rs
+44 -26
View File
@@ -46,6 +46,7 @@ use crate::{
},
model::{extract_contacts, EnvelopeWithAttachments},
schema::SchemaTools,
tokenizers::EuroTokenizer,
},
},
utc_now,
@@ -67,7 +68,7 @@ use tantivy::{
schema::{IndexRecordOption, Value},
DocAddress, Index, IndexReader, IndexWriter, Order, TantivyDocument, Term,
};
use tantivy::{query::RegexQuery, schema::Facet, Searcher};
use tantivy::{schema::Facet, Searcher};
use tokio::{
sync::{mpsc, Mutex},
task::{self, JoinHandle},
@@ -81,7 +82,6 @@ pub struct IndexManager {
index_writer: Arc<Mutex<IndexWriter>>,
sender: mpsc::Sender<TantivyDocument>,
reader: IndexReader,
query_parser: QueryParser,
handle: Mutex<Option<JoinHandle<()>>>,
}
@@ -94,6 +94,7 @@ impl IndexManager {
}
pub fn new() -> Self {
let index = Self::open_or_create_index(&DATA_DIR_MANAGER.envelope_dir);
index.tokenizers().register("euro", EuroTokenizer::new());
let mut merge_policy = LogMergePolicy::default();
merge_policy.set_min_num_segments(25);
merge_policy.set_min_layer_size(10_000);
@@ -115,8 +116,6 @@ impl IndexManager {
&DATA_DIR_MANAGER.envelope_dir, e
)
});
let mut query_parser = QueryParser::for_index(&index, SchemaTools::email_default_fields());
query_parser.set_conjunction_by_default();
let (sender, mut receiver) = mpsc::channel::<TantivyDocument>(100);
@@ -199,7 +198,6 @@ impl IndexManager {
index_writer,
sender,
reader,
query_parser,
handle: Mutex::new(Some(handler)),
}
}
@@ -277,7 +275,6 @@ impl IndexManager {
&self,
accounts: Option<HashSet<u64>>,
filter: EmailSearchFilter,
parser: QueryParser,
) -> BichonResult<Box<dyn Query>> {
let f = SchemaTools::email_fields();
let mut subqueries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
@@ -306,7 +303,10 @@ impl IndexManager {
}
if let Some(ref text) = filter.text {
let query = parser
let query_parser =
QueryParser::for_index(&self.index, SchemaTools::email_default_fields());
let query = query_parser
.parse_query(text)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, Box::new(query)));
@@ -314,16 +314,20 @@ impl IndexManager {
if let Some(ref subject_val) = filter.subject {
let query_parser = QueryParser::for_index(&self.index, vec![f.f_subject]);
if let Ok(q) = query_parser.parse_query(subject_val) {
subqueries.push((Occur::Must, q));
}
let q = query_parser
.parse_query(subject_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
println!("{:#?}", &q);
subqueries.push((Occur::Must, q));
}
if let Some(ref body_val) = filter.body {
let query_parser = QueryParser::for_index(&self.index, vec![f.f_body]);
if let Ok(q) = query_parser.parse_query(body_val) {
subqueries.push((Occur::Must, q));
}
let q = query_parser
.parse_query(body_val)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
if let Some(ref tags) = filter.tags {
@@ -347,15 +351,17 @@ impl IndexManager {
}
for (field, opt_value) in [
(f.f_from, &filter.from),
(f.f_to, &filter.to),
(f.f_cc, &filter.cc),
(f.f_bcc, &filter.bcc),
(f.f_from_text, &filter.from),
(f.f_to_text, &filter.to),
(f.f_cc_text, &filter.cc),
(f.f_bcc_text, &filter.bcc),
] {
if let Some(ref v) = opt_value {
if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) {
subqueries.push((Occur::Must, Box::new(query)));
}
let query_parser = QueryParser::for_index(&self.index, vec![field]);
let q = query_parser
.parse_query(v)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
}
@@ -378,12 +384,24 @@ impl IndexManager {
if let Some(ref name) = filter.attachment_name {
if name.contains('.') {
let term = Term::from_field_text(f.f_attachment_name_exact, name);
let query = TermQuery::new(term, IndexRecordOption::Basic);
subqueries.push((Occur::Should, Box::new(query)));
}
let exact_query = TermQuery::new(term, IndexRecordOption::Basic);
let query_parser = QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
if let Ok(q) = query_parser.parse_query(name) {
let query_parser =
QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
let q: Box<dyn Query> = query_parser
.parse_query(name)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
let query = BooleanQuery::new(vec![
(Occur::Should, Box::new(exact_query)),
(Occur::Should, Box::new(q)),
]);
subqueries.push((Occur::Must, Box::new(query)));
} else {
let query_parser =
QueryParser::for_index(&self.index, vec![f.f_attachment_name_text]);
let q = query_parser
.parse_query(name)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
subqueries.push((Occur::Must, q));
}
}
@@ -1061,7 +1079,7 @@ impl IndexManager {
) -> BichonResult<DataPage<Envelope>> {
assert!(page > 0, "Page number must be greater than 0");
assert!(page_size > 0, "Page size must be greater than 0");
let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
let query = self.filter_query(accounts, filter)?;
let searcher = self.create_searcher()?;
let total = searcher
.search(&query, &Count)
crates/core/src/store/tantivy/fields.rs
+12
View File
@@ -27,10 +27,17 @@ pub const F_SUBJECT: &str = "subject";
pub const F_BODY: &str = "body";
pub const F_PREVIEW: &str = "preview";
pub const F_CONTENT_HASH: &str = "content_hash";
pub const F_FROM: &str = "from";
pub const F_TO: &str = "to";
pub const F_CC: &str = "cc";
pub const F_BCC: &str = "bcc";
pub const F_FROM_TEXT: &str = "from_text";
pub const F_TO_TEXT: &str = "to_text";
pub const F_CC_TEXT: &str = "cc_text";
pub const F_BCC_TEXT: &str = "bcc_text";
pub const F_DATE: &str = "date";
pub const F_INTERNAL_DATE: &str = "internal_date";
pub const F_INGEST_AT: &str = "ingest_at";
@@ -73,6 +80,10 @@ pub struct EmailFields {
pub f_to: Field,
pub f_cc: Field,
pub f_bcc: Field,
pub f_from_text: Field,
pub f_to_text: Field,
pub f_cc_text: Field,
pub f_bcc_text: Field,
pub f_date: Field,
pub f_internal_date: Field,
pub f_ingest_at: Field,
@@ -97,6 +108,7 @@ pub struct AttachmentFields {
pub f_account_id: Field,
pub f_mailbox_id: Field,
pub f_from: Field,
pub f_from_text: Field,
pub f_subject: Field,
pub f_content_hash: Field,
pub f_text: Field,
crates/core/src/store/tantivy/filter.rs
+73
View File
@@ -0,0 +1,73 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use deunicode::deunicode;
use tantivy::tokenizer::{Token, TokenFilter, TokenStream, Tokenizer};
#[derive(Clone, Debug, Default)]
pub struct DeunicodeFilter;
impl TokenFilter for DeunicodeFilter {
type Tokenizer<T: Tokenizer> = DeunicodeFilterWrapper<T>;
fn transform<T: Tokenizer>(self, tokenizer: T) -> Self::Tokenizer<T> {
DeunicodeFilterWrapper { inner: tokenizer }
}
}
#[derive(Clone, Debug)]
pub struct DeunicodeFilterWrapper<T> {
inner: T,
}
impl<T: Tokenizer> Tokenizer for DeunicodeFilterWrapper<T> {
type TokenStream<'a> = DeunicodeTokenStream<T::TokenStream<'a>>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
DeunicodeTokenStream {
inner: self.inner.token_stream(text),
}
}
}
pub struct DeunicodeTokenStream<S> {
inner: S,
}
impl<S: TokenStream> TokenStream for DeunicodeTokenStream<S> {
fn advance(&mut self) -> bool {
if !self.inner.advance() {
return false;
}
let token: &mut Token = self.inner.token_mut();
// Only allocate a new String when the text actually contains
// non-ASCII characters to avoid unnecessary overhead.
if !token.text.is_ascii() {
token.text = deunicode(&token.text);
}
true
}
fn token(&self) -> &Token {
self.inner.token()
}
fn token_mut(&mut self) -> &mut Token {
self.inner.token_mut()
}
}
crates/core/src/store/tantivy/mod.rs
+2
View File
@@ -26,8 +26,10 @@ use crate::{
pub mod attachment;
pub mod envelope;
pub mod fields;
pub mod filter;
pub mod model;
pub mod schema;
pub mod tokenizers;
pub fn fatal_commit(writer: &mut IndexWriter) {
const MAX_RETRIES: usize = 3;
crates/core/src/store/tantivy/model.rs
+19 -2
View File
@@ -67,16 +67,20 @@ impl EnvelopeWithAttachments {
doc.add_text(fields.f_preview, &self.envelope.preview);
doc.add_text(fields.f_content_hash, &self.envelope.content_hash);
doc.add_text(fields.f_from, &self.envelope.from);
doc.add_text(fields.f_from_text, &self.envelope.from);
doc.add_text(fields.f_body, body_text);
for to in &self.envelope.to {
doc.add_text(fields.f_to, to);
doc.add_text(fields.f_to_text, to);
}
for cc in &self.envelope.cc {
doc.add_text(fields.f_cc, cc);
doc.add_text(fields.f_cc_text, cc);
}
for bcc in &self.envelope.bcc {
doc.add_text(fields.f_bcc, bcc);
doc.add_text(fields.f_bcc_text, bcc);
}
doc.add_i64(fields.f_date, self.envelope.date);
@@ -353,9 +357,10 @@ impl AttachmentModel {
doc.add_text(f.f_envelope_id, self.envelope_id);
doc.add_u64(f.f_account_id, self.account_id);
doc.add_u64(f.f_mailbox_id, self.mailbox_id);
doc.add_text(f.f_subject, self.subject);
doc.add_text(f.f_subject, &self.subject);
doc.add_text(f.f_content_hash, self.content_hash);
doc.add_text(f.f_from, self.from);
doc.add_text(f.f_from, &self.from);
doc.add_text(f.f_from_text, &self.from);
doc.add_i64(f.f_date, self.date);
doc.add_i64(f.f_ingest_at, self.ingest_at);
doc.add_u64(f.f_size, self.size);
@@ -385,6 +390,18 @@ impl AttachmentModel {
doc.add_text(f.f_name_exact, name);
}
if let Some(tags) = &self.tags {
for tag in tags {
doc.add_facet(f.f_tags, tag);
}
}
if let Some(tags) = &self.auto_tags {
for tag in tags {
doc.add_facet(f.f_auto_tags, tag);
}
}
doc
}
crates/core/src/store/tantivy/schema.rs
+70 -36
View File
@@ -17,17 +17,19 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::sync::{Arc, LazyLock};
use tantivy::schema::{FacetOptions, Field, INDEXED};
use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
use tantivy::schema::{
FacetOptions, Field, IndexRecordOption, TextFieldIndexing, TextOptions, INDEXED,
};
use tantivy::schema::{Schema, FAST, STORED, STRING};
use crate::store::tantivy::fields::{
AttachmentFields, EmailFields, F_ACCOUNT_ID, F_ATTACHMENTS, F_ATTACHMENT_CATEGORY,
F_ATTACHMENT_CONTENT_HASH, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_COUNT, F_ATTACHMENT_EXT,
F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BODY, F_CC,
F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_HAS_TEXT, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID, F_NAME_EXACT, F_NAME_TEXT,
F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID, F_SIZE, F_SUBJECT, F_TAGS,
F_TEXT, F_THREAD_ID, F_TO, F_UID,
F_ATTACHMENT_NAME_EXACT, F_ATTACHMENT_NAME_TEXT, F_AUTO_TAGS, F_BCC, F_BCC_TEXT, F_BODY, F_CC,
F_CC_TEXT, F_CONTENT_HASH, F_DATE, F_ENVELOPE_ID, F_FROM, F_FROM_TEXT, F_HAS_TEXT, F_ID,
F_INGEST_AT, F_INTERNAL_DATE, F_IS_INDEXED, F_IS_MESSAGE, F_IS_OCR, F_MAILBOX_ID, F_MESSAGE_ID,
F_NAME_EXACT, F_NAME_TEXT, F_PAGE_COUNT, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID,
F_SIZE, F_SUBJECT, F_TAGS, F_TEXT, F_THREAD_ID, F_TO, F_TO_TEXT, F_UID,
};
static EMAIL_FIELDS: LazyLock<Arc<EmailFields>> = LazyLock::new(|| {
@@ -58,29 +60,10 @@ impl SchemaTools {
fields.f_subject,
fields.f_body,
fields.f_attachment_name_text,
fields.f_attachment_name_exact,
fields.f_from,
fields.f_to,
]
}
pub fn attachment_schema() -> Schema {
let (schema, _) = Self::create_attachment_schema();
schema
}
pub fn attachment_fields() -> &'static AttachmentFields {
&ATTACHMENT_FIELDS
}
pub fn attachment_default_fields() -> Vec<Field> {
let fields = Self::attachment_fields();
vec![
fields.f_subject,
fields.f_text,
fields.f_name_exact,
fields.f_name_text,
fields.f_from,
fields.f_from_text,
fields.f_to_text,
fields.f_cc_text,
fields.f_bcc_text,
]
}
@@ -91,14 +74,21 @@ impl SchemaTools {
let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
let f_body = builder.add_text_field(F_BODY, TEXT);
let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
let f_body = builder.add_text_field(F_BODY, Self::text_no_store("euro"));
let f_preview = builder.add_text_field(F_PREVIEW, STORED);
let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
let f_to = builder.add_text_field(F_TO, STRING | STORED);
let f_cc = builder.add_text_field(F_CC, STRING | STORED);
let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
let f_to_text = builder.add_text_field(F_TO_TEXT, Self::text_no_store("euro"));
let f_cc_text = builder.add_text_field(F_CC_TEXT, Self::text_no_store("euro"));
let f_bcc_text = builder.add_text_field(F_BCC_TEXT, Self::text_no_store("euro"));
let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST);
let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
@@ -107,7 +97,8 @@ impl SchemaTools {
let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_regular_attachment_count =
builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST);
let f_attachment_name_text = builder.add_text_field(F_ATTACHMENT_NAME_TEXT, TEXT);
let f_attachment_name_text =
builder.add_text_field(F_ATTACHMENT_NAME_TEXT, Self::text_no_store("euro"));
let f_attachment_name_exact = builder.add_text_field(F_ATTACHMENT_NAME_EXACT, STRING);
let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED);
let f_attachment_content_hash =
@@ -133,6 +124,10 @@ impl SchemaTools {
f_to,
f_cc,
f_bcc,
f_from_text,
f_to_text,
f_cc_text,
f_bcc_text,
f_date,
f_internal_date,
f_ingest_at,
@@ -153,15 +148,35 @@ impl SchemaTools {
(builder.build(), fields)
}
pub fn attachment_schema() -> Schema {
let (schema, _) = Self::create_attachment_schema();
schema
}
pub fn attachment_fields() -> &'static AttachmentFields {
&ATTACHMENT_FIELDS
}
pub fn attachment_default_fields() -> Vec<Field> {
let fields = Self::attachment_fields();
vec![
fields.f_subject,
fields.f_text,
fields.f_name_text,
fields.f_from_text,
]
}
pub fn create_attachment_schema() -> (Schema, AttachmentFields) {
let mut builder = Schema::builder();
let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST);
let f_envelope_id = builder.add_text_field(F_ENVELOPE_ID, STRING | STORED | FAST);
let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
let f_subject = builder.add_text_field(F_SUBJECT, Self::text_store("euro"));
let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | STORED | FAST);
let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
let f_from_text = builder.add_text_field(F_FROM_TEXT, Self::text_no_store("euro"));
let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST);
let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST);
let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST);
@@ -170,13 +185,13 @@ impl SchemaTools {
let f_content_type =
builder.add_text_field(F_ATTACHMENT_CONTENT_TYPE, STRING | STORED | FAST);
let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST);
let f_text = builder.add_text_field(F_TEXT, TEXT);
let f_text = builder.add_text_field(F_TEXT, Self::text_no_store("euro"));
let f_has_text = builder.add_bool_field(F_HAS_TEXT, INDEXED | STORED | FAST);
let f_is_ocr = builder.add_bool_field(F_IS_OCR, INDEXED | STORED | FAST);
let f_page_count = builder.add_u64_field(F_PAGE_COUNT, INDEXED | STORED | FAST);
let f_is_indexed = builder.add_bool_field(F_IS_INDEXED, INDEXED | STORED | FAST);
let f_is_message = builder.add_bool_field(F_IS_MESSAGE, INDEXED | STORED | FAST);
let f_name_text = builder.add_text_field(F_NAME_TEXT, TEXT);
let f_name_text = builder.add_text_field(F_NAME_TEXT, Self::text_no_store("euro"));
let f_name_exact = builder.add_text_field(F_NAME_EXACT, STRING | STORED);
let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
let f_auto_tags =
@@ -189,6 +204,7 @@ impl SchemaTools {
f_subject,
f_content_hash,
f_from,
f_from_text,
f_date,
f_ingest_at,
f_size,
@@ -209,4 +225,22 @@ impl SchemaTools {
};
(builder.build(), fields)
}
fn text_no_store(tokenizer: &str) -> TextOptions {
TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
}
fn text_store(tokenizer: &str) -> TextOptions {
TextOptions::default()
.set_indexing_options(
TextFieldIndexing::default()
.set_tokenizer(tokenizer)
.set_index_option(IndexRecordOption::WithFreqsAndPositions),
)
.set_stored()
}
}
crates/core/src/store/tantivy/tokenizers.rs
+72
View File
@@ -0,0 +1,72 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use tantivy::tokenizer::*;
use whichlang::{detect_language, Lang};
use crate::store::tantivy::filter::DeunicodeFilter;
#[derive(Clone)]
pub struct EuroTokenizer {
en: TextAnalyzer,
fr: TextAnalyzer,
de: TextAnalyzer,
es: TextAnalyzer,
nl: TextAnalyzer,
default: TextAnalyzer,
}
impl EuroTokenizer {
pub fn new() -> Self {
fn build(lang: Language) -> TextAnalyzer {
TextAnalyzer::builder(SimpleTokenizer::default())
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser)
.filter(DeunicodeFilter)
.filter(Stemmer::new(lang))
.build()
}
Self {
en: build(Language::English),
fr: build(Language::French),
de: build(Language::German),
es: build(Language::Spanish),
nl: build(Language::Dutch),
default: TextAnalyzer::builder(SimpleTokenizer::default())
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser)
.build(),
}
}
}
impl Tokenizer for EuroTokenizer {
type TokenStream<'a> = BoxTokenStream<'a>;
fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
match detect_language(text) {
Lang::Eng => self.en.token_stream(text),
Lang::Fra => self.fr.token_stream(text),
Lang::Deu => self.de.token_stream(text),
Lang::Spa => self.es.token_stream(text),
Lang::Nld => self.nl.token_stream(text),
_ => self.default.token_stream(text), // fallback
}
}
}