initial commit

src/modules/indexer/envelope.rs

@@ -0,0 +1,194 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use crate::modules::error::code::ErrorCode;
+use crate::modules::utils::create_hash;
+use crate::modules::{error::BichonResult, indexer::schema::SchemaTools};
+use crate::raise_error;
+use poem_openapi::Object;
+use serde::{Deserialize, Serialize};
+use tantivy::schema::Facet;
+use tantivy::{doc, schema::Value, TantivyDocument};
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+pub struct Envelope {
+    pub id: u64,
+    pub message_id: String,
+    pub account_id: u64,
+    pub mailbox_id: u64,
+    pub uid: u32,
+    pub subject: String,
+    pub text: String,
+    pub from: String,
+    pub to: Vec<String>,
+    pub cc: Vec<String>,
+    pub bcc: Vec<String>,
+    pub date: i64,
+    pub internal_date: i64,
+    pub size: u32,
+    pub thread_id: u64,
+    pub attachments: Vec<String>,
+    pub tags: Option<Vec<String>>,
+}
+
+fn extract_u64_field(
+    document: &TantivyDocument,
+    field: tantivy::schema::Field,
+) -> BichonResult<u64> {
+    let value = document.get_first(field).ok_or_else(|| {
+        raise_error!(
+            format!("miss '{}' field in tantivy document", stringify!(field)),
+            ErrorCode::InternalError
+        )
+    })?;
+    value.as_u64().ok_or_else(|| {
+        raise_error!(
+            format!("'{}' field is not a u64", stringify!(field)),
+            ErrorCode::InternalError
+        )
+    })
+}
+
+fn extract_i64_field(
+    document: &TantivyDocument,
+    field: tantivy::schema::Field,
+) -> BichonResult<i64> {
+    let value = document.get_first(field).ok_or_else(|| {
+        raise_error!(
+            format!("miss '{}' field in tantivy document", stringify!(field)),
+            ErrorCode::InternalError
+        )
+    })?;
+    value.as_i64().ok_or_else(|| {
+        raise_error!(
+            format!("'{}' field is not a i64", stringify!(field)),
+            ErrorCode::InternalError
+        )
+    })
+}
+
+fn extract_string_field(
+    document: &TantivyDocument,
+    field: tantivy::schema::Field,
+) -> BichonResult<String> {
+    let value = document.get_first(field).ok_or_else(|| {
+        raise_error!(
+            format!("'{}' field not found", stringify!(field)),
+            ErrorCode::InternalError
+        )
+    })?;
+    value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+        raise_error!(
+            format!("'{}' field is not a string", stringify!(field)),
+            ErrorCode::InternalError
+        )
+    })
+}
+
+fn extract_vec_string_field(
+    document: &TantivyDocument,
+    field: tantivy::schema::Field,
+) -> BichonResult<Vec<String>> {
+    let value = document
+        .get_all(field)
+        .filter_map(|v| v.as_str().map(|s| s.to_string()))
+        .collect();
+    Ok(value)
+}
+
+impl Envelope {
+    pub fn to_document(&self, mailbox_id: u64) -> BichonResult<TantivyDocument> {
+        let fields = SchemaTools::envelope_fields();
+        let mut doc = doc!();
+        doc.add_u64(fields.f_id, self.id);
+        doc.add_text(fields.f_message_id, &self.message_id);
+        doc.add_u64(fields.f_account_id, self.account_id);
+        doc.add_u64(fields.f_mailbox_id, mailbox_id);
+        doc.add_u64(fields.f_uid, self.uid as u64);
+        doc.add_text(fields.f_subject, &self.subject);
+        doc.add_text(fields.f_text, &self.text);
+        doc.add_text(fields.f_from, &self.from);
+        for to in &self.to {
+            doc.add_text(fields.f_to, to);
+        }
+        for cc in &self.cc {
+            doc.add_text(fields.f_cc, cc);
+        }
+        for bcc in &self.bcc {
+            doc.add_text(fields.f_bcc, bcc);
+        }
+        doc.add_i64(fields.f_date, self.date);
+        doc.add_i64(fields.f_internal_date, self.internal_date);
+        doc.add_u64(fields.f_size, self.size as u64);
+        doc.add_u64(fields.f_thread_id, self.thread_id);
+        for att in &self.attachments {
+            doc.add_text(fields.f_attachments, att);
+        }
+        doc.add_bool(fields.f_has_attachment, self.attachments.len() > 0);
+        Ok(doc)
+    }
+
+    pub async fn from_tantivy_doc(doc: &TantivyDocument) -> BichonResult<Self> {
+        let fields = SchemaTools::envelope_fields();
+        let account_id = extract_u64_field(doc, fields.f_account_id)?;
+        let message_id = extract_string_field(doc, fields.f_message_id)?;
+        let mailbox_id = extract_u64_field(doc, fields.f_mailbox_id)?;
+        let id = create_hash(account_id, &message_id);
+        let full_text = extract_string_field(doc, fields.f_text)?;
+
+        // Take up to the first 120 characters as a preview;
+        let preview = if full_text.chars().count() > 120 {
+            full_text.chars().take(120).collect::<String>() + "..."
+        } else {
+            full_text
+        };
+
+        let tags: Vec<String> = doc
+            .get_all(fields.f_tags)
+            .filter_map(|value| value.as_facet())
+            .map(|facet_encoded_str| {
+                Facet::from_encoded(facet_encoded_str.as_bytes().to_vec())
+                    .ok()
+                    .map(|facet| facet.to_string())
+            })
+            .flatten()
+            .collect();
+
+        let envelope = Envelope {
+            id,
+            account_id,
+            mailbox_id,
+            message_id: extract_string_field(doc, fields.f_message_id)?,
+            uid: extract_u64_field(doc, fields.f_uid)? as u32,
+            subject: extract_string_field(doc, fields.f_subject)?,
+            text: preview,
+            from: extract_string_field(doc, fields.f_from)?,
+            to: extract_vec_string_field(doc, fields.f_to)?,
+            cc: extract_vec_string_field(doc, fields.f_cc)?,
+            bcc: extract_vec_string_field(doc, fields.f_bcc)?,
+            date: extract_i64_field(doc, fields.f_date)?,
+            internal_date: extract_i64_field(doc, fields.f_internal_date)?,
+            size: extract_u64_field(doc, fields.f_size)? as u32,
+            thread_id: extract_u64_field(doc, fields.f_thread_id)?,
+            attachments: extract_vec_string_field(doc, fields.f_attachments)?,
+            tags: Some(tags),
+        };
+        Ok(envelope)
+    }
+}

src/modules/indexer/fields.rs

@@ -0,0 +1,69 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use tantivy::schema::Field;
+
+pub const F_MESSAGE_ID: &str = "message_id";
+pub const F_ACCOUNT_ID: &str = "account_id";
+pub const F_MAILBOX_ID: &str = "mailbox_id";
+pub const F_UID: &str = "uid";
+pub const F_SUBJECT: &str = "subject";
+pub const F_TEXT: &str = "text";
+pub const F_FROM: &str = "from";
+pub const F_TO: &str = "to";
+pub const F_CC: &str = "cc";
+pub const F_BCC: &str = "bcc";
+pub const F_DATE: &str = "date";
+pub const F_INTERNAL_DATE: &str = "internal_date";
+pub const F_SIZE: &str = "size";
+pub const F_THREAD_ID: &str = "thread_id";
+pub const F_ATTACHMENTS: &str = "attachments";
+pub const F_HAS_ATTACHMENT: &str = "has_attachment";
+pub const F_TAGS: &str = "tags";
+
+pub const F_ID: &str = "id";
+pub struct EnvelopeFields {
+    pub f_id: Field,
+    pub f_message_id: Field,
+    pub f_account_id: Field,
+    pub f_mailbox_id: Field,
+    pub f_uid: Field,
+    pub f_subject: Field,
+    pub f_text: Field,
+    pub f_from: Field,
+    pub f_to: Field,
+    pub f_cc: Field,
+    pub f_bcc: Field,
+    pub f_date: Field,
+    pub f_internal_date: Field,
+    pub f_size: Field,
+    pub f_thread_id: Field,
+    pub f_attachments: Field,
+    pub f_has_attachment: Field,
+    pub f_tags: Field,
+}
+
+pub const F_EML: &str = "eml";
+
+pub struct EmlFields {
+    pub f_id: Field,
+    pub f_account_id: Field,
+    pub f_mailbox_id: Field,
+    pub f_eml: Field,
+}

src/modules/indexer/mod.rs

@@ -0,0 +1,25 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+pub mod envelope;
+pub mod fields;
+pub mod manager;
+pub mod schema;
+#[cfg(test)]
+mod tests;

src/modules/indexer/schema.rs

@@ -0,0 +1,128 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use std::sync::{Arc, LazyLock};
+
+use crate::modules::indexer::fields::{EnvelopeFields, *};
+use tantivy::schema::{FacetOptions, Field, INDEXED};
+use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT};
+
+static ENVELOPE_FIELDS: LazyLock<Arc<EnvelopeFields>> = LazyLock::new(|| {
+    let (_, fields) = SchemaTools::create_envelope_schema();
+    Arc::new(fields)
+});
+
+static EML_FIELDS: LazyLock<Arc<EmlFields>> = LazyLock::new(|| {
+    let (_, fields) = SchemaTools::create_eml_schema();
+    Arc::new(fields)
+});
+
+pub struct SchemaTools;
+
+impl SchemaTools {
+    pub fn envelope_schema() -> Schema {
+        let (schema, _) = Self::create_envelope_schema();
+        schema
+    }
+
+    pub fn eml_schema() -> Schema {
+        let (schema, _) = Self::create_eml_schema();
+        schema
+    }
+
+    pub fn envelope_fields() -> &'static EnvelopeFields {
+        &ENVELOPE_FIELDS
+    }
+
+    pub fn eml_fields() -> &'static EmlFields {
+        &EML_FIELDS
+    }
+
+    pub fn envelope_default_fields() -> Vec<Field> {
+        let fields = Self::envelope_fields();
+        vec![fields.f_subject, fields.f_text, fields.f_attachments]
+    }
+
+    pub fn create_envelope_schema() -> (Schema, EnvelopeFields) {
+        let mut builder = Schema::builder();
+        let f_id = builder.add_u64_field(F_ID, INDEXED | STORED | FAST);
+        // Account/ Mailbox IDs: numeric, for filtering/aggregation
+        let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        // UID: numeric, locate message
+        let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST);
+        // Subject/body: tokenized for full-text search
+        let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED);
+        let f_text = builder.add_text_field(F_TEXT, TEXT | STORED);
+        // Email addresses: exact match search
+        let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST);
+        let f_to = builder.add_text_field(F_TO, STRING | STORED);
+        let f_cc = builder.add_text_field(F_CC, STRING | STORED);
+        let f_bcc = builder.add_text_field(F_BCC, STRING | STORED);
+        // Date fields: numeric, range filtering
+        let f_date = builder.add_i64_field(F_DATE, STORED | FAST);
+        let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, STORED | FAST);
+        // Size: numeric, range filtering
+        let f_size = builder.add_u64_field(F_SIZE, STORED | FAST);
+        // Thread ID: numeric, filter by thread
+        let f_thread_id = builder.add_u64_field(F_THREAD_ID, INDEXED | STORED | FAST);
+        // Message-ID: unique identifier, no tokenization
+        let f_message_id = builder.add_text_field(F_MESSAGE_ID, STRING | STORED);
+        // Attachments: exact match search
+        let f_attachments = builder.add_text_field(F_ATTACHMENTS, TEXT | STORED);
+        let f_has_attachment = builder.add_bool_field(F_HAS_ATTACHMENT, INDEXED | STORED | FAST);
+        let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored());
+        let fields = EnvelopeFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_uid,
+            f_subject,
+            f_text,
+            f_from,
+            f_to,
+            f_cc,
+            f_bcc,
+            f_date,
+            f_internal_date,
+            f_size,
+            f_thread_id,
+            f_message_id,
+            f_attachments,
+            f_has_attachment,
+            f_tags,
+        };
+        (builder.build(), fields)
+    }
+
+    pub fn create_eml_schema() -> (Schema, EmlFields) {
+        let mut builder = Schema::builder();
+        let f_id = builder.add_u64_field(F_ID, INDEXED | FAST);
+        let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST);
+        let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST);
+        let f_eml = builder.add_bytes_field(F_EML, STORED);
+        let fields = EmlFields {
+            f_id,
+            f_account_id,
+            f_mailbox_id,
+            f_eml,
+        };
+        (builder.build(), fields)
+    }
+}

src/modules/indexer/tests.rs

@@ -0,0 +1,229 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+use std::{path::PathBuf, time::Duration};
+
+use chrono::Utc;
+use serde_json::json;
+use tantivy::{
+    aggregation::{
+        agg_req::Aggregations,
+        agg_result::{AggregationResult, BucketEntries, BucketResult, MetricResult},
+        AggregationCollector, Key,
+    },
+    collector::TopDocs,
+    doc,
+    indexer::UserOperation,
+    query::{AllQuery, QueryParser, TermQuery},
+    schema::{IndexRecordOption, Schema, Value},
+    Index, IndexWriter, TantivyDocument, Term,
+};
+
+use crate::{
+    modules::{
+        dashboard::TimeBucket,
+        indexer::{
+            fields::{F_FROM, F_HAS_ATTACHMENT, F_INTERNAL_DATE, F_SIZE},
+            schema::SchemaTools,
+        },
+    },
+    utc_now,
+};
+
+#[tokio::test]
+async fn test1() {
+    let index = Index::open_in_dir(PathBuf::from("E:/bichon-data/envelope")).unwrap();
+    let reader = index.reader().unwrap();
+    let mut query_parser = QueryParser::for_index(&index, SchemaTools::envelope_default_fields());
+    query_parser.set_conjunction_by_default();
+    let searcher = reader.searcher();
+
+    let now_ms = utc_now!();
+    let week_ago_ms = (Utc::now() - Duration::from_secs(60 * 60 * 24 * 7)).timestamp_millis();
+    let aggregations: Aggregations = serde_json::from_value(json!({
+        "total_size": {
+            "sum": { "field": F_SIZE }
+        },
+        "recent_7d_histogram": {
+            "histogram": {
+                "field": F_INTERNAL_DATE,
+                "interval": 86400000,
+                "hard_bounds": {
+                    "min": week_ago_ms,
+                    "max": now_ms
+                }
+            }
+        },
+        "top_from_values": {
+            "terms": {
+                "field": F_FROM,
+                "size": 10
+            }
+        },
+        "attachment_stats": {
+            "terms": {
+                "field": F_HAS_ATTACHMENT
+            }
+        }
+    }))
+    .unwrap();
+
+    let query = AllQuery;
+    let agg_collector = AggregationCollector::from_aggs(aggregations, Default::default());
+    let agg_results = searcher.search(&query, &agg_collector).unwrap();
+
+    let total_size = agg_results.0.get("total_size").unwrap();
+
+    if let AggregationResult::MetricResult(MetricResult::Sum(count)) = total_size {
+        let total_size = count.value.map(|v| v as u64).unwrap();
+        println!("{:#?}", total_size);
+    }
+
+    let recent_7d_histogram = agg_results.0.get("recent_7d_histogram").unwrap();
+
+    let mut recent_activity = Vec::with_capacity(15);
+    if let AggregationResult::BucketResult(BucketResult::Histogram { buckets, .. }) =
+        recent_7d_histogram
+    {
+        if let BucketEntries::Vec(bucket_list) = buckets {
+            for entry in bucket_list {
+                if let Key::F64(ms) = entry.key {
+                    recent_activity.push(TimeBucket {
+                        timestamp_ms: ms as i64,
+                        count: entry.doc_count,
+                    });
+                }
+            }
+        }
+    }
+    println!("recent_activity: {:#?}", recent_activity);
+
+    let top_from_values = agg_results.0.get("top_from_values").unwrap();
+
+    if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = top_from_values {
+        println!("{:#?}", buckets);
+    }
+
+    let attachment_stats = agg_results.0.get("attachment_stats").unwrap();
+
+    if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = attachment_stats {
+        println!("{:#?}", buckets);
+    }
+
+    //agg_results.
+}
+
+#[tokio::test]
+async fn test2() {
+    use tantivy::schema::{FAST, INDEXED, STORED, STRING};
+    let mut builder = Schema::builder();
+    let a = builder.add_u64_field("a", INDEXED | FAST);
+    let b = builder.add_text_field("b", STRING | STORED | FAST);
+
+    let schema = builder.build();
+    let index = Index::create_in_ram(schema);
+    let mut index_writer: IndexWriter = index.writer(50_000_000).unwrap();
+
+    let delete_term1 = Term::from_field_u64(a, 1u64);
+    let delete_term2 = Term::from_field_u64(a, 2u64);
+    let delete_term3 = Term::from_field_u64(a, 3u64);
+
+    let operations = vec![
+        //UserOperation::Delete(delete_term1),
+        UserOperation::Add(doc!(
+        a => 1u64,
+        b => "test1"
+        )),
+        //UserOperation::Delete(delete_term2),
+        UserOperation::Add(doc!(
+        a => 2u64,
+        b => "test1"
+        )),
+        //UserOperation::Delete(delete_term3),
+        UserOperation::Add(doc!(
+        a => 3u64,
+        b => "test1"
+        )),
+    ];
+
+    index_writer.run(operations).unwrap();
+    index_writer.commit().unwrap();
+
+    let reader = index.reader().unwrap();
+
+    let searcher = reader.searcher();
+
+    let tq = TermQuery::new(Term::from_field_u64(a, 3), IndexRecordOption::Basic);
+
+    let docs = searcher.search(&tq, &TopDocs::with_limit(1)).unwrap();
+
+    if let Some((_, doc_address)) = docs.first() {
+        let old_doc: TantivyDocument = searcher.doc_async(*doc_address).await.unwrap();
+
+        let mut new_doc = TantivyDocument::new();
+        for (field, value) in old_doc.field_values() {
+            if field == a {
+                new_doc.add_field_value(a, value);
+            }
+        }
+        new_doc.add_text(b, "test2");
+
+        let delete_term = Term::from_field_u64(a, 3);
+        index_writer.delete_term(delete_term);
+        index_writer.commit().unwrap();
+        index_writer.add_document(new_doc).unwrap();
+        index_writer.commit().unwrap();
+    }
+
+    reader.reload().unwrap();
+    let searcher = reader.searcher();
+    let docs = searcher.search(&tq, &TopDocs::with_limit(1)).unwrap();
+
+    if let Some((_, doc_address)) = docs.first() {
+        let doc: TantivyDocument = searcher.doc_async(*doc_address).await.unwrap();
+        for (field, value) in doc.field_values() {
+            if field == b {
+                let value = value.as_str();
+                println!("{:#?}", value);
+            }
+        }
+    } else {
+        println!("not found")
+    }
+
+    let delete_term = Term::from_field_u64(a, 3);
+    index_writer.delete_term(delete_term);
+    index_writer.commit().unwrap();
+
+    reader.reload().unwrap();
+    let searcher = reader.searcher();
+    let docs = searcher.search(&tq, &TopDocs::with_limit(1)).unwrap();
+
+    if let Some((_, doc_address)) = docs.first() {
+        let doc: TantivyDocument = searcher.doc_async(*doc_address).await.unwrap();
+        for (field, value) in doc.field_values() {
+            if field == b {
+                let value = value.as_str();
+                println!("{:#?}", value);
+            }
+        }
+    } else {
+        println!("not found")
+    }
+}