External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: use fjall to store detached emails and attachments

Browse Source
This commit is contained in:
rustmailer
2026-04-01 04:49:18 +08:00
parent c123ecb24a
commit bd10e15c65
36 changed files with 569 additions and 1542 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+180 -602
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+8 -12
View File
@@ -79,10 +79,6 @@ oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
url = { version = "2.5.8", features = ["serde"] }
sysinfo = "0.38.4"
num_cpus = "1.17.0"
cacache = { version = "13.1.0", default-features = false, features = [
"tokio-runtime",
"mmap",
] }
rand = "0.10.0"
encoding_rs = "0.8.35"
async-imap = { version = "0.11.2", default-features = false, features = [
@@ -111,8 +107,7 @@ dashmap = "6.1.0"
# Statically links OpenSSL by compiling from source, avoiding system library dependencies
openssl-sys = { version = "0.9.112", optional = true, features = ["vendored"] }
gethostname = "1.1.0"
tantivy = { version = "0.25.0", features = ["quickwit", "zstd-compression"] }
itoa = "1.0.17"
itoa = "1.0.18"
html2text = "0.16.7"
bytes = "1.11.1"
dialoguer = "0.12.0"
@@ -120,25 +115,26 @@ console = "0.16.3"
toml = "0.9.8"
memmap2 = "0.9.10"
outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" }
compressed-rtf = "1.0.0"
compressed-rtf = "1.0.1"
codepage-strings = "1.0.2"
mail-send = "0.5.2"
duckdb = { version = "1.10500.0", features = [
duckdb = { version = "1.10501.0", features = [
"chrono",
"bundled",
"r2d2",
"appender-arrow",
] }
arrow = { version = "57", features = ["ffi"] }
arrow = { version = "58", features = ["ffi"] }
r2d2 = { version = "0.8", default-features = false }
refinery = { version = "0.9", default-features = false }
refinery-core = { version = "0.9", default-features = false }
rcgen = "0.14.7"
rustls-pemfile = "2.2.0"
blake3 = "1.8.3"
uuid = { version = "1.22.0", features = ["v4", "serde"] }
blake3 = "1.8.4"
uuid = { version = "1.23.0", features = ["v4", "serde"] }
fjall = { version = "3.1.2", features = ["lz4", "metrics", "bytes_1"] }
[dev-dependencies]
#bincode = "1.3.3"
#secret-lib = "1.0.0"
tempfile = "3.27.0"
lettre = "0.11.19"
lettre = "0.11.20"
src/main.rs
+10 -4
View File
@@ -19,7 +19,9 @@
use bichon::{
bichon_version,
modules::{
common::rustls::RustMailerTls,
blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::imap::task::SYNC_TASKS,
common::rustls::BichonTls,
context::{executors::BichonContext, Initialize},
duckdb::init::DuckDBManager,
error::{code::ErrorCode, BichonResult},
@@ -64,6 +66,8 @@ async fn main() -> BichonResult<()> {
return Err(error);
}
let periodic_tasks = PeriodicTasks::setup();
let mut smtp_service: Option<SmtpServer> = None;
if SETTINGS.bichon_enable_smtp {
info!("SMTP service is enabled, starting...");
@@ -82,6 +86,7 @@ async fn main() -> BichonResult<()> {
}
start_http_server().await?;
periodic_tasks.shutdown().await;
if let Some(server) = smtp_service {
info!("Shutting down SMTP server...");
@@ -89,19 +94,20 @@ async fn main() -> BichonResult<()> {
info!("SMTP server stopped.");
}
SYNC_TASKS.shutdown().await;
ENVELOPE_INDEX_MANAGER.shutdown().await;
BLOB_MANAGER.shutdown().await;
info!("Bichon server stopped.");
Ok(())
}
/// Initialize the system by validating settings and starting necessary tasks.
async fn initialize() -> BichonResult<()> {
// SETTINGS.validate()?;
SignalManager::initialize().await?;
DataDirManager::initialize().await?;
DuckDBManager::initialize().await?;
UserManager::initialize().await?;
RustMailerTls::initialize().await?;
BichonTls::initialize().await?;
BichonContext::initialize().await?;
PeriodicTasks::start_background_tasks();
Ok(())
}
src/modules/account/migration.rs
+2 -6
View File
@@ -32,13 +32,10 @@ use crate::{
since::{DateSince, RelativeDate},
state::AccountRunningState,
},
blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::imap::mailbox::MailBox,
database::{list_all_impl, secondary_find_impl, with_transaction},
error::BichonResult,
indexer::{
attachment::ATTACHMENT_INDEX_MANAGER, eml::EML_INDEX_MANAGER,
manager::ENVELOPE_INDEX_MANAGER,
},
users::{role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel, DEFAULT_ADMIN_USER_ID},
},
utc_now,
@@ -365,8 +362,7 @@ impl AccountV4 {
let content_hashes = ENVELOPE_INDEX_MANAGER
.delete_account_envelopes(account.id)
.await?;
EML_INDEX_MANAGER.delete(&content_hashes).await?;
ATTACHMENT_INDEX_MANAGER.delete(&content_hashes).await?;
BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
Self::delete_account(account.id).await?;
info!("Sequential cleanup completed for account: {}", account.id);
Ok(())
src/modules/indexer/envelope.rs → src/modules/blob/envelope.rs
View File
src/modules/indexer/manager.rs → src/modules/blob/manager.rs
+32 -19
View File
@@ -33,39 +33,46 @@ use crate::modules::{
};
use crate::{
modules::{
blob::envelope::Envelope,
common::signal::SIGNAL_MANAGER,
dashboard::{DashboardStats, LargestEmail},
error::{code::ErrorCode, BichonResult},
indexer::envelope::Envelope,
message::search::SearchFilter,
rest::response::DataPage,
},
raise_error,
};
use tokio::{sync::mpsc, task};
use tokio::{
sync::{mpsc, Mutex},
task::{self, JoinHandle},
};
pub static ENVELOPE_INDEX_MANAGER: LazyLock<EnvelopeIndexManager> =
LazyLock::new(EnvelopeIndexManager::new);
pub const ENVELOPE_BATCH_SIZE: usize = 500;
pub const EML_BATCH_SIZE: usize = 100;
pub const ENVELOPE_BATCH_SIZE: usize = 100;
const MAX_BUFFER_DURATION: Duration = Duration::from_secs(10);
pub enum MetadataOp {
Record((Envelope, Vec<AttachmentInfo>)),
Shutdown,
}
pub struct EnvelopeIndexManager {
sender: mpsc::Sender<MetadataOp>,
sender: mpsc::Sender<(Envelope, Vec<AttachmentInfo>)>,
handle: Mutex<Option<JoinHandle<()>>>,
}
impl EnvelopeIndexManager {
pub async fn shutdown(&self) {
let mut guard = self.handle.lock().await;
if let Some(handle) = guard.take() {
tracing::info!("Waiting for EnvelopeIndexManager to sync all data...");
let _ = handle.await;
tracing::info!("EnvelopeIndexManager synchronized and closed.");
}
}
pub fn new() -> Self {
let (sender, mut receiver) = mpsc::channel::<MetadataOp>(1000);
task::spawn(async move {
let (sender, mut receiver) = mpsc::channel::<(Envelope, Vec<AttachmentInfo>)>(1000);
let handle = task::spawn(async move {
let mut buffer: Vec<(Envelope, Vec<AttachmentInfo>)> =
Vec::with_capacity(ENVELOPE_BATCH_SIZE);
let mut interval = tokio::time::interval(MAX_BUFFER_DURATION);
@@ -74,17 +81,19 @@ impl EnvelopeIndexManager {
tokio::select! {
maybe_msg = receiver.recv() => {
match maybe_msg {
Some(MetadataOp::Record(doc)) => {
Some(doc) => {
buffer.push(doc);
if buffer.len() >= ENVELOPE_BATCH_SIZE {
ENVELOPE_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
}
}
Some(MetadataOp::Shutdown) => {
None => {
if !buffer.is_empty() {
tracing::info!("Channel closed, flushing remaining {} items", buffer.len());
ENVELOPE_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
break;
}
None => break,
break;
},
}
}
_ = interval.tick() => {
@@ -93,16 +102,20 @@ impl EnvelopeIndexManager {
}
}
_ = shutdown.recv() => {
let _ = ENVELOPE_INDEX_MANAGER.sender.send(MetadataOp::Shutdown).await;
ENVELOPE_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
break;
}
}
}
});
Self { sender }
Self {
sender,
handle: Mutex::new(Some(handle)),
}
}
pub async fn add_document(&self, doc: (Envelope, Vec<AttachmentInfo>)) {
let _ = self.sender.send(MetadataOp::Record(doc)).await;
let _ = self.sender.send(doc).await;
}
async fn drain_and_commit(&self, buffer: &mut Vec<(Envelope, Vec<AttachmentInfo>)>) {
src/modules/indexer/mod.rs → src/modules/blob/mod.rs
+1 -11
View File
@@ -16,16 +16,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use tantivy::TantivyDocument;
pub mod attachment;
pub mod eml;
pub mod envelope;
pub mod fields;
pub mod manager;
pub mod schema;
pub enum DocumentOp {
Document((String, TantivyDocument)),
Shutdown,
}
pub mod storage;
src/modules/blob/storage.rs
+184
View File
@@ -0,0 +1,184 @@
use crate::modules::{
common::signal::SIGNAL_MANAGER,
envelope::extractor::reattach_eml_content,
error::{code::ErrorCode, BichonResult},
settings::dir::DATA_DIR_MANAGER,
};
use crate::raise_error;
use bytes::Bytes;
use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions};
use std::{io::Cursor, sync::LazyLock};
use tokio::{
sync::{mpsc, Mutex},
task::{self, JoinHandle},
};
pub static BLOB_MANAGER: LazyLock<BlobManager> = LazyLock::new(BlobManager::new);
pub struct DetachedEmail {
pub email: (String, Bytes),
pub attachments: Option<Vec<(String, Bytes)>>,
}
pub struct BlobManager {
sender: mpsc::Sender<DetachedEmail>,
db: Database,
email_keyspace: Keyspace,
attachments_keyspace: Keyspace,
handle: Mutex<Option<JoinHandle<()>>>,
}
impl BlobManager {
pub async fn shutdown(&self) {
let mut guard = self.handle.lock().await;
if let Some(handle) = guard.take() {
let _ = handle.await;
}
}
pub fn new() -> Self {
let db = Database::builder(&DATA_DIR_MANAGER.eml_dir)
.open()
.expect("Failed to initialize Fjall database: Check if the directory exists and has write permissions.");
let email_keyspace = db
.keyspace("email", || {
KeyspaceCreateOptions::default()
.with_kv_separation(Some(
KvSeparationOptions::default()
.separation_threshold(0)
.compression(CompressionType::Lz4)
.file_target_size(128 * 1024 * 1024)
.staleness_threshold(0.5)
.age_cutoff(0.6),
))
.max_memtable_size(64 * 1024 * 1024)
})
.expect("Failed to open 'email' keyspace: The partition metadata might be corrupted or inaccessible.");
let attachments_keyspace = db
.keyspace("attachments", || {
KeyspaceCreateOptions::default()
.with_kv_separation(Some(
KvSeparationOptions::default()
.separation_threshold(0)
.compression(CompressionType::Lz4)
.file_target_size(256 * 1024 * 1024)
.staleness_threshold(0.5)
.age_cutoff(0.6),
))
.max_memtable_size(64 * 1024 * 1024)
})
.expect("Failed to open 'attachments' keyspace: Check disk space for blob storage initialization.");
let (sender, mut receiver) = mpsc::channel::<DetachedEmail>(100);
let store = db.clone();
let email_keyspace_clone = email_keyspace.clone();
let attachments_keyspace_clone = attachments_keyspace.clone();
let handler = task::spawn(async move {
let mut shutdown = SIGNAL_MANAGER.subscribe();
loop {
tokio::select! {
res = receiver.recv() => {
match res {
Some(email) => {
let mut batch = store.batch();
batch.insert(&email_keyspace_clone, email.email.0, email.email.1);
if let Some(attachments) = email.attachments {
for a in attachments {
batch.insert(&attachments_keyspace_clone,a.0, a.1);
}
}
if let Err(e) = batch.commit() {
tracing::error!("Fjall Put Error {:?}", e);
} else {
tracing::info!("Fjall Put Success");
}
}
None => {
tracing::info!("BlobManager: All senders dropped, closing storage.");
break;
}
}
}
_ = shutdown.recv() => {
receiver.close();
let remaining = receiver.len();
tracing::info!(
"BlobManager: Shutdown signal received. Processing {} remaining tasks...",
remaining
);
while let Some(email) = receiver.recv().await {
let mut batch = store.batch();
batch.insert(&email_keyspace_clone, email.email.0, email.email.1);
if let Some(attachments) = email.attachments {
for a in attachments {
batch.insert(&attachments_keyspace_clone,a.0, a.1);
}
}
if let Err(e) = batch.commit() {
tracing::error!("Fjall Put Error {:?}", e);
} else {
tracing::info!("Fjall Put Success");
}
}
tracing::info!("BlobManager: All remaining tasks processed. Closing Fjall.");
break;
}
}
}
});
Self {
sender,
db,
email_keyspace,
attachments_keyspace,
handle: Mutex::new(Some(handler)),
}
}
pub async fn queue(&self, email: DetachedEmail) {
let _ = self.sender.send(email).await;
}
pub fn get_email(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
self.email_keyspace
.get(content_hash)
.map(|user_value| user_value.map(|s| s.into()))
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
}
pub fn get_attachment(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
self.attachments_keyspace
.get(content_hash)
.map(|user_value| user_value.map(|s| s.into()))
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
}
pub fn delete(
&self,
email_content_hashes: &[String],
attachment_content_hashes: &[String],
) -> BichonResult<()> {
let mut batch = self.db.batch();
for hash in email_content_hashes {
batch.remove(&self.email_keyspace, hash);
}
for hash in attachment_content_hashes {
batch.remove(&self.attachments_keyspace, hash);
}
batch
.commit()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(())
}
}
pub async fn get_reader(account_id: u64, eid: String) -> BichonResult<Cursor<Bytes>> {
let (_, data) = reattach_eml_content(account_id, eid).await?;
Ok(Cursor::new(data))
}
src/modules/cache/imap/sync/flow.rs
Vendored
+1 -1
View File
@@ -32,7 +32,7 @@ use crate::{
},
error::{code::ErrorCode, BichonError, BichonResult},
imap::executor::ImapExecutor,
indexer::manager::ENVELOPE_INDEX_MANAGER,
blob::manager::ENVELOPE_INDEX_MANAGER,
},
raise_error,
};
src/modules/cache/imap/sync/rebuild.rs
Vendored
+4 -9
View File
@@ -19,6 +19,7 @@
use crate::{
modules::{
account::migration::AccountModel,
blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::{
imap::{
mailbox::MailBox,
@@ -27,10 +28,6 @@ use crate::{
SEMAPHORE,
},
error::{code::ErrorCode, BichonError, BichonResult},
indexer::{
attachment::ATTACHMENT_INDEX_MANAGER, eml::EML_INDEX_MANAGER,
manager::ENVELOPE_INDEX_MANAGER,
},
},
raise_error,
};
@@ -200,10 +197,9 @@ pub async fn rebuild_mailbox_cache(
let content_hashes = ENVELOPE_INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
.await?;
// todo Distinguish these hashes to avoid mixing them
if !content_hashes.is_empty() {
EML_INDEX_MANAGER.delete(&content_hashes).await?;
ATTACHMENT_INDEX_MANAGER.delete(&content_hashes).await?;
BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
}
if remote_mailbox.exists == 0 {
@@ -236,8 +232,7 @@ pub async fn rebuild_mailbox_cache_by_date(
.await?;
if !content_hashes.is_empty() {
EML_INDEX_MANAGER.delete(&content_hashes).await?;
ATTACHMENT_INDEX_MANAGER.delete(&content_hashes).await?;
BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
}
if remote.exists == 0 {
src/modules/cache/imap/task.rs
Vendored
+40 -7
View File
@@ -16,7 +16,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::account::entity::AuthType;
use crate::modules::cache::imap::sync::execute_imap_sync;
use crate::modules::common::periodic::{PeriodicTask, TaskHandle};
@@ -26,10 +25,11 @@ use crate::modules::{
error::BichonResult,
};
use crate::utc_now;
use dashmap::DashMap;
use std::collections::HashMap;
use std::sync::atomic::{AtomicI64, Ordering};
use std::{sync::LazyLock, time::Duration};
use tracing::{error, warn};
use tokio::sync::Mutex;
use tracing::{error, info, warn};
static _DESCRIPTION: &str = "This task periodically synchronizes mailbox data for a specified account, ensuring that all local data is up-to-date.";
const TASK_INTERVAL: Duration = Duration::from_secs(10);
@@ -38,13 +38,13 @@ static LAST_WARN_TIME: AtomicI64 = AtomicI64::new(0);
const WARN_INTERVAL_MS: i64 = 600_000;
pub struct AccountSyncTask {
tasks: DashMap<u64, TaskHandle>,
tasks: Mutex<Option<HashMap<u64, TaskHandle>>>,
}
impl AccountSyncTask {
pub fn new() -> Self {
Self {
tasks: DashMap::new(),
tasks: Mutex::new(Some(HashMap::new())),
}
}
@@ -103,15 +103,48 @@ impl AccountSyncTask {
})
};
let handler = periodic_task.start(task, Some(account_id), TASK_INTERVAL, true, true);
self.tasks.insert(account_id, handler);
self.add_task(account_id, handler).await;
}
pub async fn add_task(&self, account_id: u64, handler: TaskHandle) {
let mut guard = self.tasks.lock().await;
if let Some(map) = guard.as_mut() {
map.insert(account_id, handler);
} else {
tracing::error!("Failed to add task: HashMap has been taken during shutdown.");
}
}
pub async fn stop(&self, account_id: u64) -> BichonResult<()> {
if let Some((_, handler)) = self.tasks.remove(&account_id) {
let mut guard = self.tasks.lock().await;
if let Some(map) = guard.as_mut() {
if let Some(handler) = map.remove(&account_id) {
drop(guard);
handler.cancel().await;
} else {
warn!("No sync task found for account: {}", account_id);
}
} else {
warn!(
"Stop called after global shutdown for account: {}",
account_id
);
}
Ok(())
}
pub async fn shutdown(&self) {
let mut guard = self.tasks.lock().await;
if let Some(map) = guard.take() {
drop(guard);
for (account_id, handler) in map {
info!("Shutdown: Waiting for account {} to sync...", account_id);
handler.stop().await;
}
info!("Shutdown: All sync tasks stopped successfully.");
} else {
warn!("Shutdown: Sync tasks were already shut down or never initialized.");
}
}
}
src/modules/common/periodic.rs
+15 -10
View File
@@ -16,10 +16,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::{common::signal::SIGNAL_MANAGER, error::BichonResult};
use std::{future::Future, time::Duration};
use tokio::{sync::oneshot, time::MissedTickBehavior};
use tokio::{sync::oneshot, task::JoinHandle, time::MissedTickBehavior};
use tracing::{info, warn};
pub struct PeriodicTask {
@@ -28,7 +27,7 @@ pub struct PeriodicTask {
pub struct TaskHandle {
cancel_sender: Option<oneshot::Sender<()>>,
join_handle: tokio::task::JoinHandle<()>,
join_handle: JoinHandle<()>,
}
impl TaskHandle {
@@ -38,6 +37,10 @@ impl TaskHandle {
}
let _ = self.join_handle.await;
}
pub async fn stop(self) {
let _ = self.join_handle.await;
}
}
impl PeriodicTask {
@@ -82,6 +85,14 @@ impl PeriodicTask {
let mut cancel_receiver = cancel_receiver_opt;
loop {
let cancel_fut = async {
if let Some(ref mut rx) = cancel_receiver {
rx.await.ok();
} else {
std::future::pending::<()>().await;
}
};
tokio::select! {
_ = interval.tick() => {
match task(param).await {
@@ -92,13 +103,7 @@ impl PeriodicTask {
}
}
// only enabled if cancel_receiver is Some
_ = async {
if let Some(ref mut rx) = cancel_receiver {
rx.await.ok()
} else {
futures::future::pending().await
}
} => {
_ = cancel_fut => {
info!("Task '{}' received cancellation signal", name_clone);
break;
}
src/modules/common/rustls.rs
+2 -4
View File
@@ -16,7 +16,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::{
context::Initialize,
@@ -25,9 +24,9 @@ use crate::{
raise_error,
};
pub struct RustMailerTls;
pub struct BichonTls;
impl Initialize for RustMailerTls {
impl Initialize for BichonTls {
async fn initialize() -> BichonResult<()> {
rustls::crypto::CryptoProvider::install_default(rustls::crypto::ring::default_provider())
.map_err(|_| {
@@ -38,4 +37,3 @@ impl Initialize for RustMailerTls {
})
}
}
src/modules/context/mod.rs
+3 -3
View File
@@ -16,7 +16,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::error::BichonResult;
use crate::modules::{common::periodic::TaskHandle, error::BichonResult};
pub mod controller;
pub mod executors;
@@ -27,6 +27,6 @@ pub trait Initialize {
async fn initialize() -> BichonResult<()>;
}
pub trait RustMailTask {
fn start();
pub trait BichonTask {
fn start() -> TaskHandle;
}
src/modules/dashboard/mod.rs
+1 -3
View File
@@ -25,9 +25,9 @@ use crate::{
bichon_version,
modules::{
account::migration::AccountModel,
blob::manager::ENVELOPE_INDEX_MANAGER,
common::auth::ClientContext,
error::{code::ErrorCode, BichonResult},
indexer::manager::ENVELOPE_INDEX_MANAGER,
settings::dir::DATA_DIR_MANAGER,
utils::get_total_size,
},
@@ -81,8 +81,6 @@ impl DashboardStats {
if has_all_accounts {
stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.eml_dir)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+ get_total_size(&DATA_DIR_MANAGER.attachment_dir)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
stat.index_usage_bytes = get_total_size(&DATA_DIR_MANAGER.envelope_dir)
src/modules/duckdb/build.rs
+1 -1
View File
@@ -21,7 +21,7 @@ use arrow::datatypes::{DataType, Field, Schema};
use arrow::record_batch::RecordBatch;
use std::sync::Arc;
use crate::modules::indexer::envelope::Envelope;
use crate::modules::blob::envelope::Envelope;
pub const DEFAULT_SHARD_ID: u64 = 0;
src/modules/duckdb/init.rs
+4 -15
View File
@@ -28,14 +28,11 @@ use std::{
use crate::{
modules::{
account::migration::AccountModel,
blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
context::Initialize,
dashboard::{DashboardStats, Group, LargestEmail, TimeBucket},
duckdb::{build::build_record_batch, refinery::DuckDBConnection},
error::{code::ErrorCode, BichonResult},
indexer::{
attachment::ATTACHMENT_INDEX_MANAGER, eml::EML_INDEX_MANAGER, envelope::Envelope,
manager::ENVELOPE_INDEX_MANAGER,
},
message::{
attachment::AttachmentMetadata,
content::{AttachmentDetail, AttachmentInfo},
@@ -86,7 +83,7 @@ pub struct DuckDBManager {
impl Initialize for DuckDBManager {
async fn initialize() -> BichonResult<()> {
tracing::debug!("Initializing databases");
tracing::debug!("Initializing duckdb");
if !&DATA_DIR_MANAGER.envelope_dir.exists() {
std::fs::create_dir_all(&DATA_DIR_MANAGER.envelope_dir)
@@ -1187,19 +1184,11 @@ impl DuckDBManager {
}
};
if let Err(e) = EML_INDEX_MANAGER.delete(&content_hashes).await {
if let Err(e) = BLOB_MANAGER.delete(&content_hashes, &content_hashes) {
tracing::error!(
account_id = account_id,
error = %e,
"failed to cleanup eml index"
);
}
if let Err(e) = ATTACHMENT_INDEX_MANAGER.delete(&content_hashes).await {
tracing::error!(
account_id = account_id,
error = %e,
"failed to cleanup attachment index"
"failed to cleanup eml"
);
}
});
src/modules/envelope/extractor.rs
+20 -32
View File
@@ -16,27 +16,25 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER;
use crate::modules::blob::storage::{DetachedEmail, BLOB_MANAGER};
use crate::modules::common::AddrVec;
use crate::modules::envelope::utils::normalize_subject;
use crate::modules::error::code::ErrorCode;
use crate::modules::error::BichonResult;
use crate::modules::indexer::attachment::ATTACHMENT_INDEX_MANAGER;
use crate::modules::indexer::eml::EML_INDEX_MANAGER;
use crate::modules::indexer::manager::ENVELOPE_INDEX_MANAGER;
use crate::modules::indexer::schema::SchemaTools;
use crate::modules::message::content::AttachmentInfo;
use crate::modules::utils::html::extract_text;
use crate::modules::utils::{compute_content_hash, hex_hash};
use crate::{id, modules::indexer::envelope::Envelope};
use crate::{id, modules::blob::envelope::Envelope};
use crate::{raise_error, utc_now};
use async_imap::types::Fetch;
use bytes::Bytes;
use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders};
use tantivy::doc;
use tracing::error;
use uuid::Uuid;
pub async fn extract_envelope_and_store_it(
fetch: &Fetch,
fetch: Fetch,
account_id: u64,
mailbox_id: u64,
) -> BichonResult<()> {
@@ -301,22 +299,14 @@ pub async fn detach_and_store_attachments(
.collect();
ranges.sort_by(|a, b| b.0.cmp(&a.0));
let fields = SchemaTools::fields();
let mut attachments = Vec::with_capacity(ranges.len());
for (raw_start, raw_end, att) in ranges {
// Step 2: Extract raw bytes and store them as standalone documents
let raw_bytes = &original_body[raw_start..raw_end];
let content_hash = compute_content_hash(raw_bytes);
ATTACHMENT_INDEX_MANAGER
.add_document(
content_hash.clone(),
doc!(
fields.f_id => content_hash.clone(),
fields.f_blob => raw_bytes
),
)
.await;
attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes)));
// Step 3: Replace raw attachment content with a hash-based placeholder
let placeholder = format!("<<BICHON_DETACH_HASH:{}>>", &content_hash);
let p_bytes = placeholder.as_bytes();
@@ -347,14 +337,11 @@ pub async fn detach_and_store_attachments(
attachment_infos.push(info);
}
// Step 4: Store the final stripped EML content
EML_INDEX_MANAGER
.add_document(
eml_content_hash.to_string(),
doc!(
fields.f_id => eml_content_hash.to_string(),
fields.f_blob => stripped_eml
),
)
BLOB_MANAGER
.queue(DetachedEmail {
email: (eml_content_hash.to_string(), Bytes::from(stripped_eml)),
attachments: Some(attachments),
})
.await;
attachment_infos
@@ -363,7 +350,7 @@ pub async fn detach_and_store_attachments(
pub async fn reattach_eml_content(
account_id: u64,
envelope_id: String,
) -> BichonResult<(Envelope, Vec<u8>)> {
) -> BichonResult<(Envelope, Bytes)> {
let envelope = ENVELOPE_INDEX_MANAGER
.get_envelope_by_id(account_id, envelope_id.clone())
.await?
@@ -377,9 +364,8 @@ pub async fn reattach_eml_content(
)
})?;
let mut restored_eml = EML_INDEX_MANAGER
.get(&envelope.content_hash)
.await?
let restored_eml = BLOB_MANAGER
.get_email(&envelope.content_hash)?
.ok_or_else(|| {
raise_error!(
format!(
@@ -394,6 +380,8 @@ pub async fn reattach_eml_content(
return Ok((envelope, restored_eml));
}
let mut restored_eml = restored_eml.to_vec();
let account_detail = ENVELOPE_INDEX_MANAGER
.get_attachments_by_envelope_id(account_id, envelope_id)
.await?;
@@ -432,7 +420,7 @@ pub async fn reattach_eml_content(
tasks.sort_by(|a, b| b.0.cmp(&a.0));
for (start, end, hash) in tasks {
if let Some(original_data) = ATTACHMENT_INDEX_MANAGER.get(&hash).await? {
if let Some(original_data) = BLOB_MANAGER.get_attachment(&hash)? {
let actual_hash = compute_content_hash(&original_data);
if actual_hash != hash {
error!(
@@ -447,7 +435,7 @@ pub async fn reattach_eml_content(
}
}
Ok((envelope, restored_eml))
Ok((envelope, Bytes::from(restored_eml)))
}
#[cfg(test)]
src/modules/imap/executor.rs
+2 -2
View File
@@ -202,7 +202,7 @@ impl ImapExecutor {
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
{
extract_envelope_and_store_it(&fetch, account_id, mailbox_id).await?;
extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?;
count += 1;
}
Ok(count)
@@ -229,7 +229,7 @@ impl ImapExecutor {
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
{
extract_envelope_and_store_it(&fetch, account_id, mailbox_id).await?;
extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?;
}
Ok(())
}
src/modules/import/mod.rs
-1
View File
@@ -19,7 +19,6 @@
use poem_openapi::Object;
use serde::{Deserialize, Serialize};
use tantivy::doc;
use crate::{
base64_decode_url_safe,
src/modules/indexer/attachment.rs
-299
View File
@@ -1,299 +0,0 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
collections::HashMap,
path::PathBuf,
sync::{Arc, LazyLock},
time::Duration,
};
use crate::modules::{indexer::DocumentOp, settings::cli::SETTINGS};
use crate::{
modules::{
common::signal::SIGNAL_MANAGER,
error::{code::ErrorCode, BichonResult},
indexer::schema::SchemaTools,
settings::dir::DATA_DIR_MANAGER,
},
raise_error,
};
use tantivy::indexer::{NoMergePolicy, UserOperation};
use tantivy::{
collector::TopDocs,
query::TermQuery,
schema::{IndexRecordOption, Value},
store::Compressor,
Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, TantivyDocument, Term,
};
use tokio::{
sync::{mpsc, Mutex},
task,
};
use tracing::info;
pub const ATTACHMENT_BATCH_SIZE: usize = 10;
const MAX_BUFFER_DURATION: Duration = Duration::from_secs(10);
pub static ATTACHMENT_INDEX_MANAGER: LazyLock<AttachmentManager> =
LazyLock::new(AttachmentManager::new);
pub struct AttachmentManager {
index_writer: Arc<Mutex<IndexWriter>>,
sender: mpsc::Sender<DocumentOp>,
reader: IndexReader,
}
impl AttachmentManager {
pub fn new() -> Self {
let index = Self::open_or_create_index(&DATA_DIR_MANAGER.attachment_dir);
let writer: IndexWriter<TantivyDocument> = index
.writer_with_num_threads(
SETTINGS.bichon_tantivy_threads as usize,
SETTINGS.bichon_tantivy_buffer_size,
)
.unwrap_or_else(|e| {
panic!(
"Failed to create IndexWriter (threads: {}, buffer: {}B) for {:?}: {}",
SETTINGS.bichon_tantivy_threads,
SETTINGS.bichon_tantivy_buffer_size,
DATA_DIR_MANAGER.attachment_dir,
e
)
});
writer.set_merge_policy(Box::new(NoMergePolicy));
let index_writer = Arc::new(Mutex::new(writer));
let reader = index.reader().unwrap_or_else(|e| {
panic!(
"Failed to create IndexReader for {:?}: {}",
DATA_DIR_MANAGER.eml_dir, e
)
});
let (sender, mut receiver) = mpsc::channel::<DocumentOp>(100);
task::spawn(async move {
let mut buffer: HashMap<String, TantivyDocument> =
HashMap::with_capacity(ATTACHMENT_BATCH_SIZE);
let mut interval = tokio::time::interval(MAX_BUFFER_DURATION);
let mut shutdown = SIGNAL_MANAGER.subscribe();
loop {
tokio::select! {
maybe_msg = receiver.recv() => {
match maybe_msg {
Some(DocumentOp::Document((eid, doc))) => {
buffer.insert(eid, doc);
if buffer.len() >= ATTACHMENT_BATCH_SIZE {
ATTACHMENT_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
}
}
Some(DocumentOp::Shutdown) => {
ATTACHMENT_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
break;
}
None => break,
}
}
_ = interval.tick() => {
if !buffer.is_empty() {
ATTACHMENT_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
}
}
_ = shutdown.recv() => {
let _ = ATTACHMENT_INDEX_MANAGER.sender.send(DocumentOp::Shutdown).await;
}
}
}
});
Self {
index_writer,
sender,
reader,
}
}
pub async fn add_document(&self, content_hash: String, doc: TantivyDocument) {
let _ = self
.sender
.send(DocumentOp::Document((content_hash, doc)))
.await;
}
fn open_or_create_index(index_dir: &PathBuf) -> Index {
let need_create = !index_dir.exists()
|| index_dir
.read_dir()
.map(|mut d| d.next().is_none())
.unwrap_or(true);
if need_create {
info!(
"Attachment storage not found or empty, creating new attachment storage at {}",
index_dir.display()
);
std::fs::create_dir_all(&index_dir).unwrap_or_else(|e| {
panic!("Failed to create index directory {:?}: {}", index_dir, e)
});
IndexBuilder::new()
.schema(SchemaTools::schema())
.settings(IndexSettings {
docstore_compression: Compressor::None,
docstore_compress_dedicated_thread: Default::default(),
docstore_blocksize: Default::default(),
})
.create_in_dir(&index_dir)
.unwrap_or_else(|e| panic!("Failed to create index in {:?}: {}", index_dir, e))
} else {
info!(
"Opening existing attachment data storage at {}",
index_dir.display()
);
open(&index_dir)
}
}
fn term(&self, content_hash: &str) -> Term {
Term::from_field_text(SchemaTools::fields().f_id, content_hash)
}
pub async fn get(&self, content_hash: &str) -> BichonResult<Option<Vec<u8>>> {
let searcher = self.reader.searcher();
let term = Term::from_field_text(SchemaTools::fields().f_id, content_hash);
let query = TermQuery::new(term, IndexRecordOption::Basic);
let docs = searcher
.search(&query, &TopDocs::with_limit(1))
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
if docs.is_empty() {
return Ok(None);
}
let (_, doc_address) = docs.first().unwrap();
let doc: TantivyDocument = searcher
.doc_async(*doc_address)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
let fields = SchemaTools::fields();
let value = doc.get_first(fields.f_blob).ok_or_else(|| {
raise_error!(
format!("miss '{}' field in tantivy document", stringify!(field)),
ErrorCode::InternalError
)
})?;
let bytes = value.as_bytes().ok_or_else(|| {
raise_error!(
format!("'{}' field is not a bytes", stringify!(field)),
ErrorCode::InternalError
)
})?;
Ok(Some(bytes.to_vec()))
}
pub async fn delete(
&self,
content_hashes: &Vec<String>, // HashMap<account_id, envelope_ids>
) -> BichonResult<()> {
if content_hashes.is_empty() {
tracing::warn!("deletes is empty, nothing to delete");
return Ok(());
}
let mut writer = self.index_writer.lock().await;
for hash in content_hashes {
let term = self.term(hash);
writer.delete_term(term);
}
writer
.commit()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(())
}
async fn drain_and_commit(&self, buffer: &mut HashMap<String, TantivyDocument>) {
if buffer.is_empty() {
return;
}
let mut writer = self.index_writer.lock().await;
let mut operations = Vec::new();
for (content_hash, doc) in buffer.drain() {
let delete_term = Term::from_field_text(SchemaTools::fields().f_id, &content_hash);
operations.push(UserOperation::Delete(delete_term));
operations.push(UserOperation::Add(doc));
}
if let Err(e) = writer.run(operations) {
eprintln!("[FATAL] Tantivy run failed: {e:?}");
std::process::exit(1);
}
fatal_commit(&mut writer);
}
}
fn fatal_commit(writer: &mut IndexWriter) {
const MAX_RETRIES: usize = 3;
const RETRY_DELAY_MS: u64 = 1000;
for attempt in 0..=MAX_RETRIES {
match writer.commit() {
Ok(_) => {
if attempt > 0 {
eprintln!("[INFO] Commit succeeded on attempt {}", attempt + 1);
}
return;
}
Err(e) => match &e {
tantivy::TantivyError::IoError(io_error) => {
if attempt < MAX_RETRIES {
eprintln!(
"[WARN] Commit failed (attempt {}/{}): {:?}. Retrying in {}ms...",
attempt + 1,
MAX_RETRIES + 1,
io_error,
RETRY_DELAY_MS * (attempt as u64 + 1)
);
std::thread::sleep(std::time::Duration::from_millis(
RETRY_DELAY_MS * (attempt as u64 + 1),
));
} else {
eprintln!(
"[FATAL] Tantivy commit failed after {} attempts: {:?}",
MAX_RETRIES + 1,
io_error
);
std::process::exit(1);
}
}
_ => {
eprintln!("[FATAL] Tantivy commit failed with non-IO error: {e:?}");
std::process::exit(1);
}
},
}
}
}
fn open(index_dir: &PathBuf) -> Index {
Index::open_in_dir(index_dir)
.unwrap_or_else(|e| panic!("Failed to open index in {:?}: {}", index_dir, e))
}
src/modules/indexer/eml.rs
-331
View File
@@ -1,331 +0,0 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
collections::HashMap,
path::PathBuf,
sync::{Arc, LazyLock},
time::Duration,
};
use crate::modules::{
envelope::extractor::reattach_eml_content, indexer::DocumentOp, settings::cli::SETTINGS,
};
use crate::{
modules::{
common::signal::SIGNAL_MANAGER,
error::{code::ErrorCode, BichonResult},
indexer::schema::SchemaTools,
settings::dir::DATA_DIR_MANAGER,
},
raise_error,
};
use tantivy::indexer::{NoMergePolicy, UserOperation};
use tantivy::{
collector::TopDocs,
query::TermQuery,
schema::{IndexRecordOption, Value},
store::{Compressor, ZstdCompressor},
Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, TantivyDocument, Term,
};
use tokio::{
fs::File,
io::AsyncWriteExt,
sync::{mpsc, Mutex},
task,
};
use tracing::info;
pub const EML_BATCH_SIZE: usize = 100;
const MAX_BUFFER_DURATION: Duration = Duration::from_secs(10);
pub static EML_INDEX_MANAGER: LazyLock<EmlIndexManager> = LazyLock::new(EmlIndexManager::new);
pub struct EmlIndexManager {
index_writer: Arc<Mutex<IndexWriter>>,
sender: mpsc::Sender<DocumentOp>,
reader: IndexReader,
}
impl EmlIndexManager {
pub fn new() -> Self {
let index = Self::open_or_create_index(&DATA_DIR_MANAGER.eml_dir);
let writer: IndexWriter<TantivyDocument> = index
.writer_with_num_threads(
SETTINGS.bichon_tantivy_threads as usize,
SETTINGS.bichon_tantivy_buffer_size,
)
.unwrap_or_else(|e| {
panic!(
"Failed to create IndexWriter (threads: {}, buffer: {}B) for {:?}: {}",
SETTINGS.bichon_tantivy_threads,
SETTINGS.bichon_tantivy_buffer_size,
DATA_DIR_MANAGER.eml_dir,
e
)
});
writer.set_merge_policy(Box::new(NoMergePolicy));
let index_writer = Arc::new(Mutex::new(writer));
let reader = index.reader().unwrap_or_else(|e| {
panic!(
"Failed to create IndexReader for {:?}: {}",
DATA_DIR_MANAGER.eml_dir, e
)
});
let (sender, mut receiver) = mpsc::channel::<DocumentOp>(100);
task::spawn(async move {
let mut buffer: HashMap<String, TantivyDocument> =
HashMap::with_capacity(EML_BATCH_SIZE);
let mut interval = tokio::time::interval(MAX_BUFFER_DURATION);
let mut shutdown = SIGNAL_MANAGER.subscribe();
loop {
tokio::select! {
maybe_msg = receiver.recv() => {
match maybe_msg {
Some(DocumentOp::Document((eid, doc))) => {
buffer.insert(eid, doc);
if buffer.len() >= EML_BATCH_SIZE {
EML_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
}
}
Some(DocumentOp::Shutdown) => {
EML_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
break;
}
None => break,
}
}
_ = interval.tick() => {
if !buffer.is_empty() {
EML_INDEX_MANAGER.drain_and_commit(&mut buffer).await;
}
}
_ = shutdown.recv() => {
let _ = EML_INDEX_MANAGER.sender.send(DocumentOp::Shutdown).await;
}
}
}
});
Self {
index_writer,
sender,
reader,
}
}
/// Adds a document to the indexer.
///
/// # Parameters
/// - `eid`: A hash derived from **Account ID + Message ID**.
/// This acts as a unique identifier for the EML content itself.
///
/// - `doc`: The `TantivyDocument` representing the mail body/content.
///
/// # Logical Design
/// Unlike the `envelope_id` (which is a hash of Account + Folder + Message ID),
/// this `eid` ignores the folder context. This ensures that while metadata
/// (envelopes) can be duplicated across different folders, the physical
/// EML/document storage remains de-duplicated and unique.
pub async fn add_document(&self, content_hash: String, doc: TantivyDocument) {
let _ = self
.sender
.send(DocumentOp::Document((content_hash, doc)))
.await;
}
fn open_or_create_index(index_dir: &PathBuf) -> Index {
let need_create = !index_dir.exists()
|| index_dir
.read_dir()
.map(|mut d| d.next().is_none())
.unwrap_or(true);
if need_create {
info!(
"Email storage not found or empty, creating new mail storage at {}",
index_dir.display()
);
std::fs::create_dir_all(&index_dir).unwrap_or_else(|e| {
panic!("Failed to create index directory {:?}: {}", index_dir, e)
});
IndexBuilder::new()
.schema(SchemaTools::schema())
.settings(IndexSettings {
docstore_compression: Compressor::Zstd(ZstdCompressor {
compression_level: Some(SETTINGS.bichon_eml_compression_level as i32),
}),
docstore_compress_dedicated_thread: true,
docstore_blocksize: SETTINGS.bichon_eml_blocksize,
})
.create_in_dir(&index_dir)
.unwrap_or_else(|e| panic!("Failed to create index in {:?}: {}", index_dir, e))
} else {
info!("Opening existing email storage at {}", index_dir.display());
open(&index_dir)
}
}
fn term(&self, content_hash: &str) -> Term {
Term::from_field_text(SchemaTools::fields().f_id, content_hash)
}
pub async fn get(&self, content_hash: &str) -> BichonResult<Option<Vec<u8>>> {
let searcher = self.reader.searcher();
let term = Term::from_field_text(SchemaTools::fields().f_id, content_hash);
let query = TermQuery::new(term, IndexRecordOption::Basic);
let docs = searcher
.search(&query, &TopDocs::with_limit(1))
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
if docs.is_empty() {
return Ok(None);
}
let (_, doc_address) = docs.first().unwrap();
let doc: TantivyDocument = searcher
.doc_async(*doc_address)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
let fields = SchemaTools::fields();
let value = doc.get_first(fields.f_blob).ok_or_else(|| {
raise_error!(
format!("miss '{}' field in tantivy document", stringify!(field)),
ErrorCode::InternalError
)
})?;
let bytes = value.as_bytes().ok_or_else(|| {
raise_error!(
format!("'{}' field is not a bytes", stringify!(field)),
ErrorCode::InternalError
)
})?;
Ok(Some(bytes.to_vec()))
}
pub async fn get_reader(&self, account_id: u64, eid: String) -> BichonResult<File> {
let (envelope, data) = reattach_eml_content(account_id, eid).await?;
let mut path = DATA_DIR_MANAGER.temp_dir.clone();
path.push(format!("{}.eml", envelope.content_hash));
{
let mut file = File::create(&path)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
file.write_all(&data)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
}
let file = File::open(&path)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(file)
}
pub async fn delete(
&self,
content_hashes: &Vec<String>, // HashMap<account_id, envelope_ids>
) -> BichonResult<()> {
if content_hashes.is_empty() {
tracing::warn!("delete_email_multi_account: deletes is empty, nothing to delete");
return Ok(());
}
let mut writer = self.index_writer.lock().await;
for hash in content_hashes {
let term = self.term(hash);
writer.delete_term(term);
}
writer
.commit()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(())
}
// Deduplicate directly by content_hash, regardless of the account.
async fn drain_and_commit(&self, buffer: &mut HashMap<String, TantivyDocument>) {
if buffer.is_empty() {
return;
}
let mut writer = self.index_writer.lock().await;
let mut operations = Vec::new();
for (content_hash, doc) in buffer.drain() {
let delete_term = Term::from_field_text(SchemaTools::fields().f_id, &content_hash);
operations.push(UserOperation::Delete(delete_term));
operations.push(UserOperation::Add(doc));
}
if let Err(e) = writer.run(operations) {
eprintln!("[FATAL] Tantivy run failed: {e:?}");
std::process::exit(1);
}
fatal_commit(&mut writer);
}
}
fn fatal_commit(writer: &mut IndexWriter) {
const MAX_RETRIES: usize = 3;
const RETRY_DELAY_MS: u64 = 1000;
for attempt in 0..=MAX_RETRIES {
match writer.commit() {
Ok(_) => {
if attempt > 0 {
eprintln!("[INFO] Commit succeeded on attempt {}", attempt + 1);
}
return;
}
Err(e) => match &e {
tantivy::TantivyError::IoError(io_error) => {
if attempt < MAX_RETRIES {
eprintln!(
"[WARN] Commit failed (attempt {}/{}): {:?}. Retrying in {}ms...",
attempt + 1,
MAX_RETRIES + 1,
io_error,
RETRY_DELAY_MS * (attempt as u64 + 1)
);
std::thread::sleep(std::time::Duration::from_millis(
RETRY_DELAY_MS * (attempt as u64 + 1),
));
} else {
eprintln!(
"[FATAL] Tantivy commit failed after {} attempts: {:?}",
MAX_RETRIES + 1,
io_error
);
std::process::exit(1);
}
}
_ => {
eprintln!("[FATAL] Tantivy commit failed with non-IO error: {e:?}");
std::process::exit(1);
}
},
}
}
}
fn open(index_dir: &PathBuf) -> Index {
Index::open_in_dir(index_dir)
.unwrap_or_else(|e| panic!("Failed to open index in {:?}: {}", index_dir, e))
}
src/modules/indexer/fields.rs
-27
View File
@@ -1,27 +0,0 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use tantivy::schema::Field;
pub const F_ID: &str = "id";
pub const F_BLOB: &str = "blob";
pub struct BlobFields {
pub f_id: Field,
pub f_blob: Field,
}
src/modules/indexer/schema.rs
-49
View File
@@ -1,49 +0,0 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::sync::{Arc, LazyLock};
use crate::modules::indexer::fields::*;
use tantivy::schema::STRING;
use tantivy::schema::{Schema, FAST, STORED};
static BLOB_FIELDS: LazyLock<Arc<BlobFields>> = LazyLock::new(|| {
let (_, fields) = SchemaTools::create_schema();
Arc::new(fields)
});
pub struct SchemaTools;
impl SchemaTools {
pub fn schema() -> Schema {
let (schema, _) = Self::create_schema();
schema
}
pub fn fields() -> &'static BlobFields {
&BLOB_FIELDS
}
pub fn create_schema() -> (Schema, BlobFields) {
let mut builder = Schema::builder();
let f_id = builder.add_text_field(F_ID, STRING | FAST);
let f_blob = builder.add_bytes_field(F_BLOB, STORED);
let fields = BlobFields { f_id, f_blob };
(builder.build(), fields)
}
}
src/modules/mailbox/delete.rs
+2 -6
View File
@@ -1,10 +1,7 @@
use crate::modules::{
blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::imap::mailbox::MailBox,
error::BichonResult,
indexer::{
attachment::ATTACHMENT_INDEX_MANAGER, eml::EML_INDEX_MANAGER,
manager::ENVELOPE_INDEX_MANAGER,
},
};
pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResult<()> {
@@ -33,7 +30,6 @@ pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResu
.delete_mailbox_envelopes(account_id, ids_to_delete.clone())
.await?;
EML_INDEX_MANAGER.delete(&content_hashes).await?;
ATTACHMENT_INDEX_MANAGER.delete(&content_hashes).await?;
BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
Ok(())
}
src/modules/message/attachment.rs
+7 -35
View File
@@ -1,19 +1,17 @@
use std::collections::HashSet;
use std::{collections::HashSet, io::Cursor};
use crate::{
modules::{
envelope::extractor::reattach_eml_content,
error::{code::ErrorCode, BichonResult},
settings::dir::DATA_DIR_MANAGER,
utils::compute_content_hash,
},
raise_error,
};
use bytes::Bytes;
use mail_parser::MessageParser;
use poem_openapi::Object;
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
pub struct AttachmentMetadata {
@@ -34,8 +32,8 @@ pub async fn retrieve_attachment_content(
account_id: u64,
envelope_id: String,
content_hash: &str,
) -> BichonResult<File> {
let (envelope, eml) = reattach_eml_content(account_id, envelope_id).await?;
) -> BichonResult<Cursor<Bytes>> {
let (_, eml) = reattach_eml_content(account_id, envelope_id).await?;
let message = MessageParser::default().parse(&eml).ok_or_else(|| {
raise_error!(
"Failed to parse parent EML".into(),
@@ -53,20 +51,7 @@ pub async fn retrieve_attachment_content(
ErrorCode::ResourceNotFound
)
})?;
let mut path = DATA_DIR_MANAGER.temp_dir.clone();
path.push(format!("{}.eml", envelope.content_hash));
{
let mut file = File::create(&path)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
file.write_all(attachment_content)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
}
let file = File::open(&path)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(file)
Ok(Cursor::new(Bytes::copy_from_slice(attachment_content)))
}
pub async fn retrieve_nested_attachment_content(
@@ -74,7 +59,7 @@ pub async fn retrieve_nested_attachment_content(
envelope_id: String,
content_hash: &str,
nested_content_hash: &str,
) -> BichonResult<File> {
) -> BichonResult<Cursor<Bytes>> {
let (_, eml) = reattach_eml_content(account_id, envelope_id).await?;
let parent_message = MessageParser::default().parse(&eml).ok_or_else(|| {
raise_error!(
@@ -114,18 +99,5 @@ pub async fn retrieve_nested_attachment_content(
)
})?;
let mut path = DATA_DIR_MANAGER.temp_dir.clone();
path.push(format!("{}.eml", nested_content_hash));
{
let mut file = File::create(&path)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
file.write_all(attachment_content)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
}
let file = File::open(&path)
.await
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(file)
Ok(Cursor::new(Bytes::copy_from_slice(attachment_content)))
}
src/modules/message/content.rs
+1 -1
View File
@@ -22,7 +22,7 @@ use crate::modules::envelope::extractor::{
extract_envelope_from_nested_message, reattach_eml_content,
};
use crate::modules::error::code::ErrorCode;
use crate::modules::indexer::envelope::Envelope;
use crate::modules::blob::envelope::Envelope;
use crate::modules::utils::compute_content_hash;
use crate::{modules::error::BichonResult, raise_error};
use mail_parser::{MessageParser, MimeHeaders};
src/modules/message/delete.rs
+3 -5
View File
@@ -16,10 +16,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER;
use crate::modules::blob::storage::BLOB_MANAGER;
use crate::modules::error::BichonResult;
use crate::modules::indexer::attachment::ATTACHMENT_INDEX_MANAGER;
use crate::modules::indexer::eml::EML_INDEX_MANAGER;
use crate::modules::indexer::manager::ENVELOPE_INDEX_MANAGER;
use std::collections::HashMap;
pub async fn delete_messages_impl(request: HashMap<u64, Vec<String>>) -> BichonResult<()> {
@@ -27,8 +26,7 @@ pub async fn delete_messages_impl(request: HashMap<u64, Vec<String>>) -> BichonR
.get_orphan_hashes_in_memory(request.clone())
.await?;
if !content_hashes.is_empty() {
EML_INDEX_MANAGER.delete(&content_hashes).await?;
ATTACHMENT_INDEX_MANAGER.delete(&content_hashes).await?;
BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
}
ENVELOPE_INDEX_MANAGER
.delete_envelopes_multi_account(request)
src/modules/message/list.rs
+1 -1
View File
@@ -20,7 +20,7 @@ use crate::{
modules::{
account::migration::AccountModel,
error::{code::ErrorCode, BichonResult},
indexer::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER},
blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER},
rest::response::DataPage,
},
raise_error,
src/modules/message/search.rs
+1 -1
View File
@@ -25,7 +25,7 @@ use crate::{
modules::{
duckdb::init::duckdb,
error::{code::ErrorCode, BichonResult},
indexer::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER},
blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER},
rest::response::DataPage,
},
raise_error,
src/modules/mod.rs
+1 -1
View File
@@ -29,7 +29,7 @@ pub mod envelope;
pub mod error;
pub mod imap;
pub mod import;
pub mod indexer;
pub mod blob;
pub mod logger;
pub mod mailbox;
pub mod message;
src/modules/oauth2/refresh/mod.rs
+5 -6
View File
@@ -16,9 +16,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::common::periodic::PeriodicTask;
use crate::modules::context::RustMailTask;
use crate::modules::common::periodic::{PeriodicTask, TaskHandle};
use crate::modules::context::BichonTask;
use crate::modules::oauth2::token::EXTERNAL_OAUTH_APP_ID;
use crate::modules::oauth2::{flow::OAuth2Flow, token::OAuth2AccessToken};
use crate::utc_now;
@@ -30,8 +29,8 @@ const FIFTEEN_MINUTES: Duration = Duration::from_secs(45 * 60);
///This task cleans up expired OAuth2 pending authorizations that haven't been completed by users in a timely manner.
pub struct OAuth2RefreshTask;
impl RustMailTask for OAuth2RefreshTask {
fn start() {
impl BichonTask for OAuth2RefreshTask {
fn start() -> TaskHandle {
let periodic_task = PeriodicTask::new("oauth2-token-refresh-task");
let task = move |_: Option<u64>| {
@@ -86,6 +85,6 @@ impl RustMailTask for OAuth2RefreshTask {
})
};
periodic_task.start(task, None, TASK_INTERVAL, false, true);
periodic_task.start(task, None, TASK_INTERVAL, false, true)
}
}
src/modules/oauth2/task.rs
+6 -5
View File
@@ -16,9 +16,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::{
common::periodic::PeriodicTask, context::RustMailTask, oauth2::pending::OAuth2PendingEntity,
common::periodic::{PeriodicTask, TaskHandle},
context::BichonTask,
oauth2::pending::OAuth2PendingEntity,
};
use std::time::Duration;
@@ -27,8 +28,8 @@ const TASK_INTERVAL: Duration = Duration::from_secs(6 * 60 * 60);
///This task cleans up expired OAuth2 pending authorizations that haven't been completed by users in a timely manner.
pub struct OAuth2CleanTask;
impl RustMailTask for OAuth2CleanTask {
fn start() {
impl BichonTask for OAuth2CleanTask {
fn start() -> TaskHandle {
let periodic_task = PeriodicTask::new("oauth2-pending-task-cleaner");
let task = move |_: Option<u64>| {
@@ -38,6 +39,6 @@ impl RustMailTask for OAuth2CleanTask {
})
};
periodic_task.start(task, None, TASK_INTERVAL, false, false);
periodic_task.start(task, None, TASK_INTERVAL, false, false)
}
}
src/modules/rest/api/message.rs
+5 -9
View File
@@ -17,10 +17,10 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::account::migration::AccountModel;
use crate::modules::blob::envelope::Envelope;
use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER;
use crate::modules::blob::storage::get_reader;
use crate::modules::common::auth::ClientContext;
use crate::modules::indexer::eml::EML_INDEX_MANAGER;
use crate::modules::indexer::envelope::Envelope;
use crate::modules::indexer::manager::ENVELOPE_INDEX_MANAGER;
use crate::modules::message::append::restore_emails;
use crate::modules::message::append::RestoreMessagesRequest;
use crate::modules::message::attachment::retrieve_attachment_content;
@@ -252,9 +252,7 @@ impl MessageApi {
.require_permission(Some(account_id), Permission::DATA_RAW_DOWNLOAD)
.await?;
let envelope_id = envelope_id.0;
let reader = EML_INDEX_MANAGER
.get_reader(account_id, envelope_id.clone())
.await?;
let reader = get_reader(account_id, envelope_id.clone()).await?;
let body = Body::from_async_read(reader);
let attachment = Attachment::new(body)
.attachment_type(AttachmentType::Attachment)
@@ -391,9 +389,7 @@ impl MessageApi {
.await?;
}
ENVELOPE_INDEX_MANAGER
.update_envelope_tags(req)
.await?;
ENVELOPE_INDEX_MANAGER.update_envelope_tags(req).await?;
Ok(())
}
src/modules/settings/dir.rs
+3 -10
View File
@@ -28,8 +28,7 @@ use std::sync::LazyLock;
pub const META_FILE: &str = "meta.db";
pub const MAILBOX_FILE: &str = "mailbox.db";
const ENVELOPE_DIR: &str = "envelope";
const EML_DIR: &str = "eml";
const ATTACHMENT_DIR: &str = "attachment";
const EML_DIR: &str = "bichon-emls";
const TMP_DIR: &str = "tmp";
const LOG_DIR: &str = "logs";
const TLS_CERT: &str = "cert.pem";
@@ -48,7 +47,6 @@ pub struct DataDirManager {
pub tls_key: PathBuf,
pub envelope_dir: PathBuf,
pub eml_dir: PathBuf,
pub attachment_dir: PathBuf,
pub log_dir: PathBuf,
}
@@ -60,6 +58,8 @@ impl Initialize for DataDirManager {
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
std::fs::create_dir_all(&DATA_DIR_MANAGER.temp_dir)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
std::fs::create_dir_all(&DATA_DIR_MANAGER.eml_dir)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(())
}
}
@@ -78,12 +78,6 @@ impl DataDirManager {
root_dir.join(EML_DIR)
};
let attachment_dir = if let Some(ref data_dir) = SETTINGS.bichon_data_dir {
PathBuf::from(data_dir).join(ATTACHMENT_DIR)
} else {
root_dir.join(ATTACHMENT_DIR)
};
Self {
root_dir: root_dir.clone(),
meta_db: root_dir.join(META_FILE),
@@ -94,7 +88,6 @@ impl DataDirManager {
envelope_dir,
temp_dir: root_dir.join(TMP_DIR),
eml_dir,
attachment_dir,
}
}
}
src/modules/tasks/mod.rs
+16 -6
View File
@@ -16,14 +16,24 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::context::RustMailTask;
use crate::modules::common::periodic::TaskHandle;
use crate::modules::context::BichonTask;
use crate::modules::oauth2::{refresh::OAuth2RefreshTask, task::OAuth2CleanTask};
pub struct PeriodicTasks;
pub struct PeriodicTasks {
tasks: Vec<TaskHandle>,
}
impl PeriodicTasks {
pub fn start_background_tasks() {
OAuth2CleanTask::start();
OAuth2RefreshTask::start();
pub fn setup() -> Self {
let mut tasks = Vec::new();
tasks.push(OAuth2CleanTask::start());
tasks.push(OAuth2RefreshTask::start());
Self { tasks }
}
pub async fn shutdown(self) {
for handle in self.tasks {
handle.stop().await;
}
}
}