initial commit

This commit is contained in:
rustmailer
2025-11-19 02:14:37 +08:00
commit 1a8f95117e
355 changed files with 54089 additions and 0 deletions
+207
View File
@@ -0,0 +1,207 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
decode_mailbox_name, encode_mailbox_name,
modules::{
database::{
batch_delete_impl, batch_insert_impl, batch_upsert_impl, filter_by_secondary_key_impl,
manager::DB_MANAGER,
},
error::{code::ErrorCode, BichonResult},
},
raise_error,
};
use async_imap::types::{Name, NameAttribute};
use itertools::Itertools;
use native_db::*;
use native_model::{native_model, Model};
use poem_openapi::{Enum, Object};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
#[native_model(id = 1, version = 1)]
#[native_db]
pub struct MailBox {
/// The unique identifier for the mailbox
#[primary_key]
pub id: u64,
/// The ID of the account associated with the mailbox
#[secondary_key]
pub account_id: u64,
/// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items").
/// This is the decoded name as presented to users, derived from the IMAP server's mailbox name
/// (e.g., after decoding UTF-7 or other encodings per RFC 3501).
pub name: String,
/// Optional delimiter used to separate mailbox names in a hierarchy (e.g., "/" or ".").
/// Used in IMAP to structure nested mailboxes (e.g., "INBOX/Archive").
pub delimiter: Option<String>,
/// List of attributes associated with the mailbox (e.g., `\NoSelect`, `\Deleted`).
/// These indicate special properties, such as whether the mailbox can hold messages.
pub attributes: Vec<Attribute>,
/// The number of messages that currently exist in the mailbox.
pub exists: u32,
/// Optional number of unseen messages in the mailbox (i.e., messages without the `\Seen` flag).
pub unseen: Option<u32>,
/// The next unique identifier (UID) that will be assigned to a new message in the mailbox.
/// If `None`, the IMAP server has not provided this information.
pub uid_next: Option<u32>,
/// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions.
/// If `None`, the IMAP server has not provided this information.
pub uid_validity: Option<u32>,
}
impl MailBox {
pub fn encoded_name(&self) -> String {
encode_mailbox_name!(&self.name)
}
// pub async fn batch_delete(mailboxes: Vec<MailBox>) -> BichonResult<()> {
// batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
// let mut to_deleted = Vec::new();
// for mailbox in mailboxes {
// let retrived = rw
// .get()
// .primary::<MailBox>(mailbox.id)
// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
// if let Some(retrived) = retrived {
// to_deleted.push(retrived);
// }
// }
// Ok(to_deleted)
// })
// .await?;
// Ok(())
// }
// pub async fn get(id: u64) -> RustMailerResult<MailBox> {
// let result = async_find_impl::<MailBox>(DB_MANAGER.envelope_db(), id).await?;
// Ok(result.ok_or_else(|| {
// raise_error!(
// format!("mailbox {} not found", id),
// ErrorCode::InternalError
// )
// })?)
// }
// pub async fn delete(id: u64) -> BichonResult<()> {
// delete_impl(DB_MANAGER.envelope_db(), move |rw| {
// rw.get()
// .primary::<MailBox>(id)
// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
// .ok_or_else(|| raise_error!("mailbox missing".into(), ErrorCode::InternalError))
// })
// .await
// }
pub async fn list_all(account_id: u64) -> BichonResult<Vec<MailBox>> {
filter_by_secondary_key_impl(DB_MANAGER.envelope_db(), MailBoxKey::account_id, account_id)
.await
}
pub async fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> {
batch_insert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
}
pub async fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> {
batch_upsert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
}
pub async fn clean(account_id: u64) -> BichonResult<()> {
batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
let mailboxes: Vec<MailBox> = rw
.scan()
.secondary::<MailBox>(MailBoxKey::account_id)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.start_with(account_id)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.try_collect()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(mailboxes)
})
.await?;
Ok(())
}
}
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Object)]
pub struct Attribute {
pub attr: AttributeEnum,
pub extension: Option<String>,
}
impl Attribute {
pub fn new(attr: AttributeEnum, extension: Option<String>) -> Self {
Self { attr, extension }
}
}
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Enum)]
pub enum AttributeEnum {
NoInferiors,
NoSelect,
Marked,
Unmarked,
All,
Archive,
Drafts,
Flagged,
Junk,
Sent,
Trash,
Extension,
Unknown,
}
impl From<&Name> for MailBox {
fn from(value: &Name) -> Self {
let name = decode_mailbox_name!(value.name().to_string());
let delimiter = value.delimiter().map(|f| f.to_owned());
let attributes: Vec<Attribute> = value.attributes().iter().map(|na| na.into()).collect();
//The remaining parts will be supplemented during the examine_mailbox process.
MailBox {
name,
delimiter,
attributes,
..Default::default() //has_synced is initialized to false here
}
}
}
impl From<&NameAttribute<'_>> for Attribute {
fn from(value: &NameAttribute) -> Self {
match value {
NameAttribute::NoInferiors => Attribute::new(AttributeEnum::NoInferiors, None),
NameAttribute::NoSelect => Attribute::new(AttributeEnum::NoSelect, None),
NameAttribute::Marked => Attribute::new(AttributeEnum::Marked, None),
NameAttribute::Unmarked => Attribute::new(AttributeEnum::Unmarked, None),
NameAttribute::All => Attribute::new(AttributeEnum::All, None),
NameAttribute::Archive => Attribute::new(AttributeEnum::Archive, None),
NameAttribute::Drafts => Attribute::new(AttributeEnum::Drafts, None),
NameAttribute::Flagged => Attribute::new(AttributeEnum::Flagged, None),
NameAttribute::Junk => Attribute::new(AttributeEnum::Junk, None),
NameAttribute::Sent => Attribute::new(AttributeEnum::Sent, None),
NameAttribute::Trash => Attribute::new(AttributeEnum::Trash, None),
NameAttribute::Extension(s) => {
Attribute::new(AttributeEnum::Extension, Some(s.to_string()))
}
_ => Attribute::new(AttributeEnum::Unknown, None),
}
}
}
+66
View File
@@ -0,0 +1,66 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::sync::LazyLock;
use crate::modules::{account::state::AccountRunningState, database::ModelsAdapter};
use ahash::{AHashMap, AHashSet};
use mailbox::MailBox;
use native_db::Models;
pub mod mailbox;
pub mod sync;
pub mod task;
pub static MAILBOX_MODELS: LazyLock<Models> = LazyLock::new(|| {
let mut adapter = ModelsAdapter::new();
adapter.register_model::<MailBox>();
adapter.register_model::<AccountRunningState>();
adapter.models
});
pub fn find_missing_mailboxes(
local_mailboxes: &[MailBox],
server_mailboxes: &[MailBox],
) -> Vec<MailBox> {
let local_names: AHashSet<_> = local_mailboxes.iter().map(|m| &m.name).collect();
server_mailboxes
.iter()
.filter(|m| !local_names.contains(&m.name))
.cloned()
.collect()
}
pub fn find_intersecting_mailboxes(
local_mailboxes: &[MailBox],
remote_mailboxes: &[MailBox],
) -> Vec<(MailBox, MailBox)> {
let local_map: AHashMap<_, _> = local_mailboxes
.iter()
.map(|m| (m.name.clone(), m.clone()))
.collect();
remote_mailboxes
.iter()
.filter_map(|m| {
local_map
.get(&m.name)
.map(|local_mailbox| (local_mailbox.clone(), m.clone()))
})
.collect()
}
+379
View File
@@ -0,0 +1,379 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::{
account::{migration::AccountModel, state::AccountRunningState},
cache::{
imap::{
find_intersecting_mailboxes, find_missing_mailboxes,
mailbox::MailBox,
sync::rebuild::{rebuild_mailbox_cache, rebuild_mailbox_cache_since_date},
},
SEMAPHORE,
},
context::executors::MAIL_CONTEXT,
error::{code::ErrorCode, BichonError, BichonResult},
indexer::manager::ENVELOPE_INDEX_MANAGER,
},
raise_error,
};
use std::time::Instant;
use tracing::{debug, error, info, warn};
pub const BATCH_SIZE: u32 = 50;
pub async fn fetch_and_save_since_date(
account: &AccountModel,
date: &str,
mailbox: &MailBox,
) -> BichonResult<usize> {
let account_id = account.id;
let executor = MAIL_CONTEXT.imap(account_id).await?;
let uid_list = executor
.uid_search(&mailbox.encoded_name(), format!("SINCE {date}").as_str())
.await?;
let len = uid_list.len();
if len == 0 {
return Ok(0);
}
let folder_limit = account.folder_limit;
// sort small -> bigger
let mut uid_vec: Vec<u32> = uid_list.into_iter().collect();
uid_vec.sort();
if let Some(limit) = folder_limit {
let limit = limit.max(100) as usize;
if len > limit {
uid_vec = uid_vec.split_off(len - limit as usize);
}
}
// let semaphore = Arc::new(Semaphore::new(5));
let uid_batches = generate_uid_sequence_hashset(uid_vec, BATCH_SIZE as usize, false);
AccountRunningState::set_initial_current_syncing_folder(
account_id,
mailbox.name.clone(),
uid_batches.len() as u32,
)
.await?;
for (index, batch) in uid_batches.into_iter().enumerate() {
AccountRunningState::set_current_sync_batch_number(
account_id,
mailbox.name.clone(),
(index + 1) as u32,
)
.await?;
let executor = MAIL_CONTEXT.imap(account_id).await?;
// Fetch metadata for the current batch of UIDs
executor
.uid_batch_retrieve_emails(account_id, mailbox.id, &batch, &mailbox.encoded_name())
.await?;
}
Ok(len)
}
pub async fn fetch_and_save_full_mailbox(
account: &AccountModel,
mailbox: &MailBox,
total: u32,
) -> BichonResult<usize> {
let mailbox_id = mailbox.id;
let account_id = account.id;
let folder_limit = account.folder_limit;
let total_to_fetch = match folder_limit {
Some(limit) if limit < total => total.min(limit.max(100)),
_ => total,
};
let page_size = if let Some(limit) = folder_limit {
limit.max(100).min(BATCH_SIZE as u32)
} else {
BATCH_SIZE as u32
};
let total_batches = total_to_fetch.div_ceil(page_size);
let desc = folder_limit.is_some();
let mut inserted_count = 0;
AccountRunningState::set_initial_current_syncing_folder(
account_id,
mailbox.name.clone(),
total_batches,
)
.await?;
info!(
"Starting full mailbox sync for '{}', total={}, limit={:?}, batches={}, desc={}",
mailbox.name, total, folder_limit, total_batches, desc
);
for page in 1..=total_batches {
AccountRunningState::set_current_sync_batch_number(account_id, mailbox.name.clone(), page)
.await?;
let executor = MAIL_CONTEXT.imap(account_id).await?;
let count = executor
.batch_retrieve_emails(
account_id,
mailbox_id,
page as u64,
page_size as u64,
&mailbox.encoded_name(),
desc,
)
.await?;
inserted_count += count;
info!(
"Batch insertion completed for mailbox: {}, current page: {}, inserted count: {}",
&mailbox.name, page, count
);
}
Ok(inserted_count)
}
/// # Example
///
/// ```rust
/// use std::collections::HashSet;
///
/// let mut uids = HashSet::new();
/// uids.extend([1, 2, 3, 5, 6, 7, 9, 10, 11, 15]);
///
/// let chunks = generate_uid_sequence_hashset(uids, 6, false);
/// assert_eq!(chunks, vec![
/// "1:3,5:7".to_string(),
/// "9:11,15".to_string()
/// ]);
/// ```
///
/// This splits the UIDs into chunks of 6, compresses each chunk into ranges,
/// and returns a vector like: `["1:3,5:7", "9:11,15"]`.
///
pub fn generate_uid_sequence_hashset(
unique_nums: Vec<u32>,
chunk_size: usize,
desc: bool,
) -> Vec<String> {
assert!(!unique_nums.is_empty());
// let mut nums: Vec<u32> = unique_nums.into_iter().collect();
// nums.sort();
let mut nums = unique_nums;
if desc {
nums.reverse();
}
let mut result = Vec::new();
for chunk in nums.chunks(chunk_size) {
let compressed = compress_uid_list(chunk.to_vec());
result.push(compressed);
}
result
}
pub fn compress_uid_list(nums: Vec<u32>) -> String {
if nums.is_empty() {
return String::new();
}
let mut sorted_nums = nums;
sorted_nums.sort();
let mut result = Vec::new();
let mut current_range_start = sorted_nums[0];
let mut current_range_end = sorted_nums[0];
for &n in sorted_nums.iter().skip(1) {
if n == current_range_end + 1 {
current_range_end = n;
} else {
if current_range_start == current_range_end {
result.push(current_range_start.to_string());
} else {
result.push(format!("{}:{}", current_range_start, current_range_end));
}
current_range_start = n;
current_range_end = n;
}
}
if current_range_start == current_range_end {
result.push(current_range_start.to_string());
} else {
result.push(format!("{}:{}", current_range_start, current_range_end));
}
result.join(",")
}
pub async fn reconcile_mailboxes(
account: &AccountModel,
remote_mailboxes: &[MailBox],
local_mailboxes: &[MailBox],
) -> BichonResult<()> {
let start_time = Instant::now();
let existing_mailboxes = find_intersecting_mailboxes(local_mailboxes, remote_mailboxes);
let account_id = account.id;
if !existing_mailboxes.is_empty() {
let mut mailboxes_to_update = Vec::with_capacity(existing_mailboxes.len());
for (local_mailbox, remote_mailbox) in &existing_mailboxes {
if local_mailbox.uid_validity != remote_mailbox.uid_validity {
if remote_mailbox.uid_validity.is_none() {
warn!(
"Account {}: Mailbox '{}' has invalid uid_validity (None). Skipping sync for this mailbox.",
account_id, local_mailbox.name
);
continue;
}
info!(
"Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \
The mailbox data may be invalid, resetting its envelopes and rebuilding the cache.",
account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_mailbox.uid_validity
);
match &account.date_since {
Some(date_since) => {
rebuild_mailbox_cache_since_date(
account,
local_mailbox.id,
date_since,
remote_mailbox,
)
.await?;
}
None => {
rebuild_mailbox_cache(account, local_mailbox, remote_mailbox).await?;
}
}
} else {
perform_incremental_sync(account, local_mailbox, remote_mailbox).await?;
}
if let Some(state) = AccountRunningState::get(account.id).await? {
if !state.is_initial_sync_completed {
AccountRunningState::set_folder_initial_sync_completed(
account_id,
local_mailbox.name.clone(),
)
.await?;
}
}
mailboxes_to_update.push(remote_mailbox.clone());
}
//The metadata of this mailbox must only be updated after a successful synchronization;
//otherwise, it may cause synchronization errors and result in missing emails in the local sync results.
MailBox::batch_upsert(&mailboxes_to_update).await?;
}
debug!(
"Checked mailbox folders for account ID: {}. Compared local and server folders to identify changes. Elapsed time: {} seconds",
account.id,
start_time.elapsed().as_secs()
);
let missing_mailboxes = find_missing_mailboxes(local_mailboxes, remote_mailboxes);
if !missing_mailboxes.is_empty() {
MailBox::batch_insert(&missing_mailboxes).await?;
let mut handles = Vec::new();
for mailbox in &missing_mailboxes {
if mailbox.exists > 0 {
let account = account.clone();
let mailbox = mailbox.clone();
match SEMAPHORE.clone().acquire_owned().await {
Ok(permit) => {
let handle: tokio::task::JoinHandle<Result<(), BichonError>> =
tokio::spawn(async move {
let _permit = permit;
match &account.date_since {
Some(date_since) => {
rebuild_mailbox_cache_since_date(
&account, mailbox.id, date_since, &mailbox,
)
.await
}
None => {
rebuild_mailbox_cache(&account, &mailbox, &mailbox).await
}
}
});
handles.push(handle);
}
Err(err) => {
error!("Failed to acquire semaphore permit, error: {:#?}", err);
}
}
}
}
for task in handles {
match task.await {
Ok(Ok(())) => {}
Ok(Err(err)) => return Err(err),
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
}
}
}
Ok(())
}
//only check new emails and sync
async fn perform_incremental_sync(
account: &AccountModel,
local_mailbox: &MailBox,
remote_mailbox: &MailBox,
) -> BichonResult<()> {
if remote_mailbox.exists > 0 {
let local_max_uid = ENVELOPE_INDEX_MANAGER
.get_max_uid(account.id, local_mailbox.id)
.await?;
match local_max_uid {
Some(max_uid) => {
let executor = MAIL_CONTEXT.imap(account.id).await?;
executor
.fetch_new_mail(account.id, local_mailbox, max_uid + 1)
.await?;
}
None => {
info!(
"No maximum UID found in index for mailbox, assuming local cache is missing."
);
match &account.date_since {
Some(date_since) => {
fetch_and_save_since_date(
account,
date_since.since_date()?.as_str(),
remote_mailbox,
)
.await?;
}
None => {
fetch_and_save_full_mailbox(account, remote_mailbox, remote_mailbox.exists)
.await?;
}
}
}
}
}
Ok(())
}
+113
View File
@@ -0,0 +1,113 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::{
account::{
dispatcher::STATUS_DISPATCHER,
migration::{AccountModel, AccountType},
state::AccountRunningState,
},
cache::imap::mailbox::MailBox,
error::BichonResult,
},
utc_now,
};
use flow::reconcile_mailboxes;
use rebuild::{rebuild_cache, rebuild_cache_since_date};
use std::time::Instant;
use sync_folders::get_sync_folders;
use sync_type::{determine_sync_type, SyncType};
use tracing::debug;
pub mod flow;
pub mod rebuild;
pub mod sync_folders;
pub mod sync_type;
pub async fn execute_imap_sync(account: &AccountModel) -> BichonResult<()> {
assert_eq!(account.account_type, AccountType::IMAP);
let start_time = Instant::now();
let account_id = account.id;
let sync_type = determine_sync_type(account).await?;
if matches!(sync_type, SyncType::SkipSync) {
return Ok(());
}
let remote_mailboxes = get_sync_folders(account).await?;
if matches!(sync_type, SyncType::InitialSync) {
AccountRunningState::set_initial_sync_start(account_id).await?;
let result = match &account.date_since {
Some(date_since) => {
rebuild_cache_since_date(account, &remote_mailboxes, date_since).await
}
None => rebuild_cache(account, &remote_mailboxes).await,
};
match result {
Ok(_) => {
AccountRunningState::set_initial_sync_completed(account_id).await?;
}
Err(e) => {
STATUS_DISPATCHER
.append_error(
account_id,
format!("Initial sync failed for the account, error: {:#?}", e),
)
.await;
AccountRunningState::set_initial_sync_failed(account_id).await?;
}
}
return Ok(());
}
if let Some(state) = AccountRunningState::get(account_id).await? {
let now = utc_now!();
const COOLDOWN_MS: i64 = 60 * 1000;
let mut should_skip = false;
if let Some(time) = state.initial_sync_end_time {
if now - time < COOLDOWN_MS {
should_skip = true;
}
}
if let Some(time) = state.initial_sync_failed_time {
if now - time < COOLDOWN_MS {
should_skip = true;
}
}
if should_skip {
return Ok(());
}
}
AccountRunningState::set_incremental_sync_start(account.id).await?;
let local_mailboxes = MailBox::list_all(account_id).await?;
reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes).await?;
let elapsed_time = start_time.elapsed().as_secs();
debug!(
"Account{{{}}} Incremental sync completed: {} seconds elapsed.",
account.email, elapsed_time
);
if let Some(state) = AccountRunningState::get(account.id).await? {
if !state.is_initial_sync_completed {
AccountRunningState::set_initial_sync_completed(account_id).await?;
}
}
AccountRunningState::set_incremental_sync_end(account_id).await?;
Ok(())
}
+200
View File
@@ -0,0 +1,200 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::{
account::{migration::AccountModel, since::DateSince},
cache::{
imap::{
mailbox::MailBox,
sync::flow::{fetch_and_save_full_mailbox, fetch_and_save_since_date},
},
SEMAPHORE,
},
error::{code::ErrorCode, BichonError, BichonResult},
indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER},
},
raise_error,
};
use std::time::Instant;
use tracing::{error, info};
pub async fn rebuild_cache(
account: &AccountModel,
remote_mailboxes: &[MailBox],
) -> BichonResult<()> {
let start_time = Instant::now();
let mut total_inserted = 0;
MailBox::batch_insert(remote_mailboxes).await?;
let mut handles = Vec::new();
for mailbox in remote_mailboxes {
if mailbox.exists == 0 {
info!(
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
account.id, &mailbox.name
);
continue;
}
let account = account.clone();
let mailbox = mailbox.clone();
match SEMAPHORE.clone().acquire_owned().await {
Ok(permit) => {
let handle: tokio::task::JoinHandle<Result<usize, BichonError>> =
tokio::spawn(async move {
let _permit = permit; // Ensure permit is released when task finishes
fetch_and_save_full_mailbox(&account, &mailbox, mailbox.exists).await
});
handles.push(handle);
}
Err(err) => {
error!("Failed to acquire semaphore permit, error: {:#?}", err);
}
}
}
for task in handles {
match task.await {
Ok(Ok(count)) => {
total_inserted += count;
}
Ok(Err(err)) => return Err(err),
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
}
}
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
This is a full data fetch as there was no local cache data available.",
total_inserted, elapsed_time
);
Ok(())
}
pub async fn rebuild_cache_since_date(
account: &AccountModel,
remote_mailboxes: &[MailBox],
date_since: &DateSince,
) -> BichonResult<()> {
let start_time = Instant::now();
let mut total_inserted = 0;
let date = date_since.since_date()?;
MailBox::batch_insert(remote_mailboxes).await?;
let mut handles = Vec::new();
for mailbox in remote_mailboxes {
if mailbox.exists == 0 {
info!(
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
account.id, &mailbox.name
);
continue;
}
let account = account.clone();
let mailbox = mailbox.clone();
let date = date.clone();
match SEMAPHORE.clone().acquire_owned().await {
Ok(permit) => {
let handle: tokio::task::JoinHandle<Result<usize, BichonError>> =
tokio::spawn(async move {
let _permit = permit; // Ensure permit is released when task finishes
fetch_and_save_since_date(&account, date.as_str(), &mailbox).await
});
handles.push(handle);
}
Err(err) => {
error!("Failed to acquire semaphore permit, error: {:#?}", err);
}
}
}
for task in handles {
match task.await {
Ok(Ok(count)) => {
total_inserted += count;
}
Ok(Err(err)) => return Err(err),
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
}
}
let elapsed_time = start_time.elapsed().as_secs();
info!(
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
Data fetched from server starting from the specified date: {}.",
total_inserted, elapsed_time, date
);
Ok(())
}
pub async fn rebuild_mailbox_cache(
account: &AccountModel,
local_mailbox: &MailBox,
remote_mailbox: &MailBox,
) -> BichonResult<()> {
ENVELOPE_INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
.await?;
EML_INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
.await?;
if remote_mailbox.exists == 0 {
info!(
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
account.id,
&local_mailbox.name
);
return Ok(()); // Skip if the mailbox has no emails
}
let inserted_count =
fetch_and_save_full_mailbox(account, remote_mailbox, remote_mailbox.exists).await?;
info!(
"Account {}: Successfully rebuild mailbox cache, inserted {} envelopes for mailbox '{}'.",
account.id, inserted_count, &local_mailbox.name
);
Ok(())
}
pub async fn rebuild_mailbox_cache_since_date(
account: &AccountModel,
local_mailbox_id: u64,
date_since: &DateSince,
remote: &MailBox,
) -> BichonResult<()> {
ENVELOPE_INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
.await?;
EML_INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
.await?;
if remote.exists == 0 {
info!(
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
account.id,
&remote.name
);
return Ok(()); // Skip if the mailbox has no emails
}
let count =
fetch_and_save_since_date(account, date_since.since_date()?.as_str(), remote).await?;
info!(
"Account {}: Successfully rebuild mailbox cache, inserted {} envelopes for mailbox '{}'.",
account.id, count, &remote.name
);
Ok(())
}
+184
View File
@@ -0,0 +1,184 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::collections::BTreeSet;
use crate::{
decode_mailbox_name,
modules::{
account::migration::{AccountModel, AccountType},
cache::imap::mailbox::{AttributeEnum, MailBox},
context::executors::MAIL_CONTEXT,
error::{code::ErrorCode, BichonResult},
mailbox::list::convert_names_to_mailboxes,
},
raise_error,
};
use async_imap::types::Name;
use tracing::{debug, info, warn};
pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBox>> {
assert_eq!(account.account_type, AccountType::IMAP);
let executor = MAIL_CONTEXT.imap(account.id).await?;
let names = executor.list_all_mailboxes().await?;
if names.is_empty() {
warn!(
"Account {}: No mailboxes returned from IMAP server.",
account.id
);
return Err(raise_error!(format!(
"No mailboxes returned from IMAP server for account {}. This is unexpected and may indicate an issue with the IMAP server.",
&account.id
), ErrorCode::ImapUnexpectedResult));
}
let mailboxes: Vec<(MailBox, Name)> = names.into_iter().map(|n| ((&n).into(), n)).collect();
for (mailbox, _) in &mailboxes {
debug!(
"[MAILBOX DEBUG] Account {}: mailbox='{}', attributes={:?}",
account.id, mailbox.name, mailbox.attributes
);
}
detect_mailbox_changes(
account,
mailboxes.iter().map(|(m, _)| m.name.clone()).collect(),
)
.await?;
let account = AccountModel::get(account.id).await?;
let subscribed = &account.sync_folders.unwrap_or_default();
let is_noselect = |mailbox: &MailBox| {
mailbox
.attributes
.iter()
.any(|attr| matches!(attr.attr, AttributeEnum::NoSelect))
};
let is_default_mailbox = |mailbox: &MailBox| {
mailbox.name.eq_ignore_ascii_case("INBOX")
|| mailbox
.attributes
.iter()
.any(|attr| matches!(attr.attr, AttributeEnum::Sent))
};
let mut matched_mailboxes: Vec<&Name> = if !subscribed.is_empty() {
mailboxes
.iter()
.filter(|(mailbox, _)| subscribed.contains(&mailbox.name) && !is_noselect(mailbox))
.map(|(_, name)| name)
.collect()
} else {
Vec::new()
};
if matched_mailboxes.is_empty() {
matched_mailboxes = mailboxes
.iter()
.filter(|(mailbox, _)| !is_noselect(mailbox) && is_default_mailbox(mailbox))
.map(|(_, name)| name)
.collect();
debug!(
"[MAILBOX DEBUG] Account {}: matched_mailboxes (default selection) = {:?}",
account.id,
matched_mailboxes
.iter()
.map(|n| decode_mailbox_name!(n.name().to_string()))
.collect::<Vec<_>>()
);
if !matched_mailboxes.is_empty() {
let sync_folders: Vec<String> = matched_mailboxes
.iter()
.map(|n| decode_mailbox_name!(n.name().to_string()))
.collect();
AccountModel::update_sync_folders(account.id, sync_folders).await?;
} else {
warn!(
"Account {}: No subscribed mailboxes found. This is unexpected — IMAP server should at least provide INBOX.",
account.id
);
return Err(raise_error!(format!(
"No subscribed mailboxes found for account {}. This is unexpected — IMAP server should at least provide INBOX.",
&account.id
), ErrorCode::ImapUnexpectedResult));
}
}
convert_names_to_mailboxes(account.id, matched_mailboxes).await
}
pub async fn detect_mailbox_changes(
account: &AccountModel,
all_names: BTreeSet<String>,
) -> BichonResult<()> {
if account.known_folders.is_none() {
// First time sync: just save without comparing
AccountModel::update_known_folders(account.id, all_names).await?;
return Ok(());
}
let known_folders = account.known_folders.clone().unwrap_or_default();
// Compute differences
let new_folders: Vec<String> = all_names.difference(&known_folders).cloned().collect();
let deleted_folders: Vec<String> = known_folders.difference(&all_names).cloned().collect();
let has_changes = !new_folders.is_empty() || !deleted_folders.is_empty();
let sync_folders = account.sync_folders.as_deref().unwrap_or_default();
// Handle deleted folders in sync_folders
if !deleted_folders.is_empty() {
// Check if any deleted folders are in sync_folders
let remaining_sync_folders: Vec<String> = sync_folders
.iter()
.filter(|folder| !deleted_folders.contains(folder))
.cloned()
.collect();
// If sync_folders changed, update them
if remaining_sync_folders.len() != sync_folders.len() {
let removed_count = sync_folders.len() - remaining_sync_folders.len();
info!(
"Account {}: Removed {} deleted folders from sync_folders",
account.id, removed_count
);
// Note: When all subscribed folders are deleted (remaining_sync_folders empty),
// the system's default behavior is to automatically fall back to syncing
// only the default folders (INBOX and Sent) in subsequent operations
AccountModel::update_sync_folders(account.id, remaining_sync_folders).await?;
}
info!(
"Account {}: Folders deleted: {:?}",
account.id, deleted_folders
);
}
// Fire events for new folders if needed
if !new_folders.is_empty() {
info!(
"Account {}: New folders detected: {:?}",
account.id, new_folders
);
}
// Update known folders only if there were changes
if has_changes {
AccountModel::update_known_folders(account.id, all_names).await?;
}
Ok(())
}
+68
View File
@@ -0,0 +1,68 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::{
account::{migration::AccountModel, state::AccountRunningState},
error::BichonResult,
},
utc_now,
};
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum SyncType {
/// Initial sync, used when fetching all messages for the first time.
InitialSync,
/// Incremental synchronization, typically used for updates or fetching new data since the last sync.
IncrementalSync,
/// Skip synchronization, used when it's not yet time to perform the next sync.
SkipSync,
}
pub async fn determine_sync_type(account: &AccountModel) -> BichonResult<SyncType> {
Ok(match AccountRunningState::get(account.id).await? {
Some(info) => {
let now = utc_now!();
let incremental_sync = is_time_for_incremental_sync(
now,
info.last_incremental_sync_start,
account.sync_interval_min.unwrap(),
);
if incremental_sync {
SyncType::IncrementalSync
} else {
SyncType::SkipSync
}
}
None => {
AccountRunningState::add(account.id).await?;
SyncType::InitialSync
}
})
}
/// Check if it's time for an incremental sync based on the provided interval.
fn is_time_for_incremental_sync(
now: i64,
last_incremental_sync_at: i64,
sync_interval_min: i64,
) -> bool {
now - last_incremental_sync_at > (sync_interval_min * 60 * 1000)
}
+117
View File
@@ -0,0 +1,117 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::account::entity::AuthType;
use crate::modules::cache::imap::sync::execute_imap_sync;
use crate::modules::common::periodic::{PeriodicTask, TaskHandle};
use crate::modules::oauth2::token::OAuth2AccessToken;
use crate::modules::{
account::{dispatcher::STATUS_DISPATCHER, migration::AccountModel},
error::BichonResult,
};
use crate::utc_now;
use dashmap::DashMap;
use std::sync::atomic::{AtomicI64, Ordering};
use std::{sync::LazyLock, time::Duration};
use tracing::{error, warn};
static _DESCRIPTION: &str = "This task periodically synchronizes mailbox data for a specified account, ensuring that all local data is up-to-date.";
const TASK_INTERVAL: Duration = Duration::from_secs(10);
pub static SYNC_TASKS: LazyLock<AccountSyncTask> = LazyLock::new(AccountSyncTask::new);
static LAST_WARN_TIME: AtomicI64 = AtomicI64::new(0);
const WARN_INTERVAL_MS: i64 = 600_000;
pub struct AccountSyncTask {
tasks: DashMap<u64, TaskHandle>,
}
impl AccountSyncTask {
pub fn new() -> Self {
Self {
tasks: DashMap::new(),
}
}
pub async fn start_account_sync_task(&self, account_id: u64, email: String) {
let task_name = format!("account-sync-task-{}-{}", account_id, &email);
let periodic_task = PeriodicTask::new(&task_name);
let task = move |param: Option<u64>| {
let account_id = param.unwrap();
Box::pin(async move {
let account = AccountModel::get(account_id).await.ok();
match account {
Some(account) => {
if !account.enabled {
let last = LAST_WARN_TIME.load(Ordering::Relaxed);
let now = utc_now!();
if now - last >= WARN_INTERVAL_MS {
LAST_WARN_TIME.store(now, Ordering::Relaxed);
warn!(
"Account {}: Sync aborted. Account is currently disabled.",
account_id
);
}
} else {
if let Some(imap) = &account.imap {
if let AuthType::OAuth2 = imap.auth.auth_type {
if OAuth2AccessToken::get(account.id).await?.is_none() {
if utc_now!() % 300_000 == 0 {
warn!("Account {}: Sync aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id);
}
return Ok(());
}
}
}
if let Err(e) = execute_imap_sync(&account).await {
STATUS_DISPATCHER
.append_error(
account_id,
format!("error in account sync task: {:#?}", e),
)
.await;
error!(
"Failed to synchronize mailbox data for '{}': {:?}",
account_id, e
)
}
}
}
None => {
error!(
"Account {}: Sync aborted. Account entity not found.",
account_id
);
}
}
Ok(())
})
};
let handler = periodic_task.start(task, Some(account_id), TASK_INTERVAL, true, true);
self.tasks.insert(account_id, handler);
}
pub async fn stop(&self, account_id: u64) -> BichonResult<()> {
if let Some((_, handler)) = self.tasks.remove(&account_id) {
handler.cancel().await;
} else {
warn!("No sync task found for account: {}", account_id);
}
Ok(())
}
}
+33
View File
@@ -0,0 +1,33 @@
//
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::settings::cli::SETTINGS;
use std::sync::{Arc, LazyLock};
use tokio::sync::Semaphore;
pub mod imap;
pub static SEMAPHORE: LazyLock<Arc<Semaphore>> = LazyLock::new(|| {
Arc::new(Semaphore::new(
SETTINGS
.bichon_sync_concurrency
.map(|c| c as usize)
.unwrap_or(num_cpus::get() * 2),
))
});