mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-03 07:48:34 +02:00
initial commit
This commit is contained in:
Vendored
+207
@@ -0,0 +1,207 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::{
|
||||
decode_mailbox_name, encode_mailbox_name,
|
||||
modules::{
|
||||
database::{
|
||||
batch_delete_impl, batch_insert_impl, batch_upsert_impl, filter_by_secondary_key_impl,
|
||||
manager::DB_MANAGER,
|
||||
},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
},
|
||||
raise_error,
|
||||
};
|
||||
use async_imap::types::{Name, NameAttribute};
|
||||
use itertools::Itertools;
|
||||
use native_db::*;
|
||||
use native_model::{native_model, Model};
|
||||
use poem_openapi::{Enum, Object};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
|
||||
#[native_model(id = 1, version = 1)]
|
||||
#[native_db]
|
||||
pub struct MailBox {
|
||||
/// The unique identifier for the mailbox
|
||||
#[primary_key]
|
||||
pub id: u64,
|
||||
/// The ID of the account associated with the mailbox
|
||||
#[secondary_key]
|
||||
pub account_id: u64,
|
||||
/// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items").
|
||||
/// This is the decoded name as presented to users, derived from the IMAP server's mailbox name
|
||||
/// (e.g., after decoding UTF-7 or other encodings per RFC 3501).
|
||||
pub name: String,
|
||||
/// Optional delimiter used to separate mailbox names in a hierarchy (e.g., "/" or ".").
|
||||
/// Used in IMAP to structure nested mailboxes (e.g., "INBOX/Archive").
|
||||
pub delimiter: Option<String>,
|
||||
/// List of attributes associated with the mailbox (e.g., `\NoSelect`, `\Deleted`).
|
||||
/// These indicate special properties, such as whether the mailbox can hold messages.
|
||||
pub attributes: Vec<Attribute>,
|
||||
/// The number of messages that currently exist in the mailbox.
|
||||
pub exists: u32,
|
||||
/// Optional number of unseen messages in the mailbox (i.e., messages without the `\Seen` flag).
|
||||
pub unseen: Option<u32>,
|
||||
/// The next unique identifier (UID) that will be assigned to a new message in the mailbox.
|
||||
/// If `None`, the IMAP server has not provided this information.
|
||||
pub uid_next: Option<u32>,
|
||||
/// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions.
|
||||
/// If `None`, the IMAP server has not provided this information.
|
||||
pub uid_validity: Option<u32>,
|
||||
}
|
||||
|
||||
impl MailBox {
|
||||
pub fn encoded_name(&self) -> String {
|
||||
encode_mailbox_name!(&self.name)
|
||||
}
|
||||
|
||||
// pub async fn batch_delete(mailboxes: Vec<MailBox>) -> BichonResult<()> {
|
||||
// batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
|
||||
// let mut to_deleted = Vec::new();
|
||||
// for mailbox in mailboxes {
|
||||
// let retrived = rw
|
||||
// .get()
|
||||
// .primary::<MailBox>(mailbox.id)
|
||||
// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
// if let Some(retrived) = retrived {
|
||||
// to_deleted.push(retrived);
|
||||
// }
|
||||
// }
|
||||
// Ok(to_deleted)
|
||||
// })
|
||||
// .await?;
|
||||
// Ok(())
|
||||
// }
|
||||
|
||||
// pub async fn get(id: u64) -> RustMailerResult<MailBox> {
|
||||
// let result = async_find_impl::<MailBox>(DB_MANAGER.envelope_db(), id).await?;
|
||||
// Ok(result.ok_or_else(|| {
|
||||
// raise_error!(
|
||||
// format!("mailbox {} not found", id),
|
||||
// ErrorCode::InternalError
|
||||
// )
|
||||
// })?)
|
||||
// }
|
||||
|
||||
// pub async fn delete(id: u64) -> BichonResult<()> {
|
||||
// delete_impl(DB_MANAGER.envelope_db(), move |rw| {
|
||||
// rw.get()
|
||||
// .primary::<MailBox>(id)
|
||||
// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
|
||||
// .ok_or_else(|| raise_error!("mailbox missing".into(), ErrorCode::InternalError))
|
||||
// })
|
||||
// .await
|
||||
// }
|
||||
|
||||
pub async fn list_all(account_id: u64) -> BichonResult<Vec<MailBox>> {
|
||||
filter_by_secondary_key_impl(DB_MANAGER.envelope_db(), MailBoxKey::account_id, account_id)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> {
|
||||
batch_insert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
|
||||
}
|
||||
|
||||
pub async fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> {
|
||||
batch_upsert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
|
||||
}
|
||||
|
||||
pub async fn clean(account_id: u64) -> BichonResult<()> {
|
||||
batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
|
||||
let mailboxes: Vec<MailBox> = rw
|
||||
.scan()
|
||||
.secondary::<MailBox>(MailBoxKey::account_id)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
|
||||
.start_with(account_id)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
|
||||
.try_collect()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
Ok(mailboxes)
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Object)]
|
||||
pub struct Attribute {
|
||||
pub attr: AttributeEnum,
|
||||
pub extension: Option<String>,
|
||||
}
|
||||
|
||||
impl Attribute {
|
||||
pub fn new(attr: AttributeEnum, extension: Option<String>) -> Self {
|
||||
Self { attr, extension }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize, Enum)]
|
||||
pub enum AttributeEnum {
|
||||
NoInferiors,
|
||||
NoSelect,
|
||||
Marked,
|
||||
Unmarked,
|
||||
All,
|
||||
Archive,
|
||||
Drafts,
|
||||
Flagged,
|
||||
Junk,
|
||||
Sent,
|
||||
Trash,
|
||||
Extension,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl From<&Name> for MailBox {
|
||||
fn from(value: &Name) -> Self {
|
||||
let name = decode_mailbox_name!(value.name().to_string());
|
||||
let delimiter = value.delimiter().map(|f| f.to_owned());
|
||||
let attributes: Vec<Attribute> = value.attributes().iter().map(|na| na.into()).collect();
|
||||
//The remaining parts will be supplemented during the examine_mailbox process.
|
||||
MailBox {
|
||||
name,
|
||||
delimiter,
|
||||
attributes,
|
||||
..Default::default() //has_synced is initialized to false here
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&NameAttribute<'_>> for Attribute {
|
||||
fn from(value: &NameAttribute) -> Self {
|
||||
match value {
|
||||
NameAttribute::NoInferiors => Attribute::new(AttributeEnum::NoInferiors, None),
|
||||
NameAttribute::NoSelect => Attribute::new(AttributeEnum::NoSelect, None),
|
||||
NameAttribute::Marked => Attribute::new(AttributeEnum::Marked, None),
|
||||
NameAttribute::Unmarked => Attribute::new(AttributeEnum::Unmarked, None),
|
||||
NameAttribute::All => Attribute::new(AttributeEnum::All, None),
|
||||
NameAttribute::Archive => Attribute::new(AttributeEnum::Archive, None),
|
||||
NameAttribute::Drafts => Attribute::new(AttributeEnum::Drafts, None),
|
||||
NameAttribute::Flagged => Attribute::new(AttributeEnum::Flagged, None),
|
||||
NameAttribute::Junk => Attribute::new(AttributeEnum::Junk, None),
|
||||
NameAttribute::Sent => Attribute::new(AttributeEnum::Sent, None),
|
||||
NameAttribute::Trash => Attribute::new(AttributeEnum::Trash, None),
|
||||
NameAttribute::Extension(s) => {
|
||||
Attribute::new(AttributeEnum::Extension, Some(s.to_string()))
|
||||
}
|
||||
_ => Attribute::new(AttributeEnum::Unknown, None),
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+66
@@ -0,0 +1,66 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use crate::modules::{account::state::AccountRunningState, database::ModelsAdapter};
|
||||
use ahash::{AHashMap, AHashSet};
|
||||
use mailbox::MailBox;
|
||||
use native_db::Models;
|
||||
|
||||
pub mod mailbox;
|
||||
pub mod sync;
|
||||
pub mod task;
|
||||
|
||||
pub static MAILBOX_MODELS: LazyLock<Models> = LazyLock::new(|| {
|
||||
let mut adapter = ModelsAdapter::new();
|
||||
adapter.register_model::<MailBox>();
|
||||
adapter.register_model::<AccountRunningState>();
|
||||
adapter.models
|
||||
});
|
||||
|
||||
pub fn find_missing_mailboxes(
|
||||
local_mailboxes: &[MailBox],
|
||||
server_mailboxes: &[MailBox],
|
||||
) -> Vec<MailBox> {
|
||||
let local_names: AHashSet<_> = local_mailboxes.iter().map(|m| &m.name).collect();
|
||||
server_mailboxes
|
||||
.iter()
|
||||
.filter(|m| !local_names.contains(&m.name))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn find_intersecting_mailboxes(
|
||||
local_mailboxes: &[MailBox],
|
||||
remote_mailboxes: &[MailBox],
|
||||
) -> Vec<(MailBox, MailBox)> {
|
||||
let local_map: AHashMap<_, _> = local_mailboxes
|
||||
.iter()
|
||||
.map(|m| (m.name.clone(), m.clone()))
|
||||
.collect();
|
||||
remote_mailboxes
|
||||
.iter()
|
||||
.filter_map(|m| {
|
||||
local_map
|
||||
.get(&m.name)
|
||||
.map(|local_mailbox| (local_mailbox.clone(), m.clone()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
Vendored
+379
@@ -0,0 +1,379 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::{
|
||||
modules::{
|
||||
account::{migration::AccountModel, state::AccountRunningState},
|
||||
cache::{
|
||||
imap::{
|
||||
find_intersecting_mailboxes, find_missing_mailboxes,
|
||||
mailbox::MailBox,
|
||||
sync::rebuild::{rebuild_mailbox_cache, rebuild_mailbox_cache_since_date},
|
||||
},
|
||||
SEMAPHORE,
|
||||
},
|
||||
context::executors::MAIL_CONTEXT,
|
||||
error::{code::ErrorCode, BichonError, BichonResult},
|
||||
indexer::manager::ENVELOPE_INDEX_MANAGER,
|
||||
},
|
||||
raise_error,
|
||||
};
|
||||
use std::time::Instant;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
pub const BATCH_SIZE: u32 = 50;
|
||||
|
||||
pub async fn fetch_and_save_since_date(
|
||||
account: &AccountModel,
|
||||
date: &str,
|
||||
mailbox: &MailBox,
|
||||
) -> BichonResult<usize> {
|
||||
let account_id = account.id;
|
||||
let executor = MAIL_CONTEXT.imap(account_id).await?;
|
||||
let uid_list = executor
|
||||
.uid_search(&mailbox.encoded_name(), format!("SINCE {date}").as_str())
|
||||
.await?;
|
||||
|
||||
let len = uid_list.len();
|
||||
if len == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let folder_limit = account.folder_limit;
|
||||
// sort small -> bigger
|
||||
let mut uid_vec: Vec<u32> = uid_list.into_iter().collect();
|
||||
uid_vec.sort();
|
||||
|
||||
if let Some(limit) = folder_limit {
|
||||
let limit = limit.max(100) as usize;
|
||||
if len > limit {
|
||||
uid_vec = uid_vec.split_off(len - limit as usize);
|
||||
}
|
||||
}
|
||||
|
||||
// let semaphore = Arc::new(Semaphore::new(5));
|
||||
|
||||
let uid_batches = generate_uid_sequence_hashset(uid_vec, BATCH_SIZE as usize, false);
|
||||
AccountRunningState::set_initial_current_syncing_folder(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
uid_batches.len() as u32,
|
||||
)
|
||||
.await?;
|
||||
for (index, batch) in uid_batches.into_iter().enumerate() {
|
||||
AccountRunningState::set_current_sync_batch_number(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
(index + 1) as u32,
|
||||
)
|
||||
.await?;
|
||||
let executor = MAIL_CONTEXT.imap(account_id).await?;
|
||||
// Fetch metadata for the current batch of UIDs
|
||||
executor
|
||||
.uid_batch_retrieve_emails(account_id, mailbox.id, &batch, &mailbox.encoded_name())
|
||||
.await?;
|
||||
}
|
||||
Ok(len)
|
||||
}
|
||||
|
||||
pub async fn fetch_and_save_full_mailbox(
|
||||
account: &AccountModel,
|
||||
mailbox: &MailBox,
|
||||
total: u32,
|
||||
) -> BichonResult<usize> {
|
||||
let mailbox_id = mailbox.id;
|
||||
let account_id = account.id;
|
||||
let folder_limit = account.folder_limit;
|
||||
let total_to_fetch = match folder_limit {
|
||||
Some(limit) if limit < total => total.min(limit.max(100)),
|
||||
_ => total,
|
||||
};
|
||||
let page_size = if let Some(limit) = folder_limit {
|
||||
limit.max(100).min(BATCH_SIZE as u32)
|
||||
} else {
|
||||
BATCH_SIZE as u32
|
||||
};
|
||||
|
||||
let total_batches = total_to_fetch.div_ceil(page_size);
|
||||
let desc = folder_limit.is_some();
|
||||
|
||||
let mut inserted_count = 0;
|
||||
|
||||
AccountRunningState::set_initial_current_syncing_folder(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
total_batches,
|
||||
)
|
||||
.await?;
|
||||
info!(
|
||||
"Starting full mailbox sync for '{}', total={}, limit={:?}, batches={}, desc={}",
|
||||
mailbox.name, total, folder_limit, total_batches, desc
|
||||
);
|
||||
for page in 1..=total_batches {
|
||||
AccountRunningState::set_current_sync_batch_number(account_id, mailbox.name.clone(), page)
|
||||
.await?;
|
||||
let executor = MAIL_CONTEXT.imap(account_id).await?;
|
||||
let count = executor
|
||||
.batch_retrieve_emails(
|
||||
account_id,
|
||||
mailbox_id,
|
||||
page as u64,
|
||||
page_size as u64,
|
||||
&mailbox.encoded_name(),
|
||||
desc,
|
||||
)
|
||||
.await?;
|
||||
inserted_count += count;
|
||||
info!(
|
||||
"Batch insertion completed for mailbox: {}, current page: {}, inserted count: {}",
|
||||
&mailbox.name, page, count
|
||||
);
|
||||
}
|
||||
Ok(inserted_count)
|
||||
}
|
||||
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust
|
||||
/// use std::collections::HashSet;
|
||||
///
|
||||
/// let mut uids = HashSet::new();
|
||||
/// uids.extend([1, 2, 3, 5, 6, 7, 9, 10, 11, 15]);
|
||||
///
|
||||
/// let chunks = generate_uid_sequence_hashset(uids, 6, false);
|
||||
/// assert_eq!(chunks, vec![
|
||||
/// "1:3,5:7".to_string(),
|
||||
/// "9:11,15".to_string()
|
||||
/// ]);
|
||||
/// ```
|
||||
///
|
||||
/// This splits the UIDs into chunks of 6, compresses each chunk into ranges,
|
||||
/// and returns a vector like: `["1:3,5:7", "9:11,15"]`.
|
||||
///
|
||||
pub fn generate_uid_sequence_hashset(
|
||||
unique_nums: Vec<u32>,
|
||||
chunk_size: usize,
|
||||
desc: bool,
|
||||
) -> Vec<String> {
|
||||
assert!(!unique_nums.is_empty());
|
||||
// let mut nums: Vec<u32> = unique_nums.into_iter().collect();
|
||||
// nums.sort();
|
||||
let mut nums = unique_nums;
|
||||
if desc {
|
||||
nums.reverse();
|
||||
}
|
||||
|
||||
let mut result = Vec::new();
|
||||
|
||||
for chunk in nums.chunks(chunk_size) {
|
||||
let compressed = compress_uid_list(chunk.to_vec());
|
||||
result.push(compressed);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn compress_uid_list(nums: Vec<u32>) -> String {
|
||||
if nums.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut sorted_nums = nums;
|
||||
sorted_nums.sort();
|
||||
|
||||
let mut result = Vec::new();
|
||||
let mut current_range_start = sorted_nums[0];
|
||||
let mut current_range_end = sorted_nums[0];
|
||||
|
||||
for &n in sorted_nums.iter().skip(1) {
|
||||
if n == current_range_end + 1 {
|
||||
current_range_end = n;
|
||||
} else {
|
||||
if current_range_start == current_range_end {
|
||||
result.push(current_range_start.to_string());
|
||||
} else {
|
||||
result.push(format!("{}:{}", current_range_start, current_range_end));
|
||||
}
|
||||
current_range_start = n;
|
||||
current_range_end = n;
|
||||
}
|
||||
}
|
||||
|
||||
if current_range_start == current_range_end {
|
||||
result.push(current_range_start.to_string());
|
||||
} else {
|
||||
result.push(format!("{}:{}", current_range_start, current_range_end));
|
||||
}
|
||||
|
||||
result.join(",")
|
||||
}
|
||||
|
||||
pub async fn reconcile_mailboxes(
|
||||
account: &AccountModel,
|
||||
remote_mailboxes: &[MailBox],
|
||||
local_mailboxes: &[MailBox],
|
||||
) -> BichonResult<()> {
|
||||
let start_time = Instant::now();
|
||||
let existing_mailboxes = find_intersecting_mailboxes(local_mailboxes, remote_mailboxes);
|
||||
let account_id = account.id;
|
||||
if !existing_mailboxes.is_empty() {
|
||||
let mut mailboxes_to_update = Vec::with_capacity(existing_mailboxes.len());
|
||||
for (local_mailbox, remote_mailbox) in &existing_mailboxes {
|
||||
if local_mailbox.uid_validity != remote_mailbox.uid_validity {
|
||||
if remote_mailbox.uid_validity.is_none() {
|
||||
warn!(
|
||||
"Account {}: Mailbox '{}' has invalid uid_validity (None). Skipping sync for this mailbox.",
|
||||
account_id, local_mailbox.name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \
|
||||
The mailbox data may be invalid, resetting its envelopes and rebuilding the cache.",
|
||||
account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_mailbox.uid_validity
|
||||
);
|
||||
|
||||
match &account.date_since {
|
||||
Some(date_since) => {
|
||||
rebuild_mailbox_cache_since_date(
|
||||
account,
|
||||
local_mailbox.id,
|
||||
date_since,
|
||||
remote_mailbox,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
rebuild_mailbox_cache(account, local_mailbox, remote_mailbox).await?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
perform_incremental_sync(account, local_mailbox, remote_mailbox).await?;
|
||||
}
|
||||
if let Some(state) = AccountRunningState::get(account.id).await? {
|
||||
if !state.is_initial_sync_completed {
|
||||
AccountRunningState::set_folder_initial_sync_completed(
|
||||
account_id,
|
||||
local_mailbox.name.clone(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
mailboxes_to_update.push(remote_mailbox.clone());
|
||||
}
|
||||
//The metadata of this mailbox must only be updated after a successful synchronization;
|
||||
//otherwise, it may cause synchronization errors and result in missing emails in the local sync results.
|
||||
MailBox::batch_upsert(&mailboxes_to_update).await?;
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Checked mailbox folders for account ID: {}. Compared local and server folders to identify changes. Elapsed time: {} seconds",
|
||||
account.id,
|
||||
start_time.elapsed().as_secs()
|
||||
);
|
||||
|
||||
let missing_mailboxes = find_missing_mailboxes(local_mailboxes, remote_mailboxes);
|
||||
if !missing_mailboxes.is_empty() {
|
||||
MailBox::batch_insert(&missing_mailboxes).await?;
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for mailbox in &missing_mailboxes {
|
||||
if mailbox.exists > 0 {
|
||||
let account = account.clone();
|
||||
let mailbox = mailbox.clone();
|
||||
match SEMAPHORE.clone().acquire_owned().await {
|
||||
Ok(permit) => {
|
||||
let handle: tokio::task::JoinHandle<Result<(), BichonError>> =
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
match &account.date_since {
|
||||
Some(date_since) => {
|
||||
rebuild_mailbox_cache_since_date(
|
||||
&account, mailbox.id, date_since, &mailbox,
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
rebuild_mailbox_cache(&account, &mailbox, &mailbox).await
|
||||
}
|
||||
}
|
||||
});
|
||||
handles.push(handle);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to acquire semaphore permit, error: {:#?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for task in handles {
|
||||
match task.await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => return Err(err),
|
||||
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
//only check new emails and sync
|
||||
async fn perform_incremental_sync(
|
||||
account: &AccountModel,
|
||||
local_mailbox: &MailBox,
|
||||
remote_mailbox: &MailBox,
|
||||
) -> BichonResult<()> {
|
||||
if remote_mailbox.exists > 0 {
|
||||
let local_max_uid = ENVELOPE_INDEX_MANAGER
|
||||
.get_max_uid(account.id, local_mailbox.id)
|
||||
.await?;
|
||||
match local_max_uid {
|
||||
Some(max_uid) => {
|
||||
let executor = MAIL_CONTEXT.imap(account.id).await?;
|
||||
executor
|
||||
.fetch_new_mail(account.id, local_mailbox, max_uid + 1)
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
info!(
|
||||
"No maximum UID found in index for mailbox, assuming local cache is missing."
|
||||
);
|
||||
|
||||
match &account.date_since {
|
||||
Some(date_since) => {
|
||||
fetch_and_save_since_date(
|
||||
account,
|
||||
date_since.since_date()?.as_str(),
|
||||
remote_mailbox,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
None => {
|
||||
fetch_and_save_full_mailbox(account, remote_mailbox, remote_mailbox.exists)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Vendored
+113
@@ -0,0 +1,113 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::{
|
||||
modules::{
|
||||
account::{
|
||||
dispatcher::STATUS_DISPATCHER,
|
||||
migration::{AccountModel, AccountType},
|
||||
state::AccountRunningState,
|
||||
},
|
||||
cache::imap::mailbox::MailBox,
|
||||
error::BichonResult,
|
||||
},
|
||||
utc_now,
|
||||
};
|
||||
use flow::reconcile_mailboxes;
|
||||
use rebuild::{rebuild_cache, rebuild_cache_since_date};
|
||||
use std::time::Instant;
|
||||
use sync_folders::get_sync_folders;
|
||||
use sync_type::{determine_sync_type, SyncType};
|
||||
use tracing::debug;
|
||||
|
||||
pub mod flow;
|
||||
pub mod rebuild;
|
||||
pub mod sync_folders;
|
||||
pub mod sync_type;
|
||||
|
||||
pub async fn execute_imap_sync(account: &AccountModel) -> BichonResult<()> {
|
||||
assert_eq!(account.account_type, AccountType::IMAP);
|
||||
let start_time = Instant::now();
|
||||
let account_id = account.id;
|
||||
let sync_type = determine_sync_type(account).await?;
|
||||
|
||||
if matches!(sync_type, SyncType::SkipSync) {
|
||||
return Ok(());
|
||||
}
|
||||
let remote_mailboxes = get_sync_folders(account).await?;
|
||||
if matches!(sync_type, SyncType::InitialSync) {
|
||||
AccountRunningState::set_initial_sync_start(account_id).await?;
|
||||
let result = match &account.date_since {
|
||||
Some(date_since) => {
|
||||
rebuild_cache_since_date(account, &remote_mailboxes, date_since).await
|
||||
}
|
||||
None => rebuild_cache(account, &remote_mailboxes).await,
|
||||
};
|
||||
match result {
|
||||
Ok(_) => {
|
||||
AccountRunningState::set_initial_sync_completed(account_id).await?;
|
||||
}
|
||||
Err(e) => {
|
||||
STATUS_DISPATCHER
|
||||
.append_error(
|
||||
account_id,
|
||||
format!("Initial sync failed for the account, error: {:#?}", e),
|
||||
)
|
||||
.await;
|
||||
AccountRunningState::set_initial_sync_failed(account_id).await?;
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(state) = AccountRunningState::get(account_id).await? {
|
||||
let now = utc_now!();
|
||||
const COOLDOWN_MS: i64 = 60 * 1000;
|
||||
let mut should_skip = false;
|
||||
if let Some(time) = state.initial_sync_end_time {
|
||||
if now - time < COOLDOWN_MS {
|
||||
should_skip = true;
|
||||
}
|
||||
}
|
||||
if let Some(time) = state.initial_sync_failed_time {
|
||||
if now - time < COOLDOWN_MS {
|
||||
should_skip = true;
|
||||
}
|
||||
}
|
||||
if should_skip {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
AccountRunningState::set_incremental_sync_start(account.id).await?;
|
||||
let local_mailboxes = MailBox::list_all(account_id).await?;
|
||||
reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes).await?;
|
||||
let elapsed_time = start_time.elapsed().as_secs();
|
||||
debug!(
|
||||
"Account{{{}}} Incremental sync completed: {} seconds elapsed.",
|
||||
account.email, elapsed_time
|
||||
);
|
||||
|
||||
if let Some(state) = AccountRunningState::get(account.id).await? {
|
||||
if !state.is_initial_sync_completed {
|
||||
AccountRunningState::set_initial_sync_completed(account_id).await?;
|
||||
}
|
||||
}
|
||||
AccountRunningState::set_incremental_sync_end(account_id).await?;
|
||||
Ok(())
|
||||
}
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::{
|
||||
modules::{
|
||||
account::{migration::AccountModel, since::DateSince},
|
||||
cache::{
|
||||
imap::{
|
||||
mailbox::MailBox,
|
||||
sync::flow::{fetch_and_save_full_mailbox, fetch_and_save_since_date},
|
||||
},
|
||||
SEMAPHORE,
|
||||
},
|
||||
error::{code::ErrorCode, BichonError, BichonResult},
|
||||
indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER},
|
||||
},
|
||||
raise_error,
|
||||
};
|
||||
use std::time::Instant;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub async fn rebuild_cache(
|
||||
account: &AccountModel,
|
||||
remote_mailboxes: &[MailBox],
|
||||
) -> BichonResult<()> {
|
||||
let start_time = Instant::now();
|
||||
let mut total_inserted = 0;
|
||||
MailBox::batch_insert(remote_mailboxes).await?;
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for mailbox in remote_mailboxes {
|
||||
if mailbox.exists == 0 {
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
|
||||
account.id, &mailbox.name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let account = account.clone();
|
||||
let mailbox = mailbox.clone();
|
||||
match SEMAPHORE.clone().acquire_owned().await {
|
||||
Ok(permit) => {
|
||||
let handle: tokio::task::JoinHandle<Result<usize, BichonError>> =
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit; // Ensure permit is released when task finishes
|
||||
fetch_and_save_full_mailbox(&account, &mailbox, mailbox.exists).await
|
||||
});
|
||||
handles.push(handle);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to acquire semaphore permit, error: {:#?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
for task in handles {
|
||||
match task.await {
|
||||
Ok(Ok(count)) => {
|
||||
total_inserted += count;
|
||||
}
|
||||
Ok(Err(err)) => return Err(err),
|
||||
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
|
||||
}
|
||||
}
|
||||
let elapsed_time = start_time.elapsed().as_secs();
|
||||
info!(
|
||||
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
|
||||
This is a full data fetch as there was no local cache data available.",
|
||||
total_inserted, elapsed_time
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn rebuild_cache_since_date(
|
||||
account: &AccountModel,
|
||||
remote_mailboxes: &[MailBox],
|
||||
date_since: &DateSince,
|
||||
) -> BichonResult<()> {
|
||||
let start_time = Instant::now();
|
||||
let mut total_inserted = 0;
|
||||
let date = date_since.since_date()?;
|
||||
MailBox::batch_insert(remote_mailboxes).await?;
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for mailbox in remote_mailboxes {
|
||||
if mailbox.exists == 0 {
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' on the remote server has no emails. Skipping fetch for this mailbox.",
|
||||
account.id, &mailbox.name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let account = account.clone();
|
||||
let mailbox = mailbox.clone();
|
||||
let date = date.clone();
|
||||
match SEMAPHORE.clone().acquire_owned().await {
|
||||
Ok(permit) => {
|
||||
let handle: tokio::task::JoinHandle<Result<usize, BichonError>> =
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit; // Ensure permit is released when task finishes
|
||||
fetch_and_save_since_date(&account, date.as_str(), &mailbox).await
|
||||
});
|
||||
handles.push(handle);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to acquire semaphore permit, error: {:#?}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
for task in handles {
|
||||
match task.await {
|
||||
Ok(Ok(count)) => {
|
||||
total_inserted += count;
|
||||
}
|
||||
Ok(Err(err)) => return Err(err),
|
||||
Err(e) => return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
|
||||
}
|
||||
}
|
||||
let elapsed_time = start_time.elapsed().as_secs();
|
||||
info!(
|
||||
"Rebuild account cache completed: {} envelopes inserted. {} secs elapsed. \
|
||||
Data fetched from server starting from the specified date: {}.",
|
||||
total_inserted, elapsed_time, date
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn rebuild_mailbox_cache(
|
||||
account: &AccountModel,
|
||||
local_mailbox: &MailBox,
|
||||
remote_mailbox: &MailBox,
|
||||
) -> BichonResult<()> {
|
||||
ENVELOPE_INDEX_MANAGER
|
||||
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
|
||||
.await?;
|
||||
EML_INDEX_MANAGER
|
||||
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
|
||||
.await?;
|
||||
if remote_mailbox.exists == 0 {
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
|
||||
account.id,
|
||||
&local_mailbox.name
|
||||
);
|
||||
return Ok(()); // Skip if the mailbox has no emails
|
||||
}
|
||||
|
||||
let inserted_count =
|
||||
fetch_and_save_full_mailbox(account, remote_mailbox, remote_mailbox.exists).await?;
|
||||
info!(
|
||||
"Account {}: Successfully rebuild mailbox cache, inserted {} envelopes for mailbox '{}'.",
|
||||
account.id, inserted_count, &local_mailbox.name
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn rebuild_mailbox_cache_since_date(
|
||||
account: &AccountModel,
|
||||
local_mailbox_id: u64,
|
||||
date_since: &DateSince,
|
||||
remote: &MailBox,
|
||||
) -> BichonResult<()> {
|
||||
ENVELOPE_INDEX_MANAGER
|
||||
.delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
|
||||
.await?;
|
||||
EML_INDEX_MANAGER
|
||||
.delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
|
||||
.await?;
|
||||
if remote.exists == 0 {
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
|
||||
account.id,
|
||||
&remote.name
|
||||
);
|
||||
return Ok(()); // Skip if the mailbox has no emails
|
||||
}
|
||||
|
||||
let count =
|
||||
fetch_and_save_since_date(account, date_since.since_date()?.as_str(), remote).await?;
|
||||
info!(
|
||||
"Account {}: Successfully rebuild mailbox cache, inserted {} envelopes for mailbox '{}'.",
|
||||
account.id, count, &remote.name
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crate::{
|
||||
decode_mailbox_name,
|
||||
modules::{
|
||||
account::migration::{AccountModel, AccountType},
|
||||
cache::imap::mailbox::{AttributeEnum, MailBox},
|
||||
context::executors::MAIL_CONTEXT,
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
mailbox::list::convert_names_to_mailboxes,
|
||||
},
|
||||
raise_error,
|
||||
};
|
||||
use async_imap::types::Name;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
pub async fn get_sync_folders(account: &AccountModel) -> BichonResult<Vec<MailBox>> {
|
||||
assert_eq!(account.account_type, AccountType::IMAP);
|
||||
let executor = MAIL_CONTEXT.imap(account.id).await?;
|
||||
let names = executor.list_all_mailboxes().await?;
|
||||
if names.is_empty() {
|
||||
warn!(
|
||||
"Account {}: No mailboxes returned from IMAP server.",
|
||||
account.id
|
||||
);
|
||||
return Err(raise_error!(format!(
|
||||
"No mailboxes returned from IMAP server for account {}. This is unexpected and may indicate an issue with the IMAP server.",
|
||||
&account.id
|
||||
), ErrorCode::ImapUnexpectedResult));
|
||||
}
|
||||
let mailboxes: Vec<(MailBox, Name)> = names.into_iter().map(|n| ((&n).into(), n)).collect();
|
||||
|
||||
for (mailbox, _) in &mailboxes {
|
||||
debug!(
|
||||
"[MAILBOX DEBUG] Account {}: mailbox='{}', attributes={:?}",
|
||||
account.id, mailbox.name, mailbox.attributes
|
||||
);
|
||||
}
|
||||
|
||||
detect_mailbox_changes(
|
||||
account,
|
||||
mailboxes.iter().map(|(m, _)| m.name.clone()).collect(),
|
||||
)
|
||||
.await?;
|
||||
let account = AccountModel::get(account.id).await?;
|
||||
let subscribed = &account.sync_folders.unwrap_or_default();
|
||||
let is_noselect = |mailbox: &MailBox| {
|
||||
mailbox
|
||||
.attributes
|
||||
.iter()
|
||||
.any(|attr| matches!(attr.attr, AttributeEnum::NoSelect))
|
||||
};
|
||||
|
||||
let is_default_mailbox = |mailbox: &MailBox| {
|
||||
mailbox.name.eq_ignore_ascii_case("INBOX")
|
||||
|| mailbox
|
||||
.attributes
|
||||
.iter()
|
||||
.any(|attr| matches!(attr.attr, AttributeEnum::Sent))
|
||||
};
|
||||
|
||||
let mut matched_mailboxes: Vec<&Name> = if !subscribed.is_empty() {
|
||||
mailboxes
|
||||
.iter()
|
||||
.filter(|(mailbox, _)| subscribed.contains(&mailbox.name) && !is_noselect(mailbox))
|
||||
.map(|(_, name)| name)
|
||||
.collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
if matched_mailboxes.is_empty() {
|
||||
matched_mailboxes = mailboxes
|
||||
.iter()
|
||||
.filter(|(mailbox, _)| !is_noselect(mailbox) && is_default_mailbox(mailbox))
|
||||
.map(|(_, name)| name)
|
||||
.collect();
|
||||
|
||||
debug!(
|
||||
"[MAILBOX DEBUG] Account {}: matched_mailboxes (default selection) = {:?}",
|
||||
account.id,
|
||||
matched_mailboxes
|
||||
.iter()
|
||||
.map(|n| decode_mailbox_name!(n.name().to_string()))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
if !matched_mailboxes.is_empty() {
|
||||
let sync_folders: Vec<String> = matched_mailboxes
|
||||
.iter()
|
||||
.map(|n| decode_mailbox_name!(n.name().to_string()))
|
||||
.collect();
|
||||
AccountModel::update_sync_folders(account.id, sync_folders).await?;
|
||||
} else {
|
||||
warn!(
|
||||
"Account {}: No subscribed mailboxes found. This is unexpected — IMAP server should at least provide INBOX.",
|
||||
account.id
|
||||
);
|
||||
return Err(raise_error!(format!(
|
||||
"No subscribed mailboxes found for account {}. This is unexpected — IMAP server should at least provide INBOX.",
|
||||
&account.id
|
||||
), ErrorCode::ImapUnexpectedResult));
|
||||
}
|
||||
}
|
||||
convert_names_to_mailboxes(account.id, matched_mailboxes).await
|
||||
}
|
||||
|
||||
pub async fn detect_mailbox_changes(
|
||||
account: &AccountModel,
|
||||
all_names: BTreeSet<String>,
|
||||
) -> BichonResult<()> {
|
||||
if account.known_folders.is_none() {
|
||||
// First time sync: just save without comparing
|
||||
AccountModel::update_known_folders(account.id, all_names).await?;
|
||||
return Ok(());
|
||||
}
|
||||
let known_folders = account.known_folders.clone().unwrap_or_default();
|
||||
// Compute differences
|
||||
let new_folders: Vec<String> = all_names.difference(&known_folders).cloned().collect();
|
||||
let deleted_folders: Vec<String> = known_folders.difference(&all_names).cloned().collect();
|
||||
|
||||
let has_changes = !new_folders.is_empty() || !deleted_folders.is_empty();
|
||||
let sync_folders = account.sync_folders.as_deref().unwrap_or_default();
|
||||
// Handle deleted folders in sync_folders
|
||||
if !deleted_folders.is_empty() {
|
||||
// Check if any deleted folders are in sync_folders
|
||||
let remaining_sync_folders: Vec<String> = sync_folders
|
||||
.iter()
|
||||
.filter(|folder| !deleted_folders.contains(folder))
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
// If sync_folders changed, update them
|
||||
if remaining_sync_folders.len() != sync_folders.len() {
|
||||
let removed_count = sync_folders.len() - remaining_sync_folders.len();
|
||||
info!(
|
||||
"Account {}: Removed {} deleted folders from sync_folders",
|
||||
account.id, removed_count
|
||||
);
|
||||
// Note: When all subscribed folders are deleted (remaining_sync_folders empty),
|
||||
// the system's default behavior is to automatically fall back to syncing
|
||||
// only the default folders (INBOX and Sent) in subsequent operations
|
||||
AccountModel::update_sync_folders(account.id, remaining_sync_folders).await?;
|
||||
}
|
||||
|
||||
info!(
|
||||
"Account {}: Folders deleted: {:?}",
|
||||
account.id, deleted_folders
|
||||
);
|
||||
}
|
||||
|
||||
// Fire events for new folders if needed
|
||||
if !new_folders.is_empty() {
|
||||
info!(
|
||||
"Account {}: New folders detected: {:?}",
|
||||
account.id, new_folders
|
||||
);
|
||||
}
|
||||
|
||||
// Update known folders only if there were changes
|
||||
if has_changes {
|
||||
AccountModel::update_known_folders(account.id, all_names).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::{
|
||||
modules::{
|
||||
account::{migration::AccountModel, state::AccountRunningState},
|
||||
error::BichonResult,
|
||||
},
|
||||
utc_now,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum SyncType {
|
||||
/// Initial sync, used when fetching all messages for the first time.
|
||||
InitialSync,
|
||||
/// Incremental synchronization, typically used for updates or fetching new data since the last sync.
|
||||
IncrementalSync,
|
||||
/// Skip synchronization, used when it's not yet time to perform the next sync.
|
||||
SkipSync,
|
||||
}
|
||||
|
||||
pub async fn determine_sync_type(account: &AccountModel) -> BichonResult<SyncType> {
|
||||
Ok(match AccountRunningState::get(account.id).await? {
|
||||
Some(info) => {
|
||||
let now = utc_now!();
|
||||
let incremental_sync = is_time_for_incremental_sync(
|
||||
now,
|
||||
info.last_incremental_sync_start,
|
||||
account.sync_interval_min.unwrap(),
|
||||
);
|
||||
|
||||
if incremental_sync {
|
||||
SyncType::IncrementalSync
|
||||
} else {
|
||||
SyncType::SkipSync
|
||||
}
|
||||
}
|
||||
None => {
|
||||
AccountRunningState::add(account.id).await?;
|
||||
SyncType::InitialSync
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if it's time for an incremental sync based on the provided interval.
|
||||
fn is_time_for_incremental_sync(
|
||||
now: i64,
|
||||
last_incremental_sync_at: i64,
|
||||
sync_interval_min: i64,
|
||||
) -> bool {
|
||||
now - last_incremental_sync_at > (sync_interval_min * 60 * 1000)
|
||||
}
|
||||
Vendored
+117
@@ -0,0 +1,117 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::modules::account::entity::AuthType;
|
||||
use crate::modules::cache::imap::sync::execute_imap_sync;
|
||||
use crate::modules::common::periodic::{PeriodicTask, TaskHandle};
|
||||
use crate::modules::oauth2::token::OAuth2AccessToken;
|
||||
use crate::modules::{
|
||||
account::{dispatcher::STATUS_DISPATCHER, migration::AccountModel},
|
||||
error::BichonResult,
|
||||
};
|
||||
use crate::utc_now;
|
||||
use dashmap::DashMap;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::{sync::LazyLock, time::Duration};
|
||||
use tracing::{error, warn};
|
||||
|
||||
static _DESCRIPTION: &str = "This task periodically synchronizes mailbox data for a specified account, ensuring that all local data is up-to-date.";
|
||||
const TASK_INTERVAL: Duration = Duration::from_secs(10);
|
||||
pub static SYNC_TASKS: LazyLock<AccountSyncTask> = LazyLock::new(AccountSyncTask::new);
|
||||
static LAST_WARN_TIME: AtomicI64 = AtomicI64::new(0);
|
||||
const WARN_INTERVAL_MS: i64 = 600_000;
|
||||
|
||||
pub struct AccountSyncTask {
|
||||
tasks: DashMap<u64, TaskHandle>,
|
||||
}
|
||||
|
||||
impl AccountSyncTask {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tasks: DashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn start_account_sync_task(&self, account_id: u64, email: String) {
|
||||
let task_name = format!("account-sync-task-{}-{}", account_id, &email);
|
||||
let periodic_task = PeriodicTask::new(&task_name);
|
||||
let task = move |param: Option<u64>| {
|
||||
let account_id = param.unwrap();
|
||||
Box::pin(async move {
|
||||
let account = AccountModel::get(account_id).await.ok();
|
||||
match account {
|
||||
Some(account) => {
|
||||
if !account.enabled {
|
||||
let last = LAST_WARN_TIME.load(Ordering::Relaxed);
|
||||
let now = utc_now!();
|
||||
if now - last >= WARN_INTERVAL_MS {
|
||||
LAST_WARN_TIME.store(now, Ordering::Relaxed);
|
||||
warn!(
|
||||
"Account {}: Sync aborted. Account is currently disabled.",
|
||||
account_id
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if let Some(imap) = &account.imap {
|
||||
if let AuthType::OAuth2 = imap.auth.auth_type {
|
||||
if OAuth2AccessToken::get(account.id).await?.is_none() {
|
||||
if utc_now!() % 300_000 == 0 {
|
||||
warn!("Account {}: Sync aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Err(e) = execute_imap_sync(&account).await {
|
||||
STATUS_DISPATCHER
|
||||
.append_error(
|
||||
account_id,
|
||||
format!("error in account sync task: {:#?}", e),
|
||||
)
|
||||
.await;
|
||||
error!(
|
||||
"Failed to synchronize mailbox data for '{}': {:?}",
|
||||
account_id, e
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
error!(
|
||||
"Account {}: Sync aborted. Account entity not found.",
|
||||
account_id
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
};
|
||||
let handler = periodic_task.start(task, Some(account_id), TASK_INTERVAL, true, true);
|
||||
self.tasks.insert(account_id, handler);
|
||||
}
|
||||
|
||||
pub async fn stop(&self, account_id: u64) -> BichonResult<()> {
|
||||
if let Some((_, handler)) = self.tasks.remove(&account_id) {
|
||||
handler.cancel().await;
|
||||
} else {
|
||||
warn!("No sync task found for account: {}", account_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Vendored
+33
@@ -0,0 +1,33 @@
|
||||
//
|
||||
// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use crate::modules::settings::cli::SETTINGS;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
pub mod imap;
|
||||
|
||||
pub static SEMAPHORE: LazyLock<Arc<Semaphore>> = LazyLock::new(|| {
|
||||
Arc::new(Semaphore::new(
|
||||
SETTINGS
|
||||
.bichon_sync_concurrency
|
||||
.map(|c| c as usize)
|
||||
.unwrap_or(num_cpus::get() * 2),
|
||||
))
|
||||
});
|
||||
Reference in New Issue
Block a user