External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add sync_batch_size to allow users to customize the synchronization batch size, and introduce date_before to support semantics such as downloading emails from more than one year ago. #24 #58

Browse Source
This commit is contained in:
rustmailer
2025-12-28 13:01:34 +08:00
parent b35493e4e1
commit 1f57f372d3
46 changed files with 831 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/modules/account/migration.rs
+22 -1
View File
@@ -27,7 +27,11 @@ use tracing::info;
use crate::{
encrypt,
modules::{
account::{entity::ImapConfig, since::DateSince, state::AccountRunningState},
account::{
entity::ImapConfig,
since::{DateSince, RelativeDate},
state::AccountRunningState,
},
cache::imap::mailbox::MailBox,
database::{list_all_impl, with_transaction},
error::BichonResult,
@@ -136,10 +140,12 @@ pub struct AccountV3 {
pub name: Option<String>,
pub capabilities: Option<Vec<String>>,
pub date_since: Option<DateSince>,
pub date_before: Option<RelativeDate>,
pub folder_limit: Option<u32>,
pub sync_folders: Option<Vec<String>>,
pub account_type: AccountType,
pub sync_interval_min: Option<i64>,
pub sync_batch_size: Option<u32>,
pub known_folders: Option<BTreeSet<String>>,
pub created_at: i64,
pub updated_at: i64,
@@ -174,6 +180,8 @@ impl AccountV3 {
use_dangerous: request.use_dangerous,
pgp_key: request.pgp_key,
created_by: user_id,
sync_batch_size: request.sync_batch_size,
date_before: request.date_before,
})
}
@@ -404,6 +412,12 @@ impl AccountV3 {
if let Some(date_since) = request.date_since {
new.date_since = Some(date_since);
new.date_before = None;
}
if let Some(date_before) = request.date_before {
new.date_before = Some(date_before);
new.date_since = None;
}
if let Some(folder_limit) = request.folder_limit {
@@ -439,6 +453,11 @@ impl AccountV3 {
if let Some(sync_interval_min) = &request.sync_interval_min {
new.sync_interval_min = Some(*sync_interval_min);
}
if let Some(sync_batch_size) = &request.sync_batch_size {
new.sync_batch_size = Some(*sync_batch_size);
}
if let Some(use_proxy) = request.use_proxy {
new.use_proxy = Some(use_proxy);
}
@@ -558,6 +577,8 @@ impl From<AccountV2> for AccountV3 {
use_proxy: value.use_proxy,
use_dangerous: value.use_dangerous,
pgp_key: value.pgp_key,
sync_batch_size: None,
date_before: None,
}
}
}
src/modules/account/payload.rs
+33 -1
View File
@@ -18,7 +18,7 @@
use crate::modules::account::entity::ImapConfig;
use crate::modules::account::migration::{AccountModel, AccountType};
use crate::modules::account::since::DateSince;
use crate::modules::account::since::{DateSince, RelativeDate};
use crate::modules::error::code::ErrorCode;
use crate::modules::error::BichonResult;
use crate::{raise_error, validate_email};
@@ -33,11 +33,14 @@ pub struct AccountCreateRequest {
pub imap: Option<ImapConfig>,
pub enabled: bool,
pub date_since: Option<DateSince>,
pub date_before: Option<RelativeDate>,
pub account_type: AccountType,
#[oai(validator(minimum(value = "100")))]
pub folder_limit: Option<u32>,
#[oai(validator(minimum(value = "10"), maximum(value = "480")))]
pub sync_interval_min: Option<i64>,
#[oai(validator(minimum(value = "30"), maximum(value = "200")))]
pub sync_batch_size: Option<u32>,
pub use_proxy: Option<u64>,
pub use_dangerous: bool,
pub pgp_key: Option<String>,
@@ -45,9 +48,22 @@ pub struct AccountCreateRequest {
impl AccountCreateRequest {
pub fn create_entity(self, user_id: u64) -> BichonResult<AccountModel> {
if self.date_before.is_some() && self.date_since.is_some() {
return Err(raise_error!(
"date_before and date_since are mutually exclusive; specify only one time boundary"
.into(),
ErrorCode::InvalidParameter
));
}
if let Some(date_since) = self.date_since.as_ref() {
date_since.validate()?;
}
if let Some(date_before) = self.date_before.as_ref() {
date_before.validate_date()?;
}
match self.account_type {
AccountType::IMAP => {
match &self.imap {
@@ -104,6 +120,7 @@ pub struct AccountUpdateRequest {
/// - First-time sync optimization for large accounts
/// - Reducing server load during resyncs
pub date_since: Option<DateSince>,
pub date_before: Option<RelativeDate>,
/// Max emails to sync for this folder.
/// If not set, sync all emails.
/// otherwise sync up to `n` most recent emails (min 10).
@@ -126,6 +143,8 @@ pub struct AccountUpdateRequest {
/// Incremental sync interval (seconds)
#[oai(validator(minimum(value = "10"), maximum(value = "480")))]
pub sync_interval_min: Option<i64>,
#[oai(validator(minimum(value = "30"), maximum(value = "200")))]
pub sync_batch_size: Option<u32>,
/// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook).
/// - If `None` or not provided, the client will connect directly to the API server.
/// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests.
@@ -138,9 +157,22 @@ pub struct AccountUpdateRequest {
impl AccountUpdateRequest {
pub fn validate_update_request(&self, account: &AccountModel) -> BichonResult<()> {
if self.date_before.is_some() && self.date_since.is_some() {
return Err(raise_error!(
"date_before and date_since are mutually exclusive; specify only one time boundary"
.into(),
ErrorCode::InvalidParameter
));
}
if let Some(date_since) = self.date_since.as_ref() {
date_since.validate()?;
}
if let Some(date_before) = self.date_before.as_ref() {
date_before.validate_date()?;
}
if matches!(account.account_type, AccountType::IMAP) {
if let Some(mailboxes) = self.sync_folders.as_ref() {
if mailboxes.is_empty() {
src/modules/account/since.rs
-1
View File
@@ -16,7 +16,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::error::{code::ErrorCode, BichonResult},
raise_error,
src/modules/account/view.rs
+5 -1
View File
@@ -25,7 +25,7 @@ use crate::modules::{
account::{
entity::ImapConfig,
migration::{AccountModel, AccountType},
since::DateSince,
since::{DateSince, RelativeDate},
},
users::BichonUser,
};
@@ -39,10 +39,12 @@ pub struct AccountResp {
pub name: Option<String>,
pub capabilities: Option<Vec<String>>,
pub date_since: Option<DateSince>,
pub date_before: Option<RelativeDate>,
pub folder_limit: Option<u32>,
pub sync_folders: Option<Vec<String>>,
pub account_type: AccountType,
pub sync_interval_min: Option<i64>,
pub sync_batch_size: Option<u32>,
pub known_folders: Option<BTreeSet<String>>,
pub created_at: i64,
pub updated_at: i64,
@@ -65,10 +67,12 @@ impl AccountResp {
name: account.name,
capabilities: account.capabilities,
date_since: account.date_since,
date_before: account.date_before,
folder_limit: account.folder_limit,
sync_folders: account.sync_folders,
account_type: account.account_type,
sync_interval_min: account.sync_interval_min,
sync_batch_size: account.sync_batch_size,
known_folders: account.known_folders,
created_at: account.created_at,
updated_at: account.updated_at,
src/modules/cache/imap/sync/flow.rs
Vendored
+11 -6
View File
@@ -16,7 +16,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
modules::{
account::{migration::AccountModel, state::AccountRunningState},
@@ -37,7 +36,7 @@ use crate::{
use std::time::Instant;
use tracing::{debug, error, info, warn};
pub const BATCH_SIZE: u32 = 50;
pub const DEFAULT_BATCH_SIZE: u32 = 50;
pub async fn fetch_and_save_since_date(
account: &AccountModel,
@@ -69,7 +68,11 @@ pub async fn fetch_and_save_since_date(
// let semaphore = Arc::new(Semaphore::new(5));
let uid_batches = generate_uid_sequence_hashset(uid_vec, BATCH_SIZE as usize, false);
let uid_batches = generate_uid_sequence_hashset(
uid_vec,
account.sync_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize,
false,
);
AccountRunningState::set_initial_current_syncing_folder(
account_id,
mailbox.name.clone(),
@@ -105,9 +108,11 @@ pub async fn fetch_and_save_full_mailbox(
_ => total,
};
let page_size = if let Some(limit) = folder_limit {
limit.max(100).min(BATCH_SIZE as u32)
limit
.max(100)
.min(account.sync_batch_size.unwrap_or(DEFAULT_BATCH_SIZE))
} else {
BATCH_SIZE as u32
account.sync_batch_size.unwrap_or(DEFAULT_BATCH_SIZE)
};
let total_batches = total_to_fetch.div_ceil(page_size);
@@ -349,7 +354,7 @@ async fn perform_incremental_sync(
Some(max_uid) => {
let executor = MAIL_CONTEXT.imap(account.id).await?;
executor
.fetch_new_mail(account.id, local_mailbox, max_uid + 1)
.fetch_new_mail(account, local_mailbox, max_uid + 1)
.await?;
}
None => {
src/modules/imap/executor.rs
+13 -8
View File
@@ -16,9 +16,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::modules::account::migration::AccountModel;
use crate::modules::account::state::AccountRunningState;
use crate::modules::cache::imap::mailbox::MailBox;
use crate::modules::cache::imap::sync::flow::{generate_uid_sequence_hashset, BATCH_SIZE};
use crate::modules::cache::imap::sync::flow::{generate_uid_sequence_hashset, DEFAULT_BATCH_SIZE};
use crate::modules::envelope::extractor::extract_envelope;
use crate::modules::error::code::ErrorCode;
use crate::modules::indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER};
@@ -80,7 +81,7 @@ impl ImapExecutor {
pub async fn fetch_new_mail(
&self,
account_id: u64,
account: &AccountModel,
mailbox: &MailBox,
start_uid: u64,
) -> BichonResult<()> {
@@ -98,17 +99,21 @@ impl ImapExecutor {
}
info!(
"[account {}][mailbox {}] {} envelopes need to be fetched",
account_id, mailbox.name, len
account.id, mailbox.name, len
);
let mut uid_vec: Vec<u32> = uid_list.into_iter().collect();
uid_vec.sort();
let uid_batches = generate_uid_sequence_hashset(uid_vec, BATCH_SIZE as usize, false);
let uid_batches = generate_uid_sequence_hashset(
uid_vec,
account.sync_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize,
false,
);
let too_many = len as u32 > 10 * BATCH_SIZE;
let too_many = len as u32 > 5 * account.sync_batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
if too_many {
AccountRunningState::set_initial_current_syncing_folder(
account_id,
account.id,
mailbox.name.clone(),
uid_batches.len() as u32,
)
@@ -118,13 +123,13 @@ impl ImapExecutor {
for (index, batch) in uid_batches.into_iter().enumerate() {
if too_many {
AccountRunningState::set_current_sync_batch_number(
account_id,
account.id,
mailbox.name.clone(),
(index + 1) as u32,
)
.await?;
}
self.uid_batch_retrieve_emails(account_id, mailbox.id, &batch, &mailbox.encoded_name())
self.uid_batch_retrieve_emails(account.id, mailbox.id, &batch, &mailbox.encoded_name())
.await?;
}
Ok(())