refactor: replace native_db with memdb and add tests

This commit is contained in:
rustmailer
2026-05-14 02:29:23 +08:00
parent 0abaa66a40
commit 5406c4322c
102 changed files with 7225 additions and 2735 deletions
+6 -7
View File
@@ -19,7 +19,7 @@
use std::collections::BTreeSet;
use crate::{
decode_mailbox_name,
decode_mailbox_name, raise_error,
{
account::migration::{AccountModel, AccountType},
cache::imap::mailbox::{AttributeEnum, MailBox},
@@ -27,7 +27,6 @@ use crate::{
imap::{executor::ImapExecutor, session::SessionStream},
mailbox::list::convert_names_to_mailboxes,
},
raise_error,
};
use async_imap::{types::Name, Session};
use tracing::{debug, info, warn};
@@ -62,7 +61,7 @@ pub async fn get_download_folders(
mailboxes.iter().map(|(m, _)| m.name.clone()).collect(),
)
.await?;
let account = AccountModel::async_get(account.id).await?;
let account = AccountModel::get(account.id)?;
let subscribed = &account.download_folders.unwrap_or_default();
let is_noselect = |mailbox: &MailBox| {
mailbox
@@ -110,7 +109,7 @@ pub async fn get_download_folders(
.iter()
.map(|n| decode_mailbox_name!(n.name().to_string()))
.collect();
AccountModel::update_download_folders(account.id, sync_folders).await?;
AccountModel::update_download_folders(account.id, sync_folders)?;
} else {
warn!(
"Account {}: No subscribed mailboxes found. This is unexpected — IMAP server should at least provide INBOX.",
@@ -131,7 +130,7 @@ pub async fn detect_mailbox_changes(
) -> BichonResult<()> {
if account.known_folders.is_none() {
// First time sync: just save without comparing
AccountModel::update_known_folders(account.id, all_names).await?;
AccountModel::update_known_folders(account.id, all_names)?;
return Ok(());
}
let known_folders = account.known_folders.clone().unwrap_or_default();
@@ -160,7 +159,7 @@ pub async fn detect_mailbox_changes(
// Note: When all subscribed folders are deleted (remaining_sync_folders empty),
// the system's default behavior is to automatically fall back to syncing
// only the default folders (INBOX and Sent) in subsequent operations
AccountModel::update_download_folders(account.id, remaining_sync_folders).await?;
AccountModel::update_download_folders(account.id, remaining_sync_folders)?;
}
info!(
@@ -179,7 +178,7 @@ pub async fn detect_mailbox_changes(
// Update known folders only if there were changes
if has_changes {
AccountModel::update_known_folders(account.id, all_names).await?;
AccountModel::update_known_folders(account.id, all_names)?;
}
Ok(())
}
+2 -2
View File
@@ -38,7 +38,7 @@ pub async fn decide_next_download_task(
account: &AccountModel,
trigger_type: TriggerType,
) -> BichonResult<DownloadTask> {
let state = match DownloadState::get(account.id).await? {
let state = match DownloadState::get(account.id)? {
None => {
DownloadState::init(account.id).await?;
return Ok(DownloadTask::FullFetch);
@@ -56,7 +56,7 @@ pub async fn decide_next_download_task(
};
if should_start {
DownloadState::start_new_session(account.id, trigger_type).await?;
DownloadState::start_new_session(account.id, trigger_type)?;
Ok(DownloadTask::TraceFetch)
} else {
Ok(DownloadTask::Idle)
+32 -55
View File
@@ -17,6 +17,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
raise_error,
{
account::{
migration::AccountModel,
@@ -34,7 +35,6 @@ use crate::{
imap::executor::ImapExecutor,
store::tantivy::envelope::ENVELOPE_MANAGER,
},
raise_error,
};
use std::time::Instant;
use tokio_util::sync::CancellationToken;
@@ -67,9 +67,8 @@ pub async fn fetch_and_save_by_date(
0,
FolderStatus::Failed,
Some(err_msg.clone()),
)
.await?;
DownloadState::append_session_error(account_id, err_msg).await?;
)?;
DownloadState::append_session_error(account_id, err_msg)?;
return Err(e);
}
};
@@ -93,9 +92,8 @@ pub async fn fetch_and_save_by_date(
0,
FolderStatus::Failed,
Some(err_msg.clone()),
)
.await?;
DownloadState::append_session_error(account_id, err_msg).await?;
)?;
DownloadState::append_session_error(account_id, err_msg)?;
return Err(e);
}
};
@@ -109,8 +107,7 @@ pub async fn fetch_and_save_by_date(
0,
FolderStatus::Success,
None,
)
.await?;
)?;
return Ok(());
}
@@ -145,8 +142,7 @@ pub async fn fetch_and_save_by_date(
0,
FolderStatus::Pending,
None,
)
.await?;
)?;
let mut current_processed = 0u64;
let mut has_error_or_cancel = false;
@@ -156,8 +152,7 @@ pub async fn fetch_and_save_by_date(
account_id,
DownloadStatus::Cancelled,
Some("User stopped or system shutdown".to_string()),
)
.await?;
)?;
DownloadState::update_folder_progress(
account_id,
mailbox.name.clone(),
@@ -165,8 +160,7 @@ pub async fn fetch_and_save_by_date(
current_processed,
FolderStatus::Cancelled,
None,
)
.await?;
)?;
has_error_or_cancel = true;
break;
}
@@ -189,12 +183,11 @@ pub async fn fetch_and_save_by_date(
current_processed,
FolderStatus::Downloading,
None,
)
.await?;
)?;
}
Err(e) => {
let err_msg = format!("Batch {} failed: {:#?}", index, e);
DownloadState::append_session_error(account_id, err_msg.clone()).await?;
DownloadState::append_session_error(account_id, err_msg.clone())?;
DownloadState::update_folder_progress(
account_id,
mailbox.name.clone(),
@@ -202,8 +195,7 @@ pub async fn fetch_and_save_by_date(
current_processed,
FolderStatus::Failed,
Some(err_msg),
)
.await?;
)?;
has_error_or_cancel = true;
break;
}
@@ -217,8 +209,7 @@ pub async fn fetch_and_save_by_date(
current_processed,
FolderStatus::Success,
None,
)
.await?;
)?;
}
session.logout().await.ok();
Ok(())
@@ -243,9 +234,8 @@ pub async fn fetch_and_save_full_mailbox(
0,
FolderStatus::Failed,
Some(err_msg.clone()),
)
.await?;
DownloadState::append_session_error(account_id, err_msg).await?;
)?;
DownloadState::append_session_error(account_id, err_msg)?;
return Err(e);
}
};
@@ -261,10 +251,9 @@ pub async fn fetch_and_save_full_mailbox(
0,
FolderStatus::Failed,
Some(err_msg.clone()),
)
.await?;
)?;
DownloadState::append_session_error(account_id, err_msg).await?;
DownloadState::append_session_error(account_id, err_msg)?;
session.logout().await.ok();
return Err(raise_error!(
format!("{:#?}", e),
@@ -307,8 +296,7 @@ pub async fn fetch_and_save_full_mailbox(
account_id,
DownloadStatus::Cancelled,
Some("User stopped or system shutdown".to_string()),
)
.await?;
)?;
DownloadState::update_folder_progress(
account_id,
mailbox.name.clone(),
@@ -316,8 +304,7 @@ pub async fn fetch_and_save_full_mailbox(
current_processed,
FolderStatus::Cancelled,
None,
)
.await?;
)?;
has_error_or_cancel = true;
break;
}
@@ -344,12 +331,11 @@ pub async fn fetch_and_save_full_mailbox(
current_processed,
FolderStatus::Downloading,
None,
)
.await?;
)?;
}
Err(e) => {
let err_msg = format!("Batch {} failed: {:#?}", page, e);
DownloadState::append_session_error(account_id, err_msg.clone()).await?;
DownloadState::append_session_error(account_id, err_msg.clone())?;
DownloadState::update_folder_progress(
account_id,
mailbox.name.clone(),
@@ -357,8 +343,7 @@ pub async fn fetch_and_save_full_mailbox(
current_processed,
FolderStatus::Failed,
Some(err_msg),
)
.await?;
)?;
has_error_or_cancel = true;
break;
}
@@ -373,8 +358,7 @@ pub async fn fetch_and_save_full_mailbox(
current_processed,
FolderStatus::Success,
None,
)
.await?;
)?;
}
session.logout().await.ok();
Ok(())
@@ -452,8 +436,7 @@ pub async fn reconcile_mailboxes(
DownloadState::init_folder_details(
account.id,
remote_mailboxes.iter().map(|m| m.name.clone()).collect(),
)
.await?;
)?;
for (local_mailbox, remote_mailbox) in &existing_mailboxes {
if token.is_cancelled() {
@@ -461,8 +444,7 @@ pub async fn reconcile_mailboxes(
account.id,
DownloadStatus::Cancelled,
Some("Received termination signal (User stop or System shutdown)".to_string()),
)
.await?;
)?;
break;
}
@@ -482,9 +464,8 @@ pub async fn reconcile_mailboxes(
0,
FolderStatus::Failed,
Some(err_msg.clone()),
)
.await?;
DownloadState::append_session_error(account_id, err_msg).await?;
)?;
DownloadState::append_session_error(account_id, err_msg)?;
continue;
}
info!(
@@ -500,8 +481,7 @@ pub async fn reconcile_mailboxes(
0,
FolderStatus::Downloading,
Some("UID validity changed, rebuilding...".into()),
)
.await?;
)?;
match &account.date_since {
Some(date_since) => {
@@ -547,7 +527,7 @@ pub async fn reconcile_mailboxes(
}
//The metadata of this mailbox must only be updated after a successful synchronization;
//otherwise, it may cause synchronization errors and result in missing emails in the local sync results.
MailBox::batch_upsert(&mailboxes_to_update).await?;
MailBox::batch_upsert(&mailboxes_to_update)?;
}
debug!(
@@ -559,7 +539,7 @@ pub async fn reconcile_mailboxes(
let missing_mailboxes = find_missing_mailboxes(local_mailboxes, remote_mailboxes);
//Mail folders that are not locally need to be downloaded.
if !missing_mailboxes.is_empty() {
MailBox::batch_insert(&missing_mailboxes).await?;
MailBox::batch_insert(&missing_mailboxes)?;
let mut has_error = false;
let mut last_err = None;
@@ -569,8 +549,7 @@ pub async fn reconcile_mailboxes(
account.id,
DownloadStatus::Cancelled,
Some("Received termination signal (User stop or System shutdown)".to_string()),
)
.await?;
)?;
break;
}
if mailbox.exists > 0 {
@@ -650,9 +629,7 @@ async fn perform_incremental_sync(
token: CancellationToken,
) -> BichonResult<()> {
if remote_mailbox.exists > 0 {
let local_max_uid = ENVELOPE_MANAGER
.get_max_uid(account.id, local_mailbox.id)
.await?;
let local_max_uid = ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?;
match local_max_uid {
Some(max_uid) => {
let mut session = ImapExecutor::create_connection(account.id).await?;
+15 -20
View File
@@ -57,8 +57,7 @@ pub async fn process_imap_download(
account_id,
DownloadStatus::Failed,
Some(format!("Failed to connect to IMAP server: {}", e)),
)
.await?;
)?;
return Err(e);
}
};
@@ -67,8 +66,11 @@ pub async fn process_imap_download(
Err(err) => {
let err_msg = format!("Failed to fetch mailboxes: {}", err);
warn!(account_id = account.id, error = %err, "{}", err_msg);
DownloadState::update_session_status(account_id, DownloadStatus::Failed, Some(err_msg))
.await?;
DownloadState::update_session_status(
account_id,
DownloadStatus::Failed,
Some(err_msg),
)?;
return Ok(());
}
};
@@ -101,34 +103,27 @@ pub async fn process_imap_download(
};
match result {
Ok(_) => {
DownloadState::update_session_status(account_id, DownloadStatus::Success, None)
.await?;
DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?;
}
Err(e) => {
DownloadState::update_session_status(
account_id,
DownloadStatus::Failed,
Some(format!("Email Download interrupted: {:#?}", e)),
)
.await?;
)?;
}
}
return Ok(());
}
let local_mailboxes = MailBox::list_all(account_id).await?;
let local_mailboxes = MailBox::list_all(account_id)?;
match reconcile_mailboxes(account, &remote_mailboxes, &local_mailboxes, token).await {
Ok(_) => {
DownloadState::update_session_status(account_id, DownloadStatus::Success, None).await?
}
Err(e) => {
DownloadState::update_session_status(
account_id,
DownloadStatus::Failed,
Some(format!("Email Download interrupted: {:#?}", e)),
)
.await?
}
Ok(_) => DownloadState::update_session_status(account_id, DownloadStatus::Success, None)?,
Err(e) => DownloadState::update_session_status(
account_id,
DownloadStatus::Failed,
Some(format!("Email Download interrupted: {:#?}", e)),
)?,
}
let elapsed_time = start_time.elapsed().as_secs();
debug!(
+13 -21
View File
@@ -17,6 +17,7 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
raise_error,
{
account::{
migration::AccountModel,
@@ -34,7 +35,6 @@ use crate::{
error::{code::ErrorCode, BichonResult},
store::tantivy::envelope::ENVELOPE_MANAGER,
},
raise_error,
};
use tokio_util::sync::CancellationToken;
@@ -45,12 +45,11 @@ pub async fn rebuild_cache(
remote_mailboxes: &[MailBox],
token: CancellationToken,
) -> BichonResult<()> {
MailBox::batch_insert(remote_mailboxes).await?;
MailBox::batch_insert(remote_mailboxes)?;
DownloadState::init_folder_details(
account.id,
remote_mailboxes.iter().map(|m| m.name.clone()).collect(),
)
.await?;
)?;
let mut has_error = false;
let mut last_err = None;
@@ -61,8 +60,7 @@ pub async fn rebuild_cache(
account.id,
DownloadStatus::Cancelled,
Some("Received termination signal (User stop or System shutdown)".to_string()),
)
.await?;
)?;
break;
}
if mailbox.exists == 0 {
@@ -77,8 +75,7 @@ pub async fn rebuild_cache(
0,
FolderStatus::Success,
None,
)
.await?;
)?;
continue;
}
let account = account.clone();
@@ -94,9 +91,9 @@ pub async fn rebuild_cache(
continue;
}
};
match fetch_and_save_full_mailbox(&account, &mailbox, token.clone()).await {
Ok(_) => {},
Ok(_) => {}
Err(err) => {
has_error = true;
tracing::error!("Folder sync task failed: {:#?}", err);
@@ -124,12 +121,11 @@ pub async fn rebuild_cache_by_date(
direction: FetchDirection,
token: CancellationToken,
) -> BichonResult<()> {
MailBox::batch_insert(remote_mailboxes).await?;
MailBox::batch_insert(remote_mailboxes)?;
DownloadState::init_folder_details(
account.id,
remote_mailboxes.iter().map(|m| m.name.clone()).collect(),
)
.await?;
)?;
let mut has_error = false;
let mut last_err = None;
@@ -140,8 +136,7 @@ pub async fn rebuild_cache_by_date(
account.id,
DownloadStatus::Cancelled,
Some("Received termination signal (User stop or System shutdown)".to_string()),
)
.await?;
)?;
break;
}
if mailbox.exists == 0 {
@@ -157,8 +152,7 @@ pub async fn rebuild_cache_by_date(
0,
FolderStatus::Success,
None,
)
.await?;
)?;
continue;
}
let account = account.clone();
@@ -224,8 +218,7 @@ pub async fn rebuild_mailbox_cache(
0,
FolderStatus::Success,
None,
)
.await?;
)?;
return Ok(());
}
@@ -257,8 +250,7 @@ pub async fn rebuild_mailbox_cache_by_date(
0,
FolderStatus::Success,
None,
)
.await?;
)?;
return Ok(());
}
+29 -60
View File
@@ -17,33 +17,24 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::{
decode_mailbox_name, encode_mailbox_name,
decode_mailbox_name, encode_mailbox_name, raise_error,
{
database::{
async_filter_by_secondary_key_impl, async_find_impl, batch_delete_impl,
batch_insert_impl, batch_upsert_impl, delete_impl, filter_by_secondary_key_impl,
find_impl, manager::DB_MANAGER,
batch_delete_impl, batch_insert_impl, batch_upsert_impl, delete_impl, filter_impl,
find_impl, manager::DB_MANAGER, MemDbModel,
},
error::{code::ErrorCode, BichonResult},
},
raise_error,
};
use async_imap::types::{Name, NameAttribute};
use itertools::Itertools;
use native_db::*;
use native_model::{native_model, Model};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
#[native_model(id = 1, version = 1)]
#[native_db]
pub struct MailBox {
/// The unique identifier for the mailbox
#[primary_key]
pub id: u64,
/// The ID of the account associated with the mailbox
#[secondary_key]
pub account_id: u64,
/// The unique, decoded, human-readable name of the mailbox (e.g., "INBOX", "Sent Items").
/// This is the decoded name as presented to users, derived from the IMAP server's mailbox name
@@ -67,23 +58,22 @@ pub struct MailBox {
pub uid_validity: Option<u32>,
}
impl MemDbModel for MailBox {
fn collection() -> &'static str {
"mailboxes"
}
fn key(&self) -> String {
self.id.to_string()
}
}
impl MailBox {
pub fn encoded_name(&self) -> String {
encode_mailbox_name!(&self.name)
}
pub async fn async_get(id: u64) -> BichonResult<MailBox> {
let result = async_find_impl::<MailBox>(DB_MANAGER.envelope_db(), id).await?;
Ok(result.ok_or_else(|| {
raise_error!(
format!("mailbox {} not found", id),
ErrorCode::InternalError
)
})?)
}
pub fn get(id: u64) -> BichonResult<MailBox> {
let result = find_impl::<MailBox>(DB_MANAGER.envelope_db(), id)?;
let result = find_impl::<MailBox>(DB_MANAGER.db(), &id.to_string())?;
Ok(result.ok_or_else(|| {
raise_error!(
format!("mailbox {} not found", id),
@@ -92,55 +82,34 @@ impl MailBox {
})?)
}
pub async fn delete(id: u64) -> BichonResult<()> {
delete_impl(DB_MANAGER.envelope_db(), move |rw| {
rw.get()
.primary::<MailBox>(id)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.ok_or_else(|| raise_error!("mailbox missing".into(), ErrorCode::InternalError))
})
.await
pub fn delete(id: u64) -> BichonResult<()> {
delete_impl::<MailBox>(DB_MANAGER.db(), &id.to_string())
}
pub async fn list_all(account_id: u64) -> BichonResult<Vec<MailBox>> {
async_filter_by_secondary_key_impl(
DB_MANAGER.envelope_db(),
MailBoxKey::account_id,
account_id,
)
.await
pub fn list_all(account_id: u64) -> BichonResult<Vec<MailBox>> {
filter_impl::<MailBox, _>(DB_MANAGER.db(), move |m| m.account_id == account_id)
}
pub fn find_mailbox(account_id: u64, mailbox_id: u64) -> BichonResult<Option<MailBox>> {
let all: Vec<MailBox> = filter_by_secondary_key_impl(
DB_MANAGER.envelope_db(),
MailBoxKey::account_id,
account_id,
)?;
let all = filter_impl::<MailBox, _>(DB_MANAGER.db(), move |m| m.account_id == account_id)?;
Ok(all.into_iter().find(|m| m.id == mailbox_id))
}
pub async fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> {
batch_insert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
pub fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> {
batch_insert_impl(DB_MANAGER.db(), mailboxes.to_vec())
}
pub async fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> {
batch_upsert_impl(DB_MANAGER.envelope_db(), mailboxes.to_vec()).await
pub fn batch_upsert(mailboxes: &[MailBox]) -> BichonResult<()> {
batch_upsert_impl(DB_MANAGER.db(), mailboxes.to_vec())
}
pub async fn clean(account_id: u64) -> BichonResult<()> {
batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| {
let mailboxes: Vec<MailBox> = rw
.scan()
.secondary::<MailBox>(MailBoxKey::account_id)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.start_with(account_id)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
.try_collect()
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
Ok(mailboxes)
})
.await?;
pub fn clean(account_id: u64) -> BichonResult<()> {
let mailboxes =
filter_impl::<MailBox, _>(DB_MANAGER.db(), move |m| m.account_id == account_id)?;
let keys: Vec<String> = mailboxes.iter().map(|m| m.id.to_string()).collect();
if !keys.is_empty() {
batch_delete_impl::<MailBox>(DB_MANAGER.db(), keys)?;
}
Ok(())
}
}
+1 -17
View File
@@ -16,30 +16,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use std::{
collections::{HashMap, HashSet},
sync::LazyLock,
};
use std::collections::{HashMap, HashSet};
use crate::{
account::{old_state::AccountRunningState, state::DownloadState},
database::ModelsAdapter,
};
use mailbox::MailBox;
use native_db::Models;
pub mod download;
pub mod mailbox;
pub mod task;
pub static MAILBOX_MODELS: LazyLock<Models> = LazyLock::new(|| {
let mut adapter = ModelsAdapter::new();
adapter.register_model::<MailBox>();
adapter.register_model::<AccountRunningState>();
adapter.register_model::<DownloadState>();
adapter.models
});
pub fn find_missing_mailboxes(
local_mailboxes: &[MailBox],
server_mailboxes: &[MailBox],
+5 -6
View File
@@ -99,7 +99,7 @@ impl AccountDownTask {
SYNC_TASKS.set_busy(id, false).await;
});
});
let account = AccountModel::async_get(account_id).await.ok();
let account = AccountModel::get(account_id).ok();
match account {
Some(account) => {
if !account.enabled {
@@ -115,7 +115,7 @@ impl AccountDownTask {
} else {
if let Some(imap) = &account.imap {
if let AuthType::OAuth2 = imap.auth.auth_type {
if OAuth2AccessToken::get(account.id).await?.is_none() {
if OAuth2AccessToken::get(account.id)?.is_none() {
if utc_now!() % 300_000 == 0 {
warn!("Account {}: download aborted. OAuth2 authorization not completed. Please visit the rustmailer admin page to authorize this account.", account_id);
}
@@ -133,8 +133,7 @@ impl AccountDownTask {
DownloadState::append_session_error(
account.id,
format!("error in account download task: {:#?}", e),
)
.await?;
)?;
error!(
"Failed to download mailbox data for '{}': {:?}",
account_id, e
@@ -228,7 +227,7 @@ impl AccountDownTask {
if token_clone.is_cancelled() {
return;
}
let account = match AccountModel::async_get(account_id).await {
let account = match AccountModel::get(account_id) {
Ok(acc) => acc,
Err(e) => {
error!("Failed to fetch account {}: {:?}", account_id, e);
@@ -240,7 +239,7 @@ impl AccountDownTask {
{
error!("Manual download failed for {}: {:?}", account_id, e);
let error_msg = format!("error in account download task: {:#?}", e);
let _ = DownloadState::append_session_error(account.id, error_msg).await;
let _ = DownloadState::append_session_error(account.id, error_msg);
}
});
{