Decouple folder/list sync from body prefetch

The syncer ran one loop: phase 1 (folder list + mail-id lists) then
phase 2 (body prefetch). On a large mailbox the cold prefetch pass takes
many minutes, so the next folder refresh was stuck behind it and new
folders/mail only showed up after a restart.

Split into two independent loops sharing the store: a fast list_sync_loop
(folder list + mail ids, ~every SYNC_INTERVAL) and a slow prefetch_loop
(bodies, background). Folder and new-mail refresh no longer wait on
prefetch.

Live-tested: a folder created while the bridge runs appears over IMAP in
~18s, no restart.
This commit is contained in:
Anthony
2026-05-27 16:05:11 +02:00
committed by Anthony M
parent 8f2bc4197d
commit d3b0f4a077
+58 -28
View File
@@ -13,6 +13,8 @@ use crate::tuta::{FolderInfo, MailBackend};
const INTER_REQUEST_DELAY: Duration = Duration::from_millis(150); const INTER_REQUEST_DELAY: Duration = Duration::from_millis(150);
const INTER_FOLDER_DELAY: Duration = Duration::from_millis(300); const INTER_FOLDER_DELAY: Duration = Duration::from_millis(300);
const SYNC_INTERVAL: Duration = Duration::from_secs(60); const SYNC_INTERVAL: Duration = Duration::from_secs(60);
/// Pause between background body-prefetch sweeps.
const PREFETCH_INTERVAL: Duration = Duration::from_secs(30);
const MAX_RETRIES: u32 = 3; const MAX_RETRIES: u32 = 3;
#[derive(Clone)] #[derive(Clone)]
@@ -150,7 +152,7 @@ pub async fn run_syncer(
local_store: Arc<LocalStore>, local_store: Arc<LocalStore>,
backend: Arc<dyn MailBackend>, backend: Arc<dyn MailBackend>,
sync_limit: usize, sync_limit: usize,
mut shutdown: watch::Receiver<bool>, shutdown: watch::Receiver<bool>,
) { ) {
info!( info!(
"Mail syncer started (limit={})", "Mail syncer started (limit={})",
@@ -182,12 +184,32 @@ pub async fn run_syncer(
} }
} }
let mut cycle_backoff = Duration::ZERO; // Run the fast list/folder sync and the slow body prefetch as independent
// loops: a long prefetch pass must never block folder / new-mail refresh.
tokio::join!(
list_sync_loop(&store, &local_store, &*backend, sync_limit, shutdown.clone()),
prefetch_loop(&store, &local_store, &*backend, shutdown.clone()),
);
info!("Mail syncer shutting down");
}
/// Fast loop: refresh the folder list and sync the mail-id list for every
/// folder. This is what surfaces new folders and new mail (~every
/// `SYNC_INTERVAL`), independent of the slow body prefetch.
async fn list_sync_loop(
store: &MailStore,
local_store: &LocalStore,
backend: &dyn MailBackend,
sync_limit: usize,
mut shutdown: watch::Receiver<bool>,
) {
let mut cycle_backoff = Duration::ZERO;
loop { loop {
if *shutdown.borrow() {
return;
}
let mut had_error = false; let mut had_error = false;
// Refresh the folder list each cycle (custom folders can change).
let folders = match retry(|| backend.list_folders()).await { let folders = match retry(|| backend.list_folders()).await {
Ok(folders) => { Ok(folders) => {
store.set_folder_list(folders.clone()).await; store.set_folder_list(folders.clone()).await;
@@ -200,46 +222,54 @@ pub async fn run_syncer(
} }
}; };
// Phase 1: sync mail lists for ALL folders (fast, no body loading).
for folder in &folders { for folder in &folders {
if *shutdown.borrow() { if *shutdown.borrow() {
info!("Mail syncer shutting down");
return; return;
} }
match sync_folder(&store, &local_store, &*backend, folder, sync_limit).await { if let Err(e) = sync_folder(store, local_store, backend, folder, sync_limit).await {
Ok(()) => {} warn!("Sync error for {}: {}", folder.imap_path, e);
Err(e) => { had_error = true;
warn!("Sync error for {}: {}", folder.imap_path, e);
had_error = true;
}
} }
tokio::time::sleep(INTER_FOLDER_DELAY).await; tokio::time::sleep(INTER_FOLDER_DELAY).await;
} }
// Phase 2: prefetch mail details (slow, but all folders are visible). cycle_backoff = if had_error {
for folder in &folders { backoff(cycle_backoff)
if *shutdown.borrow() {
return;
}
prefetch_details(&store, &local_store, &*backend, folder).await;
}
if had_error {
cycle_backoff = backoff(cycle_backoff);
warn!("Sync cycle had errors, backing off {:?}", cycle_backoff);
} else { } else {
cycle_backoff = Duration::ZERO; Duration::ZERO
} };
let wait = SYNC_INTERVAL + cycle_backoff; let wait = SYNC_INTERVAL + cycle_backoff;
debug!("Next sync in {:?}", wait); debug!("Next list sync in {:?}", wait);
tokio::select! { tokio::select! {
_ = tokio::time::sleep(wait) => {} _ = tokio::time::sleep(wait) => {}
_ = shutdown.changed() => { _ = shutdown.changed() => return,
info!("Mail syncer shutting down"); }
}
}
/// Slow loop: progressively prefetch mail bodies in the background, on its own
/// cadence so it never delays `list_sync_loop`.
async fn prefetch_loop(
store: &MailStore,
local_store: &LocalStore,
backend: &dyn MailBackend,
mut shutdown: watch::Receiver<bool>,
) {
loop {
if *shutdown.borrow() {
return;
}
for folder in store.list_folders().await {
if *shutdown.borrow() {
return; return;
} }
prefetch_details(store, local_store, backend, &folder).await;
}
tokio::select! {
_ = tokio::time::sleep(PREFETCH_INTERVAL) => {}
_ = shutdown.changed() => return,
} }
} }
} }