Persist stable IMAP UIDs across restarts

UIDs were assigned from an in-memory counter that reset to 1 on every
bridge restart, so the UID<->mail mapping changed each run and IMAP
clients (Thunderbird) re-downloaded the whole mailbox on reconnect.

Persist a per-folder monotonic UID in the local store (schema v3):
- mails gain a `uid` column; sync_state gains a `next_uid` counter that
  only ever advances (UIDs are never reused).
- The syncer keeps each mail's existing UID and allocates new ones for
  new mail (oldest-first, so newer mail gets higher UIDs).
- refresh_mails uses the persisted UID instead of allocating; messages
  are ordered by UID. UIDVALIDITY stays constant.

Migration v2->v3 drops the cache tables and re-syncs once (encrypted
.eml files survive). Live-tested: UID<->mail mapping is identical before
and after a restart (range 1..500 unchanged), so clients fetch only the
delta instead of re-downloading.
This commit is contained in:
Anthony
2026-05-27 16:43:44 +02:00
committed by Anthony M
parent 1630e3d860
commit 578a8357b6
3 changed files with 113 additions and 24 deletions
+17 -14
View File
@@ -577,30 +577,31 @@ impl ImapSession {
async fn refresh_mails(&mut self, folder_id: &str) -> Result<(), String> {
let stored = self.store.get_folder(folder_id).await;
let old_cache: std::collections::HashMap<String, (u32, Option<MailDetails>, Option<String>)> =
// Carry over already-loaded details/rfc for this session.
let old_cache: std::collections::HashMap<String, (Option<MailDetails>, Option<String>)> =
self.mails
.iter()
.filter_map(|m| {
let eid = m.mail._id.as_ref()?.element_id.to_string();
Some((eid, (m.uid, m.details.clone(), m.rfc2822.clone())))
Some((eid, (m.details.clone(), m.rfc2822.clone())))
})
.collect();
self.mails.clear();
// UIDs are stable and assigned by the store; sort ascending so message
// sequence order matches UID order, as IMAP clients expect.
let mut stored = stored;
stored.sort_by_key(|m| m.uid);
for sm in stored {
let elem_id = sm.mail._id.as_ref().map(|id| id.element_id.to_string());
let (uid, old_details, old_rfc) = elem_id
let (old_details, old_rfc) = elem_id
.as_ref()
.and_then(|eid| old_cache.get(eid))
.cloned()
.unwrap_or_else(|| {
let uid = self.uid_next;
self.uid_next += 1;
(uid, None, None)
});
.unwrap_or((None, None));
let uid = sm.uid;
if uid >= self.uid_next {
self.uid_next = uid + 1;
}
@@ -1339,8 +1340,8 @@ mod tests {
.set_folder(
"inbox",
vec![
StoredMail { mail: m1, details: None, rfc2822: None },
StoredMail { mail: m2, details: None, rfc2822: None },
StoredMail { mail: m1, details: None, rfc2822: None, uid: 1 },
StoredMail { mail: m2, details: None, rfc2822: None, uid: 2 },
],
)
.await;
@@ -1372,10 +1373,12 @@ mod tests {
store.set_folder_list(vec![inbox_folder()]).await;
let stored: Vec<StoredMail> = mails
.iter()
.map(|m| StoredMail {
.enumerate()
.map(|(i, m)| StoredMail {
mail: m.clone(),
details: None,
rfc2822: None,
uid: (i + 1) as u32,
})
.collect();
store.set_folder("inbox", stored).await;
@@ -1520,8 +1523,8 @@ mod tests {
let rfc2 = crate::mail::mail_to_rfc2822(&m2, Some(&d2));
store.set_folder_list(vec![inbox_folder()]).await;
store.set_folder("inbox", vec![
StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1) },
StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2) },
StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1), uid: 1 },
StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2), uid: 2 },
]).await;
let mut session = ImapSession::new(store, backend, None);
+63 -8
View File
@@ -10,7 +10,7 @@ use rusqlite::{Connection, OptionalExtension};
/// Bumped when the on-disk schema changes. A mismatch drops the cached tables
/// (mails + sync_state) and triggers a full re-sync; encrypted .eml files are
/// keyed by element id and survive the migration.
const SCHEMA_VERSION: &str = "2";
const SCHEMA_VERSION: &str = "3";
#[derive(Debug, thiserror::Error)]
pub enum StoreError {
@@ -35,6 +35,8 @@ pub struct MailMetadata {
pub received_date_ms: i64,
pub unread: bool,
pub has_details: bool,
/// Stable IMAP UID within the folder (0 = not yet assigned).
pub uid: i64,
pub mail_json: String,
}
@@ -94,13 +96,15 @@ impl LocalStore {
received_date_ms INTEGER NOT NULL,
unread INTEGER NOT NULL DEFAULT 1,
has_details INTEGER NOT NULL DEFAULT 0,
uid INTEGER NOT NULL DEFAULT 0,
mail_json TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_mails_folder
ON mails(folder_id, received_date_ms DESC);
CREATE TABLE IF NOT EXISTS sync_state (
folder_id TEXT PRIMARY KEY,
last_sync_ms INTEGER NOT NULL DEFAULT 0
last_sync_ms INTEGER NOT NULL DEFAULT 0,
next_uid INTEGER NOT NULL DEFAULT 1
);
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
))?;
@@ -150,7 +154,7 @@ impl LocalStore {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT element_id, list_id, folder_id, subject, sender_name, sender_address,
received_date_ms, unread, has_details, mail_json
received_date_ms, unread, has_details, uid, mail_json
FROM mails WHERE folder_id = ?1
ORDER BY received_date_ms DESC",
)?;
@@ -165,7 +169,8 @@ impl LocalStore {
received_date_ms: row.get(6)?,
unread: row.get::<_, i64>(7)? != 0,
has_details: row.get::<_, i64>(8)? != 0,
mail_json: row.get(9)?,
uid: row.get(9)?,
mail_json: row.get(10)?,
})
})?;
@@ -180,8 +185,8 @@ impl LocalStore {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
sender_address, received_date_ms, unread, has_details, mail_json)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
sender_address, received_date_ms, unread, has_details, uid, mail_json)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
ON CONFLICT(element_id) DO UPDATE SET
folder_id = excluded.folder_id,
subject = excluded.subject,
@@ -201,6 +206,7 @@ impl LocalStore {
meta.received_date_ms,
meta.unread as i64,
meta.has_details as i64,
meta.uid,
meta.mail_json,
],
)?;
@@ -213,8 +219,8 @@ impl LocalStore {
{
let mut stmt = conn.prepare_cached(
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
sender_address, received_date_ms, unread, has_details, mail_json)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
sender_address, received_date_ms, unread, has_details, uid, mail_json)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
ON CONFLICT(element_id) DO UPDATE SET
folder_id = excluded.folder_id,
subject = excluded.subject,
@@ -236,6 +242,7 @@ impl LocalStore {
meta.received_date_ms,
meta.unread as i64,
meta.has_details as i64,
meta.uid,
meta.mail_json,
])?;
}
@@ -244,6 +251,39 @@ impl LocalStore {
Ok(())
}
/// Assign stable, monotonic UIDs to the given (new) element ids in a folder.
/// UIDs are never reused — the per-folder counter only advances — so an IMAP
/// client's `(UIDVALIDITY, UID)` cache stays valid across bridge restarts.
/// Ids should be supplied oldest-first so newer mail gets higher UIDs.
pub fn allocate_folder_uids(
&self,
folder_id: &str,
new_element_ids: &[&str],
) -> Result<std::collections::HashMap<String, u32>, StoreError> {
let conn = self.conn.lock().unwrap();
let mut next: i64 = conn
.query_row(
"SELECT next_uid FROM sync_state WHERE folder_id = ?1",
[folder_id],
|row| row.get(0),
)
.optional()?
.unwrap_or(1);
let mut map = std::collections::HashMap::with_capacity(new_element_ids.len());
for eid in new_element_ids {
map.insert((*eid).to_string(), next as u32);
next += 1;
}
conn.execute(
"INSERT INTO sync_state(folder_id, next_uid) VALUES (?1, ?2)
ON CONFLICT(folder_id) DO UPDATE SET next_uid = excluded.next_uid",
rusqlite::params![folder_id, next],
)?;
Ok(map)
}
pub fn delete_mails_not_in(
&self,
folder_id: &str,
@@ -384,10 +424,25 @@ mod tests {
received_date_ms: received,
unread: true,
has_details: false,
uid: 0,
mail_json: "{}".into(),
}
}
#[test]
fn allocate_uids_are_monotonic_and_per_folder() {
let store = open_memory_store();
let m1 = store.allocate_folder_uids("inbox", &["a", "b"]).unwrap();
assert_eq!(m1["a"], 1);
assert_eq!(m1["b"], 2);
// Continues, never reuses, even if "a"/"b" were deleted.
let m2 = store.allocate_folder_uids("inbox", &["c"]).unwrap();
assert_eq!(m2["c"], 3);
// Each folder has its own counter.
let m3 = store.allocate_folder_uids("custom", &["x"]).unwrap();
assert_eq!(m3["x"], 1);
}
#[test]
fn test_open_and_verify() {
let store = open_memory_store();
+33 -2
View File
@@ -22,6 +22,8 @@ pub struct StoredMail {
pub mail: Mail,
pub details: Option<MailDetails>,
pub rfc2822: Option<String>,
/// Stable IMAP UID within the folder, persisted across restarts.
pub uid: u32,
}
pub struct MailStore {
@@ -309,6 +311,7 @@ async fn load_cached_folder(
mail,
details: None,
rfc2822,
uid: meta.uid as u32,
});
}
@@ -335,16 +338,42 @@ async fn sync_folder(
})
.collect();
// Allocate stable UIDs for mails we haven't seen before. `new_mails` is
// newest-first; reverse the new ones so the oldest gets the lowest UID.
let new_element_ids: Vec<String> = new_mails
.iter()
.rev()
.filter_map(|m| m._id.as_ref().map(|id| id.element_id.to_string()))
.filter(|eid| !existing_map.contains_key(eid))
.collect();
let new_uids = if new_element_ids.is_empty() {
std::collections::HashMap::new()
} else {
let refs: Vec<&str> = new_element_ids.iter().map(|s| s.as_str()).collect();
local_store
.allocate_folder_uids(&folder.id, &refs)
.unwrap_or_else(|e| {
warn!("Failed to allocate UIDs for {}: {}", folder.imap_path, e);
std::collections::HashMap::new()
})
};
let mut updated = Vec::with_capacity(new_mails.len());
let mut metas_to_upsert = Vec::with_capacity(new_mails.len());
for mail in &new_mails {
let elem_id = mail._id.as_ref().map(|id| id.element_id.to_string());
let uid = elem_id
.as_ref()
.and_then(|id| existing_map.get(id).map(|m| m.uid).or_else(|| new_uids.get(id).copied()))
.unwrap_or(0);
if let Some(existing) = elem_id.as_ref().and_then(|id| existing_map.get(id)) {
updated.push(StoredMail {
mail: mail.clone(),
details: existing.details.clone(),
rfc2822: existing.rfc2822.clone(),
uid,
});
} else {
let rfc2822 = mail_to_rfc2822(mail, None);
@@ -352,10 +381,11 @@ async fn sync_folder(
mail: mail.clone(),
details: None,
rfc2822: Some(rfc2822),
uid,
});
}
metas_to_upsert.push(mail_to_metadata(mail, &folder.id));
metas_to_upsert.push(mail_to_metadata(mail, &folder.id, uid));
}
if let Err(e) = local_store.upsert_mail_metadata_batch(&metas_to_upsert) {
@@ -474,7 +504,7 @@ where
unreachable!()
}
fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata {
fn mail_to_metadata(mail: &Mail, folder_id: &str, uid: u32) -> MailMetadata {
let (list_id, element_id) = mail
._id
.as_ref()
@@ -493,6 +523,7 @@ fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata {
received_date_ms: mail.receivedDate.as_millis() as i64,
unread: mail.unread,
has_details: false,
uid: uid as i64,
mail_json,
}
}