mirror of
https://github.com/spartanz51/tutabridge.git
synced 2026-06-24 10:54:32 +02:00
Persist stable IMAP UIDs across restarts
UIDs were assigned from an in-memory counter that reset to 1 on every bridge restart, so the UID<->mail mapping changed each run and IMAP clients (Thunderbird) re-downloaded the whole mailbox on reconnect. Persist a per-folder monotonic UID in the local store (schema v3): - mails gain a `uid` column; sync_state gains a `next_uid` counter that only ever advances (UIDs are never reused). - The syncer keeps each mail's existing UID and allocates new ones for new mail (oldest-first, so newer mail gets higher UIDs). - refresh_mails uses the persisted UID instead of allocating; messages are ordered by UID. UIDVALIDITY stays constant. Migration v2->v3 drops the cache tables and re-syncs once (encrypted .eml files survive). Live-tested: UID<->mail mapping is identical before and after a restart (range 1..500 unchanged), so clients fetch only the delta instead of re-downloading.
This commit is contained in:
@@ -577,30 +577,31 @@ impl ImapSession {
|
||||
async fn refresh_mails(&mut self, folder_id: &str) -> Result<(), String> {
|
||||
let stored = self.store.get_folder(folder_id).await;
|
||||
|
||||
let old_cache: std::collections::HashMap<String, (u32, Option<MailDetails>, Option<String>)> =
|
||||
// Carry over already-loaded details/rfc for this session.
|
||||
let old_cache: std::collections::HashMap<String, (Option<MailDetails>, Option<String>)> =
|
||||
self.mails
|
||||
.iter()
|
||||
.filter_map(|m| {
|
||||
let eid = m.mail._id.as_ref()?.element_id.to_string();
|
||||
Some((eid, (m.uid, m.details.clone(), m.rfc2822.clone())))
|
||||
Some((eid, (m.details.clone(), m.rfc2822.clone())))
|
||||
})
|
||||
.collect();
|
||||
|
||||
self.mails.clear();
|
||||
// UIDs are stable and assigned by the store; sort ascending so message
|
||||
// sequence order matches UID order, as IMAP clients expect.
|
||||
let mut stored = stored;
|
||||
stored.sort_by_key(|m| m.uid);
|
||||
|
||||
for sm in stored {
|
||||
let elem_id = sm.mail._id.as_ref().map(|id| id.element_id.to_string());
|
||||
|
||||
let (uid, old_details, old_rfc) = elem_id
|
||||
let (old_details, old_rfc) = elem_id
|
||||
.as_ref()
|
||||
.and_then(|eid| old_cache.get(eid))
|
||||
.cloned()
|
||||
.unwrap_or_else(|| {
|
||||
let uid = self.uid_next;
|
||||
self.uid_next += 1;
|
||||
(uid, None, None)
|
||||
});
|
||||
.unwrap_or((None, None));
|
||||
|
||||
let uid = sm.uid;
|
||||
if uid >= self.uid_next {
|
||||
self.uid_next = uid + 1;
|
||||
}
|
||||
@@ -1339,8 +1340,8 @@ mod tests {
|
||||
.set_folder(
|
||||
"inbox",
|
||||
vec![
|
||||
StoredMail { mail: m1, details: None, rfc2822: None },
|
||||
StoredMail { mail: m2, details: None, rfc2822: None },
|
||||
StoredMail { mail: m1, details: None, rfc2822: None, uid: 1 },
|
||||
StoredMail { mail: m2, details: None, rfc2822: None, uid: 2 },
|
||||
],
|
||||
)
|
||||
.await;
|
||||
@@ -1372,10 +1373,12 @@ mod tests {
|
||||
store.set_folder_list(vec![inbox_folder()]).await;
|
||||
let stored: Vec<StoredMail> = mails
|
||||
.iter()
|
||||
.map(|m| StoredMail {
|
||||
.enumerate()
|
||||
.map(|(i, m)| StoredMail {
|
||||
mail: m.clone(),
|
||||
details: None,
|
||||
rfc2822: None,
|
||||
uid: (i + 1) as u32,
|
||||
})
|
||||
.collect();
|
||||
store.set_folder("inbox", stored).await;
|
||||
@@ -1520,8 +1523,8 @@ mod tests {
|
||||
let rfc2 = crate::mail::mail_to_rfc2822(&m2, Some(&d2));
|
||||
store.set_folder_list(vec![inbox_folder()]).await;
|
||||
store.set_folder("inbox", vec![
|
||||
StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1) },
|
||||
StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2) },
|
||||
StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1), uid: 1 },
|
||||
StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2), uid: 2 },
|
||||
]).await;
|
||||
let mut session = ImapSession::new(store, backend, None);
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ use rusqlite::{Connection, OptionalExtension};
|
||||
/// Bumped when the on-disk schema changes. A mismatch drops the cached tables
|
||||
/// (mails + sync_state) and triggers a full re-sync; encrypted .eml files are
|
||||
/// keyed by element id and survive the migration.
|
||||
const SCHEMA_VERSION: &str = "2";
|
||||
const SCHEMA_VERSION: &str = "3";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StoreError {
|
||||
@@ -35,6 +35,8 @@ pub struct MailMetadata {
|
||||
pub received_date_ms: i64,
|
||||
pub unread: bool,
|
||||
pub has_details: bool,
|
||||
/// Stable IMAP UID within the folder (0 = not yet assigned).
|
||||
pub uid: i64,
|
||||
pub mail_json: String,
|
||||
}
|
||||
|
||||
@@ -94,13 +96,15 @@ impl LocalStore {
|
||||
received_date_ms INTEGER NOT NULL,
|
||||
unread INTEGER NOT NULL DEFAULT 1,
|
||||
has_details INTEGER NOT NULL DEFAULT 0,
|
||||
uid INTEGER NOT NULL DEFAULT 0,
|
||||
mail_json TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_mails_folder
|
||||
ON mails(folder_id, received_date_ms DESC);
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
folder_id TEXT PRIMARY KEY,
|
||||
last_sync_ms INTEGER NOT NULL DEFAULT 0
|
||||
last_sync_ms INTEGER NOT NULL DEFAULT 0,
|
||||
next_uid INTEGER NOT NULL DEFAULT 1
|
||||
);
|
||||
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
||||
))?;
|
||||
@@ -150,7 +154,7 @@ impl LocalStore {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT element_id, list_id, folder_id, subject, sender_name, sender_address,
|
||||
received_date_ms, unread, has_details, mail_json
|
||||
received_date_ms, unread, has_details, uid, mail_json
|
||||
FROM mails WHERE folder_id = ?1
|
||||
ORDER BY received_date_ms DESC",
|
||||
)?;
|
||||
@@ -165,7 +169,8 @@ impl LocalStore {
|
||||
received_date_ms: row.get(6)?,
|
||||
unread: row.get::<_, i64>(7)? != 0,
|
||||
has_details: row.get::<_, i64>(8)? != 0,
|
||||
mail_json: row.get(9)?,
|
||||
uid: row.get(9)?,
|
||||
mail_json: row.get(10)?,
|
||||
})
|
||||
})?;
|
||||
|
||||
@@ -180,8 +185,8 @@ impl LocalStore {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
conn.execute(
|
||||
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
|
||||
sender_address, received_date_ms, unread, has_details, mail_json)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
sender_address, received_date_ms, unread, has_details, uid, mail_json)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||
ON CONFLICT(element_id) DO UPDATE SET
|
||||
folder_id = excluded.folder_id,
|
||||
subject = excluded.subject,
|
||||
@@ -201,6 +206,7 @@ impl LocalStore {
|
||||
meta.received_date_ms,
|
||||
meta.unread as i64,
|
||||
meta.has_details as i64,
|
||||
meta.uid,
|
||||
meta.mail_json,
|
||||
],
|
||||
)?;
|
||||
@@ -213,8 +219,8 @@ impl LocalStore {
|
||||
{
|
||||
let mut stmt = conn.prepare_cached(
|
||||
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
|
||||
sender_address, received_date_ms, unread, has_details, mail_json)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
||||
sender_address, received_date_ms, unread, has_details, uid, mail_json)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||
ON CONFLICT(element_id) DO UPDATE SET
|
||||
folder_id = excluded.folder_id,
|
||||
subject = excluded.subject,
|
||||
@@ -236,6 +242,7 @@ impl LocalStore {
|
||||
meta.received_date_ms,
|
||||
meta.unread as i64,
|
||||
meta.has_details as i64,
|
||||
meta.uid,
|
||||
meta.mail_json,
|
||||
])?;
|
||||
}
|
||||
@@ -244,6 +251,39 @@ impl LocalStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Assign stable, monotonic UIDs to the given (new) element ids in a folder.
|
||||
/// UIDs are never reused — the per-folder counter only advances — so an IMAP
|
||||
/// client's `(UIDVALIDITY, UID)` cache stays valid across bridge restarts.
|
||||
/// Ids should be supplied oldest-first so newer mail gets higher UIDs.
|
||||
pub fn allocate_folder_uids(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
new_element_ids: &[&str],
|
||||
) -> Result<std::collections::HashMap<String, u32>, StoreError> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
let mut next: i64 = conn
|
||||
.query_row(
|
||||
"SELECT next_uid FROM sync_state WHERE folder_id = ?1",
|
||||
[folder_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.unwrap_or(1);
|
||||
|
||||
let mut map = std::collections::HashMap::with_capacity(new_element_ids.len());
|
||||
for eid in new_element_ids {
|
||||
map.insert((*eid).to_string(), next as u32);
|
||||
next += 1;
|
||||
}
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO sync_state(folder_id, next_uid) VALUES (?1, ?2)
|
||||
ON CONFLICT(folder_id) DO UPDATE SET next_uid = excluded.next_uid",
|
||||
rusqlite::params![folder_id, next],
|
||||
)?;
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
pub fn delete_mails_not_in(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
@@ -384,10 +424,25 @@ mod tests {
|
||||
received_date_ms: received,
|
||||
unread: true,
|
||||
has_details: false,
|
||||
uid: 0,
|
||||
mail_json: "{}".into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocate_uids_are_monotonic_and_per_folder() {
|
||||
let store = open_memory_store();
|
||||
let m1 = store.allocate_folder_uids("inbox", &["a", "b"]).unwrap();
|
||||
assert_eq!(m1["a"], 1);
|
||||
assert_eq!(m1["b"], 2);
|
||||
// Continues, never reuses, even if "a"/"b" were deleted.
|
||||
let m2 = store.allocate_folder_uids("inbox", &["c"]).unwrap();
|
||||
assert_eq!(m2["c"], 3);
|
||||
// Each folder has its own counter.
|
||||
let m3 = store.allocate_folder_uids("custom", &["x"]).unwrap();
|
||||
assert_eq!(m3["x"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_open_and_verify() {
|
||||
let store = open_memory_store();
|
||||
|
||||
@@ -22,6 +22,8 @@ pub struct StoredMail {
|
||||
pub mail: Mail,
|
||||
pub details: Option<MailDetails>,
|
||||
pub rfc2822: Option<String>,
|
||||
/// Stable IMAP UID within the folder, persisted across restarts.
|
||||
pub uid: u32,
|
||||
}
|
||||
|
||||
pub struct MailStore {
|
||||
@@ -309,6 +311,7 @@ async fn load_cached_folder(
|
||||
mail,
|
||||
details: None,
|
||||
rfc2822,
|
||||
uid: meta.uid as u32,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -335,16 +338,42 @@ async fn sync_folder(
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Allocate stable UIDs for mails we haven't seen before. `new_mails` is
|
||||
// newest-first; reverse the new ones so the oldest gets the lowest UID.
|
||||
let new_element_ids: Vec<String> = new_mails
|
||||
.iter()
|
||||
.rev()
|
||||
.filter_map(|m| m._id.as_ref().map(|id| id.element_id.to_string()))
|
||||
.filter(|eid| !existing_map.contains_key(eid))
|
||||
.collect();
|
||||
let new_uids = if new_element_ids.is_empty() {
|
||||
std::collections::HashMap::new()
|
||||
} else {
|
||||
let refs: Vec<&str> = new_element_ids.iter().map(|s| s.as_str()).collect();
|
||||
local_store
|
||||
.allocate_folder_uids(&folder.id, &refs)
|
||||
.unwrap_or_else(|e| {
|
||||
warn!("Failed to allocate UIDs for {}: {}", folder.imap_path, e);
|
||||
std::collections::HashMap::new()
|
||||
})
|
||||
};
|
||||
|
||||
let mut updated = Vec::with_capacity(new_mails.len());
|
||||
let mut metas_to_upsert = Vec::with_capacity(new_mails.len());
|
||||
|
||||
for mail in &new_mails {
|
||||
let elem_id = mail._id.as_ref().map(|id| id.element_id.to_string());
|
||||
let uid = elem_id
|
||||
.as_ref()
|
||||
.and_then(|id| existing_map.get(id).map(|m| m.uid).or_else(|| new_uids.get(id).copied()))
|
||||
.unwrap_or(0);
|
||||
|
||||
if let Some(existing) = elem_id.as_ref().and_then(|id| existing_map.get(id)) {
|
||||
updated.push(StoredMail {
|
||||
mail: mail.clone(),
|
||||
details: existing.details.clone(),
|
||||
rfc2822: existing.rfc2822.clone(),
|
||||
uid,
|
||||
});
|
||||
} else {
|
||||
let rfc2822 = mail_to_rfc2822(mail, None);
|
||||
@@ -352,10 +381,11 @@ async fn sync_folder(
|
||||
mail: mail.clone(),
|
||||
details: None,
|
||||
rfc2822: Some(rfc2822),
|
||||
uid,
|
||||
});
|
||||
}
|
||||
|
||||
metas_to_upsert.push(mail_to_metadata(mail, &folder.id));
|
||||
metas_to_upsert.push(mail_to_metadata(mail, &folder.id, uid));
|
||||
}
|
||||
|
||||
if let Err(e) = local_store.upsert_mail_metadata_batch(&metas_to_upsert) {
|
||||
@@ -474,7 +504,7 @@ where
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata {
|
||||
fn mail_to_metadata(mail: &Mail, folder_id: &str, uid: u32) -> MailMetadata {
|
||||
let (list_id, element_id) = mail
|
||||
._id
|
||||
.as_ref()
|
||||
@@ -493,6 +523,7 @@ fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata {
|
||||
received_date_ms: mail.receivedDate.as_millis() as i64,
|
||||
unread: mail.unread,
|
||||
has_details: false,
|
||||
uid: uid as i64,
|
||||
mail_json,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user