mirror of
https://github.com/spartanz51/tutabridge.git
synced 2026-06-24 10:54:32 +02:00
Persist stable IMAP UIDs across restarts
UIDs were assigned from an in-memory counter that reset to 1 on every bridge restart, so the UID<->mail mapping changed each run and IMAP clients (Thunderbird) re-downloaded the whole mailbox on reconnect. Persist a per-folder monotonic UID in the local store (schema v3): - mails gain a `uid` column; sync_state gains a `next_uid` counter that only ever advances (UIDs are never reused). - The syncer keeps each mail's existing UID and allocates new ones for new mail (oldest-first, so newer mail gets higher UIDs). - refresh_mails uses the persisted UID instead of allocating; messages are ordered by UID. UIDVALIDITY stays constant. Migration v2->v3 drops the cache tables and re-syncs once (encrypted .eml files survive). Live-tested: UID<->mail mapping is identical before and after a restart (range 1..500 unchanged), so clients fetch only the delta instead of re-downloading.
This commit is contained in:
@@ -577,30 +577,31 @@ impl ImapSession {
|
|||||||
async fn refresh_mails(&mut self, folder_id: &str) -> Result<(), String> {
|
async fn refresh_mails(&mut self, folder_id: &str) -> Result<(), String> {
|
||||||
let stored = self.store.get_folder(folder_id).await;
|
let stored = self.store.get_folder(folder_id).await;
|
||||||
|
|
||||||
let old_cache: std::collections::HashMap<String, (u32, Option<MailDetails>, Option<String>)> =
|
// Carry over already-loaded details/rfc for this session.
|
||||||
|
let old_cache: std::collections::HashMap<String, (Option<MailDetails>, Option<String>)> =
|
||||||
self.mails
|
self.mails
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|m| {
|
.filter_map(|m| {
|
||||||
let eid = m.mail._id.as_ref()?.element_id.to_string();
|
let eid = m.mail._id.as_ref()?.element_id.to_string();
|
||||||
Some((eid, (m.uid, m.details.clone(), m.rfc2822.clone())))
|
Some((eid, (m.details.clone(), m.rfc2822.clone())))
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
self.mails.clear();
|
self.mails.clear();
|
||||||
|
// UIDs are stable and assigned by the store; sort ascending so message
|
||||||
|
// sequence order matches UID order, as IMAP clients expect.
|
||||||
|
let mut stored = stored;
|
||||||
|
stored.sort_by_key(|m| m.uid);
|
||||||
|
|
||||||
for sm in stored {
|
for sm in stored {
|
||||||
let elem_id = sm.mail._id.as_ref().map(|id| id.element_id.to_string());
|
let elem_id = sm.mail._id.as_ref().map(|id| id.element_id.to_string());
|
||||||
|
let (old_details, old_rfc) = elem_id
|
||||||
let (uid, old_details, old_rfc) = elem_id
|
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|eid| old_cache.get(eid))
|
.and_then(|eid| old_cache.get(eid))
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_else(|| {
|
.unwrap_or((None, None));
|
||||||
let uid = self.uid_next;
|
|
||||||
self.uid_next += 1;
|
|
||||||
(uid, None, None)
|
|
||||||
});
|
|
||||||
|
|
||||||
|
let uid = sm.uid;
|
||||||
if uid >= self.uid_next {
|
if uid >= self.uid_next {
|
||||||
self.uid_next = uid + 1;
|
self.uid_next = uid + 1;
|
||||||
}
|
}
|
||||||
@@ -1339,8 +1340,8 @@ mod tests {
|
|||||||
.set_folder(
|
.set_folder(
|
||||||
"inbox",
|
"inbox",
|
||||||
vec![
|
vec![
|
||||||
StoredMail { mail: m1, details: None, rfc2822: None },
|
StoredMail { mail: m1, details: None, rfc2822: None, uid: 1 },
|
||||||
StoredMail { mail: m2, details: None, rfc2822: None },
|
StoredMail { mail: m2, details: None, rfc2822: None, uid: 2 },
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -1372,10 +1373,12 @@ mod tests {
|
|||||||
store.set_folder_list(vec![inbox_folder()]).await;
|
store.set_folder_list(vec![inbox_folder()]).await;
|
||||||
let stored: Vec<StoredMail> = mails
|
let stored: Vec<StoredMail> = mails
|
||||||
.iter()
|
.iter()
|
||||||
.map(|m| StoredMail {
|
.enumerate()
|
||||||
|
.map(|(i, m)| StoredMail {
|
||||||
mail: m.clone(),
|
mail: m.clone(),
|
||||||
details: None,
|
details: None,
|
||||||
rfc2822: None,
|
rfc2822: None,
|
||||||
|
uid: (i + 1) as u32,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
store.set_folder("inbox", stored).await;
|
store.set_folder("inbox", stored).await;
|
||||||
@@ -1520,8 +1523,8 @@ mod tests {
|
|||||||
let rfc2 = crate::mail::mail_to_rfc2822(&m2, Some(&d2));
|
let rfc2 = crate::mail::mail_to_rfc2822(&m2, Some(&d2));
|
||||||
store.set_folder_list(vec![inbox_folder()]).await;
|
store.set_folder_list(vec![inbox_folder()]).await;
|
||||||
store.set_folder("inbox", vec![
|
store.set_folder("inbox", vec![
|
||||||
StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1) },
|
StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1), uid: 1 },
|
||||||
StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2) },
|
StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2), uid: 2 },
|
||||||
]).await;
|
]).await;
|
||||||
let mut session = ImapSession::new(store, backend, None);
|
let mut session = ImapSession::new(store, backend, None);
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use rusqlite::{Connection, OptionalExtension};
|
|||||||
/// Bumped when the on-disk schema changes. A mismatch drops the cached tables
|
/// Bumped when the on-disk schema changes. A mismatch drops the cached tables
|
||||||
/// (mails + sync_state) and triggers a full re-sync; encrypted .eml files are
|
/// (mails + sync_state) and triggers a full re-sync; encrypted .eml files are
|
||||||
/// keyed by element id and survive the migration.
|
/// keyed by element id and survive the migration.
|
||||||
const SCHEMA_VERSION: &str = "2";
|
const SCHEMA_VERSION: &str = "3";
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum StoreError {
|
pub enum StoreError {
|
||||||
@@ -35,6 +35,8 @@ pub struct MailMetadata {
|
|||||||
pub received_date_ms: i64,
|
pub received_date_ms: i64,
|
||||||
pub unread: bool,
|
pub unread: bool,
|
||||||
pub has_details: bool,
|
pub has_details: bool,
|
||||||
|
/// Stable IMAP UID within the folder (0 = not yet assigned).
|
||||||
|
pub uid: i64,
|
||||||
pub mail_json: String,
|
pub mail_json: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,13 +96,15 @@ impl LocalStore {
|
|||||||
received_date_ms INTEGER NOT NULL,
|
received_date_ms INTEGER NOT NULL,
|
||||||
unread INTEGER NOT NULL DEFAULT 1,
|
unread INTEGER NOT NULL DEFAULT 1,
|
||||||
has_details INTEGER NOT NULL DEFAULT 0,
|
has_details INTEGER NOT NULL DEFAULT 0,
|
||||||
|
uid INTEGER NOT NULL DEFAULT 0,
|
||||||
mail_json TEXT NOT NULL
|
mail_json TEXT NOT NULL
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_mails_folder
|
CREATE INDEX IF NOT EXISTS idx_mails_folder
|
||||||
ON mails(folder_id, received_date_ms DESC);
|
ON mails(folder_id, received_date_ms DESC);
|
||||||
CREATE TABLE IF NOT EXISTS sync_state (
|
CREATE TABLE IF NOT EXISTS sync_state (
|
||||||
folder_id TEXT PRIMARY KEY,
|
folder_id TEXT PRIMARY KEY,
|
||||||
last_sync_ms INTEGER NOT NULL DEFAULT 0
|
last_sync_ms INTEGER NOT NULL DEFAULT 0,
|
||||||
|
next_uid INTEGER NOT NULL DEFAULT 1
|
||||||
);
|
);
|
||||||
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
||||||
))?;
|
))?;
|
||||||
@@ -150,7 +154,7 @@ impl LocalStore {
|
|||||||
let conn = self.conn.lock().unwrap();
|
let conn = self.conn.lock().unwrap();
|
||||||
let mut stmt = conn.prepare(
|
let mut stmt = conn.prepare(
|
||||||
"SELECT element_id, list_id, folder_id, subject, sender_name, sender_address,
|
"SELECT element_id, list_id, folder_id, subject, sender_name, sender_address,
|
||||||
received_date_ms, unread, has_details, mail_json
|
received_date_ms, unread, has_details, uid, mail_json
|
||||||
FROM mails WHERE folder_id = ?1
|
FROM mails WHERE folder_id = ?1
|
||||||
ORDER BY received_date_ms DESC",
|
ORDER BY received_date_ms DESC",
|
||||||
)?;
|
)?;
|
||||||
@@ -165,7 +169,8 @@ impl LocalStore {
|
|||||||
received_date_ms: row.get(6)?,
|
received_date_ms: row.get(6)?,
|
||||||
unread: row.get::<_, i64>(7)? != 0,
|
unread: row.get::<_, i64>(7)? != 0,
|
||||||
has_details: row.get::<_, i64>(8)? != 0,
|
has_details: row.get::<_, i64>(8)? != 0,
|
||||||
mail_json: row.get(9)?,
|
uid: row.get(9)?,
|
||||||
|
mail_json: row.get(10)?,
|
||||||
})
|
})
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
@@ -180,8 +185,8 @@ impl LocalStore {
|
|||||||
let conn = self.conn.lock().unwrap();
|
let conn = self.conn.lock().unwrap();
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
|
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
|
||||||
sender_address, received_date_ms, unread, has_details, mail_json)
|
sender_address, received_date_ms, unread, has_details, uid, mail_json)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||||
ON CONFLICT(element_id) DO UPDATE SET
|
ON CONFLICT(element_id) DO UPDATE SET
|
||||||
folder_id = excluded.folder_id,
|
folder_id = excluded.folder_id,
|
||||||
subject = excluded.subject,
|
subject = excluded.subject,
|
||||||
@@ -201,6 +206,7 @@ impl LocalStore {
|
|||||||
meta.received_date_ms,
|
meta.received_date_ms,
|
||||||
meta.unread as i64,
|
meta.unread as i64,
|
||||||
meta.has_details as i64,
|
meta.has_details as i64,
|
||||||
|
meta.uid,
|
||||||
meta.mail_json,
|
meta.mail_json,
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
@@ -213,8 +219,8 @@ impl LocalStore {
|
|||||||
{
|
{
|
||||||
let mut stmt = conn.prepare_cached(
|
let mut stmt = conn.prepare_cached(
|
||||||
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
|
"INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name,
|
||||||
sender_address, received_date_ms, unread, has_details, mail_json)
|
sender_address, received_date_ms, unread, has_details, uid, mail_json)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)
|
||||||
ON CONFLICT(element_id) DO UPDATE SET
|
ON CONFLICT(element_id) DO UPDATE SET
|
||||||
folder_id = excluded.folder_id,
|
folder_id = excluded.folder_id,
|
||||||
subject = excluded.subject,
|
subject = excluded.subject,
|
||||||
@@ -236,6 +242,7 @@ impl LocalStore {
|
|||||||
meta.received_date_ms,
|
meta.received_date_ms,
|
||||||
meta.unread as i64,
|
meta.unread as i64,
|
||||||
meta.has_details as i64,
|
meta.has_details as i64,
|
||||||
|
meta.uid,
|
||||||
meta.mail_json,
|
meta.mail_json,
|
||||||
])?;
|
])?;
|
||||||
}
|
}
|
||||||
@@ -244,6 +251,39 @@ impl LocalStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Assign stable, monotonic UIDs to the given (new) element ids in a folder.
|
||||||
|
/// UIDs are never reused — the per-folder counter only advances — so an IMAP
|
||||||
|
/// client's `(UIDVALIDITY, UID)` cache stays valid across bridge restarts.
|
||||||
|
/// Ids should be supplied oldest-first so newer mail gets higher UIDs.
|
||||||
|
pub fn allocate_folder_uids(
|
||||||
|
&self,
|
||||||
|
folder_id: &str,
|
||||||
|
new_element_ids: &[&str],
|
||||||
|
) -> Result<std::collections::HashMap<String, u32>, StoreError> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut next: i64 = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT next_uid FROM sync_state WHERE folder_id = ?1",
|
||||||
|
[folder_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.optional()?
|
||||||
|
.unwrap_or(1);
|
||||||
|
|
||||||
|
let mut map = std::collections::HashMap::with_capacity(new_element_ids.len());
|
||||||
|
for eid in new_element_ids {
|
||||||
|
map.insert((*eid).to_string(), next as u32);
|
||||||
|
next += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sync_state(folder_id, next_uid) VALUES (?1, ?2)
|
||||||
|
ON CONFLICT(folder_id) DO UPDATE SET next_uid = excluded.next_uid",
|
||||||
|
rusqlite::params![folder_id, next],
|
||||||
|
)?;
|
||||||
|
Ok(map)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn delete_mails_not_in(
|
pub fn delete_mails_not_in(
|
||||||
&self,
|
&self,
|
||||||
folder_id: &str,
|
folder_id: &str,
|
||||||
@@ -384,10 +424,25 @@ mod tests {
|
|||||||
received_date_ms: received,
|
received_date_ms: received,
|
||||||
unread: true,
|
unread: true,
|
||||||
has_details: false,
|
has_details: false,
|
||||||
|
uid: 0,
|
||||||
mail_json: "{}".into(),
|
mail_json: "{}".into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn allocate_uids_are_monotonic_and_per_folder() {
|
||||||
|
let store = open_memory_store();
|
||||||
|
let m1 = store.allocate_folder_uids("inbox", &["a", "b"]).unwrap();
|
||||||
|
assert_eq!(m1["a"], 1);
|
||||||
|
assert_eq!(m1["b"], 2);
|
||||||
|
// Continues, never reuses, even if "a"/"b" were deleted.
|
||||||
|
let m2 = store.allocate_folder_uids("inbox", &["c"]).unwrap();
|
||||||
|
assert_eq!(m2["c"], 3);
|
||||||
|
// Each folder has its own counter.
|
||||||
|
let m3 = store.allocate_folder_uids("custom", &["x"]).unwrap();
|
||||||
|
assert_eq!(m3["x"], 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_open_and_verify() {
|
fn test_open_and_verify() {
|
||||||
let store = open_memory_store();
|
let store = open_memory_store();
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ pub struct StoredMail {
|
|||||||
pub mail: Mail,
|
pub mail: Mail,
|
||||||
pub details: Option<MailDetails>,
|
pub details: Option<MailDetails>,
|
||||||
pub rfc2822: Option<String>,
|
pub rfc2822: Option<String>,
|
||||||
|
/// Stable IMAP UID within the folder, persisted across restarts.
|
||||||
|
pub uid: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct MailStore {
|
pub struct MailStore {
|
||||||
@@ -309,6 +311,7 @@ async fn load_cached_folder(
|
|||||||
mail,
|
mail,
|
||||||
details: None,
|
details: None,
|
||||||
rfc2822,
|
rfc2822,
|
||||||
|
uid: meta.uid as u32,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -335,16 +338,42 @@ async fn sync_folder(
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
// Allocate stable UIDs for mails we haven't seen before. `new_mails` is
|
||||||
|
// newest-first; reverse the new ones so the oldest gets the lowest UID.
|
||||||
|
let new_element_ids: Vec<String> = new_mails
|
||||||
|
.iter()
|
||||||
|
.rev()
|
||||||
|
.filter_map(|m| m._id.as_ref().map(|id| id.element_id.to_string()))
|
||||||
|
.filter(|eid| !existing_map.contains_key(eid))
|
||||||
|
.collect();
|
||||||
|
let new_uids = if new_element_ids.is_empty() {
|
||||||
|
std::collections::HashMap::new()
|
||||||
|
} else {
|
||||||
|
let refs: Vec<&str> = new_element_ids.iter().map(|s| s.as_str()).collect();
|
||||||
|
local_store
|
||||||
|
.allocate_folder_uids(&folder.id, &refs)
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
warn!("Failed to allocate UIDs for {}: {}", folder.imap_path, e);
|
||||||
|
std::collections::HashMap::new()
|
||||||
|
})
|
||||||
|
};
|
||||||
|
|
||||||
let mut updated = Vec::with_capacity(new_mails.len());
|
let mut updated = Vec::with_capacity(new_mails.len());
|
||||||
let mut metas_to_upsert = Vec::with_capacity(new_mails.len());
|
let mut metas_to_upsert = Vec::with_capacity(new_mails.len());
|
||||||
|
|
||||||
for mail in &new_mails {
|
for mail in &new_mails {
|
||||||
let elem_id = mail._id.as_ref().map(|id| id.element_id.to_string());
|
let elem_id = mail._id.as_ref().map(|id| id.element_id.to_string());
|
||||||
|
let uid = elem_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| existing_map.get(id).map(|m| m.uid).or_else(|| new_uids.get(id).copied()))
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
if let Some(existing) = elem_id.as_ref().and_then(|id| existing_map.get(id)) {
|
if let Some(existing) = elem_id.as_ref().and_then(|id| existing_map.get(id)) {
|
||||||
updated.push(StoredMail {
|
updated.push(StoredMail {
|
||||||
mail: mail.clone(),
|
mail: mail.clone(),
|
||||||
details: existing.details.clone(),
|
details: existing.details.clone(),
|
||||||
rfc2822: existing.rfc2822.clone(),
|
rfc2822: existing.rfc2822.clone(),
|
||||||
|
uid,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
let rfc2822 = mail_to_rfc2822(mail, None);
|
let rfc2822 = mail_to_rfc2822(mail, None);
|
||||||
@@ -352,10 +381,11 @@ async fn sync_folder(
|
|||||||
mail: mail.clone(),
|
mail: mail.clone(),
|
||||||
details: None,
|
details: None,
|
||||||
rfc2822: Some(rfc2822),
|
rfc2822: Some(rfc2822),
|
||||||
|
uid,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
metas_to_upsert.push(mail_to_metadata(mail, &folder.id));
|
metas_to_upsert.push(mail_to_metadata(mail, &folder.id, uid));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = local_store.upsert_mail_metadata_batch(&metas_to_upsert) {
|
if let Err(e) = local_store.upsert_mail_metadata_batch(&metas_to_upsert) {
|
||||||
@@ -474,7 +504,7 @@ where
|
|||||||
unreachable!()
|
unreachable!()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata {
|
fn mail_to_metadata(mail: &Mail, folder_id: &str, uid: u32) -> MailMetadata {
|
||||||
let (list_id, element_id) = mail
|
let (list_id, element_id) = mail
|
||||||
._id
|
._id
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@@ -493,6 +523,7 @@ fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata {
|
|||||||
received_date_ms: mail.receivedDate.as_millis() as i64,
|
received_date_ms: mail.receivedDate.as_millis() as i64,
|
||||||
unread: mail.unread,
|
unread: mail.unread,
|
||||||
has_details: false,
|
has_details: false,
|
||||||
|
uid: uid as i64,
|
||||||
mail_json,
|
mail_json,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user