diff --git a/SDK_PRS.md b/SDK_PRS.md new file mode 100644 index 0000000..7842af2 --- /dev/null +++ b/SDK_PRS.md @@ -0,0 +1,67 @@ +# SDK changes tracking + +TutaBridge depends on a few changes to the Tuta Rust SDK (`tuta-sdk/rust/sdk`, +vendored as the `tuta-repo` submodule). To stay able to switch back to Tuta's +upstream at any time, **every SDK change is kept as its own single-commit +branch off `upstream/master`**, each independently reviewable / mergeable by +Tuta. They are combined only in the `tutabridge-integration` branch, which is +what the submodule actually checks out. + +- Fork (our branches live here): `spartanz51/tutanota` +- Upstream: `tutao/tutanota` +- Integration branch (sum of the changes below): `tutabridge-integration` + +Rule: never accumulate unrelated SDK changes on one branch. One concern = one +branch = one commit, rebasable on `upstream/master`. + +## Branches + +| Branch | Summary | Upstream PR | Fork PR | Submitted to upstream | Merged | Live-tested | +|---|---|---|---|---|---|---| +| `sdk-load-multiple` | `EntityClient`/`CryptoEntityClient.load_multiple` (batch entity loading) | [tutao#10854](https://github.com/tutao/tutanota/pull/10854) | — | yes (open) | no | yes (sync 500 mails) | +| `sdk-blob-element-reading` | `BlobFacade.load_blob_element` + `MailFacade.load_mail_details_blob` (read `MailDetailsBlob`) | [tutao#10870](https://github.com/tutao/tutanota/pull/10870) | — | yes (open) | no | yes (body decrypt over IMAP) | +| `sdk-2fa-session` | Interactive 2FA: `initiate_session`, `authenticate_with_second_factor_totp`, `is_second_factor_pending`, `cancel_create_session` | [tutao#10871](https://github.com/tutao/tutanota/pull/10871) | — | yes (open) | no | yes (full TOTP login) | +| `sdk-folder-system` | Rebuild `FolderSystem` tree (system/custom/nested), add `MailSetKind` Label/Imported/Scheduled + accessors | — | [spartanz51#4](https://github.com/spartanz51/tutanota/pull/4) | no (held) | no | yes (custom folders listed + read over IMAP) | + +## Notes per branch + +### sdk-load-multiple +Additive utility, mirrors TS `EntityClient.loadMultiple`. Maintainer (charlag) +asked why it's submitted (not user-facing) and about LLM use; answered honestly. + +### sdk-blob-element-reading +Additive. Mirrors TS blob reading + `doBlobRequestWithRetry`/`tryServers`. +Returns `MailDetails` from `load_mail_details_blob` (matches TS). + +### sdk-2fa-session +Refactors `create_session` to delegate to `initiate_session`; reuses the +existing `parse_session_id`; no `clientIdentifier` change. Additive otherwise. + +### sdk-folder-system +**Held — not submitted upstream.** It modifies the existing `FolderSystem` +struct, which upstream marks as WIP (`// this structure should probably change +rather soon`), so they likely want to design it themselves. Faithful port of +`FolderSystem.ts`. Submit only if the other PRs get traction and a maintainer +signals appetite — align the API with them first. Needed locally regardless for +custom/nested folder support in the bridge. Live-tested in the bridge: custom +folders are listed and read over IMAP (the `custom-folders` bridge change keys +everything by folder id). + +## Rebasing on a newer upstream + +``` +cd tuta-repo +git fetch upstream +# rebase each SDK branch on the new master (resolve only if upstream touched +# the same files — so far it hasn't) +git rebase upstream/master sdk-load-multiple +git rebase upstream/master sdk-blob-element-reading +git rebase upstream/master sdk-2fa-session +git rebase upstream/master sdk-folder-system +# rebuild the integration branch from the rebased branches +git checkout -B tutabridge-integration upstream/master +git cherry-pick sdk-load-multiple sdk-blob-element-reading sdk-2fa-session sdk-folder-system +``` + +When an upstream PR merges, drop that branch from the cherry-pick list — the +integration branch shrinks until (ideally) it equals `upstream/master`. diff --git a/crates/bridge/src/imap/session.rs b/crates/bridge/src/imap/session.rs index fb340b0..544a500 100644 --- a/crates/bridge/src/imap/session.rs +++ b/crates/bridge/src/imap/session.rs @@ -1,12 +1,11 @@ use std::sync::Arc; use log::{info, debug}; use tutasdk::entities::generated::tutanota::{Mail, MailDetails}; -use tutasdk::folder_system::MailSetKind; use crate::mail::rfc2822::{extract_headers, format_internal_date}; use crate::mail::mail_to_rfc2822; use crate::sync::MailStore; -use crate::tuta::MailBackend; +use crate::tuta::{FolderInfo, MailBackend}; #[derive(Debug, Clone, PartialEq)] enum State { @@ -28,7 +27,7 @@ pub struct ImapSession { store: Arc, backend: Arc, state: State, - selected_folder: Option, + selected_folder: Option, mails: Vec, uid_next: u32, idle_tag: Option, @@ -122,13 +121,13 @@ impl ImapSession { } pub async fn check_new_mail(&mut self) -> Vec { - let kind = match self.selected_folder { - Some(k) => k, + let folder = match self.selected_folder.clone() { + Some(f) => f, None => return vec![], }; - let store_count = self.store.folder_count(kind).await; + let store_count = self.store.folder_count(&folder.id).await; if store_count != self.mails.len() { - if self.refresh_mails(kind).await.is_ok() { + if self.refresh_mails(&folder.id).await.is_ok() { return vec![format!("* {} EXISTS\r\n", self.mails.len())]; } } @@ -217,8 +216,12 @@ impl ImapSession { return responses; } - for (name, flags) in &self.folder_list() { - responses.push(format!("* LIST ({}) \"/\" \"{}\"\r\n", flags, name)); + for folder in self.store.list_folders().await { + let flags = folder.special_use.as_deref().unwrap_or(""); + responses.push(format!( + "* LIST ({}) \"/\" \"{}\"\r\n", + flags, folder.imap_path + )); } responses.push(format!("{} OK LIST completed\r\n", tag)); @@ -231,11 +234,17 @@ impl ImapSession { } let folder_name = args.trim().trim_matches('"'); - let kind = folder_name_to_kind(folder_name); - self.selected_folder = Some(kind); + let folder = match self.store.folder_by_imap_path(folder_name).await { + Some(f) => f, + None => { + return vec![format!("{} NO [NONEXISTENT] Mailbox does not exist\r\n", tag)]; + } + }; + let folder_id = folder.id.clone(); + self.selected_folder = Some(folder); self.state = State::Selected; - match self.refresh_mails(kind).await { + match self.refresh_mails(&folder_id).await { Ok(()) => { let count = self.mails.len(); let first_unseen = self @@ -260,7 +269,7 @@ impl ImapSession { resp } Err(e) => { - log::error!("Failed to load mails for {:?}: {}", kind, e); + log::error!("Failed to load mails for {}: {}", folder_name, e); vec![ "* 0 EXISTS\r\n".to_string(), "* 0 RECENT\r\n".to_string(), @@ -275,9 +284,10 @@ impl ImapSession { return vec![format!("{} NO Not authenticated\r\n", tag)]; } let folder_name = args.split_whitespace().next().unwrap_or("").trim_matches('"'); - let kind = folder_name_to_kind(folder_name); - - let stored = self.store.get_folder(kind).await; + let stored = match self.store.folder_by_imap_path(folder_name).await { + Some(folder) => self.store.get_folder(&folder.id).await, + None => Vec::new(), + }; let count = stored.len(); let unseen = stored.iter().filter(|m| m.mail.unread).count(); vec![ @@ -310,13 +320,10 @@ impl ImapSession { ._id .as_ref() .map(|id| id.element_id.to_string()); - let kind = self.selected_folder.unwrap_or(MailSetKind::Inbox); - // Check store — syncer may have loaded details since our snapshot - let from_store = if let Some(ref eid) = elem_id { - self.store.get_details(kind, eid).await - } else { - None + let from_store = match (&elem_id, &self.selected_folder) { + (Some(eid), Some(folder)) => self.store.get_details(&folder.id, eid).await, + _ => None, }; if let Some((details, rfc)) = from_store { @@ -491,8 +498,8 @@ impl ImapSession { responses } - async fn refresh_mails(&mut self, kind: MailSetKind) -> Result<(), String> { - let stored = self.store.get_folder(kind).await; + async fn refresh_mails(&mut self, folder_id: &str) -> Result<(), String> { + let stored = self.store.get_folder(folder_id).await; let old_cache: std::collections::HashMap, Option)> = self.mails @@ -536,7 +543,7 @@ impl ImapSession { }); } - debug!("Refreshed {} mails for {:?} from store", self.mails.len(), kind); + debug!("Refreshed {} mails for {} from store", self.mails.len(), folder_id); Ok(()) } @@ -581,16 +588,6 @@ impl ImapSession { } } - fn folder_list(&self) -> Vec<(String, String)> { - vec![ - ("INBOX".into(), "".into()), - ("Sent".into(), "\\Sent".into()), - ("Drafts".into(), "\\Drafts".into()), - ("Trash".into(), "\\Trash".into()), - ("Archive".into(), "\\Archive".into()), - ("Spam".into(), "\\Junk".into()), - ] - } } fn build_fetch_response(seq: usize, cached: &CachedMail, items: &str, uid_mode: bool) -> String { @@ -749,18 +746,6 @@ fn needs_body(items: &str) -> bool { false } -fn folder_name_to_kind(name: &str) -> MailSetKind { - match name.to_uppercase().as_str() { - "INBOX" => MailSetKind::Inbox, - "SENT" => MailSetKind::Sent, - "DRAFTS" => MailSetKind::Draft, - "TRASH" => MailSetKind::Trash, - "ARCHIVE" => MailSetKind::Archive, - "SPAM" | "JUNK" => MailSetKind::Spam, - _ => MailSetKind::Inbox, - } -} - fn parse_login_args(args: &str) -> (String, String) { let args = args.trim(); let (user, rest) = parse_imap_token(args); @@ -963,21 +948,6 @@ mod tests { assert_eq!(parse_seq_num("0", 100), 0); } - // --- folder_name_to_kind --- - - #[test] - fn test_folder_name_to_kind() { - assert_eq!(folder_name_to_kind("INBOX"), MailSetKind::Inbox); - assert_eq!(folder_name_to_kind("inbox"), MailSetKind::Inbox); - assert_eq!(folder_name_to_kind("Sent"), MailSetKind::Sent); - assert_eq!(folder_name_to_kind("Drafts"), MailSetKind::Draft); - assert_eq!(folder_name_to_kind("Trash"), MailSetKind::Trash); - assert_eq!(folder_name_to_kind("Archive"), MailSetKind::Archive); - assert_eq!(folder_name_to_kind("Spam"), MailSetKind::Spam); - assert_eq!(folder_name_to_kind("Junk"), MailSetKind::Spam); - assert_eq!(folder_name_to_kind("Unknown"), MailSetKind::Inbox); - } - // --- needs_body --- #[test] @@ -1258,8 +1228,21 @@ mod tests { } use crate::sync::StoredMail; + use crate::tuta::FolderInfo; + use tutasdk::folder_system::MailSetKind; + + fn inbox_folder() -> FolderInfo { + FolderInfo { + id: "inbox".to_string(), + entries_list_id: "inbox_entries".to_string(), + kind: MailSetKind::Inbox, + imap_path: "INBOX".to_string(), + special_use: None, + } + } async fn populate_store(store: &MailStore, mails: &[Mail]) { + store.set_folder_list(vec![inbox_folder()]).await; let stored: Vec = mails .iter() .map(|m| StoredMail { @@ -1268,7 +1251,7 @@ mod tests { rfc2822: None, }) .collect(); - store.set_folder(MailSetKind::Inbox, stored).await; + store.set_folder("inbox", stored).await; } async fn make_session(backend: Arc) -> (Arc, ImapSession) { @@ -1281,19 +1264,15 @@ mod tests { #[async_trait::async_trait] impl MailBackend for MockBackend { - async fn load_mail_ids_for_folder(&self, _kind: MailSetKind, _limit: usize) -> Result, String> { + async fn load_mail_ids_for_folder(&self, _folder: &FolderInfo, _limit: usize) -> Result, String> { Ok(self.mails.lock().unwrap().clone()) } async fn load_mail_details(&self, mail: &Mail) -> Result, String> { let key = mail._id.as_ref().map(|id| id.element_id.to_string()).unwrap_or_default(); Ok(self.details.lock().unwrap().get(&key).cloned()) } - async fn load_folder_list(&self) -> Result, String> { - Ok(vec![ - ("INBOX".to_string(), String::new()), - ("Sent".to_string(), "\\Sent".to_string()), - ("Trash".to_string(), "\\Trash".to_string()), - ]) + async fn list_folders(&self) -> Result, String> { + Ok(vec![inbox_folder()]) } async fn set_unread_status(&self, mail_ids: Vec, unread: bool) -> Result<(), String> { self.unread_calls.lock().unwrap().push((mail_ids, unread)); @@ -1408,7 +1387,8 @@ mod tests { let store = MailStore::new(); let rfc1 = crate::mail::mail_to_rfc2822(&m1, Some(&d1)); let rfc2 = crate::mail::mail_to_rfc2822(&m2, Some(&d2)); - store.set_folder(MailSetKind::Inbox, vec![ + store.set_folder_list(vec![inbox_folder()]).await; + store.set_folder("inbox", vec![ StoredMail { mail: m1, details: Some(d1), rfc2822: Some(rfc1) }, StoredMail { mail: m2, details: Some(d2), rfc2822: Some(rfc2) }, ]).await; diff --git a/crates/bridge/src/store.rs b/crates/bridge/src/store.rs index cc622c7..8268cde 100644 --- a/crates/bridge/src/store.rs +++ b/crates/bridge/src/store.rs @@ -5,8 +5,12 @@ use crypto_primitives::aes::Iv; use crypto_primitives::key::GenericAesKey; use crypto_primitives::randomizer_facade::RandomizerFacade; use log::{debug, warn}; -use rusqlite::Connection; -use tutasdk::folder_system::MailSetKind; +use rusqlite::{Connection, OptionalExtension}; + +/// Bumped when the on-disk schema changes. A mismatch drops the cached tables +/// (mails + sync_state) and triggers a full re-sync; encrypted .eml files are +/// keyed by element id and survive the migration. +const SCHEMA_VERSION: &str = "2"; #[derive(Debug, thiserror::Error)] pub enum StoreError { @@ -23,7 +27,8 @@ pub enum StoreError { pub struct MailMetadata { pub list_id: String, pub element_id: String, - pub folder_kind: i64, + /// Stable folder id (Tuta `MailSet` element id). + pub folder_id: String, pub subject: String, pub sender_name: String, pub sender_address: String, @@ -56,11 +61,33 @@ impl LocalStore { conn.pragma_update(None, "key", format!("x'{hex_key}'"))?; conn.pragma_update(None, "journal_mode", "WAL")?; + // Migrate: if the stored schema version differs, drop the cache tables. conn.execute_batch( + "CREATE TABLE IF NOT EXISTS store_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + );", + )?; + let version: Option = conn + .query_row( + "SELECT value FROM store_meta WHERE key = 'schema_version'", + [], + |row| row.get(0), + ) + .optional()?; + if version.as_deref() != Some(SCHEMA_VERSION) { + warn!( + "Local store schema {:?} != {SCHEMA_VERSION}, dropping cache tables", + version + ); + conn.execute_batch("DROP TABLE IF EXISTS mails; DROP TABLE IF EXISTS sync_state;")?; + } + + conn.execute_batch(&format!( "CREATE TABLE IF NOT EXISTS mails ( element_id TEXT PRIMARY KEY, list_id TEXT NOT NULL, - folder_kind INTEGER NOT NULL, + folder_id TEXT NOT NULL, subject TEXT NOT NULL, sender_name TEXT NOT NULL DEFAULT '', sender_address TEXT NOT NULL DEFAULT '', @@ -70,17 +97,13 @@ impl LocalStore { mail_json TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_mails_folder - ON mails(folder_kind, received_date_ms DESC); + ON mails(folder_id, received_date_ms DESC); CREATE TABLE IF NOT EXISTS sync_state ( - folder_kind INTEGER PRIMARY KEY, + folder_id TEXT PRIMARY KEY, last_sync_ms INTEGER NOT NULL DEFAULT 0 ); - CREATE TABLE IF NOT EXISTS store_meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - INSERT OR IGNORE INTO store_meta(key, value) VALUES ('schema_version', '1');", - )?; + INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');" + ))?; debug!("LocalStore opened at {}", db_path.display()); @@ -104,12 +127,12 @@ impl LocalStore { pub fn reset(&self) -> Result<(), StoreError> { warn!("Resetting local store — all cached data will be deleted"); let conn = self.conn.lock().unwrap(); - conn.execute_batch( + conn.execute_batch(&format!( "DELETE FROM mails; DELETE FROM sync_state; DELETE FROM store_meta; - INSERT INTO store_meta(key, value) VALUES ('schema_version', '1');", - )?; + INSERT INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');" + ))?; drop(conn); if self.mails_dir.exists() { @@ -123,19 +146,19 @@ impl LocalStore { Ok(()) } - pub fn load_folder_metadata(&self, kind: MailSetKind) -> Result, StoreError> { + pub fn load_folder_metadata(&self, folder_id: &str) -> Result, StoreError> { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( - "SELECT element_id, list_id, folder_kind, subject, sender_name, sender_address, + "SELECT element_id, list_id, folder_id, subject, sender_name, sender_address, received_date_ms, unread, has_details, mail_json - FROM mails WHERE folder_kind = ?1 + FROM mails WHERE folder_id = ?1 ORDER BY received_date_ms DESC", )?; - let rows = stmt.query_map([kind_to_i64(kind)], |row| { + let rows = stmt.query_map([folder_id], |row| { Ok(MailMetadata { element_id: row.get(0)?, list_id: row.get(1)?, - folder_kind: row.get(2)?, + folder_id: row.get(2)?, subject: row.get(3)?, sender_name: row.get(4)?, sender_address: row.get(5)?, @@ -156,11 +179,11 @@ impl LocalStore { pub fn upsert_mail_metadata(&self, meta: &MailMetadata) -> Result<(), StoreError> { let conn = self.conn.lock().unwrap(); conn.execute( - "INSERT INTO mails (element_id, list_id, folder_kind, subject, sender_name, + "INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name, sender_address, received_date_ms, unread, has_details, mail_json) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) ON CONFLICT(element_id) DO UPDATE SET - folder_kind = excluded.folder_kind, + folder_id = excluded.folder_id, subject = excluded.subject, sender_name = excluded.sender_name, sender_address = excluded.sender_address, @@ -171,7 +194,7 @@ impl LocalStore { rusqlite::params![ meta.element_id, meta.list_id, - meta.folder_kind, + meta.folder_id, meta.subject, meta.sender_name, meta.sender_address, @@ -189,11 +212,11 @@ impl LocalStore { conn.execute_batch("BEGIN IMMEDIATE")?; { let mut stmt = conn.prepare_cached( - "INSERT INTO mails (element_id, list_id, folder_kind, subject, sender_name, + "INSERT INTO mails (element_id, list_id, folder_id, subject, sender_name, sender_address, received_date_ms, unread, has_details, mail_json) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) ON CONFLICT(element_id) DO UPDATE SET - folder_kind = excluded.folder_kind, + folder_id = excluded.folder_id, subject = excluded.subject, sender_name = excluded.sender_name, sender_address = excluded.sender_address, @@ -206,7 +229,7 @@ impl LocalStore { stmt.execute(rusqlite::params![ meta.element_id, meta.list_id, - meta.folder_kind, + meta.folder_id, meta.subject, meta.sender_name, meta.sender_address, @@ -223,23 +246,20 @@ impl LocalStore { pub fn delete_mails_not_in( &self, - kind: MailSetKind, + folder_id: &str, element_ids: &[&str], ) -> Result, StoreError> { let conn = self.conn.lock().unwrap(); let mut deleted = Vec::new(); { - let mut stmt = conn.prepare( - "SELECT element_id FROM mails WHERE folder_kind = ?1", - )?; + let mut stmt = conn.prepare("SELECT element_id FROM mails WHERE folder_id = ?1")?; let existing: Vec = stmt - .query_map([kind_to_i64(kind)], |row| row.get(0))? + .query_map([folder_id], |row| row.get(0))? .filter_map(|r| r.ok()) .collect(); - let keep: std::collections::HashSet<&str> = - element_ids.iter().copied().collect(); + let keep: std::collections::HashSet<&str> = element_ids.iter().copied().collect(); for eid in existing { if !keep.contains(eid.as_str()) { @@ -316,11 +336,11 @@ impl LocalStore { Ok(()) } - pub fn mail_count(&self, kind: MailSetKind) -> Result { + pub fn mail_count(&self, folder_id: &str) -> Result { let conn = self.conn.lock().unwrap(); let count: i64 = conn.query_row( - "SELECT COUNT(*) FROM mails WHERE folder_kind = ?1", - [kind_to_i64(kind)], + "SELECT COUNT(*) FROM mails WHERE folder_id = ?1", + [folder_id], |row| row.get(0), )?; Ok(count as usize) @@ -334,26 +354,10 @@ impl LocalStore { } } -fn kind_to_i64(kind: MailSetKind) -> i64 { - kind as i64 -} - -pub fn kind_from_i64(v: i64) -> MailSetKind { - match v { - 0 => MailSetKind::Inbox, - 1 => MailSetKind::Sent, - 2 => MailSetKind::Trash, - 3 => MailSetKind::Archive, - 4 => MailSetKind::Spam, - 5 => MailSetKind::Draft, - _ => MailSetKind::Inbox, - } -} - #[cfg(test)] mod tests { use super::*; - use crypto_primitives::aes::{Aes256Key, AES_256_KEY_SIZE}; + use crypto_primitives::aes::Aes256Key; fn test_key() -> GenericAesKey { let randomizer = RandomizerFacade::from_core(rand_core::OsRng); @@ -369,6 +373,21 @@ mod tests { LocalStore::open(&db_path, &mails_dir, key).unwrap() } + fn meta(element_id: &str, folder_id: &str, received: i64) -> MailMetadata { + MailMetadata { + element_id: element_id.into(), + list_id: "list1".into(), + folder_id: folder_id.into(), + subject: format!("Subject {element_id}"), + sender_name: "Alice".into(), + sender_address: "alice@example.com".into(), + received_date_ms: received, + unread: true, + has_details: false, + mail_json: "{}".into(), + } + } + #[test] fn test_open_and_verify() { let store = open_memory_store(); @@ -378,50 +397,37 @@ mod tests { #[test] fn test_upsert_and_load_metadata() { let store = open_memory_store(); - let meta = MailMetadata { - element_id: "abc123".into(), - list_id: "list1".into(), - folder_kind: kind_to_i64(MailSetKind::Inbox), - subject: "Test email".into(), - sender_name: "Alice".into(), - sender_address: "alice@example.com".into(), - received_date_ms: 1700000000000, - unread: true, - has_details: false, - mail_json: "{}".into(), - }; - store.upsert_mail_metadata(&meta).unwrap(); + store.upsert_mail_metadata(&meta("abc123", "inbox", 1700000000000)).unwrap(); - let loaded = store.load_folder_metadata(MailSetKind::Inbox).unwrap(); + let loaded = store.load_folder_metadata("inbox").unwrap(); assert_eq!(loaded.len(), 1); assert_eq!(loaded[0].element_id, "abc123"); - assert_eq!(loaded[0].subject, "Test email"); + assert_eq!(loaded[0].folder_id, "inbox"); assert!(loaded[0].unread); assert!(!loaded[0].has_details); } + #[test] + fn test_metadata_is_per_folder() { + let store = open_memory_store(); + store.upsert_mail_metadata(&meta("a", "inbox", 1)).unwrap(); + store.upsert_mail_metadata(&meta("b", "custom1", 2)).unwrap(); + + assert_eq!(store.load_folder_metadata("inbox").unwrap().len(), 1); + assert_eq!(store.load_folder_metadata("custom1").unwrap().len(), 1); + assert_eq!(store.load_folder_metadata("missing").unwrap().len(), 0); + } + #[test] fn test_batch_upsert() { let store = open_memory_store(); let metas: Vec = (0..100) - .map(|i| MailMetadata { - element_id: format!("mail_{i}"), - list_id: "list1".into(), - folder_kind: kind_to_i64(MailSetKind::Inbox), - subject: format!("Subject {i}"), - sender_name: "Test".into(), - sender_address: "test@test.com".into(), - received_date_ms: 1700000000000 + i, - unread: i % 2 == 0, - has_details: false, - mail_json: "{}".into(), - }) + .map(|i| meta(&format!("mail_{i}"), "inbox", 1700000000000 + i)) .collect(); store.upsert_mail_metadata_batch(&metas).unwrap(); - let loaded = store.load_folder_metadata(MailSetKind::Inbox).unwrap(); - assert_eq!(loaded.len(), 100); - assert_eq!(store.mail_count(MailSetKind::Inbox).unwrap(), 100); + assert_eq!(store.load_folder_metadata("inbox").unwrap().len(), 100); + assert_eq!(store.mail_count("inbox").unwrap(), 100); assert_eq!(store.total_count().unwrap(), 100); } @@ -429,28 +435,16 @@ mod tests { fn test_delete_mails_not_in() { let store = open_memory_store(); let metas: Vec = (0..5) - .map(|i| MailMetadata { - element_id: format!("mail_{i}"), - list_id: "list1".into(), - folder_kind: kind_to_i64(MailSetKind::Inbox), - subject: format!("Subject {i}"), - sender_name: "Test".into(), - sender_address: "test@test.com".into(), - received_date_ms: 1700000000000 + i, - unread: false, - has_details: false, - mail_json: "{}".into(), - }) + .map(|i| meta(&format!("mail_{i}"), "inbox", 1700000000000 + i)) .collect(); store.upsert_mail_metadata_batch(&metas).unwrap(); let keep = vec!["mail_0", "mail_2", "mail_4"]; - let deleted = store.delete_mails_not_in(MailSetKind::Inbox, &keep).unwrap(); + let deleted = store.delete_mails_not_in("inbox", &keep).unwrap(); assert_eq!(deleted.len(), 2); assert!(deleted.contains(&"mail_1".to_string())); assert!(deleted.contains(&"mail_3".to_string())); - - assert_eq!(store.mail_count(MailSetKind::Inbox).unwrap(), 3); + assert_eq!(store.mail_count("inbox").unwrap(), 3); } #[test] @@ -458,16 +452,13 @@ mod tests { let store = open_memory_store(); let rfc2822 = "From: test@example.com\r\nSubject: Hello\r\n\r\nBody text here"; store.write_eml("test_mail", rfc2822).unwrap(); - - let read_back = store.read_eml("test_mail").unwrap(); - assert_eq!(read_back, Some(rfc2822.to_string())); + assert_eq!(store.read_eml("test_mail").unwrap(), Some(rfc2822.to_string())); } #[test] fn test_eml_read_nonexistent() { let store = open_memory_store(); - let result = store.read_eml("nonexistent").unwrap(); - assert_eq!(result, None); + assert_eq!(store.read_eml("nonexistent").unwrap(), None); } #[test] @@ -475,7 +466,6 @@ mod tests { let store = open_memory_store(); store.write_eml("to_delete", "content").unwrap(); assert!(store.read_eml("to_delete").unwrap().is_some()); - store.delete_eml("to_delete").unwrap(); assert!(store.read_eml("to_delete").unwrap().is_none()); } @@ -483,19 +473,7 @@ mod tests { #[test] fn test_reset() { let store = open_memory_store(); - let meta = MailMetadata { - element_id: "abc".into(), - list_id: "list1".into(), - folder_kind: kind_to_i64(MailSetKind::Inbox), - subject: "Test".into(), - sender_name: "".into(), - sender_address: "test@test.com".into(), - received_date_ms: 0, - unread: false, - has_details: true, - mail_json: "{}".into(), - }; - store.upsert_mail_metadata(&meta).unwrap(); + store.upsert_mail_metadata(&meta("abc", "inbox", 0)).unwrap(); store.write_eml("abc", "content").unwrap(); store.reset().unwrap(); @@ -507,26 +485,10 @@ mod tests { #[test] fn test_mark_has_details() { let store = open_memory_store(); - let meta = MailMetadata { - element_id: "det".into(), - list_id: "list1".into(), - folder_kind: kind_to_i64(MailSetKind::Inbox), - subject: "Test".into(), - sender_name: "".into(), - sender_address: "t@t.com".into(), - received_date_ms: 0, - unread: false, - has_details: false, - mail_json: "{}".into(), - }; - store.upsert_mail_metadata(&meta).unwrap(); - - let loaded = store.load_folder_metadata(MailSetKind::Inbox).unwrap(); - assert!(!loaded[0].has_details); + store.upsert_mail_metadata(&meta("det", "inbox", 0)).unwrap(); + assert!(!store.load_folder_metadata("inbox").unwrap()[0].has_details); store.mark_has_details("det").unwrap(); - - let loaded = store.load_folder_metadata(MailSetKind::Inbox).unwrap(); - assert!(loaded[0].has_details); + assert!(store.load_folder_metadata("inbox").unwrap()[0].has_details); } } diff --git a/crates/bridge/src/sync.rs b/crates/bridge/src/sync.rs index 9f5b901..ace4977 100644 --- a/crates/bridge/src/sync.rs +++ b/crates/bridge/src/sync.rs @@ -1,23 +1,14 @@ +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use log::{info, warn, debug}; +use log::{debug, info, warn}; use tokio::sync::{watch, RwLock}; use tutasdk::entities::generated::tutanota::{Mail, MailDetails}; -use tutasdk::folder_system::MailSetKind; use crate::mail::mail_to_rfc2822; use crate::store::{LocalStore, MailMetadata}; -use crate::tuta::MailBackend; - -const FOLDERS: &[MailSetKind] = &[ - MailSetKind::Inbox, - MailSetKind::Sent, - MailSetKind::Draft, - MailSetKind::Trash, - MailSetKind::Archive, - MailSetKind::Spam, -]; +use crate::tuta::{FolderInfo, MailBackend}; const INTER_REQUEST_DELAY: Duration = Duration::from_millis(150); const INTER_FOLDER_DELAY: Duration = Duration::from_millis(300); @@ -32,7 +23,10 @@ pub struct StoredMail { } pub struct MailStore { - folders: RwLock)>>, + /// folder id → mails in that folder. + folders: RwLock>>, + /// The folder list (system + custom), for IMAP enumeration. + folder_list: RwLock>, generation: watch::Sender, gen_counter: std::sync::atomic::AtomicU64, } @@ -41,7 +35,8 @@ impl MailStore { pub fn new() -> Arc { let (tx, _) = watch::channel(0u64); Arc::new(Self { - folders: RwLock::new(Vec::new()), + folders: RwLock::new(HashMap::new()), + folder_list: RwLock::new(Vec::new()), generation: tx, gen_counter: std::sync::atomic::AtomicU64::new(0), }) @@ -52,64 +47,79 @@ impl MailStore { } pub async fn total_mail_count(&self) -> usize { - self.folders.read().await.iter().map(|(_, v)| v.len()).sum() + self.folders.read().await.values().map(|v| v.len()).sum() } - pub async fn folder_count(&self, kind: MailSetKind) -> usize { + pub async fn folder_count(&self, folder_id: &str) -> usize { self.folders .read() .await - .iter() - .find(|(k, _)| *k == kind) - .map(|(_, v)| v.len()) + .get(folder_id) + .map(|v| v.len()) .unwrap_or(0) } - pub async fn get_folder(&self, kind: MailSetKind) -> Vec { + pub async fn get_folder(&self, folder_id: &str) -> Vec { self.folders .read() .await - .iter() - .find(|(k, _)| *k == kind) - .map(|(_, v)| v.clone()) + .get(folder_id) + .cloned() .unwrap_or_default() } - pub async fn get_details(&self, kind: MailSetKind, element_id: &str) -> Option<(MailDetails, String)> { + pub async fn get_details( + &self, + folder_id: &str, + element_id: &str, + ) -> Option<(MailDetails, String)> { let folders = self.folders.read().await; - let (_, folder) = folders.iter().find(|(k, _)| *k == kind)?; + let folder = folders.get(folder_id)?; folder.iter().find_map(|m| { let eid = m.mail._id.as_ref()?.element_id.to_string(); if eid == element_id { - let details = m.details.clone()?; - let rfc = m.rfc2822.clone()?; - Some((details, rfc)) + Some((m.details.clone()?, m.rfc2822.clone()?)) } else { None } }) } - pub(crate) async fn set_folder(&self, kind: MailSetKind, mails: Vec) { - let mut folders = self.folders.write().await; - if let Some(entry) = folders.iter_mut().find(|(k, _)| *k == kind) { - entry.1 = mails; - } else { - folders.push((kind, mails)); - } - drop(folders); + /// The current folder list (system + custom). + pub async fn list_folders(&self) -> Vec { + self.folder_list.read().await.clone() + } + + /// Look up a folder by its IMAP path (case-insensitive for INBOX). + pub async fn folder_by_imap_path(&self, path: &str) -> Option { + let list = self.folder_list.read().await; + list.iter() + .find(|f| f.imap_path == path || f.imap_path.eq_ignore_ascii_case(path)) + .cloned() + } + + pub(crate) async fn set_folder_list(&self, folders: Vec) { + *self.folder_list.write().await = folders; + self.bump_generation(); + } + + pub(crate) async fn set_folder(&self, folder_id: &str, mails: Vec) { + self.folders + .write() + .await + .insert(folder_id.to_string(), mails); self.bump_generation(); } async fn update_mail_details( &self, - kind: MailSetKind, + folder_id: &str, element_id: &str, details: MailDetails, rfc2822: String, ) { let mut folders = self.folders.write().await; - if let Some((_, folder)) = folders.iter_mut().find(|(k, _)| *k == kind) { + if let Some(folder) = folders.get_mut(folder_id) { if let Some(m) = folder.iter_mut().find(|m| { m.mail ._id @@ -127,7 +137,10 @@ impl MailStore { } fn bump_generation(&self) { - let gen = self.gen_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + let gen = self + .gen_counter + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; self.generation.send_replace(gen); } } @@ -139,16 +152,33 @@ pub async fn run_syncer( sync_limit: usize, mut shutdown: watch::Receiver, ) { - info!("Mail syncer started (limit={})", if sync_limit == 0 { "all".to_string() } else { sync_limit.to_string() }); + info!( + "Mail syncer started (limit={})", + if sync_limit == 0 { + "all".to_string() + } else { + sync_limit.to_string() + } + ); - // Phase 0: load cached mails from local store into memory - for &kind in FOLDERS { - match load_cached_folder(&store, &local_store, kind).await { + // Fetch the folder list first; everything is keyed off it. + let folders = match retry(|| backend.list_folders()).await { + Ok(folders) => folders, + Err(e) => { + warn!("Could not load folder list: {e}"); + Vec::new() + } + }; + store.set_folder_list(folders.clone()).await; + + // Phase 0: load cached mails from the local store into memory. + for folder in &folders { + match load_cached_folder(&store, &local_store, folder).await { Ok(count) if count > 0 => { - info!("Loaded {} cached mails for {:?}", count, kind); + info!("Loaded {} cached mails for {}", count, folder.imap_path); } Ok(_) => {} - Err(e) => warn!("Failed to load cache for {:?}: {}", kind, e), + Err(e) => warn!("Failed to load cache for {}: {}", folder.imap_path, e), } } @@ -157,30 +187,41 @@ pub async fn run_syncer( loop { let mut had_error = false; - // Phase 1: sync mail lists for ALL folders (fast, no body loading) - for &kind in FOLDERS { + // Refresh the folder list each cycle (custom folders can change). + let folders = match retry(|| backend.list_folders()).await { + Ok(folders) => { + store.set_folder_list(folders.clone()).await; + folders + } + Err(e) => { + warn!("Failed to refresh folder list: {e}"); + had_error = true; + store.list_folders().await + } + }; + + // Phase 1: sync mail lists for ALL folders (fast, no body loading). + for folder in &folders { if *shutdown.borrow() { info!("Mail syncer shutting down"); return; } - - match sync_folder(&store, &local_store, &*backend, kind, sync_limit).await { + match sync_folder(&store, &local_store, &*backend, folder, sync_limit).await { Ok(()) => {} Err(e) => { - warn!("Sync error for {:?}: {}", kind, e); + warn!("Sync error for {}: {}", folder.imap_path, e); had_error = true; } } - tokio::time::sleep(INTER_FOLDER_DELAY).await; } - // Phase 2: prefetch mail details (slow, but all folders are already visible) - for &kind in FOLDERS { + // Phase 2: prefetch mail details (slow, but all folders are visible). + for folder in &folders { if *shutdown.borrow() { return; } - prefetch_details(&store, &local_store, &*backend, kind).await; + prefetch_details(&store, &local_store, &*backend, folder).await; } if had_error { @@ -206,10 +247,10 @@ pub async fn run_syncer( async fn load_cached_folder( store: &MailStore, local_store: &LocalStore, - kind: MailSetKind, + folder: &FolderInfo, ) -> Result { let metas = local_store - .load_folder_metadata(kind) + .load_folder_metadata(&folder.id) .map_err(|e| format!("{e}"))?; if metas.is_empty() { @@ -242,7 +283,7 @@ async fn load_cached_folder( } let count = stored_mails.len(); - store.set_folder(kind, stored_mails).await; + store.set_folder(&folder.id, stored_mails).await; Ok(count) } @@ -250,13 +291,13 @@ async fn sync_folder( store: &MailStore, local_store: &LocalStore, backend: &dyn MailBackend, - kind: MailSetKind, + folder: &FolderInfo, limit: usize, ) -> Result<(), String> { - let new_mails = retry(|| backend.load_mail_ids_for_folder(kind, limit)).await?; + let new_mails = retry(|| backend.load_mail_ids_for_folder(folder, limit)).await?; - let existing = store.get_folder(kind).await; - let existing_map: std::collections::HashMap = existing + let existing = store.get_folder(&folder.id).await; + let existing_map: HashMap = existing .into_iter() .filter_map(|m| { let eid = m.mail._id.as_ref()?.element_id.to_string(); @@ -284,20 +325,18 @@ async fn sync_folder( }); } - metas_to_upsert.push(mail_to_metadata(mail, kind)); + metas_to_upsert.push(mail_to_metadata(mail, &folder.id)); } - // Persist metadata to local store if let Err(e) = local_store.upsert_mail_metadata_batch(&metas_to_upsert) { - warn!("Failed to persist metadata for {:?}: {}", kind, e); + warn!("Failed to persist metadata for {}: {}", folder.imap_path, e); } - // Delete mails removed from server let current_ids: Vec<&str> = new_mails .iter() .filter_map(|m| m._id.as_ref().map(|id| id.element_id.as_str())) .collect(); - match local_store.delete_mails_not_in(kind, ¤t_ids) { + match local_store.delete_mails_not_in(&folder.id, ¤t_ids) { Ok(deleted) => { for eid in &deleted { if let Err(e) = local_store.delete_eml(eid) { @@ -305,13 +344,20 @@ async fn sync_folder( } } if !deleted.is_empty() { - debug!("Removed {} deleted mails from {:?} cache", deleted.len(), kind); + debug!( + "Removed {} deleted mails from {} cache", + deleted.len(), + folder.imap_path + ); } } - Err(e) => warn!("Failed to clean up deleted mails for {:?}: {}", kind, e), + Err(e) => warn!( + "Failed to clean up deleted mails for {}: {}", + folder.imap_path, e + ), } - store.set_folder(kind, updated).await; + store.set_folder(&folder.id, updated).await; Ok(()) } @@ -320,10 +366,10 @@ async fn prefetch_details( store: &MailStore, local_store: &LocalStore, backend: &dyn MailBackend, - kind: MailSetKind, + folder: &FolderInfo, ) { - let folder = store.get_folder(kind).await; - let api_needed: Vec = folder + let mails = store.get_folder(&folder.id).await; + let api_needed: Vec = mails .into_iter() .filter(|m| m.details.is_none()) .filter_map(|m| { @@ -340,7 +386,11 @@ async fn prefetch_details( return; } - debug!("Pre-fetching {} mail details for {:?}", api_needed.len(), kind); + debug!( + "Pre-fetching {} mail details for {}", + api_needed.len(), + folder.imap_path + ); for mail in &api_needed { tokio::time::sleep(INTER_REQUEST_DELAY).await; @@ -360,7 +410,7 @@ async fn prefetch_details( } store - .update_mail_details(kind, &eid, details, rfc2822) + .update_mail_details(&folder.id, &eid, details, rfc2822) .await; } } @@ -394,7 +444,7 @@ where unreachable!() } -fn mail_to_metadata(mail: &Mail, kind: MailSetKind) -> MailMetadata { +fn mail_to_metadata(mail: &Mail, folder_id: &str) -> MailMetadata { let (list_id, element_id) = mail ._id .as_ref() @@ -406,7 +456,7 @@ fn mail_to_metadata(mail: &Mail, kind: MailSetKind) -> MailMetadata { MailMetadata { list_id, element_id, - folder_kind: kind as i64, + folder_id: folder_id.to_string(), subject: mail.subject.clone(), sender_name: mail.sender.name.clone(), sender_address: mail.sender.address.clone(), diff --git a/crates/bridge/src/tuta.rs b/crates/bridge/src/tuta.rs index 4a5c587..af95278 100644 --- a/crates/bridge/src/tuta.rs +++ b/crates/bridge/src/tuta.rs @@ -19,16 +19,61 @@ use tutasdk::{ApiCallError, CustomId, IdTupleGenerated, ListLoadDirection, Logge use crate::config::Config; use crate::mail::ParsedMessage; +/// A mail folder as seen by the bridge, keyed by its stable Tuta `MailSet` +/// element id rather than by a system folder kind (so custom/nested folders +/// are first-class). +#[derive(Clone, Debug)] +pub struct FolderInfo { + /// `MailSet` element id — the stable key used everywhere. + pub id: String, + /// `MailSet.entries` list id — used to load the mails in this folder. + pub entries_list_id: String, + pub kind: MailSetKind, + /// IMAP mailbox path, e.g. `INBOX`, `Sent`, `Work/Projects`. + pub imap_path: String, + /// RFC 6154 special-use attribute, e.g. `\Sent` (system folders only). + pub special_use: Option, +} + +/// IMAP hierarchy delimiter used to build nested folder paths. +pub const IMAP_DELIMITER: char = '/'; + #[async_trait::async_trait] pub trait MailBackend: Send + Sync { - async fn load_mail_ids_for_folder(&self, kind: MailSetKind, limit: usize) -> Result, String>; + async fn load_mail_ids_for_folder(&self, folder: &FolderInfo, limit: usize) -> Result, String>; async fn load_mail_details(&self, mail: &Mail) -> Result, String>; - async fn load_folder_list(&self) -> Result, String>; + /// Enumerate all mail folders (system + custom, with hierarchy). + async fn list_folders(&self) -> Result, String>; async fn set_unread_status(&self, mail_ids: Vec, unread: bool) -> Result<(), String>; async fn trash_mails(&self, mail_ids: Vec) -> Result<(), String>; async fn send_mail(&self, msg: &ParsedMessage) -> Result<(), String>; } +/// Canonical IMAP name for a system folder, or `None` for custom/unsupported. +fn system_imap_name(kind: MailSetKind) -> Option<&'static str> { + match kind { + MailSetKind::Inbox => Some("INBOX"), + MailSetKind::Sent => Some("Sent"), + MailSetKind::Draft => Some("Drafts"), + MailSetKind::Trash => Some("Trash"), + MailSetKind::Archive => Some("Archive"), + MailSetKind::Spam => Some("Spam"), + _ => None, + } +} + +/// RFC 6154 special-use attribute for a system folder. +fn system_special_use(kind: MailSetKind) -> Option<&'static str> { + match kind { + MailSetKind::Sent => Some("\\Sent"), + MailSetKind::Draft => Some("\\Drafts"), + MailSetKind::Trash => Some("\\Trash"), + MailSetKind::Archive => Some("\\Archive"), + MailSetKind::Spam => Some("\\Junk"), + _ => None, + } +} + pub struct TutaSession { pub logged_in: Arc, pub email: String, @@ -68,17 +113,10 @@ impl TutaSession { async fn load_mail_ids_for_folder_impl( &self, - folder_kind: MailSetKind, + entries_list_id: &tutasdk::GeneratedId, limit: usize, ) -> Result, ApiCallError> { - let mailbox = self.load_mailbox().await?; - let folders = self.load_folders(&mailbox).await?; - let folder = folders - .system_folder_by_type(folder_kind) - .ok_or_else(|| ApiCallError::internal(format!("Folder {:?} not found", folder_kind)))?; - let count = if limit == 0 { 1000 } else { limit }; - let entries_list_id = &folder.entries; let entries: Vec = self .crypto_client() .load_range( @@ -193,8 +231,9 @@ impl TutaSession { #[async_trait::async_trait] impl MailBackend for TutaSession { - async fn load_mail_ids_for_folder(&self, kind: MailSetKind, limit: usize) -> Result, String> { - self.load_mail_ids_for_folder_impl(kind, limit) + async fn load_mail_ids_for_folder(&self, folder: &FolderInfo, limit: usize) -> Result, String> { + let entries_list_id = tutasdk::GeneratedId(folder.entries_list_id.clone()); + self.load_mail_ids_for_folder_impl(&entries_list_id, limit) .await .map_err(|e| format!("{e}")) } @@ -205,24 +244,48 @@ impl MailBackend for TutaSession { .map_err(|e| format!("{e}")) } - async fn load_folder_list(&self) -> Result, String> { + async fn list_folders(&self) -> Result, String> { let mailbox = self.load_mailbox().await.map_err(|e| format!("{e}"))?; let folder_system = self.load_folders(&mailbox).await.map_err(|e| format!("{e}"))?; - let known_folders = [ - (MailSetKind::Inbox, "INBOX", ""), - (MailSetKind::Sent, "Sent", "\\Sent"), - (MailSetKind::Draft, "Drafts", "\\Drafts"), - (MailSetKind::Trash, "Trash", "\\Trash"), - (MailSetKind::Archive, "Archive", "\\Archive"), - (MailSetKind::Spam, "Spam", "\\Junk"), - ]; - let mut result = Vec::new(); - for (kind, name, flags) in &known_folders { - if folder_system.system_folder_by_type(*kind).is_some() { - result.push((name.to_string(), flags.to_string())); + for indented in folder_system.indented_list() { + let folder = indented.folder; + let kind = folder.mail_set_kind(); + + // Only expose folder types we support over IMAP. + let is_custom = kind == MailSetKind::Custom; + if !is_custom && system_imap_name(kind).is_none() { + continue; // skip Scheduled / virtual sets } + + let Some(elem_id) = folder._id.as_ref().map(|id| id.element_id.to_string()) else { + continue; + }; + + // Build the IMAP path by mapping each ancestor segment. + let mut segments: Vec = Vec::new(); + for ancestor in folder_system.path_to_folder(&tutasdk::GeneratedId(elem_id.clone())) { + let akind = ancestor.mail_set_kind(); + if let Some(name) = system_imap_name(akind) { + segments.push(name.to_string()); + } else { + // sanitize the delimiter out of custom names + segments.push(ancestor.name.replace(IMAP_DELIMITER, "_")); + } + } + let imap_path = segments.join(&IMAP_DELIMITER.to_string()); + if imap_path.is_empty() { + continue; + } + + result.push(FolderInfo { + id: elem_id, + entries_list_id: folder.entries.to_string(), + kind, + imap_path, + special_use: system_special_use(kind).map(|s| s.to_string()), + }); } Ok(result) } diff --git a/tuta-repo b/tuta-repo index e5d2bd1..cc429be 160000 --- a/tuta-repo +++ b/tuta-repo @@ -1 +1 @@ -Subproject commit e5d2bd1545d7f7bd2b4bf030977fecd1c22b6d71 +Subproject commit cc429beaedec629b92142b3c8e5e9653b1449c8a