diff --git a/crates/bridge/src/bridge.rs b/crates/bridge/src/bridge.rs index a9b2b57..ecdce32 100644 --- a/crates/bridge/src/bridge.rs +++ b/crates/bridge/src/bridge.rs @@ -365,7 +365,7 @@ impl BridgeHandle { }; let handler_handle = tokio::spawn(event_handler::run_event_handler( store.clone(), - local_store, + local_store.clone(), backend.clone(), bus_ids_for_handler, event_rx, @@ -375,6 +375,7 @@ impl BridgeHandle { imap_port, store.clone(), backend.clone(), + local_store, imap_tls, pw.clone(), )); diff --git a/crates/bridge/src/imap/mod.rs b/crates/bridge/src/imap/mod.rs index c320ee4..b52487a 100644 --- a/crates/bridge/src/imap/mod.rs +++ b/crates/bridge/src/imap/mod.rs @@ -9,6 +9,7 @@ use tokio::net::TcpListener; use tokio::sync::watch; use tokio_rustls::TlsAcceptor; +use crate::store::LocalStore; use crate::sync::MailStore; use crate::tuta::MailBackend; use session::ImapSession; @@ -17,6 +18,7 @@ pub async fn serve( port: u16, store: Arc, backend: Arc, + local_store: Arc, tls: TlsAcceptor, password_hash: Option, ) -> Result<(), Box> { @@ -28,13 +30,16 @@ pub async fn serve( debug!("IMAP connection from {}", addr); let store = store.clone(); let backend = backend.clone(); + let local_store = local_store.clone(); let tls = tls.clone(); let pw_hash = password_hash.clone(); tokio::spawn(async move { match tls.accept(stream).await { Ok(tls_stream) => { - if let Err(e) = handle_connection(tls_stream, store, backend, pw_hash).await { + if let Err(e) = + handle_connection(tls_stream, store, backend, local_store, pw_hash).await + { error!("IMAP connection error: {}", e); } } @@ -50,12 +55,13 @@ async fn handle_connection( stream: tokio_rustls::server::TlsStream, store: Arc, backend: Arc, + local_store: Arc, password_hash: Option, ) -> Result<(), Box> { let (reader, mut writer) = tokio::io::split(stream); let mut reader = BufReader::new(reader); let mut store_watch: watch::Receiver = store.subscribe(); - let mut session = ImapSession::new(store, backend, password_hash); + let mut session = ImapSession::new(store, backend, password_hash, Some(local_store)); writer .write_all(b"* OK TutaBridge IMAP4rev1 ready\r\n") diff --git a/crates/bridge/src/imap/search.rs b/crates/bridge/src/imap/search.rs index 224d76f..5b2433a 100644 --- a/crates/bridge/src/imap/search.rs +++ b/crates/bridge/src/imap/search.rs @@ -7,14 +7,17 @@ //! //! Coverage is metadata-first: subject / from / to / cc, flags, dates, sizes, //! sequence + UID sets, and boolean composition (`AND` / `OR` / `NOT`). `BODY` -//! and `TEXT` match against whatever body text the session already has decoded -//! — full-text body search over the whole mailbox is a later increment backed -//! by an on-disk index. +//! and `TEXT` are resolved against the on-disk full-text index: the session +//! queries it once per distinct term ([`collect_body_terms`]) and hands the +//! per-term hit sets to [`matches`] via a [`SearchContext`]. A body term only +//! matches messages whose body has actually been downloaded and indexed. //! //! Robustness rule: an unrecognised criterion degrades to a non-restrictive //! match (it never *hides* messages). Over-inclusion is the safe failure for //! search; silently dropping a matching mail is not. +use std::collections::{HashMap, HashSet}; + /// One element of an IMAP sequence/UID set, e.g. `1`, `3:9`, or `5:*`. #[derive(Debug, Clone, PartialEq, Eq)] struct SeqRange { @@ -107,6 +110,8 @@ pub enum SearchKey { pub struct MsgView<'a> { pub seq: u32, pub uid: u32, + /// Tuta element id — the key used to look up full-text body hits. + pub element_id: &'a str, pub subject: &'a str, /// Formatted `From` (name + address), for `FROM` substring matching. pub from: String, @@ -122,9 +127,49 @@ pub struct MsgView<'a> { pub unread: bool, pub deleted: bool, pub size: u64, - /// Decoded body text, when the session already has it. `None` means the - /// body isn't loaded; `BODY`/`TEXT` simply won't match such a message yet. - pub body: Option<&'a str>, +} + +/// Pre-resolved full-text results for one SEARCH: maps each `BODY`/`TEXT` term +/// to the set of element ids whose indexed body matched it. Built by the +/// session from the on-disk index before evaluating the query. +#[derive(Default)] +pub struct SearchContext { + pub body_hits: HashMap>, +} + +impl SearchContext { + pub fn empty() -> Self { + Self::default() + } + + fn body_matches(&self, term: &str, element_id: &str) -> bool { + self.body_hits + .get(term) + .is_some_and(|ids| ids.contains(element_id)) + } +} + +/// Collect the distinct `BODY`/`TEXT` term arguments in a parsed query, so the +/// session can resolve each against the full-text index exactly once. +pub fn collect_body_terms(key: &SearchKey) -> Vec { + let mut terms = Vec::new(); + collect_body_terms_into(key, &mut terms); + terms.sort(); + terms.dedup(); + terms +} + +fn collect_body_terms_into(key: &SearchKey, out: &mut Vec) { + match key { + SearchKey::And(keys) => keys.iter().for_each(|k| collect_body_terms_into(k, out)), + SearchKey::Or(a, b) => { + collect_body_terms_into(a, out); + collect_body_terms_into(b, out); + } + SearchKey::Not(k) => collect_body_terms_into(k, out), + SearchKey::Body(s) | SearchKey::Text(s) => out.push(s.clone()), + _ => {} + } } fn contains_ci(haystack: &str, needle: &str) -> bool { @@ -154,13 +199,14 @@ fn day_number(ms: u64) -> i64 { (ms / 86_400_000) as i64 } -/// Evaluate a parsed query against one message. -pub fn matches(key: &SearchKey, m: &MsgView) -> bool { +/// Evaluate a parsed query against one message, consulting `ctx` for the +/// full-text results of any `BODY`/`TEXT` terms. +pub fn matches(key: &SearchKey, m: &MsgView, ctx: &SearchContext) -> bool { match key { SearchKey::All => true, - SearchKey::And(keys) => keys.iter().all(|k| matches(k, m)), - SearchKey::Or(a, b) => matches(a, m) || matches(b, m), - SearchKey::Not(k) => !matches(k, m), + SearchKey::And(keys) => keys.iter().all(|k| matches(k, m, ctx)), + SearchKey::Or(a, b) => matches(a, m, ctx) || matches(b, m, ctx), + SearchKey::Not(k) => !matches(k, m, ctx), SearchKey::Seen => !m.unread, SearchKey::Unseen => m.unread, @@ -183,10 +229,8 @@ pub fn matches(key: &SearchKey, m: &MsgView) -> bool { SearchKey::To(s) => contains_ci(&m.to, s), SearchKey::Cc(s) => contains_ci(&m.cc, s), SearchKey::Bcc(s) => contains_ci(&m.bcc, s), - SearchKey::Body(s) => m.body.is_some_and(|b| contains_ci(b, s)), - SearchKey::Text(s) => { - contains_ci(&m.headers, s) || m.body.is_some_and(|b| contains_ci(b, s)) - } + SearchKey::Body(s) => ctx.body_matches(s, m.element_id), + SearchKey::Text(s) => contains_ci(&m.headers, s) || ctx.body_matches(s, m.element_id), SearchKey::Header(name, val) => header_contains(&m.headers, name, val), SearchKey::Before(d) => day_number(m.date_ms) < *d, @@ -511,6 +555,7 @@ mod tests { MsgView { seq: 1, uid: 10, + element_id: "mail1", subject: "Hello World", from: "Alice ".into(), to: "Bob ".into(), @@ -524,10 +569,22 @@ mod tests { unread: true, deleted: false, size: 5000, - body: Some("the quick brown fox"), } } + /// A context where the given terms all hit our test message ("mail1"). + fn ctx_hitting(terms: &[&str]) -> SearchContext { + let mut body_hits = HashMap::new(); + for t in terms { + body_hits.insert((*t).to_string(), HashSet::from(["mail1".to_string()])); + } + SearchContext { body_hits } + } + + fn no_ctx() -> SearchContext { + SearchContext::empty() + } + // --- tokenizer --- #[test] @@ -655,101 +712,141 @@ mod tests { #[test] fn match_subject_ci() { - assert!(matches(&SearchKey::Subject("hello".into()), &view())); - assert!(matches(&SearchKey::Subject("WORLD".into()), &view())); - assert!(!matches(&SearchKey::Subject("nope".into()), &view())); + let c = no_ctx(); + assert!(matches(&SearchKey::Subject("hello".into()), &view(), &c)); + assert!(matches(&SearchKey::Subject("WORLD".into()), &view(), &c)); + assert!(!matches(&SearchKey::Subject("nope".into()), &view(), &c)); } #[test] fn match_from_to() { - assert!(matches(&SearchKey::From("alice".into()), &view())); - assert!(matches(&SearchKey::To("bob@example".into()), &view())); - assert!(!matches(&SearchKey::Cc("anyone".into()), &view())); + let c = no_ctx(); + assert!(matches(&SearchKey::From("alice".into()), &view(), &c)); + assert!(matches(&SearchKey::To("bob@example".into()), &view(), &c)); + assert!(!matches(&SearchKey::Cc("anyone".into()), &view(), &c)); } #[test] fn match_flags_consistent_with_fetch() { let v = view(); // unread, not deleted - assert!(matches(&SearchKey::Unseen, &v)); - assert!(!matches(&SearchKey::Seen, &v)); - assert!(!matches(&SearchKey::Answered, &v)); - assert!(matches(&SearchKey::Unanswered, &v)); - assert!(!matches(&SearchKey::Flagged, &v)); - assert!(!matches(&SearchKey::Deleted, &v)); - assert!(matches(&SearchKey::Undeleted, &v)); + let c = no_ctx(); + assert!(matches(&SearchKey::Unseen, &v, &c)); + assert!(!matches(&SearchKey::Seen, &v, &c)); + assert!(!matches(&SearchKey::Answered, &v, &c)); + assert!(matches(&SearchKey::Unanswered, &v, &c)); + assert!(!matches(&SearchKey::Flagged, &v, &c)); + assert!(!matches(&SearchKey::Deleted, &v, &c)); + assert!(matches(&SearchKey::Undeleted, &v, &c)); } #[test] fn match_dates() { let v = view(); // 2022-09-22 - assert!(matches(&SearchKey::Since(day(2022, 1, 1)), &v)); - assert!(matches(&SearchKey::Before(day(2023, 1, 1)), &v)); - assert!(matches(&SearchKey::On(day(2022, 9, 22)), &v)); - assert!(!matches(&SearchKey::On(day(2022, 9, 23)), &v)); - assert!(!matches(&SearchKey::Since(day(2023, 1, 1)), &v)); + let c = no_ctx(); + assert!(matches(&SearchKey::Since(day(2022, 1, 1)), &v, &c)); + assert!(matches(&SearchKey::Before(day(2023, 1, 1)), &v, &c)); + assert!(matches(&SearchKey::On(day(2022, 9, 22)), &v, &c)); + assert!(!matches(&SearchKey::On(day(2022, 9, 23)), &v, &c)); + assert!(!matches(&SearchKey::Since(day(2023, 1, 1)), &v, &c)); } #[test] fn match_size() { let v = view(); // size 5000 - assert!(matches(&SearchKey::Larger(4000), &v)); - assert!(!matches(&SearchKey::Larger(6000), &v)); - assert!(matches(&SearchKey::Smaller(6000), &v)); + let c = no_ctx(); + assert!(matches(&SearchKey::Larger(4000), &v, &c)); + assert!(!matches(&SearchKey::Larger(6000), &v, &c)); + assert!(matches(&SearchKey::Smaller(6000), &v, &c)); } #[test] - fn match_body_and_text() { + fn match_body_uses_fts_context() { let v = view(); - assert!(matches(&SearchKey::Body("brown".into()), &v)); - assert!(!matches(&SearchKey::Body("missing".into()), &v)); - // TEXT spans headers + body. - assert!(matches(&SearchKey::Text("Message-ID".into()), &v)); - assert!(matches(&SearchKey::Text("quick".into()), &v)); + let c = ctx_hitting(&["brown"]); + assert!(matches(&SearchKey::Body("brown".into()), &v, &c)); + // A term with no FTS hit doesn't match, even though it's a real word. + assert!(!matches(&SearchKey::Body("missing".into()), &v, &c)); } #[test] - fn match_body_without_loaded_body_never_matches() { - let mut v = view(); - v.body = None; - assert!(!matches(&SearchKey::Body("brown".into()), &v)); - // TEXT still matches on headers. - assert!(matches(&SearchKey::Text("Subject".into()), &v)); + fn match_text_spans_headers_and_body() { + let v = view(); + let c = ctx_hitting(&["quick"]); + // Header hit, no body hit needed. + assert!(matches( + &SearchKey::Text("Message-ID".into()), + &v, + &no_ctx() + )); + // Body hit via the FTS context. + assert!(matches(&SearchKey::Text("quick".into()), &v, &c)); + // Neither headers nor index: no match. + assert!(!matches(&SearchKey::Text("quick".into()), &v, &no_ctx())); + } + + #[test] + fn match_body_without_index_never_matches() { + let v = view(); + assert!(!matches(&SearchKey::Body("brown".into()), &v, &no_ctx())); + // TEXT still matches on headers without any index. + assert!(matches(&SearchKey::Text("Subject".into()), &v, &no_ctx())); + } + + #[test] + fn collect_body_terms_finds_nested_terms() { + let k = parse(r#"OR BODY "alpha" (TEXT "beta" SUBJECT "x") NOT BODY "alpha""#); + // Note: top level is `OR ` then trailing keys folded into AND. + let terms = collect_body_terms(&k); + assert_eq!(terms, vec!["alpha".to_string(), "beta".to_string()]); } #[test] fn match_header() { let v = view(); + let c = no_ctx(); assert!(matches( &SearchKey::Header("message-id".into(), "abc".into()), - &v + &v, + &c )); assert!(!matches( &SearchKey::Header("message-id".into(), "zzz".into()), - &v + &v, + &c )); // Presence-only (empty value). - assert!(matches(&SearchKey::Header("subject".into(), "".into()), &v)); - assert!(!matches(&SearchKey::Header("x-nope".into(), "".into()), &v)); + assert!(matches( + &SearchKey::Header("subject".into(), "".into()), + &v, + &c + )); + assert!(!matches( + &SearchKey::Header("x-nope".into(), "".into()), + &v, + &c + )); } #[test] fn match_uid_and_seq() { let v = view(); // seq 1, uid 10 - assert!(matches(&SearchKey::Uid(parse_seqset("5:15")), &v)); - assert!(!matches(&SearchKey::Uid(parse_seqset("1:5")), &v)); - assert!(matches(&SearchKey::Sequence(parse_seqset("1")), &v)); + let c = no_ctx(); + assert!(matches(&SearchKey::Uid(parse_seqset("5:15")), &v, &c)); + assert!(!matches(&SearchKey::Uid(parse_seqset("1:5")), &v, &c)); + assert!(matches(&SearchKey::Sequence(parse_seqset("1")), &v, &c)); } #[test] fn match_boolean_composition() { let v = view(); + let c = no_ctx(); let k = parse(r#"UNSEEN SUBJECT "hello""#); - assert!(matches(&k, &v)); + assert!(matches(&k, &v, &c)); let k = parse(r#"SEEN SUBJECT "hello""#); - assert!(!matches(&k, &v)); + assert!(!matches(&k, &v, &c)); let k = parse(r#"OR SEEN SUBJECT "hello""#); - assert!(matches(&k, &v)); + assert!(matches(&k, &v, &c)); let k = parse("NOT SEEN"); - assert!(matches(&k, &v)); + assert!(matches(&k, &v, &c)); } } diff --git a/crates/bridge/src/imap/session.rs b/crates/bridge/src/imap/session.rs index 2b3da12..d1e4770 100644 --- a/crates/bridge/src/imap/session.rs +++ b/crates/bridge/src/imap/session.rs @@ -5,6 +5,7 @@ use tutasdk::entities::generated::tutanota::{Mail, MailAddress, MailDetails, Tut use crate::imap::search::{self, MsgView}; use crate::mail::mail_to_rfc2822; use crate::mail::rfc2822::{extract_headers, format_address, format_internal_date}; +use crate::store::LocalStore; use crate::sync::MailStore; use crate::tuta::{FolderInfo, MailBackend}; @@ -33,6 +34,9 @@ struct CachedMail { pub struct ImapSession { store: Arc, backend: Arc, + /// The on-disk store, used for full-text body search. `None` in unit tests + /// (body terms then simply don't match). + local_store: Option>, state: State, selected_folder: Option, mails: Vec, @@ -47,10 +51,12 @@ impl ImapSession { store: Arc, backend: Arc, password_hash: Option, + local_store: Option>, ) -> Self { Self { store, backend, + local_store, state: State::NotAuthenticated, selected_folder: None, mails: Vec::new(), @@ -465,6 +471,7 @@ impl ImapSession { } let query = search::parse(args); + let ctx = self.resolve_body_search(&query); let ids: Vec = self .mails @@ -472,7 +479,7 @@ impl ImapSession { .enumerate() .filter(|(i, cached)| { let view = Self::build_search_view(*i, cached); - search::matches(&query, &view) + search::matches(&query, &view, &ctx) }) .map(|(i, cached)| if uid_mode { cached.uid } else { (i + 1) as u32 }) .collect(); @@ -500,11 +507,6 @@ impl ImapSession { .map(extract_headers) .unwrap_or_default(); - let body = cached - .details - .as_ref() - .and_then(|d| d.body.compressedText.as_deref().or(d.body.text.as_deref())); - // To/Cc/Bcc come from details when the body is loaded; otherwise only // the envelope `firstRecipient` (rendered as To) is available. let (to, cc, bcc) = match cached.details.as_ref() { @@ -531,9 +533,17 @@ impl ImapSession { .map(|d| d.sentDate.as_millis()) .unwrap_or_else(|| cached.mail.receivedDate.as_millis()); + let element_id = cached + .mail + ._id + .as_ref() + .map(|id| id.element_id.0.as_str()) + .unwrap_or(""); + MsgView { seq: (idx + 1) as u32, uid: cached.uid, + element_id, subject: &cached.mail.subject, from: format_address(&cached.mail.sender), to, @@ -545,10 +555,28 @@ impl ImapSession { unread: cached.mail.unread, deleted: cached.deleted, size: cached.rfc2822.as_ref().map(|r| r.len() as u64).unwrap_or(0), - body, } } + /// Resolve the `BODY`/`TEXT` terms of a query against the full-text index, + /// once per distinct term. With no local store (unit tests) the context is + /// empty, so body terms match nothing. + fn resolve_body_search(&self, query: &search::SearchKey) -> search::SearchContext { + let mut ctx = search::SearchContext::empty(); + let Some(local_store) = &self.local_store else { + return ctx; + }; + for term in search::collect_body_terms(query) { + match local_store.search_body(&term) { + Ok(ids) => { + ctx.body_hits.insert(term, ids.into_iter().collect()); + } + Err(e) => log::warn!("FTS body search failed for {term:?}: {e}"), + } + } + ctx + } + async fn cmd_store(&mut self, tag: &str, args: &str, uid_mode: bool) -> Vec { if self.state != State::Selected { return vec![format!("{} NO No mailbox selected\r\n", tag)]; @@ -1528,7 +1556,7 @@ mod tests { ], ) .await; - let mut session = ImapSession::new(store, backend.clone(), None); + let mut session = ImapSession::new(store, backend.clone(), None, None); session.handle_command("a LOGIN u p").await; session.handle_command("b SELECT INBOX").await; @@ -1578,7 +1606,7 @@ mod tests { let store = MailStore::new(); let mails = backend.mails.lock().unwrap().clone(); populate_store(&store, &mails).await; - let session = ImapSession::new(store.clone(), backend, None); + let session = ImapSession::new(store.clone(), backend, None, None); (store, session) } @@ -1796,7 +1824,7 @@ mod tests { ], ) .await; - let mut session = ImapSession::new(store, backend, None); + let mut session = ImapSession::new(store, backend, None, None); // LOGIN let resp = session.handle_command("A001 LOGIN user pass").await; @@ -1840,7 +1868,7 @@ mod tests { )])); let store = MailStore::new(); populate_store(&store, &backend.mails.lock().unwrap()).await; - let mut session = ImapSession::new(store, backend.clone(), None); + let mut session = ImapSession::new(store, backend.clone(), None, None); session.handle_command("A001 LOGIN user pass").await; session.handle_command("A002 SELECT INBOX").await; @@ -1868,7 +1896,7 @@ mod tests { ])); let store = MailStore::new(); populate_store(&store, &backend.mails.lock().unwrap()).await; - let mut session = ImapSession::new(store, backend.clone(), None); + let mut session = ImapSession::new(store, backend.clone(), None, None); session.handle_command("A001 LOGIN user pass").await; session.handle_command("A002 SELECT INBOX").await; @@ -1908,7 +1936,7 @@ mod tests { ])); let store = MailStore::new(); populate_store(&store, &backend.mails.lock().unwrap()).await; - let mut session = ImapSession::new(store.clone(), backend.clone(), None); + let mut session = ImapSession::new(store.clone(), backend.clone(), None, None); session.handle_command("A001 LOGIN user pass").await; session.handle_command("A002 SELECT INBOX").await; diff --git a/crates/bridge/src/mail/mod.rs b/crates/bridge/src/mail/mod.rs index 1e8b8ee..e88cb01 100644 --- a/crates/bridge/src/mail/mod.rs +++ b/crates/bridge/src/mail/mod.rs @@ -5,3 +5,4 @@ pub(crate) mod rfc2822; pub use bodystructure::compute_bodystructure; pub use parser::{Attachment, ParsedMessage}; pub use rfc2822::mail_to_rfc2822; +pub(crate) use rfc2822::{extract_body_text, strip_html}; diff --git a/crates/bridge/src/mail/rfc2822.rs b/crates/bridge/src/mail/rfc2822.rs index f648520..ea1d7b9 100644 --- a/crates/bridge/src/mail/rfc2822.rs +++ b/crates/bridge/src/mail/rfc2822.rs @@ -225,6 +225,99 @@ pub(crate) fn extract_headers(rfc: &str) -> String { } } +/// Strip HTML markup down to readable text for full-text indexing. Drops tags, +/// the contents of `"; + assert_eq!(strip_html(html), "Visible"); + } + + #[test] + fn extract_body_text_from_single_part() { + let body = base64::engine::general_purpose::STANDARD.encode("

Quarterly invoice

"); + let rfc = format!( + "Subject: Test\r\nContent-Type: text/html; charset=UTF-8\r\n\ + Content-Transfer-Encoding: base64\r\n\r\n{body}\r\n" + ); + assert_eq!(extract_body_text(&rfc), "Quarterly invoice"); + } + + #[test] + fn extract_body_text_from_multipart_stops_at_boundary() { + let body = base64::engine::general_purpose::STANDARD.encode("

Body words here

"); + let rfc = format!( + "Content-Type: multipart/mixed; boundary=\"BND\"\r\n\r\n\ + --BND\r\nContent-Type: text/html; charset=UTF-8\r\n\ + Content-Transfer-Encoding: base64\r\n\r\n{body}\r\n\ + --BND\r\nContent-Type: application/pdf\r\n\r\nIGNOREDATTACHMENT\r\n--BND--\r\n" + ); + assert_eq!(extract_body_text(&rfc), "Body words here"); + } + #[test] fn test_days_to_ymd_known_dates() { // 2024-01-01 = day 19723 since epoch diff --git a/crates/bridge/src/store.rs b/crates/bridge/src/store.rs index b2111b9..e3b333f 100644 --- a/crates/bridge/src/store.rs +++ b/crates/bridge/src/store.rs @@ -46,6 +46,19 @@ pub struct LocalStore { mails_dir: PathBuf, } +/// Turn a free-text search term into a safe FTS5 MATCH expression. Each run of +/// alphanumeric characters becomes a prefix token (`word*`), and tokens are +/// ANDed together (FTS5's default). Because only alphanumerics survive, the +/// result can never contain FTS5 syntax, so it is injection-safe to interpolate +/// as a bound parameter. Returns an empty string when nothing is searchable. +fn fts_match_expr(term: &str) -> String { + term.split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .map(|w| format!("{w}*")) + .collect::>() + .join(" ") +} + impl LocalStore { pub fn open( db_path: &Path, @@ -85,7 +98,8 @@ impl LocalStore { conn.execute_batch( "DROP TABLE IF EXISTS mails; DROP TABLE IF EXISTS sync_state; - DROP TABLE IF EXISTS event_bus_state;", + DROP TABLE IF EXISTS event_bus_state; + DROP TABLE IF EXISTS mail_fts;", )?; } @@ -115,6 +129,11 @@ impl LocalStore { last_batch_id TEXT NOT NULL, updated_at_ms INTEGER NOT NULL DEFAULT 0 ); + CREATE VIRTUAL TABLE IF NOT EXISTS mail_fts USING fts5( + element_id UNINDEXED, + body, + tokenize = 'unicode61 remove_diacritics 2' + ); INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');" ))?; @@ -165,6 +184,7 @@ impl LocalStore { "DELETE FROM mails; DELETE FROM sync_state; DELETE FROM store_meta; + DELETE FROM mail_fts; INSERT INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');" ))?; drop(conn); @@ -405,6 +425,54 @@ impl LocalStore { Ok(()) } + /// Index (or re-index) a message's plain-text body for full-text search. + /// `body_text` should already be stripped of HTML markup. Stored inside the + /// SQLCipher database, so it is encrypted at rest like everything else. + pub fn index_body(&self, element_id: &str, body_text: &str) -> Result<(), StoreError> { + let conn = self.conn.lock().unwrap(); + // FTS5 has no UPSERT; clear any prior row for this id first. + conn.execute("DELETE FROM mail_fts WHERE element_id = ?1", [element_id])?; + conn.execute( + "INSERT INTO mail_fts(element_id, body) VALUES (?1, ?2)", + [element_id, body_text], + )?; + Ok(()) + } + + /// Drop a message from the full-text index (called when a mail is removed). + pub fn unindex_body(&self, element_id: &str) -> Result<(), StoreError> { + let conn = self.conn.lock().unwrap(); + conn.execute("DELETE FROM mail_fts WHERE element_id = ?1", [element_id])?; + Ok(()) + } + + /// Element ids whose indexed body matches a user search term. The term is a + /// free-text query (one IMAP `BODY`/`TEXT` argument); it is turned into a + /// safe FTS5 MATCH expression: each word becomes a prefix token so + /// `factur` finds `facture`/`factures`, and multi-word terms must all be + /// present. Returns an empty set when the term has no searchable tokens. + pub fn search_body(&self, term: &str) -> Result, StoreError> { + let match_expr = fts_match_expr(term); + if match_expr.is_empty() { + return Ok(Vec::new()); + } + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare("SELECT element_id FROM mail_fts WHERE body MATCH ?1")?; + let rows = stmt.query_map([&match_expr], |row| row.get::<_, String>(0))?; + let mut ids = Vec::new(); + for row in rows { + ids.push(row?); + } + Ok(ids) + } + + /// Number of bodies currently in the full-text index (diagnostics / tests). + pub fn fts_count(&self) -> Result { + let conn = self.conn.lock().unwrap(); + let count: i64 = conn.query_row("SELECT COUNT(*) FROM mail_fts", [], |row| row.get(0))?; + Ok(count as usize) + } + pub fn mail_count(&self, folder_id: &str) -> Result { let conn = self.conn.lock().unwrap(); let count: i64 = conn.query_row( @@ -539,6 +607,7 @@ impl LocalStore { conn.execute("DELETE FROM mails WHERE element_id = ?1", [element_id])?; } self.delete_eml(element_id)?; + self.unindex_body(element_id)?; Ok(()) } } @@ -579,6 +648,55 @@ mod tests { } } + #[test] + fn fts_index_and_search() { + let store = open_memory_store(); + store + .index_body("a", "the quarterly invoice is attached") + .unwrap(); + store.index_body("b", "lunch plans for friday").unwrap(); + assert_eq!(store.fts_count().unwrap(), 2); + + // Prefix match: "invoic" finds "invoice". + assert_eq!(store.search_body("invoic").unwrap(), vec!["a".to_string()]); + // Multi-word ANDs the tokens. + assert_eq!( + store.search_body("lunch friday").unwrap(), + vec!["b".to_string()] + ); + // No hit. + assert!(store.search_body("zebra").unwrap().is_empty()); + // Empty / punctuation-only term yields no FTS query. + assert!(store.search_body(" ").unwrap().is_empty()); + } + + #[test] + fn fts_reindex_replaces_prior_body() { + let store = open_memory_store(); + store.index_body("a", "first version about cats").unwrap(); + store.index_body("a", "second version about dogs").unwrap(); + assert_eq!(store.fts_count().unwrap(), 1); + assert!(store.search_body("cats").unwrap().is_empty()); + assert_eq!(store.search_body("dogs").unwrap(), vec!["a".to_string()]); + } + + #[test] + fn fts_unindex_removes_body() { + let store = open_memory_store(); + store.index_body("a", "removable content").unwrap(); + store.unindex_body("a").unwrap(); + assert!(store.search_body("removable").unwrap().is_empty()); + assert_eq!(store.fts_count().unwrap(), 0); + } + + #[test] + fn fts_match_expr_is_injection_safe() { + // FTS syntax characters never survive into the expression. + assert_eq!(fts_match_expr(r#"foo" OR bar"#), "foo* OR* bar*"); + assert_eq!(fts_match_expr("a-b.c"), "a* b* c*"); + assert_eq!(fts_match_expr(" "), ""); + } + #[test] fn allocate_uids_are_monotonic_and_per_folder() { let store = open_memory_store(); diff --git a/crates/bridge/src/sync.rs b/crates/bridge/src/sync.rs index 4b7698f..6dd0401 100644 --- a/crates/bridge/src/sync.rs +++ b/crates/bridge/src/sync.rs @@ -6,7 +6,7 @@ use log::{debug, info, warn}; use tokio::sync::{watch, RwLock}; use tutasdk::entities::generated::tutanota::{Mail, MailDetails, TutanotaFile}; -use crate::mail::mail_to_rfc2822; +use crate::mail::{extract_body_text, mail_to_rfc2822, strip_html}; use crate::store::{LocalStore, MailMetadata}; use crate::tuta::{FolderInfo, MailBackend}; @@ -426,6 +426,58 @@ pub async fn run_syncer( debug!("Skipping full-metadata sync — already complete, event bus reconciles"); } + // One-time backfill of the full-text body index from bodies that were + // already cached (downloaded before the index existed). Bodies fetched from + // here on are indexed inline by `prefetch_details`, so this only ever runs + // once per install. + const FTS_BACKFILL_MARKER: &str = "body_fts_indexed_v1"; + if local_store.get_meta(FTS_BACKFILL_MARKER).is_none() && !folders.is_empty() { + info!("Backfilling full-text body index from cached messages…"); + let mut indexed = 0usize; + let mut all_ok = true; + for folder in &folders { + if *shutdown.borrow() { + return; + } + match local_store.load_folder_metadata(&folder.id) { + Ok(metas) => { + for meta in metas.iter().filter(|m| m.has_details) { + if *shutdown.borrow() { + return; + } + match local_store.read_eml(&meta.element_id) { + Ok(Some(rfc)) => { + let text = extract_body_text(&rfc); + if let Err(e) = local_store.index_body(&meta.element_id, &text) { + warn!("FTS backfill failed for {}: {e}", meta.element_id); + all_ok = false; + } else { + indexed += 1; + } + } + Ok(None) => {} + Err(e) => { + warn!("FTS backfill read failed for {}: {e}", meta.element_id); + all_ok = false; + } + } + } + } + Err(e) => { + warn!("FTS backfill: no metadata for {}: {e}", folder.imap_path); + all_ok = false; + } + } + } + // Only mark done if nothing errored, so a partial run retries next boot. + if all_ok { + if let Err(e) = local_store.set_meta(FTS_BACKFILL_MARKER, "1") { + warn!("Could not persist FTS backfill marker: {e}"); + } + } + info!("Full-text body index backfill complete ({indexed} messages)"); + } + // From here on the syncer only owns the slow body prefetch (capped at // `sync_limit` — the body-prefetch depth). Folder / new-mail refresh is // driven by the event bus + `event_handler`. @@ -661,6 +713,9 @@ pub(crate) async fn sync_folder( if let Err(e) = local_store.delete_eml(eid) { warn!("Failed to delete cached eml {}: {}", eid, e); } + if let Err(e) = local_store.unindex_body(eid) { + warn!("Failed to unindex body {}: {}", eid, e); + } } if !deleted.is_empty() { debug!( @@ -801,6 +856,16 @@ async fn prefetch_details( if let Err(e) = local_store.mark_has_details(&eid) { warn!("Failed to mark has_details {}: {}", eid, e); } + // Index the plain-text body for full-text search. + let body_html = details + .body + .compressedText + .as_deref() + .or(details.body.text.as_deref()) + .unwrap_or(""); + if let Err(e) = local_store.index_body(&eid, &strip_html(body_html)) { + warn!("Failed to index body {}: {}", eid, e); + } if attachments_pending { attachment_retry_history.insert(eid.clone(), now); } diff --git a/src/main.rs b/src/main.rs index 9d58646..ccb1469 100644 --- a/src/main.rs +++ b/src/main.rs @@ -200,6 +200,7 @@ async fn main() -> anyhow::Result<()> { cfg.imap_port, store.clone(), backend.clone(), + local_store.clone(), imap_tls, pw.clone(), ));