Full-text body search via an encrypted FTS5 index

BODY/TEXT searches previously matched only bodies that happened to be
decoded in memory, so results were inconsistent. Add a persistent FTS5
index (a virtual table inside the SQLCipher store, encrypted at rest) over
the plain-text body of every message we download.

- store.rs: mail_fts(element_id UNINDEXED, body) with unicode61 +
  remove_diacritics; index_body / unindex_body / search_body / fts_count.
  Terms become prefix tokens ANDed together (factur -> factur*), built by
  fts_match_expr which strips everything but alphanumerics so it is
  injection-safe.
- rfc2822.rs: strip_html (drops tags + script/style + entities) and
  extract_body_text (decodes the text part of our own .eml) feed the index.
- sync.rs: index inline at prefetch; one-time backfill at boot
  (body_fts_indexed_v1) for bodies cached before the index existed;
  unindex on delete.
- search.rs: BODY/TEXT resolve through the index — the session collects the
  distinct body terms, queries the index once each, and passes the hit sets
  to matches() via a SearchContext. A body term only matches messages whose
  body has actually been downloaded (full coverage needs sync_limit = 0).
- LocalStore threaded into ImapSession (Option; None in unit tests).

Validated live: backfilled 7,687 cached bodies, then BODY/TEXT/AND/OR/NOT
queries returned coherent subsets — NOT BODY x == total - (BODY x), an
exact complement. 233 unit tests, incl. real FTS5 MATCH against the bundled
SQLCipher (confirms FTS5 is compiled in for the cross-OS release).
This commit is contained in:
Anthony
2026-06-01 10:38:47 +02:00
parent 24c3a1d908
commit 3d4ef0efad
9 changed files with 522 additions and 78 deletions
+2 -1
View File
@@ -365,7 +365,7 @@ impl BridgeHandle {
};
let handler_handle = tokio::spawn(event_handler::run_event_handler(
store.clone(),
local_store,
local_store.clone(),
backend.clone(),
bus_ids_for_handler,
event_rx,
@@ -375,6 +375,7 @@ impl BridgeHandle {
imap_port,
store.clone(),
backend.clone(),
local_store,
imap_tls,
pw.clone(),
));
+8 -2
View File
@@ -9,6 +9,7 @@ use tokio::net::TcpListener;
use tokio::sync::watch;
use tokio_rustls::TlsAcceptor;
use crate::store::LocalStore;
use crate::sync::MailStore;
use crate::tuta::MailBackend;
use session::ImapSession;
@@ -17,6 +18,7 @@ pub async fn serve(
port: u16,
store: Arc<MailStore>,
backend: Arc<dyn MailBackend>,
local_store: Arc<LocalStore>,
tls: TlsAcceptor,
password_hash: Option<String>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -28,13 +30,16 @@ pub async fn serve(
debug!("IMAP connection from {}", addr);
let store = store.clone();
let backend = backend.clone();
let local_store = local_store.clone();
let tls = tls.clone();
let pw_hash = password_hash.clone();
tokio::spawn(async move {
match tls.accept(stream).await {
Ok(tls_stream) => {
if let Err(e) = handle_connection(tls_stream, store, backend, pw_hash).await {
if let Err(e) =
handle_connection(tls_stream, store, backend, local_store, pw_hash).await
{
error!("IMAP connection error: {}", e);
}
}
@@ -50,12 +55,13 @@ async fn handle_connection(
stream: tokio_rustls::server::TlsStream<tokio::net::TcpStream>,
store: Arc<MailStore>,
backend: Arc<dyn MailBackend>,
local_store: Arc<LocalStore>,
password_hash: Option<String>,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let (reader, mut writer) = tokio::io::split(stream);
let mut reader = BufReader::new(reader);
let mut store_watch: watch::Receiver<u64> = store.subscribe();
let mut session = ImapSession::new(store, backend, password_hash);
let mut session = ImapSession::new(store, backend, password_hash, Some(local_store));
writer
.write_all(b"* OK TutaBridge IMAP4rev1 ready\r\n")
+157 -60
View File
@@ -7,14 +7,17 @@
//!
//! Coverage is metadata-first: subject / from / to / cc, flags, dates, sizes,
//! sequence + UID sets, and boolean composition (`AND` / `OR` / `NOT`). `BODY`
//! and `TEXT` match against whatever body text the session already has decoded
//! — full-text body search over the whole mailbox is a later increment backed
//! by an on-disk index.
//! and `TEXT` are resolved against the on-disk full-text index: the session
//! queries it once per distinct term ([`collect_body_terms`]) and hands the
//! per-term hit sets to [`matches`] via a [`SearchContext`]. A body term only
//! matches messages whose body has actually been downloaded and indexed.
//!
//! Robustness rule: an unrecognised criterion degrades to a non-restrictive
//! match (it never *hides* messages). Over-inclusion is the safe failure for
//! search; silently dropping a matching mail is not.
use std::collections::{HashMap, HashSet};
/// One element of an IMAP sequence/UID set, e.g. `1`, `3:9`, or `5:*`.
#[derive(Debug, Clone, PartialEq, Eq)]
struct SeqRange {
@@ -107,6 +110,8 @@ pub enum SearchKey {
pub struct MsgView<'a> {
pub seq: u32,
pub uid: u32,
/// Tuta element id — the key used to look up full-text body hits.
pub element_id: &'a str,
pub subject: &'a str,
/// Formatted `From` (name + address), for `FROM` substring matching.
pub from: String,
@@ -122,9 +127,49 @@ pub struct MsgView<'a> {
pub unread: bool,
pub deleted: bool,
pub size: u64,
/// Decoded body text, when the session already has it. `None` means the
/// body isn't loaded; `BODY`/`TEXT` simply won't match such a message yet.
pub body: Option<&'a str>,
}
/// Pre-resolved full-text results for one SEARCH: maps each `BODY`/`TEXT` term
/// to the set of element ids whose indexed body matched it. Built by the
/// session from the on-disk index before evaluating the query.
#[derive(Default)]
pub struct SearchContext {
pub body_hits: HashMap<String, HashSet<String>>,
}
impl SearchContext {
pub fn empty() -> Self {
Self::default()
}
fn body_matches(&self, term: &str, element_id: &str) -> bool {
self.body_hits
.get(term)
.is_some_and(|ids| ids.contains(element_id))
}
}
/// Collect the distinct `BODY`/`TEXT` term arguments in a parsed query, so the
/// session can resolve each against the full-text index exactly once.
pub fn collect_body_terms(key: &SearchKey) -> Vec<String> {
let mut terms = Vec::new();
collect_body_terms_into(key, &mut terms);
terms.sort();
terms.dedup();
terms
}
fn collect_body_terms_into(key: &SearchKey, out: &mut Vec<String>) {
match key {
SearchKey::And(keys) => keys.iter().for_each(|k| collect_body_terms_into(k, out)),
SearchKey::Or(a, b) => {
collect_body_terms_into(a, out);
collect_body_terms_into(b, out);
}
SearchKey::Not(k) => collect_body_terms_into(k, out),
SearchKey::Body(s) | SearchKey::Text(s) => out.push(s.clone()),
_ => {}
}
}
fn contains_ci(haystack: &str, needle: &str) -> bool {
@@ -154,13 +199,14 @@ fn day_number(ms: u64) -> i64 {
(ms / 86_400_000) as i64
}
/// Evaluate a parsed query against one message.
pub fn matches(key: &SearchKey, m: &MsgView) -> bool {
/// Evaluate a parsed query against one message, consulting `ctx` for the
/// full-text results of any `BODY`/`TEXT` terms.
pub fn matches(key: &SearchKey, m: &MsgView, ctx: &SearchContext) -> bool {
match key {
SearchKey::All => true,
SearchKey::And(keys) => keys.iter().all(|k| matches(k, m)),
SearchKey::Or(a, b) => matches(a, m) || matches(b, m),
SearchKey::Not(k) => !matches(k, m),
SearchKey::And(keys) => keys.iter().all(|k| matches(k, m, ctx)),
SearchKey::Or(a, b) => matches(a, m, ctx) || matches(b, m, ctx),
SearchKey::Not(k) => !matches(k, m, ctx),
SearchKey::Seen => !m.unread,
SearchKey::Unseen => m.unread,
@@ -183,10 +229,8 @@ pub fn matches(key: &SearchKey, m: &MsgView) -> bool {
SearchKey::To(s) => contains_ci(&m.to, s),
SearchKey::Cc(s) => contains_ci(&m.cc, s),
SearchKey::Bcc(s) => contains_ci(&m.bcc, s),
SearchKey::Body(s) => m.body.is_some_and(|b| contains_ci(b, s)),
SearchKey::Text(s) => {
contains_ci(&m.headers, s) || m.body.is_some_and(|b| contains_ci(b, s))
}
SearchKey::Body(s) => ctx.body_matches(s, m.element_id),
SearchKey::Text(s) => contains_ci(&m.headers, s) || ctx.body_matches(s, m.element_id),
SearchKey::Header(name, val) => header_contains(&m.headers, name, val),
SearchKey::Before(d) => day_number(m.date_ms) < *d,
@@ -511,6 +555,7 @@ mod tests {
MsgView {
seq: 1,
uid: 10,
element_id: "mail1",
subject: "Hello World",
from: "Alice <alice@example.com>".into(),
to: "Bob <bob@example.com>".into(),
@@ -524,10 +569,22 @@ mod tests {
unread: true,
deleted: false,
size: 5000,
body: Some("the quick brown fox"),
}
}
/// A context where the given terms all hit our test message ("mail1").
fn ctx_hitting(terms: &[&str]) -> SearchContext {
let mut body_hits = HashMap::new();
for t in terms {
body_hits.insert((*t).to_string(), HashSet::from(["mail1".to_string()]));
}
SearchContext { body_hits }
}
fn no_ctx() -> SearchContext {
SearchContext::empty()
}
// --- tokenizer ---
#[test]
@@ -655,101 +712,141 @@ mod tests {
#[test]
fn match_subject_ci() {
assert!(matches(&SearchKey::Subject("hello".into()), &view()));
assert!(matches(&SearchKey::Subject("WORLD".into()), &view()));
assert!(!matches(&SearchKey::Subject("nope".into()), &view()));
let c = no_ctx();
assert!(matches(&SearchKey::Subject("hello".into()), &view(), &c));
assert!(matches(&SearchKey::Subject("WORLD".into()), &view(), &c));
assert!(!matches(&SearchKey::Subject("nope".into()), &view(), &c));
}
#[test]
fn match_from_to() {
assert!(matches(&SearchKey::From("alice".into()), &view()));
assert!(matches(&SearchKey::To("bob@example".into()), &view()));
assert!(!matches(&SearchKey::Cc("anyone".into()), &view()));
let c = no_ctx();
assert!(matches(&SearchKey::From("alice".into()), &view(), &c));
assert!(matches(&SearchKey::To("bob@example".into()), &view(), &c));
assert!(!matches(&SearchKey::Cc("anyone".into()), &view(), &c));
}
#[test]
fn match_flags_consistent_with_fetch() {
let v = view(); // unread, not deleted
assert!(matches(&SearchKey::Unseen, &v));
assert!(!matches(&SearchKey::Seen, &v));
assert!(!matches(&SearchKey::Answered, &v));
assert!(matches(&SearchKey::Unanswered, &v));
assert!(!matches(&SearchKey::Flagged, &v));
assert!(!matches(&SearchKey::Deleted, &v));
assert!(matches(&SearchKey::Undeleted, &v));
let c = no_ctx();
assert!(matches(&SearchKey::Unseen, &v, &c));
assert!(!matches(&SearchKey::Seen, &v, &c));
assert!(!matches(&SearchKey::Answered, &v, &c));
assert!(matches(&SearchKey::Unanswered, &v, &c));
assert!(!matches(&SearchKey::Flagged, &v, &c));
assert!(!matches(&SearchKey::Deleted, &v, &c));
assert!(matches(&SearchKey::Undeleted, &v, &c));
}
#[test]
fn match_dates() {
let v = view(); // 2022-09-22
assert!(matches(&SearchKey::Since(day(2022, 1, 1)), &v));
assert!(matches(&SearchKey::Before(day(2023, 1, 1)), &v));
assert!(matches(&SearchKey::On(day(2022, 9, 22)), &v));
assert!(!matches(&SearchKey::On(day(2022, 9, 23)), &v));
assert!(!matches(&SearchKey::Since(day(2023, 1, 1)), &v));
let c = no_ctx();
assert!(matches(&SearchKey::Since(day(2022, 1, 1)), &v, &c));
assert!(matches(&SearchKey::Before(day(2023, 1, 1)), &v, &c));
assert!(matches(&SearchKey::On(day(2022, 9, 22)), &v, &c));
assert!(!matches(&SearchKey::On(day(2022, 9, 23)), &v, &c));
assert!(!matches(&SearchKey::Since(day(2023, 1, 1)), &v, &c));
}
#[test]
fn match_size() {
let v = view(); // size 5000
assert!(matches(&SearchKey::Larger(4000), &v));
assert!(!matches(&SearchKey::Larger(6000), &v));
assert!(matches(&SearchKey::Smaller(6000), &v));
let c = no_ctx();
assert!(matches(&SearchKey::Larger(4000), &v, &c));
assert!(!matches(&SearchKey::Larger(6000), &v, &c));
assert!(matches(&SearchKey::Smaller(6000), &v, &c));
}
#[test]
fn match_body_and_text() {
fn match_body_uses_fts_context() {
let v = view();
assert!(matches(&SearchKey::Body("brown".into()), &v));
assert!(!matches(&SearchKey::Body("missing".into()), &v));
// TEXT spans headers + body.
assert!(matches(&SearchKey::Text("Message-ID".into()), &v));
assert!(matches(&SearchKey::Text("quick".into()), &v));
let c = ctx_hitting(&["brown"]);
assert!(matches(&SearchKey::Body("brown".into()), &v, &c));
// A term with no FTS hit doesn't match, even though it's a real word.
assert!(!matches(&SearchKey::Body("missing".into()), &v, &c));
}
#[test]
fn match_body_without_loaded_body_never_matches() {
let mut v = view();
v.body = None;
assert!(!matches(&SearchKey::Body("brown".into()), &v));
// TEXT still matches on headers.
assert!(matches(&SearchKey::Text("Subject".into()), &v));
fn match_text_spans_headers_and_body() {
let v = view();
let c = ctx_hitting(&["quick"]);
// Header hit, no body hit needed.
assert!(matches(
&SearchKey::Text("Message-ID".into()),
&v,
&no_ctx()
));
// Body hit via the FTS context.
assert!(matches(&SearchKey::Text("quick".into()), &v, &c));
// Neither headers nor index: no match.
assert!(!matches(&SearchKey::Text("quick".into()), &v, &no_ctx()));
}
#[test]
fn match_body_without_index_never_matches() {
let v = view();
assert!(!matches(&SearchKey::Body("brown".into()), &v, &no_ctx()));
// TEXT still matches on headers without any index.
assert!(matches(&SearchKey::Text("Subject".into()), &v, &no_ctx()));
}
#[test]
fn collect_body_terms_finds_nested_terms() {
let k = parse(r#"OR BODY "alpha" (TEXT "beta" SUBJECT "x") NOT BODY "alpha""#);
// Note: top level is `OR <a> <b>` then trailing keys folded into AND.
let terms = collect_body_terms(&k);
assert_eq!(terms, vec!["alpha".to_string(), "beta".to_string()]);
}
#[test]
fn match_header() {
let v = view();
let c = no_ctx();
assert!(matches(
&SearchKey::Header("message-id".into(), "abc".into()),
&v
&v,
&c
));
assert!(!matches(
&SearchKey::Header("message-id".into(), "zzz".into()),
&v
&v,
&c
));
// Presence-only (empty value).
assert!(matches(&SearchKey::Header("subject".into(), "".into()), &v));
assert!(!matches(&SearchKey::Header("x-nope".into(), "".into()), &v));
assert!(matches(
&SearchKey::Header("subject".into(), "".into()),
&v,
&c
));
assert!(!matches(
&SearchKey::Header("x-nope".into(), "".into()),
&v,
&c
));
}
#[test]
fn match_uid_and_seq() {
let v = view(); // seq 1, uid 10
assert!(matches(&SearchKey::Uid(parse_seqset("5:15")), &v));
assert!(!matches(&SearchKey::Uid(parse_seqset("1:5")), &v));
assert!(matches(&SearchKey::Sequence(parse_seqset("1")), &v));
let c = no_ctx();
assert!(matches(&SearchKey::Uid(parse_seqset("5:15")), &v, &c));
assert!(!matches(&SearchKey::Uid(parse_seqset("1:5")), &v, &c));
assert!(matches(&SearchKey::Sequence(parse_seqset("1")), &v, &c));
}
#[test]
fn match_boolean_composition() {
let v = view();
let c = no_ctx();
let k = parse(r#"UNSEEN SUBJECT "hello""#);
assert!(matches(&k, &v));
assert!(matches(&k, &v, &c));
let k = parse(r#"SEEN SUBJECT "hello""#);
assert!(!matches(&k, &v));
assert!(!matches(&k, &v, &c));
let k = parse(r#"OR SEEN SUBJECT "hello""#);
assert!(matches(&k, &v));
assert!(matches(&k, &v, &c));
let k = parse("NOT SEEN");
assert!(matches(&k, &v));
assert!(matches(&k, &v, &c));
}
}
+41 -13
View File
@@ -5,6 +5,7 @@ use tutasdk::entities::generated::tutanota::{Mail, MailAddress, MailDetails, Tut
use crate::imap::search::{self, MsgView};
use crate::mail::mail_to_rfc2822;
use crate::mail::rfc2822::{extract_headers, format_address, format_internal_date};
use crate::store::LocalStore;
use crate::sync::MailStore;
use crate::tuta::{FolderInfo, MailBackend};
@@ -33,6 +34,9 @@ struct CachedMail {
pub struct ImapSession {
store: Arc<MailStore>,
backend: Arc<dyn MailBackend>,
/// The on-disk store, used for full-text body search. `None` in unit tests
/// (body terms then simply don't match).
local_store: Option<Arc<LocalStore>>,
state: State,
selected_folder: Option<FolderInfo>,
mails: Vec<CachedMail>,
@@ -47,10 +51,12 @@ impl ImapSession {
store: Arc<MailStore>,
backend: Arc<dyn MailBackend>,
password_hash: Option<String>,
local_store: Option<Arc<LocalStore>>,
) -> Self {
Self {
store,
backend,
local_store,
state: State::NotAuthenticated,
selected_folder: None,
mails: Vec::new(),
@@ -465,6 +471,7 @@ impl ImapSession {
}
let query = search::parse(args);
let ctx = self.resolve_body_search(&query);
let ids: Vec<u32> = self
.mails
@@ -472,7 +479,7 @@ impl ImapSession {
.enumerate()
.filter(|(i, cached)| {
let view = Self::build_search_view(*i, cached);
search::matches(&query, &view)
search::matches(&query, &view, &ctx)
})
.map(|(i, cached)| if uid_mode { cached.uid } else { (i + 1) as u32 })
.collect();
@@ -500,11 +507,6 @@ impl ImapSession {
.map(extract_headers)
.unwrap_or_default();
let body = cached
.details
.as_ref()
.and_then(|d| d.body.compressedText.as_deref().or(d.body.text.as_deref()));
// To/Cc/Bcc come from details when the body is loaded; otherwise only
// the envelope `firstRecipient` (rendered as To) is available.
let (to, cc, bcc) = match cached.details.as_ref() {
@@ -531,9 +533,17 @@ impl ImapSession {
.map(|d| d.sentDate.as_millis())
.unwrap_or_else(|| cached.mail.receivedDate.as_millis());
let element_id = cached
.mail
._id
.as_ref()
.map(|id| id.element_id.0.as_str())
.unwrap_or("");
MsgView {
seq: (idx + 1) as u32,
uid: cached.uid,
element_id,
subject: &cached.mail.subject,
from: format_address(&cached.mail.sender),
to,
@@ -545,10 +555,28 @@ impl ImapSession {
unread: cached.mail.unread,
deleted: cached.deleted,
size: cached.rfc2822.as_ref().map(|r| r.len() as u64).unwrap_or(0),
body,
}
}
/// Resolve the `BODY`/`TEXT` terms of a query against the full-text index,
/// once per distinct term. With no local store (unit tests) the context is
/// empty, so body terms match nothing.
fn resolve_body_search(&self, query: &search::SearchKey) -> search::SearchContext {
let mut ctx = search::SearchContext::empty();
let Some(local_store) = &self.local_store else {
return ctx;
};
for term in search::collect_body_terms(query) {
match local_store.search_body(&term) {
Ok(ids) => {
ctx.body_hits.insert(term, ids.into_iter().collect());
}
Err(e) => log::warn!("FTS body search failed for {term:?}: {e}"),
}
}
ctx
}
async fn cmd_store(&mut self, tag: &str, args: &str, uid_mode: bool) -> Vec<String> {
if self.state != State::Selected {
return vec![format!("{} NO No mailbox selected\r\n", tag)];
@@ -1528,7 +1556,7 @@ mod tests {
],
)
.await;
let mut session = ImapSession::new(store, backend.clone(), None);
let mut session = ImapSession::new(store, backend.clone(), None, None);
session.handle_command("a LOGIN u p").await;
session.handle_command("b SELECT INBOX").await;
@@ -1578,7 +1606,7 @@ mod tests {
let store = MailStore::new();
let mails = backend.mails.lock().unwrap().clone();
populate_store(&store, &mails).await;
let session = ImapSession::new(store.clone(), backend, None);
let session = ImapSession::new(store.clone(), backend, None, None);
(store, session)
}
@@ -1796,7 +1824,7 @@ mod tests {
],
)
.await;
let mut session = ImapSession::new(store, backend, None);
let mut session = ImapSession::new(store, backend, None, None);
// LOGIN
let resp = session.handle_command("A001 LOGIN user pass").await;
@@ -1840,7 +1868,7 @@ mod tests {
)]));
let store = MailStore::new();
populate_store(&store, &backend.mails.lock().unwrap()).await;
let mut session = ImapSession::new(store, backend.clone(), None);
let mut session = ImapSession::new(store, backend.clone(), None, None);
session.handle_command("A001 LOGIN user pass").await;
session.handle_command("A002 SELECT INBOX").await;
@@ -1868,7 +1896,7 @@ mod tests {
]));
let store = MailStore::new();
populate_store(&store, &backend.mails.lock().unwrap()).await;
let mut session = ImapSession::new(store, backend.clone(), None);
let mut session = ImapSession::new(store, backend.clone(), None, None);
session.handle_command("A001 LOGIN user pass").await;
session.handle_command("A002 SELECT INBOX").await;
@@ -1908,7 +1936,7 @@ mod tests {
]));
let store = MailStore::new();
populate_store(&store, &backend.mails.lock().unwrap()).await;
let mut session = ImapSession::new(store.clone(), backend.clone(), None);
let mut session = ImapSession::new(store.clone(), backend.clone(), None, None);
session.handle_command("A001 LOGIN user pass").await;
session.handle_command("A002 SELECT INBOX").await;
+1
View File
@@ -5,3 +5,4 @@ pub(crate) mod rfc2822;
pub use bodystructure::compute_bodystructure;
pub use parser::{Attachment, ParsedMessage};
pub use rfc2822::mail_to_rfc2822;
pub(crate) use rfc2822::{extract_body_text, strip_html};
+127
View File
@@ -225,6 +225,99 @@ pub(crate) fn extract_headers(rfc: &str) -> String {
}
}
/// Strip HTML markup down to readable text for full-text indexing. Drops tags,
/// the contents of `<script>`/`<style>` blocks, and decodes the handful of
/// entities that actually show up in mail bodies, then collapses whitespace.
/// This is lossy by design — it only needs to be good enough that a body
/// search matches the words a human would see, not the markup around them.
pub(crate) fn strip_html(html: &str) -> String {
let mut out = String::with_capacity(html.len());
let mut chars = html.chars().peekable();
while let Some(c) = chars.next() {
if c == '<' {
// Capture the tag name to detect script/style blocks we must skip
// wholesale (their text content is not human-visible).
let mut tag = String::new();
for t in chars.clone().take(6) {
if t == '>' || t.is_whitespace() {
break;
}
tag.push(t.to_ascii_lowercase());
}
let skip_block = matches!(tag.trim_start_matches('/'), "script" | "style");
// Consume up to and including the closing '>'.
for t in chars.by_ref() {
if t == '>' {
break;
}
}
if skip_block && !tag.starts_with('/') {
// Swallow everything until the matching closing tag.
let close = format!("</{tag}");
let mut window = String::new();
for t in chars.by_ref() {
window.push(t.to_ascii_lowercase());
if window.ends_with(&close) {
// Drop the rest of the closing tag.
for t2 in chars.by_ref() {
if t2 == '>' {
break;
}
}
break;
}
}
}
out.push(' ');
} else {
out.push(c);
}
}
let decoded = decode_entities(&out);
decoded.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn decode_entities(s: &str) -> String {
s.replace("&nbsp;", " ")
.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&apos;", "'")
}
/// Extract the readable body text from one of our own RFC 2822 messages, for
/// full-text indexing: locate the first `text/*` part, base64-decode it, and
/// strip HTML. Works for both the single-part and multipart layouts produced
/// by [`mail_to_rfc2822`]. Returns an empty string when no text part is found.
pub(crate) fn extract_body_text(rfc: &str) -> String {
// Find the first textual MIME part.
let lower = rfc.to_lowercase();
let part_start = lower.find("text/html").or_else(|| lower.find("text/plain"));
let Some(part_start) = part_start else {
return String::new();
};
// Body begins after that part's header/body separator.
let Some(sep) = rfc[part_start..].find("\r\n\r\n") else {
return String::new();
};
let body_start = part_start + sep + 4;
// Body ends at the next MIME boundary delimiter (multipart) or end of input.
let body_end = rfc[body_start..]
.find("\r\n--")
.map(|i| body_start + i)
.unwrap_or(rfc.len());
let raw = &rfc[body_start..body_end];
let stripped_b64: String = raw.split_whitespace().collect();
match base64::engine::general_purpose::STANDARD.decode(stripped_b64) {
Ok(bytes) => strip_html(&String::from_utf8_lossy(&bytes)),
// Not base64 (shouldn't happen for our own messages) — treat as text.
Err(_) => strip_html(raw),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -238,6 +331,40 @@ mod tests {
assert_eq!(days_to_ymd(0), (1970, 1, 1));
}
#[test]
fn strip_html_drops_tags_and_decodes_entities() {
let html = "<p>Hello&nbsp;<b>World</b> &amp; goodbye</p>";
assert_eq!(strip_html(html), "Hello World & goodbye");
}
#[test]
fn strip_html_skips_script_and_style() {
let html = "<style>.a{color:red}</style><div>Visible</div><script>alert(1)</script>";
assert_eq!(strip_html(html), "Visible");
}
#[test]
fn extract_body_text_from_single_part() {
let body = base64::engine::general_purpose::STANDARD.encode("<p>Quarterly invoice</p>");
let rfc = format!(
"Subject: Test\r\nContent-Type: text/html; charset=UTF-8\r\n\
Content-Transfer-Encoding: base64\r\n\r\n{body}\r\n"
);
assert_eq!(extract_body_text(&rfc), "Quarterly invoice");
}
#[test]
fn extract_body_text_from_multipart_stops_at_boundary() {
let body = base64::engine::general_purpose::STANDARD.encode("<p>Body words here</p>");
let rfc = format!(
"Content-Type: multipart/mixed; boundary=\"BND\"\r\n\r\n\
--BND\r\nContent-Type: text/html; charset=UTF-8\r\n\
Content-Transfer-Encoding: base64\r\n\r\n{body}\r\n\
--BND\r\nContent-Type: application/pdf\r\n\r\nIGNOREDATTACHMENT\r\n--BND--\r\n"
);
assert_eq!(extract_body_text(&rfc), "Body words here");
}
#[test]
fn test_days_to_ymd_known_dates() {
// 2024-01-01 = day 19723 since epoch
+119 -1
View File
@@ -46,6 +46,19 @@ pub struct LocalStore {
mails_dir: PathBuf,
}
/// Turn a free-text search term into a safe FTS5 MATCH expression. Each run of
/// alphanumeric characters becomes a prefix token (`word*`), and tokens are
/// ANDed together (FTS5's default). Because only alphanumerics survive, the
/// result can never contain FTS5 syntax, so it is injection-safe to interpolate
/// as a bound parameter. Returns an empty string when nothing is searchable.
fn fts_match_expr(term: &str) -> String {
term.split(|c: char| !c.is_alphanumeric())
.filter(|w| !w.is_empty())
.map(|w| format!("{w}*"))
.collect::<Vec<_>>()
.join(" ")
}
impl LocalStore {
pub fn open(
db_path: &Path,
@@ -85,7 +98,8 @@ impl LocalStore {
conn.execute_batch(
"DROP TABLE IF EXISTS mails;
DROP TABLE IF EXISTS sync_state;
DROP TABLE IF EXISTS event_bus_state;",
DROP TABLE IF EXISTS event_bus_state;
DROP TABLE IF EXISTS mail_fts;",
)?;
}
@@ -115,6 +129,11 @@ impl LocalStore {
last_batch_id TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL DEFAULT 0
);
CREATE VIRTUAL TABLE IF NOT EXISTS mail_fts USING fts5(
element_id UNINDEXED,
body,
tokenize = 'unicode61 remove_diacritics 2'
);
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
))?;
@@ -165,6 +184,7 @@ impl LocalStore {
"DELETE FROM mails;
DELETE FROM sync_state;
DELETE FROM store_meta;
DELETE FROM mail_fts;
INSERT INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
))?;
drop(conn);
@@ -405,6 +425,54 @@ impl LocalStore {
Ok(())
}
/// Index (or re-index) a message's plain-text body for full-text search.
/// `body_text` should already be stripped of HTML markup. Stored inside the
/// SQLCipher database, so it is encrypted at rest like everything else.
pub fn index_body(&self, element_id: &str, body_text: &str) -> Result<(), StoreError> {
let conn = self.conn.lock().unwrap();
// FTS5 has no UPSERT; clear any prior row for this id first.
conn.execute("DELETE FROM mail_fts WHERE element_id = ?1", [element_id])?;
conn.execute(
"INSERT INTO mail_fts(element_id, body) VALUES (?1, ?2)",
[element_id, body_text],
)?;
Ok(())
}
/// Drop a message from the full-text index (called when a mail is removed).
pub fn unindex_body(&self, element_id: &str) -> Result<(), StoreError> {
let conn = self.conn.lock().unwrap();
conn.execute("DELETE FROM mail_fts WHERE element_id = ?1", [element_id])?;
Ok(())
}
/// Element ids whose indexed body matches a user search term. The term is a
/// free-text query (one IMAP `BODY`/`TEXT` argument); it is turned into a
/// safe FTS5 MATCH expression: each word becomes a prefix token so
/// `factur` finds `facture`/`factures`, and multi-word terms must all be
/// present. Returns an empty set when the term has no searchable tokens.
pub fn search_body(&self, term: &str) -> Result<Vec<String>, StoreError> {
let match_expr = fts_match_expr(term);
if match_expr.is_empty() {
return Ok(Vec::new());
}
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare("SELECT element_id FROM mail_fts WHERE body MATCH ?1")?;
let rows = stmt.query_map([&match_expr], |row| row.get::<_, String>(0))?;
let mut ids = Vec::new();
for row in rows {
ids.push(row?);
}
Ok(ids)
}
/// Number of bodies currently in the full-text index (diagnostics / tests).
pub fn fts_count(&self) -> Result<usize, StoreError> {
let conn = self.conn.lock().unwrap();
let count: i64 = conn.query_row("SELECT COUNT(*) FROM mail_fts", [], |row| row.get(0))?;
Ok(count as usize)
}
pub fn mail_count(&self, folder_id: &str) -> Result<usize, StoreError> {
let conn = self.conn.lock().unwrap();
let count: i64 = conn.query_row(
@@ -539,6 +607,7 @@ impl LocalStore {
conn.execute("DELETE FROM mails WHERE element_id = ?1", [element_id])?;
}
self.delete_eml(element_id)?;
self.unindex_body(element_id)?;
Ok(())
}
}
@@ -579,6 +648,55 @@ mod tests {
}
}
#[test]
fn fts_index_and_search() {
let store = open_memory_store();
store
.index_body("a", "the quarterly invoice is attached")
.unwrap();
store.index_body("b", "lunch plans for friday").unwrap();
assert_eq!(store.fts_count().unwrap(), 2);
// Prefix match: "invoic" finds "invoice".
assert_eq!(store.search_body("invoic").unwrap(), vec!["a".to_string()]);
// Multi-word ANDs the tokens.
assert_eq!(
store.search_body("lunch friday").unwrap(),
vec!["b".to_string()]
);
// No hit.
assert!(store.search_body("zebra").unwrap().is_empty());
// Empty / punctuation-only term yields no FTS query.
assert!(store.search_body(" ").unwrap().is_empty());
}
#[test]
fn fts_reindex_replaces_prior_body() {
let store = open_memory_store();
store.index_body("a", "first version about cats").unwrap();
store.index_body("a", "second version about dogs").unwrap();
assert_eq!(store.fts_count().unwrap(), 1);
assert!(store.search_body("cats").unwrap().is_empty());
assert_eq!(store.search_body("dogs").unwrap(), vec!["a".to_string()]);
}
#[test]
fn fts_unindex_removes_body() {
let store = open_memory_store();
store.index_body("a", "removable content").unwrap();
store.unindex_body("a").unwrap();
assert!(store.search_body("removable").unwrap().is_empty());
assert_eq!(store.fts_count().unwrap(), 0);
}
#[test]
fn fts_match_expr_is_injection_safe() {
// FTS syntax characters never survive into the expression.
assert_eq!(fts_match_expr(r#"foo" OR bar"#), "foo* OR* bar*");
assert_eq!(fts_match_expr("a-b.c"), "a* b* c*");
assert_eq!(fts_match_expr(" "), "");
}
#[test]
fn allocate_uids_are_monotonic_and_per_folder() {
let store = open_memory_store();
+66 -1
View File
@@ -6,7 +6,7 @@ use log::{debug, info, warn};
use tokio::sync::{watch, RwLock};
use tutasdk::entities::generated::tutanota::{Mail, MailDetails, TutanotaFile};
use crate::mail::mail_to_rfc2822;
use crate::mail::{extract_body_text, mail_to_rfc2822, strip_html};
use crate::store::{LocalStore, MailMetadata};
use crate::tuta::{FolderInfo, MailBackend};
@@ -426,6 +426,58 @@ pub async fn run_syncer(
debug!("Skipping full-metadata sync — already complete, event bus reconciles");
}
// One-time backfill of the full-text body index from bodies that were
// already cached (downloaded before the index existed). Bodies fetched from
// here on are indexed inline by `prefetch_details`, so this only ever runs
// once per install.
const FTS_BACKFILL_MARKER: &str = "body_fts_indexed_v1";
if local_store.get_meta(FTS_BACKFILL_MARKER).is_none() && !folders.is_empty() {
info!("Backfilling full-text body index from cached messages…");
let mut indexed = 0usize;
let mut all_ok = true;
for folder in &folders {
if *shutdown.borrow() {
return;
}
match local_store.load_folder_metadata(&folder.id) {
Ok(metas) => {
for meta in metas.iter().filter(|m| m.has_details) {
if *shutdown.borrow() {
return;
}
match local_store.read_eml(&meta.element_id) {
Ok(Some(rfc)) => {
let text = extract_body_text(&rfc);
if let Err(e) = local_store.index_body(&meta.element_id, &text) {
warn!("FTS backfill failed for {}: {e}", meta.element_id);
all_ok = false;
} else {
indexed += 1;
}
}
Ok(None) => {}
Err(e) => {
warn!("FTS backfill read failed for {}: {e}", meta.element_id);
all_ok = false;
}
}
}
}
Err(e) => {
warn!("FTS backfill: no metadata for {}: {e}", folder.imap_path);
all_ok = false;
}
}
}
// Only mark done if nothing errored, so a partial run retries next boot.
if all_ok {
if let Err(e) = local_store.set_meta(FTS_BACKFILL_MARKER, "1") {
warn!("Could not persist FTS backfill marker: {e}");
}
}
info!("Full-text body index backfill complete ({indexed} messages)");
}
// From here on the syncer only owns the slow body prefetch (capped at
// `sync_limit` — the body-prefetch depth). Folder / new-mail refresh is
// driven by the event bus + `event_handler`.
@@ -661,6 +713,9 @@ pub(crate) async fn sync_folder(
if let Err(e) = local_store.delete_eml(eid) {
warn!("Failed to delete cached eml {}: {}", eid, e);
}
if let Err(e) = local_store.unindex_body(eid) {
warn!("Failed to unindex body {}: {}", eid, e);
}
}
if !deleted.is_empty() {
debug!(
@@ -801,6 +856,16 @@ async fn prefetch_details(
if let Err(e) = local_store.mark_has_details(&eid) {
warn!("Failed to mark has_details {}: {}", eid, e);
}
// Index the plain-text body for full-text search.
let body_html = details
.body
.compressedText
.as_deref()
.or(details.body.text.as_deref())
.unwrap_or("");
if let Err(e) = local_store.index_body(&eid, &strip_html(body_html)) {
warn!("Failed to index body {}: {}", eid, e);
}
if attachments_pending {
attachment_retry_history.insert(eid.clone(), now);
}
+1
View File
@@ -200,6 +200,7 @@ async fn main() -> anyhow::Result<()> {
cfg.imap_port,
store.clone(),
backend.clone(),
local_store.clone(),
imap_tls,
pw.clone(),
));