mirror of
https://github.com/spartanz51/tutabridge.git
synced 2026-06-24 10:54:32 +02:00
Full-text body search via an encrypted FTS5 index
BODY/TEXT searches previously matched only bodies that happened to be decoded in memory, so results were inconsistent. Add a persistent FTS5 index (a virtual table inside the SQLCipher store, encrypted at rest) over the plain-text body of every message we download. - store.rs: mail_fts(element_id UNINDEXED, body) with unicode61 + remove_diacritics; index_body / unindex_body / search_body / fts_count. Terms become prefix tokens ANDed together (factur -> factur*), built by fts_match_expr which strips everything but alphanumerics so it is injection-safe. - rfc2822.rs: strip_html (drops tags + script/style + entities) and extract_body_text (decodes the text part of our own .eml) feed the index. - sync.rs: index inline at prefetch; one-time backfill at boot (body_fts_indexed_v1) for bodies cached before the index existed; unindex on delete. - search.rs: BODY/TEXT resolve through the index — the session collects the distinct body terms, queries the index once each, and passes the hit sets to matches() via a SearchContext. A body term only matches messages whose body has actually been downloaded (full coverage needs sync_limit = 0). - LocalStore threaded into ImapSession (Option; None in unit tests). Validated live: backfilled 7,687 cached bodies, then BODY/TEXT/AND/OR/NOT queries returned coherent subsets — NOT BODY x == total - (BODY x), an exact complement. 233 unit tests, incl. real FTS5 MATCH against the bundled SQLCipher (confirms FTS5 is compiled in for the cross-OS release).
This commit is contained in:
@@ -365,7 +365,7 @@ impl BridgeHandle {
|
|||||||
};
|
};
|
||||||
let handler_handle = tokio::spawn(event_handler::run_event_handler(
|
let handler_handle = tokio::spawn(event_handler::run_event_handler(
|
||||||
store.clone(),
|
store.clone(),
|
||||||
local_store,
|
local_store.clone(),
|
||||||
backend.clone(),
|
backend.clone(),
|
||||||
bus_ids_for_handler,
|
bus_ids_for_handler,
|
||||||
event_rx,
|
event_rx,
|
||||||
@@ -375,6 +375,7 @@ impl BridgeHandle {
|
|||||||
imap_port,
|
imap_port,
|
||||||
store.clone(),
|
store.clone(),
|
||||||
backend.clone(),
|
backend.clone(),
|
||||||
|
local_store,
|
||||||
imap_tls,
|
imap_tls,
|
||||||
pw.clone(),
|
pw.clone(),
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ use tokio::net::TcpListener;
|
|||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
use tokio_rustls::TlsAcceptor;
|
use tokio_rustls::TlsAcceptor;
|
||||||
|
|
||||||
|
use crate::store::LocalStore;
|
||||||
use crate::sync::MailStore;
|
use crate::sync::MailStore;
|
||||||
use crate::tuta::MailBackend;
|
use crate::tuta::MailBackend;
|
||||||
use session::ImapSession;
|
use session::ImapSession;
|
||||||
@@ -17,6 +18,7 @@ pub async fn serve(
|
|||||||
port: u16,
|
port: u16,
|
||||||
store: Arc<MailStore>,
|
store: Arc<MailStore>,
|
||||||
backend: Arc<dyn MailBackend>,
|
backend: Arc<dyn MailBackend>,
|
||||||
|
local_store: Arc<LocalStore>,
|
||||||
tls: TlsAcceptor,
|
tls: TlsAcceptor,
|
||||||
password_hash: Option<String>,
|
password_hash: Option<String>,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
@@ -28,13 +30,16 @@ pub async fn serve(
|
|||||||
debug!("IMAP connection from {}", addr);
|
debug!("IMAP connection from {}", addr);
|
||||||
let store = store.clone();
|
let store = store.clone();
|
||||||
let backend = backend.clone();
|
let backend = backend.clone();
|
||||||
|
let local_store = local_store.clone();
|
||||||
let tls = tls.clone();
|
let tls = tls.clone();
|
||||||
let pw_hash = password_hash.clone();
|
let pw_hash = password_hash.clone();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
match tls.accept(stream).await {
|
match tls.accept(stream).await {
|
||||||
Ok(tls_stream) => {
|
Ok(tls_stream) => {
|
||||||
if let Err(e) = handle_connection(tls_stream, store, backend, pw_hash).await {
|
if let Err(e) =
|
||||||
|
handle_connection(tls_stream, store, backend, local_store, pw_hash).await
|
||||||
|
{
|
||||||
error!("IMAP connection error: {}", e);
|
error!("IMAP connection error: {}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -50,12 +55,13 @@ async fn handle_connection(
|
|||||||
stream: tokio_rustls::server::TlsStream<tokio::net::TcpStream>,
|
stream: tokio_rustls::server::TlsStream<tokio::net::TcpStream>,
|
||||||
store: Arc<MailStore>,
|
store: Arc<MailStore>,
|
||||||
backend: Arc<dyn MailBackend>,
|
backend: Arc<dyn MailBackend>,
|
||||||
|
local_store: Arc<LocalStore>,
|
||||||
password_hash: Option<String>,
|
password_hash: Option<String>,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
let (reader, mut writer) = tokio::io::split(stream);
|
let (reader, mut writer) = tokio::io::split(stream);
|
||||||
let mut reader = BufReader::new(reader);
|
let mut reader = BufReader::new(reader);
|
||||||
let mut store_watch: watch::Receiver<u64> = store.subscribe();
|
let mut store_watch: watch::Receiver<u64> = store.subscribe();
|
||||||
let mut session = ImapSession::new(store, backend, password_hash);
|
let mut session = ImapSession::new(store, backend, password_hash, Some(local_store));
|
||||||
|
|
||||||
writer
|
writer
|
||||||
.write_all(b"* OK TutaBridge IMAP4rev1 ready\r\n")
|
.write_all(b"* OK TutaBridge IMAP4rev1 ready\r\n")
|
||||||
|
|||||||
@@ -7,14 +7,17 @@
|
|||||||
//!
|
//!
|
||||||
//! Coverage is metadata-first: subject / from / to / cc, flags, dates, sizes,
|
//! Coverage is metadata-first: subject / from / to / cc, flags, dates, sizes,
|
||||||
//! sequence + UID sets, and boolean composition (`AND` / `OR` / `NOT`). `BODY`
|
//! sequence + UID sets, and boolean composition (`AND` / `OR` / `NOT`). `BODY`
|
||||||
//! and `TEXT` match against whatever body text the session already has decoded
|
//! and `TEXT` are resolved against the on-disk full-text index: the session
|
||||||
//! — full-text body search over the whole mailbox is a later increment backed
|
//! queries it once per distinct term ([`collect_body_terms`]) and hands the
|
||||||
//! by an on-disk index.
|
//! per-term hit sets to [`matches`] via a [`SearchContext`]. A body term only
|
||||||
|
//! matches messages whose body has actually been downloaded and indexed.
|
||||||
//!
|
//!
|
||||||
//! Robustness rule: an unrecognised criterion degrades to a non-restrictive
|
//! Robustness rule: an unrecognised criterion degrades to a non-restrictive
|
||||||
//! match (it never *hides* messages). Over-inclusion is the safe failure for
|
//! match (it never *hides* messages). Over-inclusion is the safe failure for
|
||||||
//! search; silently dropping a matching mail is not.
|
//! search; silently dropping a matching mail is not.
|
||||||
|
|
||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
/// One element of an IMAP sequence/UID set, e.g. `1`, `3:9`, or `5:*`.
|
/// One element of an IMAP sequence/UID set, e.g. `1`, `3:9`, or `5:*`.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
struct SeqRange {
|
struct SeqRange {
|
||||||
@@ -107,6 +110,8 @@ pub enum SearchKey {
|
|||||||
pub struct MsgView<'a> {
|
pub struct MsgView<'a> {
|
||||||
pub seq: u32,
|
pub seq: u32,
|
||||||
pub uid: u32,
|
pub uid: u32,
|
||||||
|
/// Tuta element id — the key used to look up full-text body hits.
|
||||||
|
pub element_id: &'a str,
|
||||||
pub subject: &'a str,
|
pub subject: &'a str,
|
||||||
/// Formatted `From` (name + address), for `FROM` substring matching.
|
/// Formatted `From` (name + address), for `FROM` substring matching.
|
||||||
pub from: String,
|
pub from: String,
|
||||||
@@ -122,9 +127,49 @@ pub struct MsgView<'a> {
|
|||||||
pub unread: bool,
|
pub unread: bool,
|
||||||
pub deleted: bool,
|
pub deleted: bool,
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
/// Decoded body text, when the session already has it. `None` means the
|
}
|
||||||
/// body isn't loaded; `BODY`/`TEXT` simply won't match such a message yet.
|
|
||||||
pub body: Option<&'a str>,
|
/// Pre-resolved full-text results for one SEARCH: maps each `BODY`/`TEXT` term
|
||||||
|
/// to the set of element ids whose indexed body matched it. Built by the
|
||||||
|
/// session from the on-disk index before evaluating the query.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct SearchContext {
|
||||||
|
pub body_hits: HashMap<String, HashSet<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SearchContext {
|
||||||
|
pub fn empty() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn body_matches(&self, term: &str, element_id: &str) -> bool {
|
||||||
|
self.body_hits
|
||||||
|
.get(term)
|
||||||
|
.is_some_and(|ids| ids.contains(element_id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Collect the distinct `BODY`/`TEXT` term arguments in a parsed query, so the
|
||||||
|
/// session can resolve each against the full-text index exactly once.
|
||||||
|
pub fn collect_body_terms(key: &SearchKey) -> Vec<String> {
|
||||||
|
let mut terms = Vec::new();
|
||||||
|
collect_body_terms_into(key, &mut terms);
|
||||||
|
terms.sort();
|
||||||
|
terms.dedup();
|
||||||
|
terms
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_body_terms_into(key: &SearchKey, out: &mut Vec<String>) {
|
||||||
|
match key {
|
||||||
|
SearchKey::And(keys) => keys.iter().for_each(|k| collect_body_terms_into(k, out)),
|
||||||
|
SearchKey::Or(a, b) => {
|
||||||
|
collect_body_terms_into(a, out);
|
||||||
|
collect_body_terms_into(b, out);
|
||||||
|
}
|
||||||
|
SearchKey::Not(k) => collect_body_terms_into(k, out),
|
||||||
|
SearchKey::Body(s) | SearchKey::Text(s) => out.push(s.clone()),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn contains_ci(haystack: &str, needle: &str) -> bool {
|
fn contains_ci(haystack: &str, needle: &str) -> bool {
|
||||||
@@ -154,13 +199,14 @@ fn day_number(ms: u64) -> i64 {
|
|||||||
(ms / 86_400_000) as i64
|
(ms / 86_400_000) as i64
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Evaluate a parsed query against one message.
|
/// Evaluate a parsed query against one message, consulting `ctx` for the
|
||||||
pub fn matches(key: &SearchKey, m: &MsgView) -> bool {
|
/// full-text results of any `BODY`/`TEXT` terms.
|
||||||
|
pub fn matches(key: &SearchKey, m: &MsgView, ctx: &SearchContext) -> bool {
|
||||||
match key {
|
match key {
|
||||||
SearchKey::All => true,
|
SearchKey::All => true,
|
||||||
SearchKey::And(keys) => keys.iter().all(|k| matches(k, m)),
|
SearchKey::And(keys) => keys.iter().all(|k| matches(k, m, ctx)),
|
||||||
SearchKey::Or(a, b) => matches(a, m) || matches(b, m),
|
SearchKey::Or(a, b) => matches(a, m, ctx) || matches(b, m, ctx),
|
||||||
SearchKey::Not(k) => !matches(k, m),
|
SearchKey::Not(k) => !matches(k, m, ctx),
|
||||||
|
|
||||||
SearchKey::Seen => !m.unread,
|
SearchKey::Seen => !m.unread,
|
||||||
SearchKey::Unseen => m.unread,
|
SearchKey::Unseen => m.unread,
|
||||||
@@ -183,10 +229,8 @@ pub fn matches(key: &SearchKey, m: &MsgView) -> bool {
|
|||||||
SearchKey::To(s) => contains_ci(&m.to, s),
|
SearchKey::To(s) => contains_ci(&m.to, s),
|
||||||
SearchKey::Cc(s) => contains_ci(&m.cc, s),
|
SearchKey::Cc(s) => contains_ci(&m.cc, s),
|
||||||
SearchKey::Bcc(s) => contains_ci(&m.bcc, s),
|
SearchKey::Bcc(s) => contains_ci(&m.bcc, s),
|
||||||
SearchKey::Body(s) => m.body.is_some_and(|b| contains_ci(b, s)),
|
SearchKey::Body(s) => ctx.body_matches(s, m.element_id),
|
||||||
SearchKey::Text(s) => {
|
SearchKey::Text(s) => contains_ci(&m.headers, s) || ctx.body_matches(s, m.element_id),
|
||||||
contains_ci(&m.headers, s) || m.body.is_some_and(|b| contains_ci(b, s))
|
|
||||||
}
|
|
||||||
SearchKey::Header(name, val) => header_contains(&m.headers, name, val),
|
SearchKey::Header(name, val) => header_contains(&m.headers, name, val),
|
||||||
|
|
||||||
SearchKey::Before(d) => day_number(m.date_ms) < *d,
|
SearchKey::Before(d) => day_number(m.date_ms) < *d,
|
||||||
@@ -511,6 +555,7 @@ mod tests {
|
|||||||
MsgView {
|
MsgView {
|
||||||
seq: 1,
|
seq: 1,
|
||||||
uid: 10,
|
uid: 10,
|
||||||
|
element_id: "mail1",
|
||||||
subject: "Hello World",
|
subject: "Hello World",
|
||||||
from: "Alice <alice@example.com>".into(),
|
from: "Alice <alice@example.com>".into(),
|
||||||
to: "Bob <bob@example.com>".into(),
|
to: "Bob <bob@example.com>".into(),
|
||||||
@@ -524,10 +569,22 @@ mod tests {
|
|||||||
unread: true,
|
unread: true,
|
||||||
deleted: false,
|
deleted: false,
|
||||||
size: 5000,
|
size: 5000,
|
||||||
body: Some("the quick brown fox"),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A context where the given terms all hit our test message ("mail1").
|
||||||
|
fn ctx_hitting(terms: &[&str]) -> SearchContext {
|
||||||
|
let mut body_hits = HashMap::new();
|
||||||
|
for t in terms {
|
||||||
|
body_hits.insert((*t).to_string(), HashSet::from(["mail1".to_string()]));
|
||||||
|
}
|
||||||
|
SearchContext { body_hits }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn no_ctx() -> SearchContext {
|
||||||
|
SearchContext::empty()
|
||||||
|
}
|
||||||
|
|
||||||
// --- tokenizer ---
|
// --- tokenizer ---
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -655,101 +712,141 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_subject_ci() {
|
fn match_subject_ci() {
|
||||||
assert!(matches(&SearchKey::Subject("hello".into()), &view()));
|
let c = no_ctx();
|
||||||
assert!(matches(&SearchKey::Subject("WORLD".into()), &view()));
|
assert!(matches(&SearchKey::Subject("hello".into()), &view(), &c));
|
||||||
assert!(!matches(&SearchKey::Subject("nope".into()), &view()));
|
assert!(matches(&SearchKey::Subject("WORLD".into()), &view(), &c));
|
||||||
|
assert!(!matches(&SearchKey::Subject("nope".into()), &view(), &c));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_from_to() {
|
fn match_from_to() {
|
||||||
assert!(matches(&SearchKey::From("alice".into()), &view()));
|
let c = no_ctx();
|
||||||
assert!(matches(&SearchKey::To("bob@example".into()), &view()));
|
assert!(matches(&SearchKey::From("alice".into()), &view(), &c));
|
||||||
assert!(!matches(&SearchKey::Cc("anyone".into()), &view()));
|
assert!(matches(&SearchKey::To("bob@example".into()), &view(), &c));
|
||||||
|
assert!(!matches(&SearchKey::Cc("anyone".into()), &view(), &c));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_flags_consistent_with_fetch() {
|
fn match_flags_consistent_with_fetch() {
|
||||||
let v = view(); // unread, not deleted
|
let v = view(); // unread, not deleted
|
||||||
assert!(matches(&SearchKey::Unseen, &v));
|
let c = no_ctx();
|
||||||
assert!(!matches(&SearchKey::Seen, &v));
|
assert!(matches(&SearchKey::Unseen, &v, &c));
|
||||||
assert!(!matches(&SearchKey::Answered, &v));
|
assert!(!matches(&SearchKey::Seen, &v, &c));
|
||||||
assert!(matches(&SearchKey::Unanswered, &v));
|
assert!(!matches(&SearchKey::Answered, &v, &c));
|
||||||
assert!(!matches(&SearchKey::Flagged, &v));
|
assert!(matches(&SearchKey::Unanswered, &v, &c));
|
||||||
assert!(!matches(&SearchKey::Deleted, &v));
|
assert!(!matches(&SearchKey::Flagged, &v, &c));
|
||||||
assert!(matches(&SearchKey::Undeleted, &v));
|
assert!(!matches(&SearchKey::Deleted, &v, &c));
|
||||||
|
assert!(matches(&SearchKey::Undeleted, &v, &c));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_dates() {
|
fn match_dates() {
|
||||||
let v = view(); // 2022-09-22
|
let v = view(); // 2022-09-22
|
||||||
assert!(matches(&SearchKey::Since(day(2022, 1, 1)), &v));
|
let c = no_ctx();
|
||||||
assert!(matches(&SearchKey::Before(day(2023, 1, 1)), &v));
|
assert!(matches(&SearchKey::Since(day(2022, 1, 1)), &v, &c));
|
||||||
assert!(matches(&SearchKey::On(day(2022, 9, 22)), &v));
|
assert!(matches(&SearchKey::Before(day(2023, 1, 1)), &v, &c));
|
||||||
assert!(!matches(&SearchKey::On(day(2022, 9, 23)), &v));
|
assert!(matches(&SearchKey::On(day(2022, 9, 22)), &v, &c));
|
||||||
assert!(!matches(&SearchKey::Since(day(2023, 1, 1)), &v));
|
assert!(!matches(&SearchKey::On(day(2022, 9, 23)), &v, &c));
|
||||||
|
assert!(!matches(&SearchKey::Since(day(2023, 1, 1)), &v, &c));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_size() {
|
fn match_size() {
|
||||||
let v = view(); // size 5000
|
let v = view(); // size 5000
|
||||||
assert!(matches(&SearchKey::Larger(4000), &v));
|
let c = no_ctx();
|
||||||
assert!(!matches(&SearchKey::Larger(6000), &v));
|
assert!(matches(&SearchKey::Larger(4000), &v, &c));
|
||||||
assert!(matches(&SearchKey::Smaller(6000), &v));
|
assert!(!matches(&SearchKey::Larger(6000), &v, &c));
|
||||||
|
assert!(matches(&SearchKey::Smaller(6000), &v, &c));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_body_and_text() {
|
fn match_body_uses_fts_context() {
|
||||||
let v = view();
|
let v = view();
|
||||||
assert!(matches(&SearchKey::Body("brown".into()), &v));
|
let c = ctx_hitting(&["brown"]);
|
||||||
assert!(!matches(&SearchKey::Body("missing".into()), &v));
|
assert!(matches(&SearchKey::Body("brown".into()), &v, &c));
|
||||||
// TEXT spans headers + body.
|
// A term with no FTS hit doesn't match, even though it's a real word.
|
||||||
assert!(matches(&SearchKey::Text("Message-ID".into()), &v));
|
assert!(!matches(&SearchKey::Body("missing".into()), &v, &c));
|
||||||
assert!(matches(&SearchKey::Text("quick".into()), &v));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_body_without_loaded_body_never_matches() {
|
fn match_text_spans_headers_and_body() {
|
||||||
let mut v = view();
|
let v = view();
|
||||||
v.body = None;
|
let c = ctx_hitting(&["quick"]);
|
||||||
assert!(!matches(&SearchKey::Body("brown".into()), &v));
|
// Header hit, no body hit needed.
|
||||||
// TEXT still matches on headers.
|
assert!(matches(
|
||||||
assert!(matches(&SearchKey::Text("Subject".into()), &v));
|
&SearchKey::Text("Message-ID".into()),
|
||||||
|
&v,
|
||||||
|
&no_ctx()
|
||||||
|
));
|
||||||
|
// Body hit via the FTS context.
|
||||||
|
assert!(matches(&SearchKey::Text("quick".into()), &v, &c));
|
||||||
|
// Neither headers nor index: no match.
|
||||||
|
assert!(!matches(&SearchKey::Text("quick".into()), &v, &no_ctx()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn match_body_without_index_never_matches() {
|
||||||
|
let v = view();
|
||||||
|
assert!(!matches(&SearchKey::Body("brown".into()), &v, &no_ctx()));
|
||||||
|
// TEXT still matches on headers without any index.
|
||||||
|
assert!(matches(&SearchKey::Text("Subject".into()), &v, &no_ctx()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn collect_body_terms_finds_nested_terms() {
|
||||||
|
let k = parse(r#"OR BODY "alpha" (TEXT "beta" SUBJECT "x") NOT BODY "alpha""#);
|
||||||
|
// Note: top level is `OR <a> <b>` then trailing keys folded into AND.
|
||||||
|
let terms = collect_body_terms(&k);
|
||||||
|
assert_eq!(terms, vec!["alpha".to_string(), "beta".to_string()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_header() {
|
fn match_header() {
|
||||||
let v = view();
|
let v = view();
|
||||||
|
let c = no_ctx();
|
||||||
assert!(matches(
|
assert!(matches(
|
||||||
&SearchKey::Header("message-id".into(), "abc".into()),
|
&SearchKey::Header("message-id".into(), "abc".into()),
|
||||||
&v
|
&v,
|
||||||
|
&c
|
||||||
));
|
));
|
||||||
assert!(!matches(
|
assert!(!matches(
|
||||||
&SearchKey::Header("message-id".into(), "zzz".into()),
|
&SearchKey::Header("message-id".into(), "zzz".into()),
|
||||||
&v
|
&v,
|
||||||
|
&c
|
||||||
));
|
));
|
||||||
// Presence-only (empty value).
|
// Presence-only (empty value).
|
||||||
assert!(matches(&SearchKey::Header("subject".into(), "".into()), &v));
|
assert!(matches(
|
||||||
assert!(!matches(&SearchKey::Header("x-nope".into(), "".into()), &v));
|
&SearchKey::Header("subject".into(), "".into()),
|
||||||
|
&v,
|
||||||
|
&c
|
||||||
|
));
|
||||||
|
assert!(!matches(
|
||||||
|
&SearchKey::Header("x-nope".into(), "".into()),
|
||||||
|
&v,
|
||||||
|
&c
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_uid_and_seq() {
|
fn match_uid_and_seq() {
|
||||||
let v = view(); // seq 1, uid 10
|
let v = view(); // seq 1, uid 10
|
||||||
assert!(matches(&SearchKey::Uid(parse_seqset("5:15")), &v));
|
let c = no_ctx();
|
||||||
assert!(!matches(&SearchKey::Uid(parse_seqset("1:5")), &v));
|
assert!(matches(&SearchKey::Uid(parse_seqset("5:15")), &v, &c));
|
||||||
assert!(matches(&SearchKey::Sequence(parse_seqset("1")), &v));
|
assert!(!matches(&SearchKey::Uid(parse_seqset("1:5")), &v, &c));
|
||||||
|
assert!(matches(&SearchKey::Sequence(parse_seqset("1")), &v, &c));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn match_boolean_composition() {
|
fn match_boolean_composition() {
|
||||||
let v = view();
|
let v = view();
|
||||||
|
let c = no_ctx();
|
||||||
let k = parse(r#"UNSEEN SUBJECT "hello""#);
|
let k = parse(r#"UNSEEN SUBJECT "hello""#);
|
||||||
assert!(matches(&k, &v));
|
assert!(matches(&k, &v, &c));
|
||||||
let k = parse(r#"SEEN SUBJECT "hello""#);
|
let k = parse(r#"SEEN SUBJECT "hello""#);
|
||||||
assert!(!matches(&k, &v));
|
assert!(!matches(&k, &v, &c));
|
||||||
let k = parse(r#"OR SEEN SUBJECT "hello""#);
|
let k = parse(r#"OR SEEN SUBJECT "hello""#);
|
||||||
assert!(matches(&k, &v));
|
assert!(matches(&k, &v, &c));
|
||||||
let k = parse("NOT SEEN");
|
let k = parse("NOT SEEN");
|
||||||
assert!(matches(&k, &v));
|
assert!(matches(&k, &v, &c));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ use tutasdk::entities::generated::tutanota::{Mail, MailAddress, MailDetails, Tut
|
|||||||
use crate::imap::search::{self, MsgView};
|
use crate::imap::search::{self, MsgView};
|
||||||
use crate::mail::mail_to_rfc2822;
|
use crate::mail::mail_to_rfc2822;
|
||||||
use crate::mail::rfc2822::{extract_headers, format_address, format_internal_date};
|
use crate::mail::rfc2822::{extract_headers, format_address, format_internal_date};
|
||||||
|
use crate::store::LocalStore;
|
||||||
use crate::sync::MailStore;
|
use crate::sync::MailStore;
|
||||||
use crate::tuta::{FolderInfo, MailBackend};
|
use crate::tuta::{FolderInfo, MailBackend};
|
||||||
|
|
||||||
@@ -33,6 +34,9 @@ struct CachedMail {
|
|||||||
pub struct ImapSession {
|
pub struct ImapSession {
|
||||||
store: Arc<MailStore>,
|
store: Arc<MailStore>,
|
||||||
backend: Arc<dyn MailBackend>,
|
backend: Arc<dyn MailBackend>,
|
||||||
|
/// The on-disk store, used for full-text body search. `None` in unit tests
|
||||||
|
/// (body terms then simply don't match).
|
||||||
|
local_store: Option<Arc<LocalStore>>,
|
||||||
state: State,
|
state: State,
|
||||||
selected_folder: Option<FolderInfo>,
|
selected_folder: Option<FolderInfo>,
|
||||||
mails: Vec<CachedMail>,
|
mails: Vec<CachedMail>,
|
||||||
@@ -47,10 +51,12 @@ impl ImapSession {
|
|||||||
store: Arc<MailStore>,
|
store: Arc<MailStore>,
|
||||||
backend: Arc<dyn MailBackend>,
|
backend: Arc<dyn MailBackend>,
|
||||||
password_hash: Option<String>,
|
password_hash: Option<String>,
|
||||||
|
local_store: Option<Arc<LocalStore>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
store,
|
store,
|
||||||
backend,
|
backend,
|
||||||
|
local_store,
|
||||||
state: State::NotAuthenticated,
|
state: State::NotAuthenticated,
|
||||||
selected_folder: None,
|
selected_folder: None,
|
||||||
mails: Vec::new(),
|
mails: Vec::new(),
|
||||||
@@ -465,6 +471,7 @@ impl ImapSession {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let query = search::parse(args);
|
let query = search::parse(args);
|
||||||
|
let ctx = self.resolve_body_search(&query);
|
||||||
|
|
||||||
let ids: Vec<u32> = self
|
let ids: Vec<u32> = self
|
||||||
.mails
|
.mails
|
||||||
@@ -472,7 +479,7 @@ impl ImapSession {
|
|||||||
.enumerate()
|
.enumerate()
|
||||||
.filter(|(i, cached)| {
|
.filter(|(i, cached)| {
|
||||||
let view = Self::build_search_view(*i, cached);
|
let view = Self::build_search_view(*i, cached);
|
||||||
search::matches(&query, &view)
|
search::matches(&query, &view, &ctx)
|
||||||
})
|
})
|
||||||
.map(|(i, cached)| if uid_mode { cached.uid } else { (i + 1) as u32 })
|
.map(|(i, cached)| if uid_mode { cached.uid } else { (i + 1) as u32 })
|
||||||
.collect();
|
.collect();
|
||||||
@@ -500,11 +507,6 @@ impl ImapSession {
|
|||||||
.map(extract_headers)
|
.map(extract_headers)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let body = cached
|
|
||||||
.details
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|d| d.body.compressedText.as_deref().or(d.body.text.as_deref()));
|
|
||||||
|
|
||||||
// To/Cc/Bcc come from details when the body is loaded; otherwise only
|
// To/Cc/Bcc come from details when the body is loaded; otherwise only
|
||||||
// the envelope `firstRecipient` (rendered as To) is available.
|
// the envelope `firstRecipient` (rendered as To) is available.
|
||||||
let (to, cc, bcc) = match cached.details.as_ref() {
|
let (to, cc, bcc) = match cached.details.as_ref() {
|
||||||
@@ -531,9 +533,17 @@ impl ImapSession {
|
|||||||
.map(|d| d.sentDate.as_millis())
|
.map(|d| d.sentDate.as_millis())
|
||||||
.unwrap_or_else(|| cached.mail.receivedDate.as_millis());
|
.unwrap_or_else(|| cached.mail.receivedDate.as_millis());
|
||||||
|
|
||||||
|
let element_id = cached
|
||||||
|
.mail
|
||||||
|
._id
|
||||||
|
.as_ref()
|
||||||
|
.map(|id| id.element_id.0.as_str())
|
||||||
|
.unwrap_or("");
|
||||||
|
|
||||||
MsgView {
|
MsgView {
|
||||||
seq: (idx + 1) as u32,
|
seq: (idx + 1) as u32,
|
||||||
uid: cached.uid,
|
uid: cached.uid,
|
||||||
|
element_id,
|
||||||
subject: &cached.mail.subject,
|
subject: &cached.mail.subject,
|
||||||
from: format_address(&cached.mail.sender),
|
from: format_address(&cached.mail.sender),
|
||||||
to,
|
to,
|
||||||
@@ -545,10 +555,28 @@ impl ImapSession {
|
|||||||
unread: cached.mail.unread,
|
unread: cached.mail.unread,
|
||||||
deleted: cached.deleted,
|
deleted: cached.deleted,
|
||||||
size: cached.rfc2822.as_ref().map(|r| r.len() as u64).unwrap_or(0),
|
size: cached.rfc2822.as_ref().map(|r| r.len() as u64).unwrap_or(0),
|
||||||
body,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolve the `BODY`/`TEXT` terms of a query against the full-text index,
|
||||||
|
/// once per distinct term. With no local store (unit tests) the context is
|
||||||
|
/// empty, so body terms match nothing.
|
||||||
|
fn resolve_body_search(&self, query: &search::SearchKey) -> search::SearchContext {
|
||||||
|
let mut ctx = search::SearchContext::empty();
|
||||||
|
let Some(local_store) = &self.local_store else {
|
||||||
|
return ctx;
|
||||||
|
};
|
||||||
|
for term in search::collect_body_terms(query) {
|
||||||
|
match local_store.search_body(&term) {
|
||||||
|
Ok(ids) => {
|
||||||
|
ctx.body_hits.insert(term, ids.into_iter().collect());
|
||||||
|
}
|
||||||
|
Err(e) => log::warn!("FTS body search failed for {term:?}: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx
|
||||||
|
}
|
||||||
|
|
||||||
async fn cmd_store(&mut self, tag: &str, args: &str, uid_mode: bool) -> Vec<String> {
|
async fn cmd_store(&mut self, tag: &str, args: &str, uid_mode: bool) -> Vec<String> {
|
||||||
if self.state != State::Selected {
|
if self.state != State::Selected {
|
||||||
return vec![format!("{} NO No mailbox selected\r\n", tag)];
|
return vec![format!("{} NO No mailbox selected\r\n", tag)];
|
||||||
@@ -1528,7 +1556,7 @@ mod tests {
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let mut session = ImapSession::new(store, backend.clone(), None);
|
let mut session = ImapSession::new(store, backend.clone(), None, None);
|
||||||
session.handle_command("a LOGIN u p").await;
|
session.handle_command("a LOGIN u p").await;
|
||||||
session.handle_command("b SELECT INBOX").await;
|
session.handle_command("b SELECT INBOX").await;
|
||||||
|
|
||||||
@@ -1578,7 +1606,7 @@ mod tests {
|
|||||||
let store = MailStore::new();
|
let store = MailStore::new();
|
||||||
let mails = backend.mails.lock().unwrap().clone();
|
let mails = backend.mails.lock().unwrap().clone();
|
||||||
populate_store(&store, &mails).await;
|
populate_store(&store, &mails).await;
|
||||||
let session = ImapSession::new(store.clone(), backend, None);
|
let session = ImapSession::new(store.clone(), backend, None, None);
|
||||||
(store, session)
|
(store, session)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1796,7 +1824,7 @@ mod tests {
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let mut session = ImapSession::new(store, backend, None);
|
let mut session = ImapSession::new(store, backend, None, None);
|
||||||
|
|
||||||
// LOGIN
|
// LOGIN
|
||||||
let resp = session.handle_command("A001 LOGIN user pass").await;
|
let resp = session.handle_command("A001 LOGIN user pass").await;
|
||||||
@@ -1840,7 +1868,7 @@ mod tests {
|
|||||||
)]));
|
)]));
|
||||||
let store = MailStore::new();
|
let store = MailStore::new();
|
||||||
populate_store(&store, &backend.mails.lock().unwrap()).await;
|
populate_store(&store, &backend.mails.lock().unwrap()).await;
|
||||||
let mut session = ImapSession::new(store, backend.clone(), None);
|
let mut session = ImapSession::new(store, backend.clone(), None, None);
|
||||||
|
|
||||||
session.handle_command("A001 LOGIN user pass").await;
|
session.handle_command("A001 LOGIN user pass").await;
|
||||||
session.handle_command("A002 SELECT INBOX").await;
|
session.handle_command("A002 SELECT INBOX").await;
|
||||||
@@ -1868,7 +1896,7 @@ mod tests {
|
|||||||
]));
|
]));
|
||||||
let store = MailStore::new();
|
let store = MailStore::new();
|
||||||
populate_store(&store, &backend.mails.lock().unwrap()).await;
|
populate_store(&store, &backend.mails.lock().unwrap()).await;
|
||||||
let mut session = ImapSession::new(store, backend.clone(), None);
|
let mut session = ImapSession::new(store, backend.clone(), None, None);
|
||||||
|
|
||||||
session.handle_command("A001 LOGIN user pass").await;
|
session.handle_command("A001 LOGIN user pass").await;
|
||||||
session.handle_command("A002 SELECT INBOX").await;
|
session.handle_command("A002 SELECT INBOX").await;
|
||||||
@@ -1908,7 +1936,7 @@ mod tests {
|
|||||||
]));
|
]));
|
||||||
let store = MailStore::new();
|
let store = MailStore::new();
|
||||||
populate_store(&store, &backend.mails.lock().unwrap()).await;
|
populate_store(&store, &backend.mails.lock().unwrap()).await;
|
||||||
let mut session = ImapSession::new(store.clone(), backend.clone(), None);
|
let mut session = ImapSession::new(store.clone(), backend.clone(), None, None);
|
||||||
|
|
||||||
session.handle_command("A001 LOGIN user pass").await;
|
session.handle_command("A001 LOGIN user pass").await;
|
||||||
session.handle_command("A002 SELECT INBOX").await;
|
session.handle_command("A002 SELECT INBOX").await;
|
||||||
|
|||||||
@@ -5,3 +5,4 @@ pub(crate) mod rfc2822;
|
|||||||
pub use bodystructure::compute_bodystructure;
|
pub use bodystructure::compute_bodystructure;
|
||||||
pub use parser::{Attachment, ParsedMessage};
|
pub use parser::{Attachment, ParsedMessage};
|
||||||
pub use rfc2822::mail_to_rfc2822;
|
pub use rfc2822::mail_to_rfc2822;
|
||||||
|
pub(crate) use rfc2822::{extract_body_text, strip_html};
|
||||||
|
|||||||
@@ -225,6 +225,99 @@ pub(crate) fn extract_headers(rfc: &str) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Strip HTML markup down to readable text for full-text indexing. Drops tags,
|
||||||
|
/// the contents of `<script>`/`<style>` blocks, and decodes the handful of
|
||||||
|
/// entities that actually show up in mail bodies, then collapses whitespace.
|
||||||
|
/// This is lossy by design — it only needs to be good enough that a body
|
||||||
|
/// search matches the words a human would see, not the markup around them.
|
||||||
|
pub(crate) fn strip_html(html: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(html.len());
|
||||||
|
let mut chars = html.chars().peekable();
|
||||||
|
while let Some(c) = chars.next() {
|
||||||
|
if c == '<' {
|
||||||
|
// Capture the tag name to detect script/style blocks we must skip
|
||||||
|
// wholesale (their text content is not human-visible).
|
||||||
|
let mut tag = String::new();
|
||||||
|
for t in chars.clone().take(6) {
|
||||||
|
if t == '>' || t.is_whitespace() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tag.push(t.to_ascii_lowercase());
|
||||||
|
}
|
||||||
|
let skip_block = matches!(tag.trim_start_matches('/'), "script" | "style");
|
||||||
|
// Consume up to and including the closing '>'.
|
||||||
|
for t in chars.by_ref() {
|
||||||
|
if t == '>' {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if skip_block && !tag.starts_with('/') {
|
||||||
|
// Swallow everything until the matching closing tag.
|
||||||
|
let close = format!("</{tag}");
|
||||||
|
let mut window = String::new();
|
||||||
|
for t in chars.by_ref() {
|
||||||
|
window.push(t.to_ascii_lowercase());
|
||||||
|
if window.ends_with(&close) {
|
||||||
|
// Drop the rest of the closing tag.
|
||||||
|
for t2 in chars.by_ref() {
|
||||||
|
if t2 == '>' {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push(' ');
|
||||||
|
} else {
|
||||||
|
out.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let decoded = decode_entities(&out);
|
||||||
|
decoded.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_entities(s: &str) -> String {
|
||||||
|
s.replace(" ", " ")
|
||||||
|
.replace("&", "&")
|
||||||
|
.replace("<", "<")
|
||||||
|
.replace(">", ">")
|
||||||
|
.replace(""", "\"")
|
||||||
|
.replace("'", "'")
|
||||||
|
.replace("'", "'")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the readable body text from one of our own RFC 2822 messages, for
|
||||||
|
/// full-text indexing: locate the first `text/*` part, base64-decode it, and
|
||||||
|
/// strip HTML. Works for both the single-part and multipart layouts produced
|
||||||
|
/// by [`mail_to_rfc2822`]. Returns an empty string when no text part is found.
|
||||||
|
pub(crate) fn extract_body_text(rfc: &str) -> String {
|
||||||
|
// Find the first textual MIME part.
|
||||||
|
let lower = rfc.to_lowercase();
|
||||||
|
let part_start = lower.find("text/html").or_else(|| lower.find("text/plain"));
|
||||||
|
let Some(part_start) = part_start else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
|
// Body begins after that part's header/body separator.
|
||||||
|
let Some(sep) = rfc[part_start..].find("\r\n\r\n") else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
|
let body_start = part_start + sep + 4;
|
||||||
|
// Body ends at the next MIME boundary delimiter (multipart) or end of input.
|
||||||
|
let body_end = rfc[body_start..]
|
||||||
|
.find("\r\n--")
|
||||||
|
.map(|i| body_start + i)
|
||||||
|
.unwrap_or(rfc.len());
|
||||||
|
let raw = &rfc[body_start..body_end];
|
||||||
|
|
||||||
|
let stripped_b64: String = raw.split_whitespace().collect();
|
||||||
|
match base64::engine::general_purpose::STANDARD.decode(stripped_b64) {
|
||||||
|
Ok(bytes) => strip_html(&String::from_utf8_lossy(&bytes)),
|
||||||
|
// Not base64 (shouldn't happen for our own messages) — treat as text.
|
||||||
|
Err(_) => strip_html(raw),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -238,6 +331,40 @@ mod tests {
|
|||||||
assert_eq!(days_to_ymd(0), (1970, 1, 1));
|
assert_eq!(days_to_ymd(0), (1970, 1, 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strip_html_drops_tags_and_decodes_entities() {
|
||||||
|
let html = "<p>Hello <b>World</b> & goodbye</p>";
|
||||||
|
assert_eq!(strip_html(html), "Hello World & goodbye");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strip_html_skips_script_and_style() {
|
||||||
|
let html = "<style>.a{color:red}</style><div>Visible</div><script>alert(1)</script>";
|
||||||
|
assert_eq!(strip_html(html), "Visible");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_body_text_from_single_part() {
|
||||||
|
let body = base64::engine::general_purpose::STANDARD.encode("<p>Quarterly invoice</p>");
|
||||||
|
let rfc = format!(
|
||||||
|
"Subject: Test\r\nContent-Type: text/html; charset=UTF-8\r\n\
|
||||||
|
Content-Transfer-Encoding: base64\r\n\r\n{body}\r\n"
|
||||||
|
);
|
||||||
|
assert_eq!(extract_body_text(&rfc), "Quarterly invoice");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extract_body_text_from_multipart_stops_at_boundary() {
|
||||||
|
let body = base64::engine::general_purpose::STANDARD.encode("<p>Body words here</p>");
|
||||||
|
let rfc = format!(
|
||||||
|
"Content-Type: multipart/mixed; boundary=\"BND\"\r\n\r\n\
|
||||||
|
--BND\r\nContent-Type: text/html; charset=UTF-8\r\n\
|
||||||
|
Content-Transfer-Encoding: base64\r\n\r\n{body}\r\n\
|
||||||
|
--BND\r\nContent-Type: application/pdf\r\n\r\nIGNOREDATTACHMENT\r\n--BND--\r\n"
|
||||||
|
);
|
||||||
|
assert_eq!(extract_body_text(&rfc), "Body words here");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_days_to_ymd_known_dates() {
|
fn test_days_to_ymd_known_dates() {
|
||||||
// 2024-01-01 = day 19723 since epoch
|
// 2024-01-01 = day 19723 since epoch
|
||||||
|
|||||||
+119
-1
@@ -46,6 +46,19 @@ pub struct LocalStore {
|
|||||||
mails_dir: PathBuf,
|
mails_dir: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Turn a free-text search term into a safe FTS5 MATCH expression. Each run of
|
||||||
|
/// alphanumeric characters becomes a prefix token (`word*`), and tokens are
|
||||||
|
/// ANDed together (FTS5's default). Because only alphanumerics survive, the
|
||||||
|
/// result can never contain FTS5 syntax, so it is injection-safe to interpolate
|
||||||
|
/// as a bound parameter. Returns an empty string when nothing is searchable.
|
||||||
|
fn fts_match_expr(term: &str) -> String {
|
||||||
|
term.split(|c: char| !c.is_alphanumeric())
|
||||||
|
.filter(|w| !w.is_empty())
|
||||||
|
.map(|w| format!("{w}*"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
impl LocalStore {
|
impl LocalStore {
|
||||||
pub fn open(
|
pub fn open(
|
||||||
db_path: &Path,
|
db_path: &Path,
|
||||||
@@ -85,7 +98,8 @@ impl LocalStore {
|
|||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
"DROP TABLE IF EXISTS mails;
|
"DROP TABLE IF EXISTS mails;
|
||||||
DROP TABLE IF EXISTS sync_state;
|
DROP TABLE IF EXISTS sync_state;
|
||||||
DROP TABLE IF EXISTS event_bus_state;",
|
DROP TABLE IF EXISTS event_bus_state;
|
||||||
|
DROP TABLE IF EXISTS mail_fts;",
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,6 +129,11 @@ impl LocalStore {
|
|||||||
last_batch_id TEXT NOT NULL,
|
last_batch_id TEXT NOT NULL,
|
||||||
updated_at_ms INTEGER NOT NULL DEFAULT 0
|
updated_at_ms INTEGER NOT NULL DEFAULT 0
|
||||||
);
|
);
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS mail_fts USING fts5(
|
||||||
|
element_id UNINDEXED,
|
||||||
|
body,
|
||||||
|
tokenize = 'unicode61 remove_diacritics 2'
|
||||||
|
);
|
||||||
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
INSERT OR REPLACE INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
||||||
))?;
|
))?;
|
||||||
|
|
||||||
@@ -165,6 +184,7 @@ impl LocalStore {
|
|||||||
"DELETE FROM mails;
|
"DELETE FROM mails;
|
||||||
DELETE FROM sync_state;
|
DELETE FROM sync_state;
|
||||||
DELETE FROM store_meta;
|
DELETE FROM store_meta;
|
||||||
|
DELETE FROM mail_fts;
|
||||||
INSERT INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
INSERT INTO store_meta(key, value) VALUES ('schema_version', '{SCHEMA_VERSION}');"
|
||||||
))?;
|
))?;
|
||||||
drop(conn);
|
drop(conn);
|
||||||
@@ -405,6 +425,54 @@ impl LocalStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Index (or re-index) a message's plain-text body for full-text search.
|
||||||
|
/// `body_text` should already be stripped of HTML markup. Stored inside the
|
||||||
|
/// SQLCipher database, so it is encrypted at rest like everything else.
|
||||||
|
pub fn index_body(&self, element_id: &str, body_text: &str) -> Result<(), StoreError> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
// FTS5 has no UPSERT; clear any prior row for this id first.
|
||||||
|
conn.execute("DELETE FROM mail_fts WHERE element_id = ?1", [element_id])?;
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO mail_fts(element_id, body) VALUES (?1, ?2)",
|
||||||
|
[element_id, body_text],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop a message from the full-text index (called when a mail is removed).
|
||||||
|
pub fn unindex_body(&self, element_id: &str) -> Result<(), StoreError> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
conn.execute("DELETE FROM mail_fts WHERE element_id = ?1", [element_id])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Element ids whose indexed body matches a user search term. The term is a
|
||||||
|
/// free-text query (one IMAP `BODY`/`TEXT` argument); it is turned into a
|
||||||
|
/// safe FTS5 MATCH expression: each word becomes a prefix token so
|
||||||
|
/// `factur` finds `facture`/`factures`, and multi-word terms must all be
|
||||||
|
/// present. Returns an empty set when the term has no searchable tokens.
|
||||||
|
pub fn search_body(&self, term: &str) -> Result<Vec<String>, StoreError> {
|
||||||
|
let match_expr = fts_match_expr(term);
|
||||||
|
if match_expr.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let mut stmt = conn.prepare("SELECT element_id FROM mail_fts WHERE body MATCH ?1")?;
|
||||||
|
let rows = stmt.query_map([&match_expr], |row| row.get::<_, String>(0))?;
|
||||||
|
let mut ids = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
ids.push(row?);
|
||||||
|
}
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of bodies currently in the full-text index (diagnostics / tests).
|
||||||
|
pub fn fts_count(&self) -> Result<usize, StoreError> {
|
||||||
|
let conn = self.conn.lock().unwrap();
|
||||||
|
let count: i64 = conn.query_row("SELECT COUNT(*) FROM mail_fts", [], |row| row.get(0))?;
|
||||||
|
Ok(count as usize)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn mail_count(&self, folder_id: &str) -> Result<usize, StoreError> {
|
pub fn mail_count(&self, folder_id: &str) -> Result<usize, StoreError> {
|
||||||
let conn = self.conn.lock().unwrap();
|
let conn = self.conn.lock().unwrap();
|
||||||
let count: i64 = conn.query_row(
|
let count: i64 = conn.query_row(
|
||||||
@@ -539,6 +607,7 @@ impl LocalStore {
|
|||||||
conn.execute("DELETE FROM mails WHERE element_id = ?1", [element_id])?;
|
conn.execute("DELETE FROM mails WHERE element_id = ?1", [element_id])?;
|
||||||
}
|
}
|
||||||
self.delete_eml(element_id)?;
|
self.delete_eml(element_id)?;
|
||||||
|
self.unindex_body(element_id)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -579,6 +648,55 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fts_index_and_search() {
|
||||||
|
let store = open_memory_store();
|
||||||
|
store
|
||||||
|
.index_body("a", "the quarterly invoice is attached")
|
||||||
|
.unwrap();
|
||||||
|
store.index_body("b", "lunch plans for friday").unwrap();
|
||||||
|
assert_eq!(store.fts_count().unwrap(), 2);
|
||||||
|
|
||||||
|
// Prefix match: "invoic" finds "invoice".
|
||||||
|
assert_eq!(store.search_body("invoic").unwrap(), vec!["a".to_string()]);
|
||||||
|
// Multi-word ANDs the tokens.
|
||||||
|
assert_eq!(
|
||||||
|
store.search_body("lunch friday").unwrap(),
|
||||||
|
vec!["b".to_string()]
|
||||||
|
);
|
||||||
|
// No hit.
|
||||||
|
assert!(store.search_body("zebra").unwrap().is_empty());
|
||||||
|
// Empty / punctuation-only term yields no FTS query.
|
||||||
|
assert!(store.search_body(" ").unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fts_reindex_replaces_prior_body() {
|
||||||
|
let store = open_memory_store();
|
||||||
|
store.index_body("a", "first version about cats").unwrap();
|
||||||
|
store.index_body("a", "second version about dogs").unwrap();
|
||||||
|
assert_eq!(store.fts_count().unwrap(), 1);
|
||||||
|
assert!(store.search_body("cats").unwrap().is_empty());
|
||||||
|
assert_eq!(store.search_body("dogs").unwrap(), vec!["a".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fts_unindex_removes_body() {
|
||||||
|
let store = open_memory_store();
|
||||||
|
store.index_body("a", "removable content").unwrap();
|
||||||
|
store.unindex_body("a").unwrap();
|
||||||
|
assert!(store.search_body("removable").unwrap().is_empty());
|
||||||
|
assert_eq!(store.fts_count().unwrap(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fts_match_expr_is_injection_safe() {
|
||||||
|
// FTS syntax characters never survive into the expression.
|
||||||
|
assert_eq!(fts_match_expr(r#"foo" OR bar"#), "foo* OR* bar*");
|
||||||
|
assert_eq!(fts_match_expr("a-b.c"), "a* b* c*");
|
||||||
|
assert_eq!(fts_match_expr(" "), "");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn allocate_uids_are_monotonic_and_per_folder() {
|
fn allocate_uids_are_monotonic_and_per_folder() {
|
||||||
let store = open_memory_store();
|
let store = open_memory_store();
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use log::{debug, info, warn};
|
|||||||
use tokio::sync::{watch, RwLock};
|
use tokio::sync::{watch, RwLock};
|
||||||
use tutasdk::entities::generated::tutanota::{Mail, MailDetails, TutanotaFile};
|
use tutasdk::entities::generated::tutanota::{Mail, MailDetails, TutanotaFile};
|
||||||
|
|
||||||
use crate::mail::mail_to_rfc2822;
|
use crate::mail::{extract_body_text, mail_to_rfc2822, strip_html};
|
||||||
use crate::store::{LocalStore, MailMetadata};
|
use crate::store::{LocalStore, MailMetadata};
|
||||||
use crate::tuta::{FolderInfo, MailBackend};
|
use crate::tuta::{FolderInfo, MailBackend};
|
||||||
|
|
||||||
@@ -426,6 +426,58 @@ pub async fn run_syncer(
|
|||||||
debug!("Skipping full-metadata sync — already complete, event bus reconciles");
|
debug!("Skipping full-metadata sync — already complete, event bus reconciles");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One-time backfill of the full-text body index from bodies that were
|
||||||
|
// already cached (downloaded before the index existed). Bodies fetched from
|
||||||
|
// here on are indexed inline by `prefetch_details`, so this only ever runs
|
||||||
|
// once per install.
|
||||||
|
const FTS_BACKFILL_MARKER: &str = "body_fts_indexed_v1";
|
||||||
|
if local_store.get_meta(FTS_BACKFILL_MARKER).is_none() && !folders.is_empty() {
|
||||||
|
info!("Backfilling full-text body index from cached messages…");
|
||||||
|
let mut indexed = 0usize;
|
||||||
|
let mut all_ok = true;
|
||||||
|
for folder in &folders {
|
||||||
|
if *shutdown.borrow() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match local_store.load_folder_metadata(&folder.id) {
|
||||||
|
Ok(metas) => {
|
||||||
|
for meta in metas.iter().filter(|m| m.has_details) {
|
||||||
|
if *shutdown.borrow() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
match local_store.read_eml(&meta.element_id) {
|
||||||
|
Ok(Some(rfc)) => {
|
||||||
|
let text = extract_body_text(&rfc);
|
||||||
|
if let Err(e) = local_store.index_body(&meta.element_id, &text) {
|
||||||
|
warn!("FTS backfill failed for {}: {e}", meta.element_id);
|
||||||
|
all_ok = false;
|
||||||
|
} else {
|
||||||
|
indexed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("FTS backfill read failed for {}: {e}", meta.element_id);
|
||||||
|
all_ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
warn!("FTS backfill: no metadata for {}: {e}", folder.imap_path);
|
||||||
|
all_ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Only mark done if nothing errored, so a partial run retries next boot.
|
||||||
|
if all_ok {
|
||||||
|
if let Err(e) = local_store.set_meta(FTS_BACKFILL_MARKER, "1") {
|
||||||
|
warn!("Could not persist FTS backfill marker: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info!("Full-text body index backfill complete ({indexed} messages)");
|
||||||
|
}
|
||||||
|
|
||||||
// From here on the syncer only owns the slow body prefetch (capped at
|
// From here on the syncer only owns the slow body prefetch (capped at
|
||||||
// `sync_limit` — the body-prefetch depth). Folder / new-mail refresh is
|
// `sync_limit` — the body-prefetch depth). Folder / new-mail refresh is
|
||||||
// driven by the event bus + `event_handler`.
|
// driven by the event bus + `event_handler`.
|
||||||
@@ -661,6 +713,9 @@ pub(crate) async fn sync_folder(
|
|||||||
if let Err(e) = local_store.delete_eml(eid) {
|
if let Err(e) = local_store.delete_eml(eid) {
|
||||||
warn!("Failed to delete cached eml {}: {}", eid, e);
|
warn!("Failed to delete cached eml {}: {}", eid, e);
|
||||||
}
|
}
|
||||||
|
if let Err(e) = local_store.unindex_body(eid) {
|
||||||
|
warn!("Failed to unindex body {}: {}", eid, e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !deleted.is_empty() {
|
if !deleted.is_empty() {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -801,6 +856,16 @@ async fn prefetch_details(
|
|||||||
if let Err(e) = local_store.mark_has_details(&eid) {
|
if let Err(e) = local_store.mark_has_details(&eid) {
|
||||||
warn!("Failed to mark has_details {}: {}", eid, e);
|
warn!("Failed to mark has_details {}: {}", eid, e);
|
||||||
}
|
}
|
||||||
|
// Index the plain-text body for full-text search.
|
||||||
|
let body_html = details
|
||||||
|
.body
|
||||||
|
.compressedText
|
||||||
|
.as_deref()
|
||||||
|
.or(details.body.text.as_deref())
|
||||||
|
.unwrap_or("");
|
||||||
|
if let Err(e) = local_store.index_body(&eid, &strip_html(body_html)) {
|
||||||
|
warn!("Failed to index body {}: {}", eid, e);
|
||||||
|
}
|
||||||
if attachments_pending {
|
if attachments_pending {
|
||||||
attachment_retry_history.insert(eid.clone(), now);
|
attachment_retry_history.insert(eid.clone(), now);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -200,6 +200,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
cfg.imap_port,
|
cfg.imap_port,
|
||||||
store.clone(),
|
store.clone(),
|
||||||
backend.clone(),
|
backend.clone(),
|
||||||
|
local_store.clone(),
|
||||||
imap_tls,
|
imap_tls,
|
||||||
pw.clone(),
|
pw.clone(),
|
||||||
));
|
));
|
||||||
|
|||||||
Reference in New Issue
Block a user