From 286ae160577936f68fd75cdee5c1b86924b4d500 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 5 Apr 2026 16:06:07 +0800 Subject: [PATCH] update --- Cargo.lock | 1207 ++++-------- Cargo.toml | 17 +- src/main.rs | 6 +- src/modules/account/migration.rs | 7 +- src/modules/blob/envelope.rs | 121 -- src/modules/blob/manager.rs | 349 ---- src/modules/cache/imap/mailbox.rs | 14 +- src/modules/cache/imap/sync/flow.rs | 4 +- src/modules/cache/imap/sync/rebuild.rs | 15 +- src/modules/dashboard/mod.rs | 67 +- src/modules/database/mod.rs | 15 + src/modules/duckdb/build.rs | 152 -- src/modules/duckdb/init.rs | 1716 ----------------- src/modules/duckdb/migrations/V1__initial.sql | 69 - src/modules/duckdb/refinery.rs | 93 - src/modules/duckdb/tests.rs | 28 - src/modules/envelope/extractor.rs | 66 +- src/modules/mailbox/delete.rs | 28 +- src/modules/message/content.rs | 26 +- src/modules/message/delete.rs | 13 +- src/modules/message/list.rs | 46 +- src/modules/message/search.rs | 26 +- src/modules/mod.rs | 3 +- src/modules/rest/api/message.rs | 58 +- src/modules/settings/dir.rs | 7 +- src/modules/store/envelope.rs | 53 + src/modules/{blob => store}/mod.rs | 2 +- src/modules/{blob => store}/storage.rs | 87 +- src/modules/store/tantivy/fields.rs | 80 + src/modules/store/tantivy/manager.rs | 1479 ++++++++++++++ src/modules/{duckdb => store/tantivy}/mod.rs | 9 +- src/modules/store/tantivy/model.rs | 241 +++ src/modules/store/tantivy/schema.rs | 130 ++ web/src/api/index.ts | 2 +- web/src/features/dashboard/index.tsx | 9 +- web/src/features/search/mail-list-table.tsx | 6 +- web/src/features/search/thread-dialog.tsx | 4 +- web/tsconfig.app.json | 19 +- 38 files changed, 2634 insertions(+), 3640 deletions(-) delete mode 100644 src/modules/blob/envelope.rs delete mode 100644 src/modules/blob/manager.rs delete mode 100644 src/modules/duckdb/build.rs delete mode 100644 src/modules/duckdb/init.rs delete mode 100644 src/modules/duckdb/migrations/V1__initial.sql delete mode 100644 src/modules/duckdb/refinery.rs delete mode 100644 src/modules/duckdb/tests.rs create mode 100644 src/modules/store/envelope.rs rename src/modules/{blob => store}/mod.rs (98%) rename src/modules/{blob => store}/storage.rs (67%) create mode 100644 src/modules/store/tantivy/fields.rs create mode 100644 src/modules/store/tantivy/manager.rs rename src/modules/{duckdb => store/tantivy}/mod.rs (91%) create mode 100644 src/modules/store/tantivy/model.rs create mode 100644 src/modules/store/tantivy/schema.rs diff --git a/Cargo.lock b/Cargo.lock index 41ef030..c238532 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,17 +73,6 @@ dependencies = [ "opaque-debug", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.12" @@ -91,7 +80,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -202,6 +190,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -214,225 +211,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "arrow" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", -] - -[[package]] -name = "arrow-arith" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "num-traits", -] - -[[package]] -name = "arrow-array" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "hashbrown 0.16.1", - "num-complex", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-buffer" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" -dependencies = [ - "bytes 1.11.1", - "half", - "num-bigint", - "num-traits", -] - -[[package]] -name = "arrow-cast" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num-traits", - "ryu", -] - -[[package]] -name = "arrow-csv" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" -dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "csv", - "csv-core", - "regex", -] - -[[package]] -name = "arrow-data" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num-integer", - "num-traits", -] - -[[package]] -name = "arrow-ipc" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "flatbuffers", -] - -[[package]] -name = "arrow-json" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "indexmap", - "itoa", - "lexical-core", - "memchr", - "num-traits", - "ryu", - "serde_core", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-ord" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", -] - -[[package]] -name = "arrow-row" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "half", -] - -[[package]] -name = "arrow-schema" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" -dependencies = [ - "bitflags", -] - -[[package]] -name = "arrow-select" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" -dependencies = [ - "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num-traits", -] - -[[package]] -name = "arrow-string" -version = "58.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num-traits", - "regex", - "regex-syntax", -] - [[package]] name = "asn1-rs" version = "0.7.1" @@ -632,15 +410,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -713,8 +482,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" name = "bichon" version = "1.0.0" dependencies = [ - "ahash 0.8.12", - "arrow", + "ahash", "async-imap", "autoconfig", "base64 0.22.1", @@ -727,7 +495,6 @@ dependencies = [ "console", "dashmap", "dialoguer", - "duckdb", "email_address", "encoding_rs", "fjall", @@ -757,11 +524,8 @@ dependencies = [ "poem", "poem-derive", "poem-openapi", - "r2d2", "rand 0.10.0", "rcgen", - "refinery", - "refinery-core", "regex", "reqwest", "ring", @@ -769,11 +533,12 @@ dependencies = [ "rustls", "rustls-pemfile", "rustls-pki-types", - "semver 1.0.27", + "semver 1.0.28", "serde", "serde_json", "snafu", "sysinfo", + "tantivy", "tempfile", "time 0.3.47", "timeago", @@ -808,15 +573,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] -name = "bitvec" -version = "1.0.1" +name = "bitpacking" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" dependencies = [ - "funty", - "radium", - "tap", - "wyz", + "crunchy", ] [[package]] @@ -865,25 +627,27 @@ dependencies = [ ] [[package]] -name = "borsh" -version = "1.6.0" +name = "bon" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" dependencies = [ - "borsh-derive", - "cfg_aliases", + "bon-macros", + "rustversion", ] [[package]] -name = "borsh-derive" -version = "1.6.0" +name = "bon-macros" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "once_cell", - "proc-macro-crate", + "darling", + "ident_case", + "prettyplease", "proc-macro2", "quote", + "rustversion", "syn 2.0.117", ] @@ -914,28 +678,6 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytecount" version = "0.6.9" @@ -1007,17 +749,11 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.27", + "semver 1.0.28", "serde", "serde_json", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.2.47" @@ -1030,6 +766,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cfg-if" version = "1.0.4" @@ -1161,17 +903,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" -[[package]] -name = "comfy-table" -version = "7.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" -dependencies = [ - "strum 0.26.3", - "strum_macros 0.26.4", - "unicode-width", -] - [[package]] name = "compare" version = "0.0.6" @@ -1229,26 +960,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "tiny-keccak", -] - [[package]] name = "const_fn" version = "0.4.11" @@ -1363,6 +1074,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -1414,27 +1135,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "csv" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde_core", -] - -[[package]] -name = "csv-core" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" -dependencies = [ - "memchr", -] - [[package]] name = "ctr" version = "0.6.0" @@ -1455,7 +1155,7 @@ dependencies = [ "openssl-probe 0.1.6", "openssl-sys", "schannel", - "socket2 0.6.1", + "socket2 0.6.3", "windows-sys 0.59.0", ] @@ -1530,6 +1230,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "deflate64" version = "0.1.10" @@ -1557,6 +1263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -1641,24 +1348,10 @@ dependencies = [ ] [[package]] -name = "duckdb" -version = "1.10501.0" +name = "downcast-rs" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13bc6d6487032fc2825a62ef8b4924b2378a2eb3166e132e5f3141ae9dd633f" -dependencies = [ - "arrow", - "cast", - "chrono", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libduckdb-sys", - "num", - "num-integer", - "r2d2", - "rust_decimal", - "strum 0.27.2", -] +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "dunce" @@ -1748,6 +1441,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1795,16 +1499,10 @@ dependencies = [ ] [[package]] -name = "fallible-iterator" -version = "0.3.0" +name = "fastdivide" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" @@ -1821,17 +1519,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "filetime" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" -dependencies = [ - "cfg-if", - "libc", - "libredox", -] - [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -1855,16 +1542,6 @@ dependencies = [ "xxhash-rust", ] -[[package]] -name = "flatbuffers" -version = "25.12.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" -dependencies = [ - "bitflags", - "rustc_version 0.4.1", -] - [[package]] name = "flate2" version = "1.1.5" @@ -1938,18 +1615,22 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.32" @@ -2214,27 +1895,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "num-traits", - "zerocopy", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -2272,15 +1932,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "headers" version = "0.4.1" @@ -2445,6 +2096,12 @@ dependencies = [ "markup5ever", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.12" @@ -2595,7 +2252,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2804,6 +2461,15 @@ dependencies = [ "compare", ] +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -2928,9 +2594,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "lettre" -version = "0.11.20" +version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "471816f3e24b85e820dee02cde962379ea1a669e5242f19c61bcbcffedf4c4fb" +checksum = "dabda5859ee7c06b995b9d1165aa52c39110e079ef609db97178d86aeb051fa7" dependencies = [ "base64 0.22.1", "email-encoding", @@ -2945,67 +2611,16 @@ dependencies = [ "nom 8.0.0", "percent-encoding", "quoted_printable", - "socket2 0.6.1", + "socket2 0.6.3", "tokio", "url", ] [[package]] -name = "lexical-core" -version = "1.0.6" +name = "levenshtein_automata" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" [[package]] name = "libbz2-rs-sys" @@ -3015,32 +2630,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" - -[[package]] -name = "libduckdb-sys" -version = "1.10501.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248" -dependencies = [ - "cc", - "flate2", - "pkg-config", - "reqwest", - "serde", - "serde_json", - "tar", - "vcpkg", - "zip", -] - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "libmimalloc-sys" @@ -3062,17 +2654,6 @@ dependencies = [ "libc", ] -[[package]] -name = "libredox" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" -dependencies = [ - "bitflags", - "libc", - "redox_syscall 0.7.0", -] - [[package]] name = "libz-rs-sys" version = "0.5.5" @@ -3202,7 +2783,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b7da45f78cc525d3750b623c967ae21c0cd28b2e6a9a2ee4b536a7cce3b21ce" dependencies = [ - "ahash 0.8.12", + "ahash", "flate2", "hashify", "hickory-resolver", @@ -3289,6 +2870,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + [[package]] name = "memchr" version = "2.7.6" @@ -3347,9 +2937,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3397,6 +2987,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "native-tls" version = "0.2.18" @@ -3424,7 +3020,7 @@ dependencies = [ "native_model", "redb 1.5.1", "redb 2.6.3", - "semver 1.0.27", + "semver 1.0.28", "serde", "skeptic", "thiserror 1.0.69", @@ -3528,20 +3124,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -3552,15 +3134,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.0" @@ -3576,28 +3149,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -3605,7 +3156,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -3672,7 +3222,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a95602f5a6eec5b15394516448c0b6fc2f2ef4ca2a9ab2951a91a18cec516bf0" dependencies = [ - "ahash 0.8.12", + "ahash", "lazy_static", ] @@ -3701,6 +3251,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -3767,6 +3323,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "outlook-pst" version = "1.1.0" @@ -3777,6 +3342,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.2.1" @@ -3801,7 +3375,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -4140,26 +3714,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "pulldown-cmark" version = "0.9.6" @@ -4211,7 +3765,7 @@ version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" dependencies = [ - "ahash 0.8.12", + "ahash", "equivalent", "hashbrown 0.16.1", "parking_lot", @@ -4230,7 +3784,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.1", + "socket2 0.6.3", "thiserror 2.0.17", "tokio", "tracing", @@ -4267,7 +3821,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -4293,23 +3847,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "rand" version = "0.7.3" @@ -4436,6 +3973,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.14.7" @@ -4477,55 +4034,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" -dependencies = [ - "bitflags", -] - -[[package]] -name = "refinery" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c427f2572afe5c6cbfa2b1bf40071c89bf1a8539e958ea582842f6f38dcfae" -dependencies = [ - "refinery-core", - "refinery-macros", -] - -[[package]] -name = "refinery-core" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702655abfc67f93a6f735e9fa4ace7d2e580633f8961f28acbfd7583ddce936c" -dependencies = [ - "async-trait", - "cfg-if", - "log", - "regex", - "siphasher", - "thiserror 2.0.17", - "time 0.3.47", - "url", - "walkdir", -] - -[[package]] -name = "refinery-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5145756cdf293b5089dc6b4f103f1a1229cc55d67082c866f8c8289531c4b983" -dependencies = [ - "proc-macro2", - "quote", - "refinery-core", - "regex", - "syn 2.0.117", -] - [[package]] name = "regex" version = "1.12.3" @@ -4555,15 +4063,6 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - [[package]] name = "reqwest" version = "0.12.24" @@ -4635,35 +4134,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" -dependencies = [ - "bitvec", - "bytecheck", - "bytes 1.11.1", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "rust-embed" version = "8.11.0" @@ -4699,19 +4169,13 @@ dependencies = [ ] [[package]] -name = "rust_decimal" -version = "1.40.0" +name = "rust-stemmers" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" dependencies = [ - "arrayvec", - "borsh", - "bytes 1.11.1", - "num-traits", - "rand 0.8.5", - "rkyv", "serde", - "serde_json", + "serde_derive", ] [[package]] @@ -4729,15 +4193,6 @@ dependencies = [ "semver 0.9.0", ] -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver 1.0.27", -] - [[package]] name = "rusticata-macros" version = "4.1.0" @@ -4837,27 +4292,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "security-framework" version = "3.6.0" @@ -4898,9 +4338,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -5120,12 +4560,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] name = "siphasher" version = "1.0.1" @@ -5147,6 +4581,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sketches-ddsketch" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.11" @@ -5209,12 +4652,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5266,7 +4709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version 0.2.3", + "rustc_version", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -5350,46 +4793,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" -dependencies = [ - "strum_macros 0.27.2", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.117", -] - -[[package]] -name = "strum_macros" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "subtle" version = "2.6.1" @@ -5482,20 +4885,152 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] -name = "tap" -version = "1.0.1" +name = "tantivy" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +checksum = "778da245841522199d512d19511b041425d8cff3a8f262b4e1516fceb050289a" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64 0.22.1", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.17", + "time 0.3.47", + "typetag", + "uuid", + "winapi", + "zstd", +] [[package]] -name = "tar" -version = "0.4.44" +name = "tantivy-bitpacker" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" dependencies = [ - "filetime", - "libc", - "xattr", + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time 0.3.47", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom 7.1.3", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", ] [[package]] @@ -5651,15 +5186,6 @@ dependencies = [ "isolang", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.2" @@ -5687,9 +5213,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" dependencies = [ "bytes 1.11.1", "libc", @@ -5697,7 +5223,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] @@ -5714,9 +5240,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -6019,12 +5545,42 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "typetag" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "uncased" version = "0.9.10" @@ -6131,6 +5687,12 @@ dependencies = [ "regex", ] +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6342,7 +5904,7 @@ dependencies = [ "bitflags", "hashbrown 0.15.5", "indexmap", - "semver 1.0.27", + "semver 1.0.28", ] [[package]] @@ -6875,7 +6437,7 @@ dependencies = [ "id-arena", "indexmap", "log", - "semver 1.0.27", + "semver 1.0.28", "serde", "serde_derive", "serde_json", @@ -6889,15 +6451,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - [[package]] name = "x509-parser" version = "0.18.1" @@ -6916,16 +6469,6 @@ dependencies = [ "time 0.3.47", ] -[[package]] -name = "xattr" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" -dependencies = [ - "libc", - "rustix", -] - [[package]] name = "xml-rs" version = "0.8.28" diff --git a/Cargo.toml b/Cargo.toml index cd7ea22..38dee68 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ poem-openapi = { version = "5.1.16", features = [ ring = { version = "0.17.14", features = ["std"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" -tokio = { version = "1.50.0", features = ["full"] } +tokio = { version = "1.51.0", features = ["full"] } tracing = "0.1.44" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] } @@ -89,7 +89,7 @@ webpki-roots = "1.0.6" rustls = { version = "0.23.37", default-features = false, features = ["ring"] } rustls-pki-types = "1.14.0" tokio-io-timeout = "1.2.1" -semver = "1.0.27" +semver = "1.0.28" governor = "0.10.4" lru = "0.16.3" mime_guess = "2.0.5" @@ -118,23 +118,14 @@ outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch compressed-rtf = "1.0.1" codepage-strings = "1.0.2" mail-send = "0.5.2" -duckdb = { version = "1.10501.0", features = [ - "chrono", - "bundled", - "r2d2", - "appender-arrow", -] } -arrow = { version = "58", features = ["ffi"] } -r2d2 = { version = "0.8", default-features = false } -refinery = { version = "0.9", default-features = false } -refinery-core = { version = "0.9", default-features = false } rcgen = "0.14.7" rustls-pemfile = "2.2.0" blake3 = "1.8.4" uuid = { version = "1.23.0", features = ["v4", "serde"] } fjall = { version = "3.1.2", features = ["lz4", "metrics", "bytes_1"] } +tantivy = { version = "0.26.0", features = ["zstd-compression"] } [dev-dependencies] #bincode = "1.3.3" #secret-lib = "1.0.0" tempfile = "3.27.0" -lettre = "0.11.20" +lettre = "0.11.21" diff --git a/src/main.rs b/src/main.rs index 08f3119..934e0cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,16 +19,15 @@ use bichon::{ bichon_version, modules::{ - blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER}, cache::imap::task::SYNC_TASKS, common::rustls::BichonTls, context::{executors::BichonContext, Initialize}, - duckdb::init::DuckDBManager, error::{code::ErrorCode, BichonResult}, logger, rest::start_http_server, settings::cli::SETTINGS, smtp::{start_smtp_server, SmtpServer}, + store::{storage::BLOB_MANAGER, tantivy::manager::INDEX_MANAGER}, tasks::PeriodicTasks, }, raise_error, @@ -95,7 +94,7 @@ async fn main() -> BichonResult<()> { } SYNC_TASKS.shutdown().await; - ENVELOPE_INDEX_MANAGER.shutdown().await; + INDEX_MANAGER.shutdown().await; BLOB_MANAGER.shutdown().await; info!("Bichon server stopped."); Ok(()) @@ -105,7 +104,6 @@ async fn main() -> BichonResult<()> { async fn initialize() -> BichonResult<()> { SignalManager::initialize().await?; DataDirManager::initialize().await?; - DuckDBManager::initialize().await?; UserManager::initialize().await?; BichonTls::initialize().await?; BichonContext::initialize().await?; diff --git a/src/modules/account/migration.rs b/src/modules/account/migration.rs index 59891c6..d7ca695 100644 --- a/src/modules/account/migration.rs +++ b/src/modules/account/migration.rs @@ -32,10 +32,10 @@ use crate::{ since::{DateSince, RelativeDate}, state::AccountRunningState, }, - blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER}, cache::imap::mailbox::MailBox, database::{list_all_impl, secondary_find_impl, with_transaction}, error::BichonResult, + store::tantivy::manager::INDEX_MANAGER, users::{role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel, DEFAULT_ADMIN_USER_ID}, }, utc_now, @@ -359,10 +359,7 @@ impl AccountV4 { OAuth2AccessToken::try_delete(account.id).await?; UserModel::cleanup_account(account.id).await?; MailBox::clean(account.id).await?; - let content_hashes = ENVELOPE_INDEX_MANAGER - .delete_account_envelopes(account.id) - .await?; - BLOB_MANAGER.delete(&content_hashes, &content_hashes)?; + INDEX_MANAGER.delete_account_envelopes(account.id).await?; Self::delete_account(account.id).await?; info!("Sequential cleanup completed for account: {}", account.id); Ok(()) diff --git a/src/modules/blob/envelope.rs b/src/modules/blob/envelope.rs deleted file mode 100644 index 8b05f3b..0000000 --- a/src/modules/blob/envelope.rs +++ /dev/null @@ -1,121 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use duckdb::types::Value; -use poem_openapi::Object; -use serde::{Deserialize, Serialize}; - -use crate::modules::{account::migration::AccountModel, cache::imap::mailbox::MailBox}; - -#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)] -pub struct Envelope { - pub id: String, - pub message_id: String, - pub account_id: u64, - pub account_email: Option, - pub mailbox_id: u64, - pub mailbox_name: Option, - pub uid: u32, - pub subject: String, - pub text: String, - pub from: String, - pub to: Vec, - pub cc: Vec, - pub bcc: Vec, - pub date: i64, - pub internal_date: i64, - pub size: u32, - pub thread_id: String, - pub attachment_count: usize, - pub regular_attachment_count: usize, - pub tags: Option>, - /// Hash of the content. - pub content_hash: String, -} - -impl Envelope { - pub fn has_any_attachments(&self) -> bool { - self.attachment_count > 0 - } - - pub fn from_row(row: &duckdb::Row) -> duckdb::Result { - let get_list = |col_name: &str| -> Vec { - row.get::<_, Value>(col_name) - .map(|v| { - if let Value::List(inner_list) = v { - inner_list - .into_iter() - .filter_map(|item| { - if let Value::Text(s) = item { - Some(s) - } else { - None - } - }) - .collect() - } else { - vec![] - } - }) - .unwrap_or_default() - }; - let account_id = row.get("account_id")?; - let mailbox_id = row.get("mailbox_id")?; - let email = match AccountModel::get(account_id) { - Ok(account) => account.email, - Err(_) => "unknown".to_string(), - }; - - let mailbox_name = MailBox::find_mailbox(account_id, mailbox_id) - .ok() - .and_then(|m| m) - .map(|m| m.name) - .unwrap_or_else(|| "unknown".to_string()); - - Ok(Self { - id: row.get("id")?, - message_id: row.get("message_id").unwrap_or_default(), - account_id, - account_email: Some(email), - mailbox_id, - mailbox_name: Some(mailbox_name), - uid: row.get::<_, u64>("uid")? as u32, - subject: row.get("subject").unwrap_or_default(), - text: row.get("body").unwrap_or_default(), - from: row.get("sender").unwrap_or_default(), - to: get_list("recipients"), - cc: get_list("cc"), - bcc: get_list("bcc"), - date: row.get("sent_at").unwrap_or(0), - internal_date: row.get("received_at").unwrap_or(0), - size: row.get::<_, u64>("size_bytes")? as u32, - thread_id: row.get("thread_id")?, - attachment_count: row.get::<_, i32>("attachment_count")? as usize, - regular_attachment_count: row.get::<_, i32>("regular_attachment_count")? as usize, - tags: { - let t = get_list("tags"); - if t.is_empty() { - None - } else { - Some(t) - } - }, - content_hash: row.get("content_hash")?, - }) - } -} diff --git a/src/modules/blob/manager.rs b/src/modules/blob/manager.rs deleted file mode 100644 index f042ac9..0000000 --- a/src/modules/blob/manager.rs +++ /dev/null @@ -1,349 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::{ - collections::{HashMap, HashSet}, - sync::LazyLock, - time::Duration, -}; - -use crate::modules::{ - duckdb::init::duckdb, - message::{ - attachment::AttachmentMetadata, - content::{AttachmentDetail, AttachmentInfo}, - search::SortBy, - tags::{TagCount, TagsRequest}, - }, -}; -use crate::{ - modules::{ - blob::envelope::Envelope, - common::signal::SIGNAL_MANAGER, - dashboard::{DashboardStats, LargestEmail}, - error::{code::ErrorCode, BichonResult}, - message::search::SearchFilter, - rest::response::DataPage, - }, - raise_error, -}; - -use tokio::{ - sync::{mpsc, Mutex}, - task::{self, JoinHandle}, -}; - -pub static ENVELOPE_INDEX_MANAGER: LazyLock = - LazyLock::new(EnvelopeIndexManager::new); - -pub const ENVELOPE_BATCH_SIZE: usize = 100; - -const MAX_BUFFER_DURATION: Duration = Duration::from_secs(10); - -pub struct EnvelopeIndexManager { - sender: mpsc::Sender<(Envelope, Vec)>, - handle: Mutex>>, -} - -impl EnvelopeIndexManager { - pub async fn shutdown(&self) { - let mut guard = self.handle.lock().await; - if let Some(handle) = guard.take() { - tracing::info!("Waiting for EnvelopeIndexManager to sync all data..."); - let _ = handle.await; - tracing::info!("EnvelopeIndexManager synchronized and closed."); - } - } - - pub fn new() -> Self { - let (sender, mut receiver) = mpsc::channel::<(Envelope, Vec)>(1000); - let handle = task::spawn(async move { - let mut buffer: Vec<(Envelope, Vec)> = - Vec::with_capacity(ENVELOPE_BATCH_SIZE); - let mut interval = tokio::time::interval(MAX_BUFFER_DURATION); - let mut shutdown = SIGNAL_MANAGER.subscribe(); - loop { - tokio::select! { - maybe_msg = receiver.recv() => { - match maybe_msg { - Some(doc) => { - buffer.push(doc); - if buffer.len() >= ENVELOPE_BATCH_SIZE { - ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await; - } - } - None => { - if !buffer.is_empty() { - tracing::info!("Channel closed, flushing remaining {} items", buffer.len()); - ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await; - } - break; - }, - } - } - _ = interval.tick() => { - if !buffer.is_empty() { - ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await; - } - } - _ = shutdown.recv() => { - ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await; - break; - } - } - } - }); - Self { - sender, - handle: Mutex::new(Some(handle)), - } - } - - pub async fn queue(&self, doc: (Envelope, Vec)) { - let _ = self.sender.send(doc).await; - } - - async fn flush(&self, buffer: &mut Vec<(Envelope, Vec)>) { - if buffer.is_empty() { - return; - } - - let items: Vec<(Envelope, Vec)> = buffer.drain(..).collect(); - - let result = (|| -> BichonResult<()> { - duckdb()?.append_envelopes_with_attachments(&items)?; - Ok(()) - })(); - - if let Err(e) = result { - tracing::error!("Failed to drain and commit envelopes to DuckDB: {:#?}", e); - } - } - - pub async fn total_emails(&self, accounts: Option>) -> BichonResult { - tokio::task::spawn_blocking(move || duckdb()?.total_emails(accounts)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn delete_account_envelopes(&self, account_id: u64) -> BichonResult> { - let content_hashes = tokio::task::spawn_blocking(move || { - duckdb()?.delete_account_envelopes_with_orphans(account_id) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))??; - Ok(content_hashes) - } - - pub async fn delete_mailbox_envelopes( - &self, - account_id: u64, - mailbox_ids: Vec, - ) -> BichonResult> { - if mailbox_ids.is_empty() { - tracing::warn!("delete_mailbox_envelopes: mailbox_ids is empty, nothing to delete"); - return Ok(vec![]); - } - - let content_hashes = tokio::task::spawn_blocking(move || { - duckdb()?.delete_mailbox_envelopes_with_orphans(account_id, mailbox_ids) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))??; - Ok(content_hashes) - } - - pub async fn get_all_tags( - &self, - accounts: Option>, - ) -> BichonResult> { - tokio::task::spawn_blocking(move || duckdb()?.get_all_tags(accounts)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_all_contacts( - &self, - accounts: Option>, - ) -> BichonResult> { - tokio::task::spawn_blocking(move || duckdb()?.get_all_contacts(accounts)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_attachment_metadata( - &self, - accounts: Option>, - ) -> BichonResult { - tokio::task::spawn_blocking(move || duckdb()?.get_attachment_metadata(accounts)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_orphan_hashes_in_memory( - &self, - deletes: HashMap>, - ) -> BichonResult> { - tokio::task::spawn_blocking(move || duckdb()?.get_orphan_hashes_in_memory(deletes)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn delete_envelopes_multi_account( - &self, - deletes: HashMap>, // HashMap - ) -> BichonResult<()> { - if deletes.is_empty() { - tracing::warn!("delete_envelopes_multi_account: deletes is empty, nothing to delete"); - return Ok(()); - } - tokio::task::spawn_blocking(move || duckdb()?.delete_envelopes_multi_account(deletes)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn update_envelope_tags(&self, request: TagsRequest) -> BichonResult<()> { - if request.updates.is_empty() { - tracing::warn!("update_envelope_tags: request is empty, nothing to update"); - return Ok(()); - } - tokio::task::spawn_blocking(move || duckdb()?.update_envelope_tags(request)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn search( - &self, - accounts: Option>, - filter: SearchFilter, - page: u64, - page_size: u64, - desc: bool, - sort_by: SortBy, - ) -> BichonResult> { - assert!(page > 0, "Page number must be greater than 0"); - assert!(page_size > 0, "Page size must be greater than 0"); - tokio::task::spawn_blocking(move || { - duckdb()?.search(accounts, filter, page, page_size, desc, sort_by) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn list_mailbox_envelopes( - &self, - account_id: u64, - mailbox_id: u64, - page: u64, - page_size: u64, - desc: bool, - ) -> BichonResult> { - assert!(page > 0, "Page number must be greater than 0"); - assert!(page_size > 0, "Page size must be greater than 0"); - tokio::task::spawn_blocking(move || { - duckdb()?.list_mailbox_envelopes(account_id, mailbox_id, page, page_size, desc) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn list_thread_envelopes( - &self, - account_id: u64, - thread_id: String, - page: u64, - page_size: u64, - desc: bool, - ) -> BichonResult> { - assert!(page > 0, "Page number must be greater than 0"); - assert!(page_size > 0, "Page size must be greater than 0"); - tokio::task::spawn_blocking(move || { - duckdb()?.list_thread_envelopes(account_id, thread_id, page, page_size, desc) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_envelope_by_id( - &self, - account_id: u64, - envelope_id: String, - ) -> BichonResult> { - tokio::task::spawn_blocking(move || duckdb()?.get_envelope_by_id(account_id, envelope_id)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_attachments_by_envelope_id( - &self, - account_id: u64, - envelope_id: String, - ) -> BichonResult> { - tokio::task::spawn_blocking(move || { - duckdb()?.get_attachments_by_envelope_id(account_id, envelope_id) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn top_10_largest_emails( - &self, - accounts: Option>, - ) -> BichonResult> { - tokio::task::spawn_blocking(move || duckdb()?.top_10_largest_emails(accounts)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { - tokio::task::spawn_blocking(move || duckdb()?.get_max_uid(account_id, mailbox_id)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn num_messages_in_mailbox( - &self, - account_id: u64, - mailbox_id: u64, - ) -> BichonResult { - tokio::task::spawn_blocking(move || { - duckdb()?.num_messages_in_mailbox(account_id, mailbox_id) - }) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn num_messages_in_thread( - &self, - account_id: u64, - thread_id: String, - ) -> BichonResult { - tokio::task::spawn_blocking(move || duckdb()?.num_messages_in_thread(account_id, thread_id)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } - - pub async fn get_dashboard_stats( - &self, - accounts: Option>, - ) -> BichonResult { - tokio::task::spawn_blocking(move || duckdb()?.get_dashboard_stats(accounts)) - .await - .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? - } -} diff --git a/src/modules/cache/imap/mailbox.rs b/src/modules/cache/imap/mailbox.rs index 7cd4a67..4ff48d7 100644 --- a/src/modules/cache/imap/mailbox.rs +++ b/src/modules/cache/imap/mailbox.rs @@ -22,7 +22,7 @@ use crate::{ database::{ async_filter_by_secondary_key_impl, async_find_impl, batch_delete_impl, batch_insert_impl, batch_upsert_impl, delete_impl, filter_by_secondary_key_impl, - manager::DB_MANAGER, + find_impl, manager::DB_MANAGER, }, error::{code::ErrorCode, BichonResult}, }, @@ -72,7 +72,7 @@ impl MailBox { encode_mailbox_name!(&self.name) } - pub async fn get(id: u64) -> BichonResult { + pub async fn async_get(id: u64) -> BichonResult { let result = async_find_impl::(DB_MANAGER.envelope_db(), id).await?; Ok(result.ok_or_else(|| { raise_error!( @@ -82,6 +82,16 @@ impl MailBox { })?) } + pub fn get(id: u64) -> BichonResult { + let result = find_impl::(DB_MANAGER.envelope_db(), id)?; + Ok(result.ok_or_else(|| { + raise_error!( + format!("mailbox {} not found", id), + ErrorCode::InternalError + ) + })?) + } + pub async fn delete(id: u64) -> BichonResult<()> { delete_impl(DB_MANAGER.envelope_db(), move |rw| { rw.get() diff --git a/src/modules/cache/imap/sync/flow.rs b/src/modules/cache/imap/sync/flow.rs index 5e71338..eb02d0e 100644 --- a/src/modules/cache/imap/sync/flow.rs +++ b/src/modules/cache/imap/sync/flow.rs @@ -32,7 +32,7 @@ use crate::{ }, error::{code::ErrorCode, BichonError, BichonResult}, imap::executor::ImapExecutor, - blob::manager::ENVELOPE_INDEX_MANAGER, + store::tantivy::manager::INDEX_MANAGER, }, raise_error, }; @@ -426,7 +426,7 @@ async fn perform_incremental_sync( remote_mailbox: &MailBox, ) -> BichonResult<()> { if remote_mailbox.exists > 0 { - let local_max_uid = ENVELOPE_INDEX_MANAGER + let local_max_uid = INDEX_MANAGER .get_max_uid(account.id, local_mailbox.id) .await?; match local_max_uid { diff --git a/src/modules/cache/imap/sync/rebuild.rs b/src/modules/cache/imap/sync/rebuild.rs index d1c5e97..e3320fb 100644 --- a/src/modules/cache/imap/sync/rebuild.rs +++ b/src/modules/cache/imap/sync/rebuild.rs @@ -19,7 +19,6 @@ use crate::{ modules::{ account::migration::AccountModel, - blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER}, cache::{ imap::{ mailbox::MailBox, @@ -28,6 +27,7 @@ use crate::{ SEMAPHORE, }, error::{code::ErrorCode, BichonError, BichonResult}, + store::tantivy::manager::INDEX_MANAGER, }, raise_error, }; @@ -194,13 +194,9 @@ pub async fn rebuild_mailbox_cache( local_mailbox: &MailBox, remote_mailbox: &MailBox, ) -> BichonResult<()> { - let content_hashes = ENVELOPE_INDEX_MANAGER + INDEX_MANAGER .delete_mailbox_envelopes(account.id, vec![local_mailbox.id]) .await?; - // todo Distinguish these hashes to avoid mixing them - if !content_hashes.is_empty() { - BLOB_MANAGER.delete(&content_hashes, &content_hashes)?; - } if remote_mailbox.exists == 0 { info!( @@ -227,14 +223,9 @@ pub async fn rebuild_mailbox_cache_by_date( remote: &MailBox, direction: FetchDirection, ) -> BichonResult<()> { - let content_hashes = ENVELOPE_INDEX_MANAGER + INDEX_MANAGER .delete_mailbox_envelopes(account.id, vec![local_mailbox_id]) .await?; - - if !content_hashes.is_empty() { - BLOB_MANAGER.delete(&content_hashes, &content_hashes)?; - } - if remote.exists == 0 { info!( "Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.", diff --git a/src/modules/dashboard/mod.rs b/src/modules/dashboard/mod.rs index 80bc746..64fdc84 100644 --- a/src/modules/dashboard/mod.rs +++ b/src/modules/dashboard/mod.rs @@ -16,16 +16,19 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::modules::users::permissions::Permission; +use crate::modules::{ + store::tantivy::{manager::INDEX_MANAGER, schema::SchemaTools}, + users::permissions::Permission, +}; use poem_openapi::Object; use serde::{Deserialize, Serialize}; use std::collections::HashSet; +use tantivy::{schema::Value, TantivyDocument}; use crate::{ bichon_version, modules::{ account::migration::AccountModel, - blob::manager::ENVELOPE_INDEX_MANAGER, common::auth::ClientContext, error::{code::ErrorCode, BichonResult}, settings::dir::DATA_DIR_MANAGER, @@ -63,13 +66,9 @@ impl DashboardStats { Some(context.user.account_access_map.keys().cloned().collect()) }; - let mut stat = ENVELOPE_INDEX_MANAGER - .get_dashboard_stats(authorized_ids.clone()) - .await?; + let mut stat = INDEX_MANAGER.get_dashboard_stats(&authorized_ids).await?; - stat.top_largest_emails = ENVELOPE_INDEX_MANAGER - .top_10_largest_emails(authorized_ids.clone()) - .await?; + stat.top_largest_emails = INDEX_MANAGER.top_10_largest_emails(&authorized_ids).await?; stat.account_count = if has_all_accounts { AccountModel::count().await? @@ -77,15 +76,14 @@ impl DashboardStats { authorized_ids.as_ref().map(|ids| ids.len()).unwrap_or(0) }; - stat.email_count = ENVELOPE_INDEX_MANAGER.total_emails(authorized_ids).await?; + stat.email_count = INDEX_MANAGER.total_emails(&authorized_ids)?; if has_all_accounts { stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.eml_dir) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - stat.index_usage_bytes = - get_total_size(&DATA_DIR_MANAGER.envelope_dir.join("envelopes.db")) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.tantivy_dir) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; } else { stat.storage_usage_bytes = 0; stat.index_usage_bytes = 0; @@ -116,3 +114,48 @@ pub struct LargestEmail { pub size_bytes: u64, // Email size in bytes pub id: String, } + +impl LargestEmail { + pub fn from_tantivy_doc(document: &TantivyDocument) -> BichonResult { + let fields = SchemaTools::email_fields(); + let value = document.get_first(fields.f_size).ok_or_else(|| { + raise_error!( + "miss 'size' field in tantivy document".into(), + ErrorCode::InternalError + ) + })?; + let size_bytes = value.as_u64().ok_or_else(|| { + raise_error!("'size' field is not a u64".into(), ErrorCode::InternalError) + })?; + let value = document.get_first(fields.f_subject).ok_or_else(|| { + raise_error!("'subject' field not found".into(), ErrorCode::InternalError) + })?; + let subject = value.as_str().map(|s| s.to_string()).ok_or_else(|| { + raise_error!( + "'subject' field is not a string".into(), + ErrorCode::InternalError + ) + })?; + + let value = document.get_first(fields.f_id).ok_or_else(|| { + raise_error!( + format!("'{}' field not found", stringify!(field)), + ErrorCode::InternalError + ) + })?; + let id = value.as_str().map(|s| s.to_string()).ok_or_else(|| { + raise_error!( + format!("'{}' field is not a string", stringify!(field)), + ErrorCode::InternalError + ) + })?; + + let envelope = LargestEmail { + subject, + size_bytes, + id, + }; + + Ok(envelope) + } +} diff --git a/src/modules/database/mod.rs b/src/modules/database/mod.rs index cd4325a..12b3b10 100644 --- a/src/modules/database/mod.rs +++ b/src/modules/database/mod.rs @@ -208,6 +208,21 @@ pub async fn async_find_impl( .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? } +pub fn find_impl( + database: &Arc>, + key: impl ToKey + Send + 'static, +) -> BichonResult> { + let db = database.clone(); + let r_transaction = db + .r_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let entity: Option = r_transaction + .get() + .primary(key) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(entity) +} + pub async fn delete_impl( database: &Arc>, delete: impl FnOnce(&RwTransaction) -> BichonResult + Send + 'static, diff --git a/src/modules/duckdb/build.rs b/src/modules/duckdb/build.rs deleted file mode 100644 index a31a289..0000000 --- a/src/modules/duckdb/build.rs +++ /dev/null @@ -1,152 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use arrow::array::{Int32Array, Int64Array, ListBuilder, StringBuilder, UInt64Array}; -use arrow::datatypes::{DataType, Field, Schema}; -use arrow::record_batch::RecordBatch; -use std::sync::Arc; - -use crate::modules::blob::envelope::Envelope; - -pub const DEFAULT_SHARD_ID: u64 = 0; - -pub fn build_record_batch(items: &[Envelope]) -> RecordBatch { - let capacity = items.len(); - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("account_id", DataType::UInt64, false), - Field::new("mailbox_id", DataType::UInt64, false), - Field::new("uid", DataType::UInt64, false), - Field::new("content_hash", DataType::Utf8, false), - Field::new("subject", DataType::Utf8, true), - Field::new("body", DataType::Utf8, true), - Field::new("sender", DataType::Utf8, true), - Field::new( - "recipients", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - ), - Field::new( - "cc", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - ), - Field::new( - "bcc", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - ), - Field::new("sent_at", DataType::Int64, true), - Field::new("received_at", DataType::Int64, true), - Field::new("size_bytes", DataType::UInt64, true), - Field::new("thread_id", DataType::Utf8, true), - Field::new("message_id", DataType::Utf8, true), - Field::new("attachment_count", DataType::Int32, false), - Field::new("regular_attachment_count", DataType::Int32, false), - Field::new( - "tags", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - ), - Field::new("shard_id", DataType::UInt64, false), - ])); - - let mut id_b = StringBuilder::with_capacity(capacity, capacity * 20); - let mut account_id_b = UInt64Array::builder(capacity); - let mut mailbox_id_b = UInt64Array::builder(capacity); - let mut uid_b = UInt64Array::builder(capacity); - - let mut content_hash_b = StringBuilder::with_capacity(capacity, capacity * 64); - let mut subject_b = StringBuilder::with_capacity(capacity, capacity * 20); - let mut body_b = StringBuilder::with_capacity(capacity, capacity * 100); - let mut from_b = StringBuilder::with_capacity(capacity, capacity * 20); - - let mut to_b = ListBuilder::new(StringBuilder::new()); - let mut cc_b = ListBuilder::new(StringBuilder::new()); - let mut bcc_b = ListBuilder::new(StringBuilder::new()); - - let mut date_b = Int64Array::builder(capacity); - let mut internal_date_b = Int64Array::builder(capacity); - let mut size_b = UInt64Array::builder(capacity); - let mut thread_id_b = StringBuilder::with_capacity(capacity, capacity * 20); - let mut msg_id_b = StringBuilder::with_capacity(capacity, capacity * 30); - let mut att_count_b = Int32Array::builder(capacity); - let mut regular_att_count_b = Int32Array::builder(capacity); - let mut tags_b = ListBuilder::new(StringBuilder::new()); - let mut shard_id_b = UInt64Array::builder(capacity); - - for e in items { - id_b.append_value(&e.id); - account_id_b.append_value(e.account_id); - mailbox_id_b.append_value(e.mailbox_id); - uid_b.append_value(e.uid as u64); - content_hash_b.append_value(&e.content_hash); - subject_b.append_value(&e.subject); - body_b.append_value(&e.text); - from_b.append_value(&e.from); - for addr in &e.to { - to_b.values().append_value(addr); - } - to_b.append(true); - for addr in &e.cc { - cc_b.values().append_value(addr); - } - cc_b.append(true); - for addr in &e.bcc { - bcc_b.values().append_value(addr); - } - bcc_b.append(true); - date_b.append_value(e.date); - internal_date_b.append_value(e.internal_date); - size_b.append_value(e.size as u64); - thread_id_b.append_value(&e.thread_id); - msg_id_b.append_value(&e.message_id); - att_count_b.append_value(e.attachment_count as i32); - regular_att_count_b.append_value(e.regular_attachment_count as i32); - tags_b.append(true); - shard_id_b.append_value(DEFAULT_SHARD_ID); - } - - RecordBatch::try_new( - schema, - vec![ - Arc::new(id_b.finish()), - Arc::new(account_id_b.finish()), - Arc::new(mailbox_id_b.finish()), - Arc::new(uid_b.finish()), - Arc::new(content_hash_b.finish()), - Arc::new(subject_b.finish()), - Arc::new(body_b.finish()), - Arc::new(from_b.finish()), - Arc::new(to_b.finish()), - Arc::new(cc_b.finish()), - Arc::new(bcc_b.finish()), - Arc::new(date_b.finish()), - Arc::new(internal_date_b.finish()), - Arc::new(size_b.finish()), - Arc::new(thread_id_b.finish()), - Arc::new(msg_id_b.finish()), - Arc::new(att_count_b.finish()), - Arc::new(regular_att_count_b.finish()), - Arc::new(tags_b.finish()), - Arc::new(shard_id_b.finish()), - ], - ) - .expect("Failed to build RecordBatch") -} diff --git a/src/modules/duckdb/init.rs b/src/modules/duckdb/init.rs deleted file mode 100644 index a75ad34..0000000 --- a/src/modules/duckdb/init.rs +++ /dev/null @@ -1,1716 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use chrono::{NaiveDateTime, Utc}; -use duckdb::{params, types::Value, DuckdbConnectionManager}; -use refinery::Runner; -use std::{ - collections::{HashMap, HashSet}, - path::PathBuf, - sync::OnceLock, -}; - -use crate::{ - modules::{ - account::migration::AccountModel, - blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER}, - context::Initialize, - dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, - duckdb::{build::build_record_batch, refinery::DuckDBConnection}, - error::{code::ErrorCode, BichonResult}, - message::{ - attachment::AttachmentMetadata, - content::{AttachmentDetail, AttachmentInfo}, - search::{SearchFilter, SortBy}, - tags::{TagAction, TagCount, TagsRequest}, - }, - rest::response::DataPage, - settings::{cli::SETTINGS, dir::DATA_DIR_MANAGER}, - }, - raise_error, -}; - -pub type DuckDBConn = r2d2::PooledConnection; - -pub static DUCKDBMANAGER: OnceLock = OnceLock::new(); - -pub mod duckdb_tables { - refinery::embed_migrations!("src/modules/duckdb/migrations"); -} - -pub fn duckdb() -> BichonResult<&'static DuckDBManager> { - DUCKDBMANAGER.get().ok_or_else(|| { - raise_error!( - "DuckDB manager is not initialized".into(), - ErrorCode::InternalError - ) - }) -} - -fn debug_sql(sql: &str, args: &[Value]) -> String { - let mut result = String::new(); - let mut parts = sql.split('?'); - - for (i, part) in parts.by_ref().enumerate() { - result.push_str(part); - - if i < args.len() { - result.push_str(&format!("{:?}", args[i])); - } - } - - result -} - -pub struct DuckDBManager { - pool: r2d2::Pool, -} - -impl Initialize for DuckDBManager { - async fn initialize() -> BichonResult<()> { - tracing::debug!("Initializing duckdb"); - - if !&DATA_DIR_MANAGER.envelope_dir.exists() { - std::fs::create_dir_all(&DATA_DIR_MANAGER.envelope_dir) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - - let pool = init( - &DATA_DIR_MANAGER.envelope_dir.join("envelopes.db"), - duckdb_tables::migrations::runner(), - )?; - let _ = DUCKDBMANAGER.set(DuckDBManager { pool }); - Ok(()) - } -} - -impl DuckDBManager { - pub fn conn(&self) -> BichonResult { - Ok(self - .pool - .get() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?) - } - - pub fn shutdown(&self) -> BichonResult<()> { - self.conn()? - .execute("FORCE CHECKPOINT", []) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - tracing::info!("Shutting down"); - Ok(()) - } - - pub fn validate_regex(&self, pattern: &str) -> BichonResult<()> { - let conn = self.conn()?; - let check_sql = "SELECT regexp_matches('', ?)"; - if let Err(e) = conn - .prepare(check_sql) - .and_then(|mut stmt| stmt.execute([pattern])) - { - return Err(raise_error!( - format!("Invalid Regular Expression for DuckDB: {}", e).into(), - ErrorCode::InvalidParameter - )); - } - Ok(()) - } - - pub fn delete_account_envelopes_with_orphans( - &self, - account_id: u64, - ) -> BichonResult> { - let mut conn = self.conn()?; - let tx = conn - .transaction() - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let orphan_sql = r#" - WITH target_hashes AS ( - SELECT content_hash FROM envelopes WHERE account_id = ? - UNION - SELECT content_hash FROM envelope_attachments WHERE account_id = ? - ), - active_hashes AS ( - SELECT content_hash FROM envelopes WHERE account_id != ? - UNION - SELECT content_hash FROM envelope_attachments WHERE account_id != ? - ) - SELECT content_hash FROM target_hashes - EXCEPT - SELECT content_hash FROM active_hashes - "#; - - let mut stmt = tx - .prepare(orphan_sql) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let orphan_hashes: Vec = stmt - .query_map([account_id, account_id, account_id, account_id], |row| { - row.get::<_, String>(0) - }) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - tx.execute( - "DELETE FROM envelope_attachments WHERE account_id = ?", - [account_id], - ) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let count = tx - .execute("DELETE FROM envelopes WHERE account_id = ?", [account_id]) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - tx.commit() - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - if count > 0 { - tracing::info!( - "Account {} data cleared. Deleted {} envelopes, {} orphan hashes identified.", - account_id, - count, - orphan_hashes.len() - ); - } - - Ok(orphan_hashes) - } - - pub fn delete_mailbox_envelopes_with_orphans( - &self, - account_id: u64, - mailbox_ids: Vec, - ) -> BichonResult> { - if mailbox_ids.is_empty() { - return Ok(vec![]); - } - - let mut conn = self.conn()?; - let tx = conn - .transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let placeholders = mailbox_ids - .iter() - .map(|_| "?") - .collect::>() - .join(", "); - let mut sql_params: Vec = vec![account_id.into()]; - sql_params.extend(mailbox_ids.iter().map(|&id| duckdb::types::Value::from(id))); - let param_iter = duckdb::params_from_iter(sql_params); - - let orphan_sql = format!( - r#" - WITH target_hashes AS ( - SELECT content_hash FROM envelopes WHERE account_id = ? AND mailbox_id IN ({0}) - UNION - SELECT content_hash FROM envelope_attachments WHERE account_id = ? AND mailbox_id IN ({0}) - ), - active_hashes AS ( - SELECT content_hash FROM envelopes WHERE NOT (account_id = ? AND mailbox_id IN ({0})) - UNION - SELECT content_hash FROM envelope_attachments WHERE NOT (account_id = ? AND mailbox_id IN ({0})) - ) - SELECT content_hash FROM target_hashes - EXCEPT - SELECT content_hash FROM active_hashes - "#, - placeholders - ); - - let mut query_params: Vec = Vec::new(); - for _ in 0..4 { - query_params.push(account_id.into()); - query_params.extend(mailbox_ids.iter().map(|&id| duckdb::types::Value::from(id))); - } - - let mut stmt = tx - .prepare(&orphan_sql) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - let orphan_hashes: Vec = stmt - .query_map(duckdb::params_from_iter(query_params), |row| { - row.get::<_, String>(0) - }) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let delete_att_sql = format!( - "DELETE FROM envelope_attachments WHERE account_id = ? AND mailbox_id IN ({})", - placeholders - ); - tx.execute(&delete_att_sql, param_iter.clone()) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let delete_env_sql = format!( - "DELETE FROM envelopes WHERE account_id = ? AND mailbox_id IN ({})", - placeholders - ); - let count = tx - .execute(&delete_env_sql, param_iter) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - tx.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - if count > 0 { - tracing::info!( - "Deleted {} emails. Found {} orphan hashes to clean up.", - count, - orphan_hashes.len() - ); - } - - Ok(orphan_hashes) - } - - pub fn append_envelopes_with_attachments( - &self, - items: &[(Envelope, Vec)], - ) -> BichonResult<()> { - let mut conn = self.conn()?; - let tx = conn - .transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - { - let mut env_appender = tx - .appender("envelopes") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let mut att_appender = tx - .appender("envelope_attachments") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - for (env, atts) in items { - env_appender - .append_record_batch(build_record_batch(std::slice::from_ref(env))) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - for att in atts { - att_appender - .append_row(params![ - env.id, - env.account_id, - env.mailbox_id, - att.filename, - att.is_message, - att.inline, - att.content_id, - att.get_extension(), - att.get_category(), - att.file_type.to_ascii_lowercase(), - att.size as u64, - att.content_hash, - 0 - ]) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - } - env_appender - .flush() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - att_appender - .flush() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - tx.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - } - - pub fn total_emails( - &self, - accounts: Option>, - ) -> BichonResult { - let conn = self.conn()?; - let total: u64 = match accounts { - None => conn - .query_row("SELECT COUNT(*) FROM envelopes", [], |row| row.get(0)) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?, - Some(accts) if !accts.is_empty() => { - let placeholders = accts.iter().map(|_| "?").collect::>().join(", "); - let sql = format!( - "SELECT COUNT(*) FROM envelopes WHERE account_id IN ({})", - placeholders - ); - - conn.query_row(&sql, duckdb::params_from_iter(accts), |row| row.get(0)) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - } - _ => 0, - }; - - Ok(total) - } - - pub fn num_messages_in_thread(&self, account_id: u64, thread_id: String) -> BichonResult { - let conn = self.conn()?; - let count: u64 = conn - .query_row( - "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND thread_id = ?;", - params![account_id, thread_id], - |row| row.get(0), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(count) - } - - pub fn num_messages_in_mailbox(&self, account_id: u64, mailbox_id: u64) -> BichonResult { - let conn = self.conn()?; - let count: u64 = conn - .query_row( - "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND mailbox_id = ?;", - params![account_id, mailbox_id], - |row| row.get(0), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(count) - } - - pub fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { - let conn = self.conn()?; - let max_uid: Option = conn - .query_row( - "SELECT MAX(uid) FROM envelopes WHERE account_id = ? AND mailbox_id = ?;", - params![account_id, mailbox_id], - |row| row.get(0), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(max_uid) - } - - pub fn get_attachments_by_envelope_id( - &self, - account_id: u64, - envelope_id: String, - ) -> BichonResult> { - let conn = self.conn()?; - - let mut stmt = conn - .prepare( - "SELECT * FROM envelope_attachments - WHERE account_id = ? AND envelope_id = ?;", - ) - .map_err(|e| { - raise_error!( - format!("Prepare failed: {:#?}", e), - ErrorCode::InternalError - ) - })?; - - let attachment_iter = stmt - .query_map(params![account_id, envelope_id], |row| { - AttachmentDetail::from_row(row) - }) - .map_err(|e| { - raise_error!(format!("Query failed: {:#?}", e), ErrorCode::InternalError) - })?; - - let mut attachments = Vec::new(); - for att_res in attachment_iter { - attachments.push(att_res.map_err(|e| { - raise_error!( - format!("Row mapping failed: {:#?}", e), - ErrorCode::InternalError - ) - })?); - } - - Ok(attachments) - } - - pub fn get_envelope_by_id( - &self, - account_id: u64, - envelope_id: String, - ) -> BichonResult> { - let conn = self.conn()?; - let mut stmt = conn - .prepare("SELECT * FROM envelopes WHERE account_id = ? AND id = ? LIMIT 1;") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let envelope_res = stmt.query_row(params![account_id, envelope_id], |row| { - Envelope::from_row(row) - }); - - match envelope_res { - Ok(env) => Ok(Some(env)), - Err(duckdb::Error::QueryReturnedNoRows) => Ok(None), - Err(e) => Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)), - } - } - - pub fn get_envelopes_by_ids( - &self, - account_id: u64, - envelope_ids: &[&str], - ) -> BichonResult> { - if envelope_ids.is_empty() { - return Ok(vec![]); - } - let conn = self.conn()?; - let placeholders = vec!["?"; envelope_ids.len()].join(", "); - - let query = format!( - "SELECT * FROM envelopes WHERE account_id = ? AND id IN ({})", - placeholders - ); - - let mut stmt = conn.prepare(&query).map_err(|e| { - raise_error!(format!("Prepare error: {:#?}", e), ErrorCode::InternalError) - })?; - - let mut params: Vec> = Vec::new(); - params.push(Box::new(account_id)); - for id in envelope_ids { - params.push(Box::new(id.to_string())); - } - - let param_refs: Vec<&dyn duckdb::ToSql> = params.iter().map(|p| p.as_ref()).collect(); - - let rows = stmt - .query_map(duckdb::params_from_iter(param_refs), |row| { - Envelope::from_row(row) - }) - .map_err(|e| { - raise_error!(format!("Query error: {:#?}", e), ErrorCode::InternalError) - })?; - - let mut result = Vec::new(); - for row in rows { - result.push(row.map_err(|e| { - raise_error!(format!("Row error: {:#?}", e), ErrorCode::InternalError) - })?); - } - - Ok(result) - } - - pub fn get_all_tags(&self, accounts: Option>) -> BichonResult> { - let conn = self.conn()?; - let mut sql = " - SELECT t.tag, COUNT(*) as count - FROM ( - SELECT unnest(tags) as tag, account_id - FROM envelopes - WHERE len(tags) > 0 - ) t - " - .to_string(); - - let mut params_vec: Vec = Vec::new(); - if let Some(ref acc_set) = accounts { - if !acc_set.is_empty() { - let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", "); - sql.push_str(&format!(" WHERE t.account_id IN ({})", placeholders)); - for id in acc_set { - params_vec.push((*id).into()); - } - } - } - - sql.push_str(" GROUP BY t.tag ORDER BY count DESC, t.tag ASC;"); - - let mut stmt = conn - .prepare(&sql) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let tag_counts = stmt - .query_map(duckdb::params_from_iter(params_vec), |row| { - Ok(TagCount { - tag: row.get(0)?, - count: row.get(1)?, - }) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(tag_counts) - } - - pub fn get_all_contacts( - &self, - accounts: Option>, - ) -> BichonResult> { - let conn = self.conn()?; - let mut sql = r#" - SELECT DISTINCT LOWER(contact) AS contact - FROM ( - SELECT sender AS contact, account_id - FROM envelopes - WHERE sender IS NOT NULL AND sender != '' - - UNION ALL - - SELECT unnest(recipients) AS contact, account_id - FROM envelopes - WHERE length(recipients) > 0 - - UNION ALL - - SELECT unnest(cc) AS contact, account_id - FROM envelopes - WHERE length(cc) > 0 - - UNION ALL - - SELECT unnest(bcc) AS contact, account_id - FROM envelopes - WHERE length(bcc) > 0 - ) t - WHERE contact IS NOT NULL AND contact != '' - "# - .to_string(); - - let mut params_vec: Vec = Vec::new(); - - if let Some(ref acc_set) = accounts { - if !acc_set.is_empty() { - let placeholders = vec!["?"; acc_set.len()].join(", "); - sql.push_str(&format!(" AND account_id IN ({})", placeholders)); - - for &id in acc_set { - params_vec.push(id.into()); - } - } - } - - sql.push_str(" ORDER BY contact;"); - - let mut stmt = conn - .prepare(&sql) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let contacts: HashSet = stmt - .query_map(duckdb::params_from_iter(params_vec), |row| { - let email: String = row.get(0)?; - Ok(email) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(contacts) - } - - pub fn update_envelope_tags(&self, request: TagsRequest) -> BichonResult<()> { - let mut conn = self.conn()?; - let tags_json = serde_json::to_string(&request.tags) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let set_clause = match &request.action { - TagAction::Overwrite => "SET tags = ?::JSON::VARCHAR[]", - TagAction::Add => "SET tags = list_distinct(list_concat(tags, ?::JSON::VARCHAR[]))", - TagAction::Remove => { - "SET tags = list_filter(tags, x -> NOT list_contains(?::JSON::VARCHAR[], x))" - } - }; - - let tx = conn - .transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - for (account_id, ids) in request.updates { - if ids.is_empty() { - continue; - } - - for chunk in ids.chunks(100) { - let placeholders = vec!["?"; chunk.len()].join(", "); - let query = format!( - "UPDATE envelopes - {} - WHERE account_id = ? - AND id IN ({})", - set_clause, placeholders - ); - - let mut params: Vec> = Vec::with_capacity(chunk.len() + 2); - - params.push(Box::new(tags_json.clone())); - params.push(Box::new(account_id)); - - for id in chunk { - params.push(Box::new(id.clone())); - } - - let param_refs: Vec<&dyn duckdb::ToSql> = - params.iter().map(|p| p.as_ref()).collect(); - - tx.execute(&query, duckdb::params_from_iter(param_refs)) - .map_err(|e| { - raise_error!( - format!("Update failed. Account: {}, Error: {}", account_id, e), - ErrorCode::InternalError - ) - })?; - } - } - - tx.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - } - - pub fn get_attachment_metadata( - &self, - accounts: Option>, - ) -> BichonResult { - let conn = self.conn()?; - let mut sql = r#" - SELECT - COALESCE(CAST(array_agg(DISTINCT extension) FILTER (WHERE extension IS NOT NULL) AS JSON), '[]') AS extensions, - COALESCE(CAST(array_agg(DISTINCT ext_category) AS JSON), '[]') AS categories, - COALESCE(CAST(array_agg(DISTINCT content_type) AS JSON), '[]') AS content_types - FROM envelope_attachments - "# - .to_string(); - - let mut params_vec: Vec = Vec::new(); - if let Some(ref acc_set) = accounts { - if !acc_set.is_empty() { - let placeholders = vec!["?"; acc_set.len()].join(", "); - sql.push_str(&format!(" WHERE account_id IN ({})", placeholders)); - - for &id in acc_set { - params_vec.push(id.into()); - } - } - } - - let mut stmt = conn - .prepare(&sql) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let result = stmt - .query_row(duckdb::params_from_iter(params_vec), |row| { - let exts_raw: String = row.get(0).unwrap_or_else(|_| "[]".to_string()); - let cats_raw: String = row.get(1).unwrap_or_else(|_| "[]".to_string()); - let ctypes_raw: String = row.get(2).unwrap_or_else(|_| "[]".to_string()); - - let exts: Vec = serde_json::from_str(&exts_raw).unwrap_or_default(); - let cats: Vec = serde_json::from_str(&cats_raw).unwrap_or_default(); - let ctypes: Vec = serde_json::from_str(&ctypes_raw).unwrap_or_default(); - - Ok(AttachmentMetadata { - extensions: exts.into_iter().collect(), - categories: cats.into_iter().collect(), - content_types: ctypes.into_iter().collect(), - }) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(result) - } - - pub fn delete_envelopes_multi_account( - &self, - deletes: HashMap>, - ) -> BichonResult<()> { - let mut conn = self.conn()?; - let tx = conn - .transaction() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - for (account_id, ids) in deletes { - if ids.is_empty() { - continue; - } - - for chunk in ids.chunks(100) { - let placeholders = vec!["?"; chunk.len()].join(", "); - let mut params: Vec> = Vec::new(); - params.push(Box::new(account_id)); - for id in chunk { - params.push(Box::new(id.clone())); - } - - let param_refs: Vec<&dyn duckdb::ToSql> = - params.iter().map(|p| p.as_ref()).collect(); - let query_params = duckdb::params_from_iter(param_refs); - - let del_attachments_query = format!( - "DELETE FROM envelope_attachments - WHERE account_id = ? - AND envelope_id IN ({})", - placeholders - ); - - tx.execute(&del_attachments_query, query_params.clone()) - .map_err(|e| { - raise_error!( - format!("Delete attachments fail: {:#?}", e), - ErrorCode::InternalError - ) - })?; - - let query = format!( - "DELETE FROM envelopes - WHERE account_id = ? - AND id IN ({})", - placeholders - ); - - tx.execute(&query, query_params) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - } - - tx.commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(()) - } - - pub fn get_orphan_hashes_in_memory( - &self, - deletes: HashMap>, - ) -> BichonResult> { - let conn = self.conn()?; - let all_delete_ids: Vec = deletes.values().flatten().cloned().collect(); - if all_delete_ids.is_empty() { - return Ok(vec![]); - } - - if all_delete_ids.len() > 100 { - return Err(raise_error!( - "Too many IDs for batch delete, please shrink the batch".into(), - ErrorCode::InvalidParameter - )); - } - - let mut target_hashes = HashSet::new(); - let placeholders = vec!["?"; all_delete_ids.len()].join(", "); - let params = duckdb::params_from_iter(&all_delete_ids); - - let mut stmt = conn - .prepare(&format!( - "SELECT content_hash FROM envelopes WHERE id IN ({})", - placeholders - )) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let rows = stmt - .query_map(params.clone(), |r| r.get::<_, String>(0)) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - for h in rows { - target_hashes - .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?); - } - - let mut stmt = conn - .prepare(&format!( - "SELECT content_hash FROM envelope_attachments WHERE envelope_id IN ({})", - placeholders - )) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let rows = stmt - .query_map(params, |r| r.get::<_, String>(0)) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - for h in rows { - target_hashes - .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?); - } - - if target_hashes.is_empty() { - return Ok(vec![]); - } - - let hash_placeholders = vec!["?"; target_hashes.len()].join(", "); - let mut still_used_hashes = HashSet::new(); - - let mut check_params: Vec> = Vec::new(); - for id in &all_delete_ids { - check_params.push(Box::new(id.clone())); - } - for hash in &target_hashes { - check_params.push(Box::new(hash.clone())); - } - let check_params_refs: Vec<&dyn duckdb::ToSql> = - check_params.iter().map(|p| p.as_ref()).collect(); - - let mut stmt = conn - .prepare(&format!( - "SELECT DISTINCT content_hash FROM envelopes WHERE id NOT IN ({}) AND content_hash IN ({})", - placeholders, hash_placeholders - )) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let rows = stmt - .query_map(duckdb::params_from_iter(&check_params_refs), |r| { - r.get::<_, String>(0) - }) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - for h in rows { - still_used_hashes - .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?); - } - - let mut stmt = conn.prepare(&format!( - "SELECT DISTINCT content_hash FROM envelope_attachments WHERE envelope_id NOT IN ({}) AND content_hash IN ({})", - placeholders, hash_placeholders - )).map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - - let rows = stmt - .query_map(duckdb::params_from_iter(&check_params_refs), |r| { - r.get::<_, String>(0) - }) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?; - for h in rows { - still_used_hashes - .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?); - } - - let orphans: Vec = target_hashes - .into_iter() - .filter(|h| !still_used_hashes.contains(h)) - .collect(); - - Ok(orphans) - } - - pub fn top_10_largest_emails( - &self, - accounts: Option>, - ) -> BichonResult> { - let conn = self.conn()?; - let mut sql = r#" - SELECT id, subject, size_bytes - FROM envelopes - "# - .to_string(); - - let mut params_vec: Vec = Vec::new(); - if let Some(acc_set) = accounts { - if !acc_set.is_empty() { - let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", "); - sql.push_str(&format!(" WHERE account_id IN ({})", placeholders)); - - for id in acc_set { - params_vec.push(id.into()); - } - } - } - - sql.push_str(" ORDER BY size_bytes DESC LIMIT 10;"); - let mut stmt = conn - .prepare(&sql) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let results = stmt - .query_map(duckdb::params_from_iter(params_vec), |row| { - Ok(LargestEmail { - id: row.get(0)?, - subject: row.get(1)?, - size_bytes: row.get(2)?, - }) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(results) - } - - pub fn get_dashboard_stats( - &self, - accounts: Option>, - ) -> BichonResult { - let conn = self.conn()?; - - let mut account_filter = String::new(); - let mut params: Vec = Vec::new(); - - if let Some(acc_set) = accounts { - if !acc_set.is_empty() { - let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", "); - account_filter = format!("WHERE account_id IN ({})", placeholders); - for id in acc_set { - params.push(id.into()); - } - } - } - - let mut stats = DashboardStats::default(); - - { - let sql = format!( - r#" - SELECT COALESCE(SUM(size_bytes), 0) AS total_size - FROM envelopes - {account_filter} - "# - ); - - let mut stmt = conn.prepare(&sql).map_err(|e| { - raise_error!( - format!("Prepare total_size failed: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut rows = stmt - .query(duckdb::params_from_iter(params.clone())) - .map_err(|e| { - raise_error!( - format!("Failed to execute total_size query: {}", e), - ErrorCode::InternalError - ) - })?; - - if let Some(row) = rows.next().map_err(|e| { - raise_error!( - format!("Failed to fetch row from total_size query: {}", e), - ErrorCode::InternalError - ) - })? { - stats.total_size_bytes = row - .get(0) - .map_err(|e| { - raise_error!( - format!("Failed to read total_size value: {}", e), - ErrorCode::InternalError - ) - }) - .unwrap_or(0); - } - } - - { - let thirty_days_ago = Utc::now() - chrono::Duration::days(30); - let thirty_days_ago_ms = thirty_days_ago.timestamp_millis(); - - let sql = format!( - r#" - SELECT - date_trunc('day', to_timestamp(sent_at / 1000)) AS day, - COUNT(*) AS cnt - FROM envelopes - {account_filter} - WHERE sent_at >= ? - GROUP BY day - ORDER BY day ASC - "# - ); - - let mut stmt = conn.prepare(&sql).map_err(|e| { - raise_error!( - format!("Prepare recent_activity failed: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut query_params = params.clone(); - query_params.push(thirty_days_ago_ms.into()); - - let mut rows = stmt - .query(duckdb::params_from_iter(query_params)) - .map_err(|e| { - raise_error!( - format!("Failed to execute recent_activity query: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut recent_activity = Vec::new(); - - while let Some(row) = rows.next().map_err(|e| { - raise_error!( - format!("Failed to fetch row from recent_activity query: {}", e), - ErrorCode::InternalError - ) - })? { - let day: NaiveDateTime = row.get(0).map_err(|e| { - raise_error!( - format!("Failed to read recent_activity value: {}", e), - ErrorCode::InternalError - ) - })?; - let timestamp_ms = day.and_utc().timestamp_millis(); - - recent_activity.push(TimeBucket { - timestamp_ms, - count: row.get(1).map_err(|e| { - raise_error!( - format!("Failed to read recent_activity value: {}", e), - ErrorCode::InternalError - ) - })?, - }); - } - - stats.recent_activity = recent_activity; - } - - { - let sql = format!( - r#" - SELECT sender, COUNT(*) AS cnt - FROM envelopes - {account_filter} - WHERE sender IS NOT NULL AND sender != '' - GROUP BY sender - ORDER BY cnt DESC, sender ASC - LIMIT 10 - "# - ); - - let mut stmt = conn.prepare(&sql).map_err(|e| { - raise_error!( - format!("Prepare top_senders failed: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut rows = stmt - .query(duckdb::params_from_iter(params.clone())) - .map_err(|e| { - raise_error!( - format!("Failed to execute top_senders query: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut top_senders = Vec::new(); - - while let Some(row) = rows.next().map_err(|e| { - raise_error!( - format!("Failed to fetch row from top_senders query: {}", e), - ErrorCode::InternalError - ) - })? { - top_senders.push(Group { - key: row.get(0).map_err(|e| { - raise_error!( - format!("Failed to read top_senders value: {}", e), - ErrorCode::InternalError - ) - })?, - count: row.get(1).map_err(|e| { - raise_error!( - format!("Failed to read top_senders value: {}", e), - ErrorCode::InternalError - ) - })?, - }); - } - - stats.top_senders = top_senders; - } - { - let sql = format!( - r#" - SELECT account_id, COUNT(*) AS cnt - FROM envelopes - {account_filter} - GROUP BY account_id - ORDER BY cnt DESC - LIMIT 10 - "# - ); - - let mut stmt = conn.prepare(&sql).map_err(|e| { - raise_error!( - format!("Prepare top_accounts failed: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut rows = stmt - .query(duckdb::params_from_iter(params.clone())) - .map_err(|e| { - raise_error!( - format!("Failed to execute top_accounts query: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut top_accounts = Vec::new(); - - while let Some(row) = rows.next().map_err(|e| { - raise_error!( - format!("Failed to fetch row from top_accounts query: {}", e), - ErrorCode::InternalError - ) - })? { - let account_id: u64 = row.get(0).map_err(|e| { - raise_error!( - format!("Failed to read top_accounts value: {}", e), - ErrorCode::InternalError - ) - })?; - let count: u64 = row.get(1).map_err(|e| { - raise_error!( - format!("Failed to read top_accounts value: {}", e), - ErrorCode::InternalError - ) - })?; - - if let Ok(account) = AccountModel::get(account_id) { - top_accounts.push(Group { - key: account.email, - count, - }); - } else { - tracing::warn!(account_id, "account not found in top accounts query"); - tokio::spawn(async move { - let content_hashes = match ENVELOPE_INDEX_MANAGER - .delete_account_envelopes(account_id) - .await - { - Ok(content_hashes) => content_hashes, - Err(e) => { - tracing::error!( - account_id = account_id, - error = %e, - "failed to cleanup envelope index" - ); - return; - } - }; - - if let Err(e) = BLOB_MANAGER.delete(&content_hashes, &content_hashes) { - tracing::error!( - account_id = account_id, - error = %e, - "failed to cleanup eml" - ); - } - }); - } - } - - stats.top_accounts = top_accounts; - } - - { - let sql = format!( - r#" - SELECT (regular_attachment_count > 0) AS has_attachment, COUNT(*) AS cnt - FROM envelopes - {account_filter} - GROUP BY has_attachment - "# - ); - - let mut stmt = conn.prepare(&sql).map_err(|e| { - raise_error!( - format!("Prepare attachment_stats failed: {}", e), - ErrorCode::InternalError - ) - })?; - - let mut rows = stmt.query(duckdb::params_from_iter(params)).map_err(|e| { - raise_error!( - format!("Failed to execute attachment_stats query: {}", e), - ErrorCode::InternalError - ) - })?; - - while let Some(row) = rows.next().map_err(|e| { - raise_error!( - format!("Failed to fetch row from attachment_stats query: {}", e), - ErrorCode::InternalError - ) - })? { - let has_attachment: bool = row.get(0).map_err(|e| { - raise_error!( - format!("Failed to read attachment_stats value: {}", e), - ErrorCode::InternalError - ) - })?; - let cnt: u64 = row.get(1).map_err(|e| { - raise_error!( - format!("Failed to read attachment_stats value: {}", e), - ErrorCode::InternalError - ) - })?; - - if has_attachment { - stats.with_attachment_count = cnt; - } else { - stats.without_attachment_count = cnt; - } - } - } - - Ok(stats) - } - - pub fn list_mailbox_envelopes( - &self, - account_id: u64, - mailbox_id: u64, - page: u64, - page_size: u64, - desc: bool, - ) -> BichonResult> { - let conn = self.conn()?; - let current_page = page.max(1); - - let total_items: u64 = conn - .query_row( - "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND mailbox_id = ?", - params![account_id, mailbox_id], - |row| row.get(0), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let total_pages = if total_items == 0 { - 0 - } else { - (total_items + page_size - 1) / page_size - }; - - let offset = (current_page - 1) * page_size; - let order_direction = if desc { "DESC" } else { "ASC" }; - let query = format!( - "SELECT * FROM envelopes - WHERE account_id = ? AND mailbox_id = ? - ORDER BY sent_at {} - LIMIT ? OFFSET ?", - order_direction - ); - - let mut stmt = conn - .prepare(&query) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let items = stmt - .query_map(params![account_id, mailbox_id, page_size, offset], |row| { - Envelope::from_row(row) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(DataPage { - current_page: Some(current_page), - page_size: Some(page_size), - total_items, - items, - total_pages: Some(total_pages), - }) - } - - pub fn list_thread_envelopes( - &self, - account_id: u64, - thread_id: String, - page: u64, - page_size: u64, - desc: bool, - ) -> BichonResult> { - let conn = self.conn()?; - let current_page = page.max(1); - - let total_items: u64 = conn - .query_row( - "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND thread_id = ?", - params![account_id, thread_id], - |row| row.get(0), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let total_pages = if total_items == 0 { - 0 - } else { - (total_items + page_size - 1) / page_size - }; - - let offset = (current_page - 1) * page_size; - - let order_direction = if desc { "DESC" } else { "ASC" }; - let query = format!( - "SELECT * FROM envelopes - WHERE account_id = ? AND thread_id = ? - ORDER BY sent_at {} - LIMIT ? OFFSET ?", - order_direction - ); - - let mut stmt = conn - .prepare(&query) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let items = stmt - .query_map(params![account_id, thread_id, page_size, offset], |row| { - Envelope::from_row(row) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(DataPage { - current_page: Some(current_page), - page_size: Some(page_size), - total_items, - items, - total_pages: Some(total_pages), - }) - } - - pub fn search( - &self, - accounts: Option>, - filter: SearchFilter, - page: u64, - page_size: u64, - desc: bool, - sort_by: SortBy, - ) -> BichonResult> { - let page = page.max(1); - let page_size = page_size.clamp(1, 500); - let offset = (page - 1) * page_size; - - let allowed_accounts: Option> = match (accounts, filter.account_ids.clone()) { - (Some(user_set), Some(filter_set)) => { - let intersection: HashSet = - user_set.intersection(&filter_set).cloned().collect(); - Some(intersection) - } - (Some(user_set), None) => Some(user_set), - (None, Some(filter_set)) => Some(filter_set), - (None, None) => None, - }; - - if let Some(ref set) = allowed_accounts { - if set.is_empty() { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: 0, - items: Vec::new(), - total_pages: Some(0), - }); - } - } - - let mut base_sql = String::from(" FROM envelopes e "); - - let need_join_attachment = filter.attachment_name.is_some() - || filter.attachment_extension.is_some() - || filter.attachment_category.is_some() - || filter.attachment_content_type.is_some(); - - if need_join_attachment { - base_sql.push_str( - " - LEFT JOIN envelope_attachments a - ON a.account_id = e.account_id - AND a.mailbox_id = e.mailbox_id - AND a.envelope_id = e.id - ", - ); - } - - base_sql.push_str(" WHERE 1=1 "); - let mut args: Vec = Vec::new(); - - if let Some(set) = allowed_accounts { - base_sql.push_str(" AND e.account_id IN ("); - for (i, _) in set.iter().enumerate() { - if i > 0 { - base_sql.push_str(","); - } - base_sql.push_str("?"); - } - base_sql.push_str(")"); - for id in set { - args.push(id.into()); - } - } - - if let Some(mailboxes) = filter.mailbox_ids { - if !mailboxes.is_empty() { - base_sql.push_str(" AND e.mailbox_id IN ("); - for (i, _) in mailboxes.iter().enumerate() { - if i > 0 { - base_sql.push_str(","); - } - base_sql.push_str("?"); - } - base_sql.push_str(")"); - for id in mailboxes { - args.push(id.into()); - } - } - } - - if let Some(subject) = filter.subject { - let pattern = format!("(?i){}", subject); - base_sql.push_str(" AND regexp_matches(coalesce(e.subject, ''), ?)"); - args.push(pattern.into()); - } - - if let Some(body) = filter.body { - let pattern = format!("(?i){}", body); - base_sql.push_str(" AND regexp_matches(coalesce(e.body, ''), ?)"); - args.push(pattern.into()); - } - - if let Some(text) = filter.text { - let pattern = format!("(?i){}", text); - base_sql.push_str( - " - AND ( - regexp_matches(coalesce(e.subject, ''), ?) - OR regexp_matches(coalesce(e.body, ''), ?) - OR regexp_matches(coalesce(e.sender, ''), ?) - OR regexp_matches(array_to_string(e.recipients, ','), ?) - ) - ", - ); - args.push(pattern.clone().into()); - args.push(pattern.clone().into()); - args.push(pattern.clone().into()); - args.push(pattern.into()); - } - - if let Some(from) = filter.from { - base_sql.push_str(" AND e.sender ILIKE ? "); - args.push(format!("%{}%", from).into()); - } - - if let Some(to) = filter.to { - base_sql.push_str(" AND array_to_string(e.recipients, ',') ILIKE ?"); - args.push(format!("%{}%", to).into()); - } - - if let Some(cc) = filter.cc { - base_sql.push_str(" AND array_to_string(e.cc, ',') ILIKE ?"); - args.push(format!("%{}%", cc).into()); - } - - if let Some(bcc) = filter.bcc { - base_sql.push_str(" AND array_to_string(e.bcc, ',') ILIKE ?"); - args.push(format!("%{}%", bcc).into()); - } - - if let Some(since) = filter.since { - base_sql.push_str(" AND e.sent_at >= ? "); - args.push(since.into()); - } - - if let Some(before) = filter.before { - base_sql.push_str(" AND e.sent_at <= ? "); - args.push(before.into()); - } - - if let Some(min) = filter.min_size { - base_sql.push_str(" AND e.size_bytes >= ? "); - args.push(min.into()); - } - - if let Some(max) = filter.max_size { - base_sql.push_str(" AND e.size_bytes <= ? "); - args.push(max.into()); - } - - if let Some(mid) = filter.message_id { - base_sql.push_str(" AND e.message_id ILIKE ? "); - args.push(format!("%{}%", mid).into()); - } - - if let Some(has) = filter.has_attachment { - if has { - base_sql.push_str(" AND e.regular_attachment_count > 0 "); - } else { - base_sql.push_str(" AND e.regular_attachment_count = 0 "); - } - } - - if let Some(tags) = filter.tags { - if !tags.is_empty() { - base_sql.push_str(" AND ("); - for i in 0..tags.len() { - if i > 0 { - base_sql.push_str(" OR "); - } - base_sql.push_str("list_contains(e.tags, ?)"); - } - base_sql.push_str(") "); - for tag in tags { - args.push(tag.into()); - } - } - } - - if let Some(name) = filter.attachment_name { - base_sql.push_str(" AND a.filename ILIKE ? "); - args.push(format!("%{}%", name).into()); - } - // Normalized to lowercase at write-time. LIKE is sufficient here instead of ILIKE. - if let Some(ext) = filter.attachment_extension { - base_sql.push_str(" AND a.extension LIKE ? "); - args.push(format!("%{}%", ext).into()); - } - // Normalized to lowercase at write-time. LIKE is sufficient here instead of ILIKE. - if let Some(cat) = filter.attachment_category { - base_sql.push_str(" AND a.ext_category LIKE ? "); - args.push(format!("%{}%", cat).into()); - } - // Normalized to lowercase at write-time. LIKE is sufficient here instead of ILIKE. - if let Some(ctype) = filter.attachment_content_type { - base_sql.push_str(" AND a.content_type LIKE ? "); - args.push(format!("%{}%", ctype).into()); - } - - let count_sql = if need_join_attachment { - format!("SELECT COUNT(DISTINCT e.id) {}", base_sql) - } else { - format!("SELECT COUNT(*) {}", base_sql) - }; - - println!("================ COUNT SQL ================"); - println!("FINAL: {}", debug_sql(&count_sql, &args)); - println!("==========================================="); - - let total_items: u64 = self - .conn()? - .query_row(&count_sql, duckdb::params_from_iter(args.iter()), |row| { - row.get(0) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - if total_items == 0 { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: 0, - items: Vec::new(), - total_pages: Some(0), - }); - } - - let sort_column = match sort_by { - SortBy::DATE => "e.sent_at", - SortBy::SIZE => "e.size_bytes", - }; - - let order = if desc { "DESC" } else { "ASC" }; - - let select_prefix = if need_join_attachment { - "SELECT DISTINCT e.*" - } else { - "SELECT e.*" - }; - - let data_sql = format!( - "{}{}ORDER BY {} {} - LIMIT ? OFFSET ? - ", - select_prefix, base_sql, sort_column, order - ); - - let mut data_args = args.clone(); - data_args.push(page_size.into()); - data_args.push(offset.into()); - let conn = self.conn()?; - let mut stmt = conn - .prepare(&data_sql) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let items = stmt - .query_map(duckdb::params_from_iter(data_args.iter()), |row| { - Envelope::from_row(row) - }) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .collect::, _>>() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let total_pages = if total_items == 0 { - Some(0) - } else { - Some((total_items + page_size - 1) / page_size) - }; - - Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items, - items, - total_pages, - }) - } -} - -pub fn init( - path: &PathBuf, - mut migrations_runner: Runner, -) -> BichonResult> { - let mut flags = duckdb::Config::default() - .enable_autoload_extension(true) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .access_mode(duckdb::AccessMode::ReadWrite) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .with("enable_fsst_vectors", "true") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - .with("allocator_background_threads", "true") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - if let Some(memory_limit) = &SETTINGS.bichon_duckdb_max_memory { - flags = flags - .max_memory(memory_limit) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - - if let Some(threads) = &SETTINGS.bichon_duckdb_threads { - flags = flags - .threads(*threads) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - - let conn = DuckdbConnectionManager::file_with_flags(path, flags) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let pool = r2d2::Pool::new(conn) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - { - let conn = pool - .get() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - conn.execute("PRAGMA enable_checkpoint_on_shutdown", []) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - conn.pragma_update(None, "autoload_known_extensions", &"true") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - conn.pragma_update(None, "allow_community_extensions", &"false") - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - - { - let conn = pool - .get() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - migrations_runner.set_migration_table_name("migrations"); - for migration in migrations_runner.run_iter(&mut DuckDBConnection(conn)) { - match migration { - Ok(migration) => { - tracing::info!("Applied migration: {}", migration); - } - Err(err) => { - return Err(raise_error!( - format!("Failed to apply migration: {}", err), - ErrorCode::InternalError - )); - } - } - } - } - - Ok(pool) -} diff --git a/src/modules/duckdb/migrations/V1__initial.sql b/src/modules/duckdb/migrations/V1__initial.sql deleted file mode 100644 index 4dcd397..0000000 --- a/src/modules/duckdb/migrations/V1__initial.sql +++ /dev/null @@ -1,69 +0,0 @@ --- ========================= --- envelopes (main table) --- ========================= -CREATE TABLE IF NOT EXISTS envelopes ( - -- internal id (tantivy f_id) - id UUID PRIMARY KEY, - - -- account / mailbox / uid - account_id UBIGINT NOT NULL, - mailbox_id UBIGINT NOT NULL, - uid UBIGINT NOT NULL, - - -- headers / content - content_hash VARCHAR(64) NOT NULL, - subject TEXT, - body TEXT, - - sender TEXT, - recipients VARCHAR[], - cc VARCHAR[], - bcc VARCHAR[], - - -- dates - sent_at BIGINT, - received_at BIGINT, - - -- size - size_bytes UBIGINT, - - -- thread - thread_id VARCHAR NOT NULL, - - -- message-id - message_id TEXT, - - -- attachment summary - attachment_count INTEGER NOT NULL CHECK (attachment_count >= 0), - regular_attachment_count INTEGER NOT NULL CHECK (regular_attachment_count >= 0), - tags VARCHAR[], - shard_id UBIGINT NOT NULL -); - --- ========================= --- envelope_attachments --- --- Normalized attachment metadata for fast filtering and UI display --- ========================= -CREATE TABLE IF NOT EXISTS envelope_attachments ( - -- Reference to envelopes.id - envelope_id UUID NOT NULL, - account_id UBIGINT NOT NULL, - mailbox_id UBIGINT NOT NULL, - -- Original attachment filename (for display) - filename TEXT, - is_message BOOLEAN NOT NULL DEFAULT FALSE, - is_inline BOOLEAN NOT NULL DEFAULT FALSE, - cid TEXT, - -- Normalized file extension (lowercase, without dot) - extension TEXT, - - -- Extension category (document / image / archive / ...) - ext_category TEXT NOT NULL, - content_type TEXT NOT NULL, - -- Attachment size in bytes - -- 0 if unknown - size_bytes UBIGINT NOT NULL, - content_hash VARCHAR(64) NOT NULL, - shard_id UINTEGER NOT NULL -); \ No newline at end of file diff --git a/src/modules/duckdb/refinery.rs b/src/modules/duckdb/refinery.rs deleted file mode 100644 index 2a70aa8..0000000 --- a/src/modules/duckdb/refinery.rs +++ /dev/null @@ -1,93 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - - -use chrono::{DateTime, Utc}; -use refinery::{error::WrapMigrationError, Migration}; -use refinery_core::{ - traits::sync::{Query, Transaction}, - Migrate, -}; -use std::{ops::DerefMut, time::SystemTime}; - -pub struct DuckDBConnection>(pub T); - -impl> From for DuckDBConnection { - fn from(conn: T) -> Self { - Self(conn) - } -} - -impl> Transaction for DuckDBConnection { - type Error = duckdb::Error; - fn execute<'a, T: Iterator>( - &mut self, - mut queries: T, - ) -> std::result::Result { - let transaction = self.0.transaction()?; - let count = queries.try_fold(0, |count, query| { - transaction.execute_batch(query)?; - Ok::<_, Self::Error>(count + 1) - })?; - transaction.commit()?; - Ok(count) - } -} - -impl> Query> for DuckDBConnection { - fn query(&mut self, query: &str) -> Result, Self::Error> { - let mut stmt = self.0.prepare(query)?; - let applied: Vec = stmt - .query_map([], |row| { - let version = row.get(0)?; - let name: String = row.get(1)?; - let applied_on: DateTime = row.get(2)?; - let checksum: u64 = row.get(3)?; - let applied_on: SystemTime = applied_on.into(); - - Ok(Migration::applied( - version, - name, - applied_on.into(), - checksum, - )) - })? - .collect::, _>>()?; - Ok(applied) - } -} - -impl> Migrate for DuckDBConnection { - fn assert_migrations_table( - &mut self, - migration_table_name: &str, - ) -> std::result::Result { - let query = format!( - "CREATE TABLE IF NOT EXISTS {migration_table_name} ( - version INT primary key, - name TEXT not null, - applied_on TIMESTAMP not null, - checksum TEXT not null - )" - ); - - self.execute(std::iter::once(query.as_str())) - .migration_err("failed to create or verify DuckDB migrations table", None)?; - Ok(0) - } -} diff --git a/src/modules/duckdb/tests.rs b/src/modules/duckdb/tests.rs deleted file mode 100644 index 84c0bcc..0000000 --- a/src/modules/duckdb/tests.rs +++ /dev/null @@ -1,28 +0,0 @@ -use duckdb::Connection; - -#[test] -fn test_get_envelopes_with_params_iter() { - let conn = Connection::open_in_memory().unwrap(); - conn.execute("CREATE TABLE envelopes (id VARCHAR, account_id UBIGINT)", []).unwrap(); - - conn.execute("INSERT INTO envelopes VALUES ('mail_1', 1), ('mail_2', 1), ('mail_3', 2)", []).unwrap(); - - let search_ids = vec!["mail_1", "mail_2"]; - let acc_id = 1u64; - - let placeholders = search_ids.iter().map(|_| "?").collect::>().join(","); - let query = format!("SELECT id FROM envelopes WHERE account_id = ? AND id IN ({})", placeholders); - - let mut params: Vec<&dyn duckdb::ToSql> = Vec::new(); - params.push(&acc_id); - for id in &search_ids { - params.push(id); - } - - let mut stmt = conn.prepare(&query).unwrap(); - let res: Vec = stmt.query_map(duckdb::params_from_iter(params), |r| r.get(0)).unwrap() - .map(|r| r.unwrap()).collect(); - println!("{:#?}", &res); - assert_eq!(res.len(), 2); - assert!(res.contains(&"mail_1".to_string())); -} \ No newline at end of file diff --git a/src/modules/envelope/extractor.rs b/src/modules/envelope/extractor.rs index b0435ef..c452300 100644 --- a/src/modules/envelope/extractor.rs +++ b/src/modules/envelope/extractor.rs @@ -16,16 +16,17 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER; -use crate::modules::blob::storage::{DetachedEmail, BLOB_MANAGER}; use crate::modules::common::AddrVec; use crate::modules::envelope::utils::normalize_subject; use crate::modules::error::code::ErrorCode; use crate::modules::error::BichonResult; use crate::modules::message::content::AttachmentInfo; +use crate::modules::store::storage::{DetachedEmail, BLOB_MANAGER}; +use crate::modules::store::tantivy::manager::INDEX_MANAGER; +use crate::modules::store::tantivy::model::EnvelopeWithAttachments; use crate::modules::utils::html::extract_text; use crate::modules::utils::{compute_content_hash, hex_hash}; -use crate::{id, modules::blob::envelope::Envelope}; +use crate::{id, modules::store::envelope::Envelope}; use crate::{raise_error, utc_now}; use async_imap::types::Fetch; use bytes::Bytes; @@ -90,6 +91,7 @@ async fn extract_envelope_core( ) })?; + let preview_limit = 100; let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) { text } else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) { @@ -98,6 +100,16 @@ async fn extract_envelope_core( String::new() }; + let text = text.split_whitespace().collect::>().join(" "); + + let preview = if text.chars().count() > preview_limit { + text.chars().take(preview_limit).collect::() + "..." + } else { + text.clone() + }; + + let body_text = text; + let message_id = message .message_id() .map(String::from) @@ -153,13 +165,14 @@ async fn extract_envelope_core( mailbox_id, uid, subject, - text: String::new(),//for test + preview, from, to, cc, bcc, date, internal_date, + ingest_at: utc_now!(), size, thread_id, attachment_count, @@ -169,7 +182,12 @@ async fn extract_envelope_core( mailbox_name: None, content_hash: email_content_hash, }; - ENVELOPE_INDEX_MANAGER.queue((envelope, attachments)).await; + let ea = EnvelopeWithAttachments { + envelope, + attachments: Some(attachments), + }; + let doc = ea.to_document(&body_text, 0)?; + INDEX_MANAGER.queue(doc).await; Ok(()) } @@ -230,13 +248,14 @@ pub fn extract_envelope_from_nested_message( mailbox_id: Default::default(), uid: Default::default(), subject, - text, + preview: text, from, to, cc, bcc, date, internal_date: Default::default(), + ingest_at: Default::default(), size: Default::default(), thread_id, attachment_count: Default::default(), @@ -349,8 +368,8 @@ pub async fn reattach_eml_content( account_id: u64, envelope_id: String, ) -> BichonResult<(Envelope, Bytes)> { - let envelope = ENVELOPE_INDEX_MANAGER - .get_envelope_by_id(account_id, envelope_id.clone()) + let e = INDEX_MANAGER + .get_envelope_by_id(account_id, &envelope_id) .await? .ok_or_else(|| { raise_error!( @@ -363,38 +382,37 @@ pub async fn reattach_eml_content( })?; let restored_eml = BLOB_MANAGER - .get_email(&envelope.content_hash)? + .get_email(&e.envelope.content_hash)? .ok_or_else(|| { raise_error!( format!( "Original email content not found: account_id={} envelope_id={} content_hash={}", - account_id, &envelope_id, &envelope.content_hash + account_id, &envelope_id, &e.envelope.content_hash ), ErrorCode::ResourceNotFound ) })?; - if !envelope.has_any_attachments() { - return Ok((envelope, restored_eml)); + if !e.envelope.has_any_attachments() { + return Ok((e.envelope, restored_eml)); } let mut restored_eml = restored_eml.to_vec(); - - let account_detail = ENVELOPE_INDEX_MANAGER - .get_attachments_by_envelope_id(account_id, envelope_id) - .await?; - - if envelope.attachment_count != account_detail.len() { + let actual_count = e.attachments.as_ref().map(|a| a.len()).unwrap_or(0); + if e.envelope.attachment_count != actual_count { return Err(raise_error!( - "Consistency check failed: attachment_count does not match account_detail length" - .into(), + format!( + "Consistency check failed: envelope.attachment_count ({}) does not match attachments.len ({})", + e.envelope.attachment_count, + actual_count + ), ErrorCode::InternalError )); } let mut tasks = Vec::new(); - for detail in account_detail { - let placeholder_str = format!("<>", &detail.info.content_hash); + for detail in e.attachments.unwrap() { + let placeholder_str = format!("<>", &detail.content_hash); let pattern = placeholder_str.as_bytes(); let pattern_len = pattern.len(); @@ -409,7 +427,7 @@ pub async fn reattach_eml_content( tasks.push(( absolute_start, absolute_end, - detail.info.content_hash.clone(), + detail.content_hash.clone(), )); search_cursor = absolute_end; } @@ -433,7 +451,7 @@ pub async fn reattach_eml_content( } } - Ok((envelope, Bytes::from(restored_eml))) + Ok((e.envelope, Bytes::from(restored_eml))) } #[cfg(test)] diff --git a/src/modules/mailbox/delete.rs b/src/modules/mailbox/delete.rs index 5209b36..ab7c303 100644 --- a/src/modules/mailbox/delete.rs +++ b/src/modules/mailbox/delete.rs @@ -1,11 +1,27 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use crate::modules::{ - blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER}, - cache::imap::mailbox::MailBox, - error::BichonResult, + cache::imap::mailbox::MailBox, error::BichonResult, store::tantivy::manager::INDEX_MANAGER, }; pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResult<()> { - let mailbox = MailBox::get(mailbox_id).await?; + let mailbox = MailBox::async_get(mailbox_id).await?; let name = mailbox.name; let delimiter = mailbox.delimiter.unwrap_or("/".to_owned()); @@ -26,10 +42,8 @@ pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResu MailBox::delete(*id).await?; } - let content_hashes = ENVELOPE_INDEX_MANAGER + INDEX_MANAGER .delete_mailbox_envelopes(account_id, ids_to_delete.clone()) .await?; - - BLOB_MANAGER.delete(&content_hashes, &content_hashes)?; Ok(()) } diff --git a/src/modules/message/content.rs b/src/modules/message/content.rs index 71f4594..4769fe8 100644 --- a/src/modules/message/content.rs +++ b/src/modules/message/content.rs @@ -22,7 +22,7 @@ use crate::modules::envelope::extractor::{ extract_envelope_from_nested_message, reattach_eml_content, }; use crate::modules::error::code::ErrorCode; -use crate::modules::blob::envelope::Envelope; +use crate::modules::store::envelope::Envelope; use crate::modules::utils::compute_content_hash; use crate::{modules::error::BichonResult, raise_error}; use mail_parser::{MessageParser, MimeHeaders}; @@ -121,30 +121,6 @@ impl AttachmentInfo { "other" } } - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct AttachmentDetail { - pub shard_id: usize, - pub info: AttachmentInfo, -} - -impl AttachmentDetail { - pub fn from_row(row: &duckdb::Row) -> duckdb::Result { - Ok(Self { - shard_id: row.get("shard_id")?, - info: AttachmentInfo { - file_type: row.get("content_type")?, - inline: row.get("is_inline")?, - filename: row.get("filename")?, - size: row.get("size_bytes")?, - content_id: row.get("cid")?, - content_hash: row.get("content_hash")?, - is_message: row.get("is_message")?, - }, - }) - } -} - /// Represents the content of an email message in both plain text and HTML formats. /// /// This struct contains optional fields for plain text and HTML versions of diff --git a/src/modules/message/delete.rs b/src/modules/message/delete.rs index a01a67b..8dfbb24 100644 --- a/src/modules/message/delete.rs +++ b/src/modules/message/delete.rs @@ -16,19 +16,10 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER; -use crate::modules::blob::storage::BLOB_MANAGER; use crate::modules::error::BichonResult; +use crate::modules::store::tantivy::manager::INDEX_MANAGER; use std::collections::HashMap; pub async fn delete_messages_impl(request: HashMap>) -> BichonResult<()> { - let content_hashes = ENVELOPE_INDEX_MANAGER - .get_orphan_hashes_in_memory(request.clone()) - .await?; - if !content_hashes.is_empty() { - BLOB_MANAGER.delete(&content_hashes, &content_hashes)?; - } - ENVELOPE_INDEX_MANAGER - .delete_envelopes_multi_account(request) - .await + INDEX_MANAGER.delete_envelopes_multi_account(request).await } diff --git a/src/modules/message/list.rs b/src/modules/message/list.rs index d468854..5fba652 100644 --- a/src/modules/message/list.rs +++ b/src/modules/message/list.rs @@ -16,53 +16,21 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::{ - modules::{ - account::migration::AccountModel, - error::{code::ErrorCode, BichonResult}, - blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER}, - rest::response::DataPage, - }, - raise_error, +use crate::modules::{ + account::migration::AccountModel, + error::BichonResult, + rest::response::DataPage, + store::{envelope::Envelope, tantivy::manager::INDEX_MANAGER}, }; -pub async fn list_messages_impl( - account_id: u64, - mailbox_id: u64, - page: u64, - page_size: u64, -) -> BichonResult> { - AccountModel::check_account_exists(account_id).await?; - validate_pagination_params(page, page_size)?; - ENVELOPE_INDEX_MANAGER - .list_mailbox_envelopes(account_id, mailbox_id, page, page_size, true) - .await -} - -fn validate_pagination_params(page: u64, page_size: u64) -> BichonResult<()> { - if page == 0 || page_size == 0 { - return Err(raise_error!( - "Both page and page_size must be greater than 0.".into(), - ErrorCode::InvalidParameter - )); - } - if page_size > 500 { - return Err(raise_error!( - "The page_size exceeds the maximum allowed limit of 500.".into(), - ErrorCode::InvalidParameter - )); - } - Ok(()) -} - pub async fn get_thread_messages( account_id: u64, - thread_id: String, + thread_id: &str, page: u64, page_size: u64, ) -> BichonResult> { AccountModel::check_account_exists(account_id).await?; - ENVELOPE_INDEX_MANAGER + INDEX_MANAGER .list_thread_envelopes(account_id, thread_id, page, page_size, true) .await } diff --git a/src/modules/message/search.rs b/src/modules/message/search.rs index fd4cb37..995dcb9 100644 --- a/src/modules/message/search.rs +++ b/src/modules/message/search.rs @@ -16,17 +16,15 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::collections::HashSet; - use poem_openapi::{Enum, Object}; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use crate::{ modules::{ - duckdb::init::duckdb, error::{code::ErrorCode, BichonResult}, - blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER}, rest::response::DataPage, + store::{envelope::Envelope, tantivy::manager::INDEX_MANAGER}, }, raise_error, }; @@ -85,24 +83,6 @@ impl SearchRequest { )); } - let validate = |pattern: &Option| -> BichonResult<()> { - if let Some(ref p) = pattern { - if duckdb()?.validate_regex(p).is_err() { - return Err(raise_error!( - "Invalid search pattern: The regular expression is not supported by DuckDB.".into(), - ErrorCode::InvalidParameter - )); - } - } - Ok(()) - }; - - validate(&self.filter.text)?; - validate(&self.filter.subject)?; - validate(&self.filter.body)?; - validate(&self.filter.from)?; - validate(&self.filter.to)?; - Ok(()) } } @@ -112,7 +92,7 @@ pub async fn search_messages_impl( request: SearchRequest, ) -> BichonResult> { request.validate()?; - ENVELOPE_INDEX_MANAGER + INDEX_MANAGER .search( accounts, request.filter, diff --git a/src/modules/mod.rs b/src/modules/mod.rs index 6d249de..938bbd0 100644 --- a/src/modules/mod.rs +++ b/src/modules/mod.rs @@ -24,12 +24,10 @@ pub mod common; pub mod context; pub mod dashboard; pub mod database; -pub mod duckdb; pub mod envelope; pub mod error; pub mod imap; pub mod import; -pub mod blob; pub mod logger; pub mod mailbox; pub mod message; @@ -37,6 +35,7 @@ pub mod oauth2; pub mod rest; pub mod settings; pub mod smtp; +pub mod store; pub mod tasks; pub mod token; pub mod users; diff --git a/src/modules/rest/api/message.rs b/src/modules/rest/api/message.rs index 1be717c..8dc6a0d 100644 --- a/src/modules/rest/api/message.rs +++ b/src/modules/rest/api/message.rs @@ -17,9 +17,6 @@ // along with this program. If not, see . use crate::modules::account::migration::AccountModel; -use crate::modules::blob::envelope::Envelope; -use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER; -use crate::modules::blob::storage::get_reader; use crate::modules::common::auth::ClientContext; use crate::modules::message::append::restore_emails; use crate::modules::message::append::RestoreMessagesRequest; @@ -30,7 +27,7 @@ use crate::modules::message::content::retrieve_nested_eml_content; use crate::modules::message::content::FullNestedMessageContent; use crate::modules::message::content::{retrieve_email_content, FullMessageContent}; use crate::modules::message::delete::delete_messages_impl; -use crate::modules::message::list::{get_thread_messages, list_messages_impl}; +use crate::modules::message::list::get_thread_messages; use crate::modules::message::search::{search_messages_impl, SearchRequest}; use crate::modules::message::tags::TagCount; use crate::modules::message::tags::TagsRequest; @@ -38,6 +35,9 @@ use crate::modules::rest::api::ApiTags; use crate::modules::rest::response::DataPage; use crate::modules::rest::ApiResult; use crate::modules::rest::ErrorCode; +use crate::modules::store::envelope::Envelope; +use crate::modules::store::storage::get_reader; +use crate::modules::store::tantivy::manager::INDEX_MANAGER; use crate::modules::users::permissions::Permission; use crate::modules::utils::validate_tag; use crate::raise_error; @@ -73,32 +73,6 @@ impl MessageApi { Ok(delete_messages_impl(request).await?) } - /// Lists messages in a mailbox. Requires `mailbox_id`, `page`, and `page_size` query parameters. - #[oai( - path = "/list-messages/:account_id", - method = "get", - operation_id = "list_messages" - )] - async fn list_messages( - &self, - /// The ID of the account. - account_id: Path, - /// The ID of the mailbox to list messages from. - mailbox_id: Query, - page: Query, - page_size: Query, - context: ClientContext, - ) -> ApiResult>> { - let account_id = account_id.0; - let mailbox_id = mailbox_id.0; - context - .require_permission(Some(account_id), Permission::DATA_READ) - .await?; - Ok(Json( - list_messages_impl(account_id, mailbox_id, page.0, page_size.0).await?, - )) - } - /// Searches messages across all mailboxes using various filter criteria. /// The search filters are provided in the request body. #[oai( @@ -141,7 +115,7 @@ impl MessageApi { context: ClientContext, ) -> ApiResult>> { let account_id = account_id.0; - let thread_id = thread_id.0; + let thread_id = thread_id.0.trim(); context .require_permission(Some(account_id), Permission::DATA_READ) .await?; @@ -217,8 +191,8 @@ impl MessageApi { .require_permission(Some(account_id), Permission::DATA_READ) .await?; let envelope_id = envelope_id.0; - let envelope = ENVELOPE_INDEX_MANAGER - .get_envelope_by_id(account_id, envelope_id.clone()) + let e = INDEX_MANAGER + .get_envelope_by_id(account_id, &envelope_id) .await? .ok_or_else(|| { raise_error!( @@ -229,7 +203,7 @@ impl MessageApi { ErrorCode::ResourceNotFound ) })?; - Ok(Json(envelope)) + Ok(Json(e.envelope)) } /// Downloads the raw EML file of a specific email. @@ -361,9 +335,7 @@ impl MessageApi { } else { Some(context.user.account_access_map.keys().cloned().collect()) }; - Ok(Json( - ENVELOPE_INDEX_MANAGER.get_all_tags(authorized_ids).await?, - )) + Ok(Json(INDEX_MANAGER.get_all_tags(authorized_ids).await?)) } /// Adds or removes facet tags for multiple emails across accounts. @@ -389,7 +361,7 @@ impl MessageApi { .await?; } - ENVELOPE_INDEX_MANAGER.update_envelope_tags(req).await?; + INDEX_MANAGER.update_envelope_tags(req).await?; Ok(()) } @@ -408,11 +380,7 @@ impl MessageApi { } else { Some(context.user.account_access_map.keys().cloned().collect()) }; - Ok(Json( - ENVELOPE_INDEX_MANAGER - .get_all_contacts(authorized_ids) - .await?, - )) + Ok(Json(INDEX_MANAGER.get_all_contacts(authorized_ids).await?)) } /// Retrieves unique metadata for all attachments across authorized accounts. @@ -434,9 +402,7 @@ impl MessageApi { Some(context.user.account_access_map.keys().cloned().collect()) }; Ok(Json( - ENVELOPE_INDEX_MANAGER - .get_attachment_metadata(authorized_ids) - .await?, + INDEX_MANAGER.collect_attachment_metadata(authorized_ids)?, )) } } diff --git a/src/modules/settings/dir.rs b/src/modules/settings/dir.rs index 7fbff74..0217a00 100644 --- a/src/modules/settings/dir.rs +++ b/src/modules/settings/dir.rs @@ -31,6 +31,7 @@ const ENVELOPE_DIR: &str = "envelope"; const EML_DIR: &str = "bichon-emls"; const TMP_DIR: &str = "tmp"; const LOG_DIR: &str = "logs"; +const TANTIVY_DIR: &str = "tantivy"; const TLS_CERT: &str = "cert.pem"; const TLS_KEY: &str = "key.pem"; @@ -45,7 +46,7 @@ pub struct DataDirManager { pub temp_dir: PathBuf, pub tls_cert: PathBuf, pub tls_key: PathBuf, - pub envelope_dir: PathBuf, + pub tantivy_dir: PathBuf, pub eml_dir: PathBuf, pub log_dir: PathBuf, } @@ -66,7 +67,7 @@ impl Initialize for DataDirManager { impl DataDirManager { pub fn new(root_dir: PathBuf) -> Self { - let envelope_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir { + let index_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir { PathBuf::from(index_dir) } else { root_dir.join(ENVELOPE_DIR) @@ -85,7 +86,7 @@ impl DataDirManager { tls_key: root_dir.join(TLS_KEY), tls_cert: root_dir.join(TLS_CERT), log_dir: root_dir.join(LOG_DIR), - envelope_dir, + tantivy_dir: index_dir.join(TANTIVY_DIR), temp_dir: root_dir.join(TMP_DIR), eml_dir, } diff --git a/src/modules/store/envelope.rs b/src/modules/store/envelope.rs new file mode 100644 index 0000000..ac02919 --- /dev/null +++ b/src/modules/store/envelope.rs @@ -0,0 +1,53 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use poem_openapi::Object; +use serde::{Deserialize, Serialize}; +use tantivy::doc; + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)] +pub struct Envelope { + pub id: String, + pub message_id: String, + pub account_id: u64, + pub account_email: Option, + pub mailbox_id: u64, + pub mailbox_name: Option, + pub uid: u32, + pub subject: String, + pub preview: String, + pub from: String, + pub to: Vec, + pub cc: Vec, + pub bcc: Vec, + pub date: i64, + pub internal_date: i64, + pub ingest_at: i64, + pub size: u32, + pub thread_id: String, + pub attachment_count: usize, + pub regular_attachment_count: usize, + pub tags: Option>, + pub content_hash: String, +} + +impl Envelope { + pub fn has_any_attachments(&self) -> bool { + self.attachment_count > 0 + } +} diff --git a/src/modules/blob/mod.rs b/src/modules/store/mod.rs similarity index 98% rename from src/modules/blob/mod.rs rename to src/modules/store/mod.rs index 0aab906..89220a3 100644 --- a/src/modules/blob/mod.rs +++ b/src/modules/store/mod.rs @@ -17,5 +17,5 @@ // along with this program. If not, see . pub mod envelope; -pub mod manager; pub mod storage; +pub mod tantivy; diff --git a/src/modules/blob/storage.rs b/src/modules/store/storage.rs similarity index 67% rename from src/modules/blob/storage.rs rename to src/modules/store/storage.rs index f943cb7..7708676 100644 --- a/src/modules/blob/storage.rs +++ b/src/modules/store/storage.rs @@ -1,3 +1,21 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + use crate::modules::{ common::signal::SIGNAL_MANAGER, envelope::extractor::reattach_eml_content, @@ -6,7 +24,7 @@ use crate::modules::{ }; use crate::raise_error; use bytes::Bytes; -use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions}; +use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions, config::{BlockSizePolicy, CompressionPolicy}}; use std::{io::Cursor, sync::LazyLock}; use tokio::{ @@ -78,35 +96,46 @@ impl BlobManager { pub fn new() -> Self { let db = Database::builder(&DATA_DIR_MANAGER.eml_dir) + .cache_size(64 * 1024 * 1024) + .max_cached_files(Some(400)) + .max_journaling_size(256 * 1024 * 1024) .open() .expect("Failed to initialize Fjall database: Check if the directory exists and has write permissions."); let email_keyspace = db .keyspace("email", || { KeyspaceCreateOptions::default() - .with_kv_separation(Some( - KvSeparationOptions::default() - .separation_threshold(0) - .compression(CompressionType::Lz4) - .file_target_size(128 * 1024 * 1024) - .staleness_threshold(0.5) - .age_cutoff(0.6), - )) - .max_memtable_size(64 * 1024 * 1024) + .max_memtable_size(16 * 1024 * 1024) + .data_block_size_policy(BlockSizePolicy::all(4 * 1024)) + .data_block_compression_policy( + CompressionPolicy::all(CompressionType::Lz4) + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(0) + .compression(CompressionType::Lz4) + .file_target_size(128 * 1024 * 1024) + .staleness_threshold(0.5) + .age_cutoff(0.6), + )) }) .expect("Failed to open 'email' keyspace: The partition metadata might be corrupted or inaccessible."); let attachments_keyspace = db .keyspace("attachments", || { KeyspaceCreateOptions::default() - .with_kv_separation(Some( - KvSeparationOptions::default() - .separation_threshold(0) - .compression(CompressionType::Lz4) - .file_target_size(256 * 1024 * 1024) - .staleness_threshold(0.5) - .age_cutoff(0.6), - )) - .max_memtable_size(64 * 1024 * 1024) + .data_block_size_policy(BlockSizePolicy::all(4 * 1024)) + .data_block_compression_policy( + CompressionPolicy::all(CompressionType::Lz4) + ) + .with_kv_separation(Some( + KvSeparationOptions::default() + .separation_threshold(0) + .compression(CompressionType::Lz4) + .file_target_size(256 * 1024 * 1024) + .staleness_threshold(0.5) + .age_cutoff(0.6), + )) + .max_memtable_size(16 * 1024 * 1024) }) .expect("Failed to open 'attachments' keyspace: Check disk space for blob storage initialization."); @@ -123,6 +152,9 @@ impl BlobManager { match res { Some(eml) => { Self::process_detached_email(eml, &store, &email_ks, &attach_ks); + while let Ok(next_eml) = receiver.try_recv() { + Self::process_detached_email(next_eml, &store, &email_ks, &attach_ks); + } } None => { tracing::info!("BlobManager: All senders dropped, closing storage."); @@ -176,17 +208,22 @@ impl BlobManager { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError)) } - pub fn delete( + pub fn delete( &self, - email_content_hashes: &[String], - attachment_content_hashes: &[String], - ) -> BichonResult<()> { + email_content_hashes: I1, + attachment_content_hashes: I2, + ) -> BichonResult<()> + where + I1: IntoIterator, + I1::Item: AsRef, + I2: IntoIterator, + I2::Item: AsRef { let mut batch = self.db.batch(); for hash in email_content_hashes { - batch.remove(&self.email_keyspace, hash); + batch.remove(&self.email_keyspace, hash.as_ref()); } for hash in attachment_content_hashes { - batch.remove(&self.attachments_keyspace, hash); + batch.remove(&self.attachments_keyspace, hash.as_ref()); } batch .commit() diff --git a/src/modules/store/tantivy/fields.rs b/src/modules/store/tantivy/fields.rs new file mode 100644 index 0000000..267a0b7 --- /dev/null +++ b/src/modules/store/tantivy/fields.rs @@ -0,0 +1,80 @@ +// +// Copyright (c) 2025 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use tantivy::schema::Field; + +pub const F_ID: &str = "id"; +pub const F_MESSAGE_ID: &str = "message_id"; +pub const F_ACCOUNT_ID: &str = "account_id"; +pub const F_MAILBOX_ID: &str = "mailbox_id"; +pub const F_UID: &str = "uid"; +pub const F_SUBJECT: &str = "subject"; +pub const F_BODY: &str = "body"; +pub const F_PREVIEW: &str = "preview"; +pub const F_CONTENT_HASH: &str = "content_hash"; +pub const F_FROM: &str = "from"; +pub const F_TO: &str = "to"; +pub const F_CC: &str = "cc"; +pub const F_BCC: &str = "bcc"; +pub const F_DATE: &str = "date"; +pub const F_INTERNAL_DATE: &str = "internal_date"; +pub const F_INGEST_AT: &str = "ingest_at"; +pub const F_SIZE: &str = "size"; +pub const F_THREAD_ID: &str = "thread_id"; +pub const F_ATTACHMENT_COUNT: &str = "attachment_count"; +pub const F_REGULAR_ATTACHMENT_COUNT: &str = "regular_attachment_count"; +pub const F_ATTACHMENTS: &str = "attachments"; +pub const F_ATTACHMENT_GLUE: &str = "attachment_glue"; +pub const F_ATTACHMENT_CONTENT_HASH: &str = "attachment_content_hash"; +pub const F_ATTACHMENT_EXT: &str = "attachment_ext"; +pub const F_ATTACHMENT_CATEGORY: &str = "attachment_category"; +pub const F_ATTACHMENT_CONTENT_TYPE: &str = "attachment_content_type"; + +pub const F_TAGS: &str = "tags"; +pub const F_SHARD_ID: &str = "shard_id"; + +pub struct EmailFields { + pub f_id: Field, + pub f_message_id: Field, + pub f_account_id: Field, + pub f_mailbox_id: Field, + pub f_uid: Field, + pub f_subject: Field, + pub f_body: Field, + pub f_preview: Field, + pub f_content_hash: Field, + pub f_from: Field, + pub f_to: Field, + pub f_cc: Field, + pub f_bcc: Field, + pub f_date: Field, + pub f_internal_date: Field, + pub f_ingest_at: Field, + pub f_size: Field, + pub f_thread_id: Field, + pub f_attachment_count: Field, + pub f_regular_attachment_count: Field, + pub f_attachments: Field, + pub f_attachment_glue: Field, + pub f_attachment_content_hash: Field, + pub f_attachment_ext: Field, + pub f_attachment_category: Field, + pub f_attachment_content_type: Field, + pub f_tags: Field, + pub f_shard_id: Field, +} diff --git a/src/modules/store/tantivy/manager.rs b/src/modules/store/tantivy/manager.rs new file mode 100644 index 0000000..5494b38 --- /dev/null +++ b/src/modules/store/tantivy/manager.rs @@ -0,0 +1,1479 @@ +// +// Copyright (c) 2025 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{ + collections::{HashMap, HashSet}, + ops::Bound, + path::PathBuf, + sync::{Arc, LazyLock}, + time::Duration, +}; + +use crate::{ + modules::{ + account::migration::AccountModel, + common::signal::SIGNAL_MANAGER, + dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, + error::{code::ErrorCode, BichonResult}, + message::{ + attachment::AttachmentMetadata, + search::{SearchFilter, SortBy}, + tags::{TagAction, TagCount, TagsRequest}, + }, + rest::response::DataPage, + settings::dir::DATA_DIR_MANAGER, + store::{ + envelope::Envelope, + storage::BLOB_MANAGER, + tantivy::{ + fields::{ + F_ACCOUNT_ID, F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, + F_ATTACHMENT_EXT, F_DATE, F_FROM, F_REGULAR_ATTACHMENT_COUNT, F_SIZE, F_TAGS, + F_THREAD_ID, F_UID, + }, + model::{extract_contacts, EnvelopeWithAttachments}, + schema::SchemaTools, + }, + }, + }, + raise_error, utc_now, +}; + +use chrono::Utc; +use serde_json::json; +use tantivy::{ + aggregation::{ + agg_req::Aggregations, + agg_result::{ + AggregationResult, AggregationResults, BucketEntries, BucketResult, MetricResult, + }, + AggregationCollector, Key, + }, + collector::{Count, DocSetCollector, FacetCollector, TopDocs}, + indexer::{LogMergePolicy, UserOperation}, + query::{AllQuery, BooleanQuery, EmptyQuery, Occur, Query, QueryParser, RangeQuery, TermQuery}, + schema::{IndexRecordOption, Value}, + DocAddress, Index, IndexReader, IndexWriter, Order, TantivyDocument, Term, +}; +use tantivy::{query::RegexQuery, schema::Facet, Searcher}; +use tokio::{ + sync::{mpsc, Mutex}, + task::{self, JoinHandle}, +}; +use tracing::{info, warn}; + +pub static INDEX_MANAGER: LazyLock = LazyLock::new(IndexManager::new); + +pub struct IndexManager { + index_writer: Arc>, + sender: mpsc::Sender, + reader: IndexReader, + query_parser: QueryParser, + handle: Mutex>>, +} + +impl IndexManager { + pub async fn shutdown(&self) { + let mut guard = self.handle.lock().await; + if let Some(handle) = guard.take() { + let _ = handle.await; + } + } + pub fn new() -> Self { + let index = Self::open_or_create_index(&DATA_DIR_MANAGER.tantivy_dir); + let mut merge_policy = LogMergePolicy::default(); + merge_policy.set_min_num_segments(25); + merge_policy.set_min_layer_size(10_000); + merge_policy.set_max_docs_before_merge(100_000); + + let index_writer = index + .writer_with_num_threads(4, 134_217_728) + .unwrap_or_else(|e| { + panic!( + "Failed to create IndexWriter with 4 threads and 128MB buffer for {:?}: {}", + &DATA_DIR_MANAGER.tantivy_dir, e + ) + }); + index_writer.set_merge_policy(Box::new(merge_policy)); + let index_writer = Arc::new(Mutex::new(index_writer)); + let reader = index.reader().unwrap_or_else(|e| { + panic!( + "Failed to create IndexReader for {:?}: {}", + &DATA_DIR_MANAGER.tantivy_dir, e + ) + }); + let mut query_parser = QueryParser::for_index(&index, SchemaTools::email_default_fields()); + query_parser.set_conjunction_by_default(); + + let (sender, mut receiver) = mpsc::channel::(100); + + let writer = index_writer.clone(); + let handler = task::spawn(async move { + let mut shutdown = SIGNAL_MANAGER.subscribe(); + let mut commit_interval = tokio::time::interval(Duration::from_secs(30)); + loop { + tokio::select! { + maybe_msg = receiver.recv() => { + match maybe_msg { + Some(doc) => { + let writer = writer.lock().await; + if let Err(e) = writer.add_document(doc) { + eprintln!("[ERROR] Failed to add document: {e:?}"); + tracing::error!("Tantivy: Failed to add document: {e:?}"); + } + while let Ok(next_doc) = receiver.try_recv() { + let _ = writer.add_document(next_doc); + } + } + None => { + tracing::info!("Tantivy: Receiver closed. Finalizing..."); + let mut writer = writer.lock().await; + fatal_commit(&mut writer); + break; + }, + } + } + _ = commit_interval.tick() => { + let mut writer = writer.lock().await; + fatal_commit(&mut writer); + } + _ = shutdown.recv() => { + tracing::info!("Tantivy: Shutdown signal received. Performing final commit..."); + let mut writer = writer.lock().await; + fatal_commit(&mut writer); + tracing::info!("Tantivy: Shutdown cleanup complete."); + break; + } + } + } + }); + Self { + index_writer, + sender, + reader, + query_parser, + handle: Mutex::new(Some(handler)), + } + } + + pub async fn queue(&self, doc: TantivyDocument) { + let _ = self.sender.send(doc).await; + } + + fn open_or_create_index(index_dir: &PathBuf) -> Index { + let need_create = !index_dir.exists() + || index_dir + .read_dir() + .map(|mut d| d.next().is_none()) + .unwrap_or(true); + if need_create { + info!( + "Email index not found or empty, creating new index at {}", + index_dir.display() + ); + std::fs::create_dir_all(&index_dir).unwrap_or_else(|e| { + panic!("Failed to create index directory {:?}: {}", index_dir, e) + }); + Index::create_in_dir(&index_dir, SchemaTools::email_schema()) + .unwrap_or_else(|e| panic!("Failed to create index in {:?}: {}", index_dir, e)) + } else { + info!("Opening existing email index at {}", index_dir.display()); + Self::open(&index_dir) + } + } + + fn open(index_dir: &PathBuf) -> Index { + Index::open_in_dir(index_dir) + .unwrap_or_else(|e| panic!("Failed to open index in {:?}: {}", index_dir, e)) + } + + fn account_query(&self, account_id: u64) -> Box { + let account_term = + Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id); + Box::new(TermQuery::new(account_term, IndexRecordOption::Basic)) + } + + fn mailbox_query(&self, account_id: u64, mailbox_id: u64) -> Box { + let account_query = TermQuery::new( + Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id), + IndexRecordOption::Basic, + ); + let mailbox_query = TermQuery::new( + Term::from_field_u64(SchemaTools::email_fields().f_mailbox_id, mailbox_id), + IndexRecordOption::Basic, + ); + let boolean_query = BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_query)), + (Occur::Must, Box::new(mailbox_query)), + ]); + Box::new(boolean_query) + } + + fn envelope_query(&self, account_id: u64, eid: &str) -> Box { + let account_id_query = TermQuery::new( + Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id), + IndexRecordOption::Basic, + ); + let envelope_id_query = TermQuery::new( + Term::from_field_text(SchemaTools::email_fields().f_id, eid), + IndexRecordOption::Basic, + ); + let boolean_query = BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_id_query)), + (Occur::Must, Box::new(envelope_id_query)), + ]); + Box::new(boolean_query) + } + + fn filter_query( + &self, + accounts: Option>, + filter: SearchFilter, + parser: QueryParser, + ) -> BichonResult> { + let f = SchemaTools::email_fields(); + let mut subqueries: Vec<(Occur, Box)> = Vec::new(); + + if let Some(authorized_ids) = accounts { + if authorized_ids.is_empty() { + let term = Term::from_field_u64(f.f_account_id, u64::MAX); + subqueries.push(( + Occur::Must, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } else { + let mut account_must_queries = Vec::new(); + for id in authorized_ids { + let term = Term::from_field_u64(f.f_account_id, id); + account_must_queries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + subqueries.push(( + Occur::Must, + Box::new(BooleanQuery::new(account_must_queries)), + )); + } + } + + if let Some(ref text) = filter.text { + let query = parser + .parse_query(text) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + subqueries.push((Occur::Must, Box::new(query))); + } + + if let Some(ref subject_val) = filter.subject { + let term = Term::from_field_text(f.f_subject, subject_val); + let query = TermQuery::new(term, IndexRecordOption::Basic); + subqueries.push((Occur::Must, Box::new(query))); + } + + if let Some(ref body_val) = filter.body { + let term = Term::from_field_text(f.f_body, body_val); + let query = TermQuery::new(term, IndexRecordOption::Basic); + subqueries.push((Occur::Must, Box::new(query))); + } + + if let Some(ref tags) = filter.tags { + if !tags.is_empty() { + let mut should_queries: Vec<(Occur, Box)> = Vec::new(); + + for tag in tags { + let facet = Facet::from_text(tag).map_err(|e| { + raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter) + })?; + + let term = Term::from_facet(f.f_tags, &facet); + + should_queries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries)))); + } + } + + for (field, opt_value) in [ + (f.f_from, &filter.from), + (f.f_to, &filter.to), + (f.f_cc, &filter.cc), + (f.f_bcc, &filter.bcc), + ] { + if let Some(ref v) = opt_value { + if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) { + subqueries.push((Occur::Must, Box::new(query))); + } + } + } + + if let Some(has) = filter.has_attachment { + let lower: Bound; + let upper: Bound; + + if has { + lower = Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, 1)); + upper = + Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, u64::MAX)); + } else { + lower = Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, 0)); + upper = Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, 0)); + } + + subqueries.push((Occur::Must, Box::new(RangeQuery::new(lower, upper)))); + } + + if let Some(ref name) = filter.attachment_name { + if let Ok(query) = RegexQuery::from_pattern(name.as_str(), f.f_attachment_glue) { + subqueries.push((Occur::Must, Box::new(query))); + } + } + + if let Some(ref extension) = filter.attachment_extension { + if let Ok(query) = RegexQuery::from_pattern(extension.as_str(), f.f_attachment_glue) { + subqueries.push((Occur::Must, Box::new(query))); + } + } + + if let Some(ref category) = filter.attachment_category { + if let Ok(query) = RegexQuery::from_pattern(category.as_str(), f.f_attachment_glue) { + subqueries.push((Occur::Must, Box::new(query))); + } + } + + if let Some(ref content_type) = filter.attachment_content_type { + if let Ok(query) = RegexQuery::from_pattern(content_type.as_str(), f.f_attachment_glue) + { + subqueries.push((Occur::Must, Box::new(query))); + } + } + + let start_bound = if let Some(from) = filter.since { + Bound::Included(Term::from_field_i64(f.f_date, from)) + } else { + Bound::Unbounded + }; + + let end_bound = if let Some(to) = filter.before { + Bound::Included(Term::from_field_i64(f.f_date, to)) + } else { + Bound::Unbounded + }; + + if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded { + let q = RangeQuery::new(start_bound, end_bound); + subqueries.push((Occur::Must, Box::new(q))); + } + + if let Some(account_ids) = filter.account_ids { + let mut should_queries: Vec<(Occur, Box)> = Vec::new(); + for id in account_ids { + let term = Term::from_field_u64(f.f_account_id, id); + should_queries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries)))); + } + + if let Some(mailbox_ids) = filter.mailbox_ids { + let mut should_queries: Vec<(Occur, Box)> = Vec::new(); + for id in mailbox_ids { + let term = Term::from_field_u64(f.f_mailbox_id, id); + should_queries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries)))); + } + + let start_bound = if let Some(from) = filter.min_size { + Bound::Included(Term::from_field_u64(f.f_size, from)) + } else { + Bound::Unbounded + }; + + let end_bound = if let Some(to) = filter.max_size { + Bound::Included(Term::from_field_u64(f.f_size, to)) + } else { + Bound::Unbounded + }; + + if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded { + let q = RangeQuery::new(start_bound, end_bound); + subqueries.push((Occur::Must, Box::new(q))); + } + + if let Some(ref msg_id) = filter.message_id { + let term = Term::from_field_text(f.f_message_id, msg_id); + subqueries.push(( + Occur::Must, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + )); + } + + if subqueries.is_empty() { + return Ok(Box::new(AllQuery)); + } + + Ok(Box::new(BooleanQuery::new(subqueries))) + } + + fn thread_query(&self, account_id: u64, thread_id: &str) -> Box { + let account_query = TermQuery::new( + Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id), + IndexRecordOption::Basic, + ); + let thread_query = TermQuery::new( + Term::from_field_text(SchemaTools::email_fields().f_thread_id, thread_id), + IndexRecordOption::Basic, + ); + let boolean_query = BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_query)), + (Occur::Must, Box::new(thread_query)), + ]); + Box::new(boolean_query) + } + + pub async fn get_envelope_by_id( + &self, + account_id: u64, + envelope_id: &str, + ) -> BichonResult> { + let searcher = self.create_searcher()?; + let f = SchemaTools::email_fields(); + + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, account_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_id, envelope_id), + IndexRecordOption::Basic, + )), + ), + ]); + + let docs: Vec<(f32, DocAddress)> = searcher + .search(&query, &TopDocs::with_limit(1).order_by_score()) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if let Some((_, doc_address)) = docs.first() { + let doc: TantivyDocument = searcher + .doc(*doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let envelope = EnvelopeWithAttachments::from_tantivy_doc(&doc)?; + Ok(Some(envelope)) + } else { + Ok(None) + } + } + + pub async fn top_10_largest_emails( + &self, + accounts: &Option>, + ) -> BichonResult> { + self.reader + .reload() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let searcher = self.reader.searcher(); + + let query: Box = match accounts { + Some(ref ids) if !ids.is_empty() => { + let mut subqueries = Vec::new(); + for &id in ids { + let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id); + subqueries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + Box::new(BooleanQuery::new(subqueries)) + } + Some(_) => Box::new(EmptyQuery), + None => Box::new(AllQuery), + }; + + let mailbox_docs: Vec<(Option, DocAddress)> = searcher + .search( + &query, + &TopDocs::with_limit(10).order_by_fast_field(F_SIZE, Order::Desc), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut result = Vec::new(); + + for (_, doc_address) in mailbox_docs { + let doc: TantivyDocument = searcher + .doc(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let envelope = LargestEmail::from_tantivy_doc(&doc)?; + result.push(envelope); + } + Ok(result) + } + + pub fn total_emails(&self, accounts: &Option>) -> BichonResult { + let searcher = self.create_searcher()?; + + match accounts { + Some(ref ids) if !ids.is_empty() => { + let mut subqueries = Vec::new(); + for &id in ids { + let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id); + subqueries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + let query = Box::new(BooleanQuery::new(subqueries)) as Box; + let count = searcher + .search(&query, &Count) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(count as u64) + } + Some(_) => Ok(0), + None => Ok(searcher.num_docs()), + } + } + + pub async fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { + let searcher = self.create_searcher()?; + + let query = self.mailbox_query(account_id, mailbox_id); + let agg_req: Aggregations = serde_json::from_value(json!({ + "max_uid": { + "max": { + "field": F_UID + } + } + })) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let collector = AggregationCollector::from_aggs(agg_req, Default::default()); + + let agg_res = searcher + .search(query.as_ref(), &collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(Self::extract_max_uid(&agg_res)) + } + + fn extract_max_uid(agg_res: &AggregationResults) -> Option { + agg_res.0.get("max_uid").and_then(|result| match result { + AggregationResult::MetricResult(MetricResult::Max(max)) => { + max.value.and_then(|value| { + (value >= 0.0 && value <= u64::MAX as f64).then(|| value.trunc() as u64) + }) + } + _ => None, + }) + } + + pub async fn delete_account_envelopes(&self, account_id: u64) -> BichonResult<()> { + let query = self.account_query(account_id); + let (eml_content_hashes, attachments_content_hashes) = + self.collect_content_hashes(query).await?; + + let query = self.account_query(account_id); + + let mut writer = self.index_writer.lock().await; + writer + .delete_query(query) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + writer + .commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { + self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes) + .await?; + } + Ok(()) + } + + pub async fn delete_mailbox_envelopes( + &self, + account_id: u64, + mailbox_ids: Vec, + ) -> BichonResult<()> { + if mailbox_ids.is_empty() { + return Ok(()); + } + + let mut eml_content_hashes: HashSet = HashSet::new(); + let mut attachments_content_hashes: HashSet = HashSet::new(); + + for mailbox_id in &mailbox_ids { + let query = self.mailbox_query(account_id, *mailbox_id); + let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query).await?; + eml_content_hashes.extend(eml_hashes); + attachments_content_hashes.extend(attachment_hashes); + } + + let mut queries: Vec> = Vec::with_capacity(mailbox_ids.len()); + for mailbox_id in mailbox_ids { + queries.push(self.mailbox_query(account_id, mailbox_id)); + } + let mut writer = self.index_writer.lock().await; + for query in queries { + writer + .delete_query(query) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + writer + .commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { + self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes) + .await?; + } + Ok(()) + } + + async fn collect_content_hashes( + &self, + query: Box, + ) -> BichonResult<(HashSet, HashSet)> { + let mut eml_content_hashes = HashSet::new(); + let mut attachments_content_hashes = HashSet::new(); + + let fields = SchemaTools::email_fields(); + let searcher = self.create_searcher()?; + + let docs = searcher + .search(&query, &DocSetCollector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + for doc_address in docs { + let doc = searcher + .doc::(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + // Extract content_hash + if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) { + if let Some(str) = content_hash_value.as_str() { + eml_content_hashes.insert(str.to_string()); + } + } + + // Extract attachment_content_hash + let attachment_hash_values = doc.get_all(fields.f_attachment_content_hash); + for hash_value in attachment_hash_values { + if let Some(str) = hash_value.as_str() { + attachments_content_hashes.insert(str.to_string()); + } + } + } + + Ok((eml_content_hashes, attachments_content_hashes)) + } + + async fn cleanup_unused_content( + &self, + eml_content_hashes: HashSet, + attachments_content_hashes: HashSet, + ) -> BichonResult<()> { + let searcher = self.create_searcher()?; + let fields = SchemaTools::email_fields(); + let mut eml: HashSet = HashSet::new(); + for content_hash in eml_content_hashes { + // Check if any other emails still reference this content hash + let hash_term = Term::from_field_text(fields.f_content_hash, &content_hash); + let hash_query = TermQuery::new(hash_term, IndexRecordOption::Basic); + let count = searcher + .search(&hash_query, &Count) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + // If no references found, delete from KV store + if count == 0 { + eml.insert(content_hash); + } + } + + let mut attachments: HashSet = HashSet::new(); + for content_hash in attachments_content_hashes { + // Check if any other emails still reference this content hash + let hash_term = Term::from_field_text(fields.f_content_hash, &content_hash); + let hash_query = TermQuery::new(hash_term, IndexRecordOption::Basic); + let count = searcher + .search(&hash_query, &Count) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + // If no references found, delete from KV store + if count == 0 { + attachments.insert(content_hash); + } + } + + BLOB_MANAGER.delete(&eml, &attachments) + } + + pub async fn delete_envelopes_multi_account( + &self, + deletes: HashMap>, + ) -> BichonResult<()> { + if deletes.is_empty() { + tracing::warn!("delete_envelopes_multi_account: deletes is empty, nothing to delete"); + return Ok(()); + } + + let mut eml_content_hashes: HashSet = HashSet::new(); + let mut attachments_content_hashes: HashSet = HashSet::new(); + + for (account_id, envelope_ids) in &deletes { + let unique_ids: HashSet<&String> = envelope_ids.iter().collect(); + if unique_ids.is_empty() { + continue; + } + + for eid in unique_ids { + let query = self.envelope_query(*account_id, eid); + let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query).await?; + eml_content_hashes.extend(eml_hashes); + attachments_content_hashes.extend(attachment_hashes); + } + } + + let mut writer = self.index_writer.lock().await; + + for (account_id, envelope_ids) in deletes { + let unique_ids: HashSet<&String> = envelope_ids.iter().collect(); + if unique_ids.is_empty() { + continue; + } + for eid in unique_ids { + let query = self.envelope_query(account_id, eid); + writer + .delete_query(query) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + } + writer + .commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { + self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes) + .await?; + } + + Ok(()) + } + + fn collect_facets_recursive( + query: &dyn Query, + searcher: &Searcher, + parent_facet: &str, + all_facets: &mut Vec, + ) -> BichonResult<()> { + let mut facet_collector = FacetCollector::for_field(F_TAGS); + facet_collector.add_facet(parent_facet); + + let facet_counts = searcher + .search(query, &facet_collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + for (facet, count) in facet_counts.get(parent_facet) { + all_facets.push(TagCount { + tag: facet.to_string(), + count, + }); + Self::collect_facets_recursive(query, searcher, &facet.to_string(), all_facets)?; + } + + Ok(()) + } + + pub async fn get_all_tags( + &self, + accounts: Option>, + ) -> BichonResult> { + let searcher = self.reader.searcher(); + + let query: Box = match accounts { + Some(ref ids) if !ids.is_empty() => { + let mut subqueries = Vec::new(); + for &id in ids { + let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id); + subqueries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + Box::new(BooleanQuery::new(subqueries)) + } + Some(_) => Box::new(EmptyQuery), + None => Box::new(AllQuery), + }; + + let mut all_facets = Vec::new(); + Self::collect_facets_recursive(&query, &searcher, "/", &mut all_facets)?; + Ok(all_facets) + } + + pub async fn get_all_contacts( + &self, + accounts: Option>, + ) -> BichonResult> { + let searcher = self.create_searcher()?; + + let query: Box = match accounts { + Some(ref ids) if !ids.is_empty() => { + let mut subqueries = Vec::new(); + for &id in ids { + let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id); + subqueries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + Box::new(BooleanQuery::new(subqueries)) + } + Some(_) => Box::new(EmptyQuery), + None => Box::new(AllQuery), + }; + + let mut contacts_set: HashSet = HashSet::new(); + + let top_docs = searcher + .search(&query, &TopDocs::with_limit(1_000_000).order_by_score()) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + for (_score, doc_address) in top_docs { + let doc: TantivyDocument = searcher + .doc(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let contacts = extract_contacts(&doc).await?; + for value in contacts { + contacts_set.insert(value); + } + } + Ok(contacts_set) + } + + pub async fn update_envelope_tags(&self, request: TagsRequest) -> BichonResult<()> { + if request.updates.is_empty() { + tracing::warn!("update_envelope_tags: request is empty, nothing to update"); + return Ok(()); + } + let searcher = self.create_searcher()?; + let mut writer = self.index_writer.lock().await; + + let f_tags = SchemaTools::email_fields().f_tags; + let f_id = SchemaTools::email_fields().f_id; + let deduplicated_updates: HashMap> = request + .updates + .into_iter() + .map(|(account_id, envelope_ids)| (account_id, envelope_ids.into_iter().collect())) + .collect(); + + let mut operations = Vec::new(); + + for (account_id, envelope_ids) in &deduplicated_updates { + for eid in envelope_ids { + let query = self.envelope_query(*account_id, eid); + let docs = searcher + .search(query.as_ref(), &TopDocs::with_limit(1).order_by_score()) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if let Some((_, doc_address)) = docs.first() { + let old_doc: TantivyDocument = searcher + .doc(*doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut current_tags: HashSet = old_doc + .get_all(f_tags) + .filter_map(|val| val.as_facet()) + .map(|facet| facet.to_string()) + .collect(); + + match request.action { + TagAction::Add => { + for tag in &request.tags { + current_tags.insert(tag.clone()); + } + } + TagAction::Remove => { + for tag in &request.tags { + current_tags.remove(tag); + } + } + TagAction::Overwrite => { + current_tags = request.tags.iter().cloned().collect(); + } + } + + let mut new_doc = TantivyDocument::new(); + + for (field, value) in old_doc.field_values() { + if field != f_tags { + new_doc.add_field_value(field, value); + } + } + for tag in current_tags { + new_doc.add_facet(f_tags, &tag); + } + + let delete_term = Term::from_field_text(f_id, eid); + operations.push(UserOperation::Delete(delete_term)); + operations.push(UserOperation::Add(new_doc)); + } + } + } + + writer + .run(operations) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + // commit + writer + .commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(()) + } + + pub async fn search( + &self, + accounts: Option>, + filter: SearchFilter, + page: u64, + page_size: u64, + desc: bool, + sort_by: SortBy, + ) -> BichonResult> { + assert!(page > 0, "Page number must be greater than 0"); + assert!(page_size > 0, "Page size must be greater than 0"); + let query = self.filter_query(accounts, filter, self.query_parser.clone())?; + let searcher = self.create_searcher()?; + let total = searcher + .search(&query, &Count) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + as u64; + + if total == 0 { + return Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: 0, + items: vec![], + total_pages: Some(0), + }); + } + let offset = (page - 1) * page_size; + let total_pages = total.div_ceil(page_size); + if offset > total { + return Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: total, + items: vec![], + total_pages: Some(total_pages), + }); + } + + let order = if desc { Order::Desc } else { Order::Asc }; + let mailbox_docs: Vec; + + match sort_by { + SortBy::DATE => { + let date_docs: Vec<(Option, DocAddress)> = searcher + .search( + &query, + &TopDocs::with_limit(page_size as usize) + .and_offset(offset as usize) + .order_by_fast_field(F_DATE, order), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + mailbox_docs = date_docs.into_iter().map(|(_, addr)| addr).collect(); + } + SortBy::SIZE => { + let size_docs: Vec<(Option, DocAddress)> = searcher + .search( + &query, + &TopDocs::with_limit(page_size as usize) + .and_offset(offset as usize) + .order_by_fast_field(F_SIZE, order), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + mailbox_docs = size_docs.into_iter().map(|(_, addr)| addr).collect(); + } + } + + let mut result = Vec::new(); + + for doc_address in mailbox_docs { + let doc: TantivyDocument = searcher + .doc(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let envelope = EnvelopeWithAttachments::from_tantivy_doc(&doc)?.envelope; + result.push(envelope); + } + Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: total, + items: result, + total_pages: Some(total_pages), + }) + } + + fn create_searcher(&self) -> BichonResult { + self.reader + .reload() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(self.reader.searcher()) + } + + pub async fn num_messages_in_thread( + &self, + searcher: &Searcher, + account_id: u64, + thread_id: &str, + ) -> BichonResult { + let query = self.thread_query(account_id, thread_id); + + let agg_req: Aggregations = serde_json::from_value(json!({ + "thread_count": { + "value_count": { + "field": F_THREAD_ID + } + } + })) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let collector = AggregationCollector::from_aggs(agg_req, Default::default()); + + let agg_res = searcher + .search(query.as_ref(), &collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Self::extract_thread_count(&agg_res) + } + + fn extract_thread_count(agg_res: &AggregationResults) -> BichonResult { + let Some(result) = agg_res.0.get("thread_count") else { + return Err(raise_error!( + "Missing aggregation result: thread_count".into(), + ErrorCode::InternalError + )); + }; + + match result { + AggregationResult::MetricResult(MetricResult::Count(count)) => { + Ok(count.value.map(|v| v as u64).ok_or_else(|| { + raise_error!( + "Failed to get count value from aggregation result: value is None".into(), + ErrorCode::InternalError + ) + })?) + } + other => Err(raise_error!( + format!("Unexpected aggregation result type: {other:?}"), + ErrorCode::InternalError + )), + } + } + + pub async fn list_thread_envelopes( + &self, + account_id: u64, + thread_id: &str, + page: u64, + page_size: u64, + desc: bool, + ) -> BichonResult> { + assert!(page > 0, "Page number must be greater than 0"); + assert!(page_size > 0, "Page size must be greater than 0"); + let searcher = self.create_searcher()?; + let total = self + .num_messages_in_thread(&searcher, account_id, thread_id) + .await?; + if total == 0 { + return Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: 0, + items: vec![], + total_pages: Some(0), + }); + } + let offset = (page - 1) * page_size; + let total_pages = total.div_ceil(page_size); + if offset > total { + return Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: total, + items: vec![], + total_pages: Some(total_pages), + }); + } + + let query = self.thread_query(account_id, thread_id); + + let order = if desc { Order::Desc } else { Order::Asc }; + let thread_docs: Vec<(Option, DocAddress)> = searcher + .search( + query.as_ref(), + &TopDocs::with_limit(page_size as usize) + .and_offset(offset as usize) + .order_by_fast_field(F_DATE, order), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut result = Vec::new(); + + for (_, doc_address) in thread_docs { + let doc: TantivyDocument = searcher + .doc(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let envelope = EnvelopeWithAttachments::from_tantivy_doc(&doc)?.envelope; + result.push(envelope); + } + Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: total, + items: result, + total_pages: Some(total_pages), + }) + } + + pub fn collect_attachment_metadata( + &self, + accounts: Option>, + ) -> BichonResult { + let searcher = self.create_searcher()?; + + let query: Box = match accounts { + Some(ref ids) if !ids.is_empty() => { + let mut subqueries = Vec::new(); + for &id in ids { + let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id); + subqueries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + Box::new(BooleanQuery::new(subqueries)) + } + Some(_) => Box::new(EmptyQuery), + None => Box::new(AllQuery), + }; + + // Extension collector + let mut ext_collector = FacetCollector::for_field(F_ATTACHMENT_EXT); + ext_collector.add_facet("/"); + + // Category collector + let mut cat_collector = FacetCollector::for_field(F_ATTACHMENT_CATEGORY); + cat_collector.add_facet("/"); + + // Content-Type collector + let mut type_collector = FacetCollector::for_field(F_ATTACHMENT_CONTENT_TYPE); + type_collector.add_facet("/"); + + let ext_counts = searcher + .search(&query, &ext_collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let cat_counts = searcher + .search(&query, &cat_collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let type_counts = searcher + .search(&query, &type_collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut metadata = AttachmentMetadata { + extensions: HashSet::new(), + categories: HashSet::new(), + content_types: HashSet::new(), + }; + + for (facet, _) in ext_counts.get("/") { + let val = facet.to_path_string().trim_start_matches('/').to_string(); + metadata.extensions.insert(val); + } + + for (facet, _) in cat_counts.get("/") { + let val = facet.to_path_string().trim_start_matches('/').to_string(); + metadata.categories.insert(val); + } + + for (facet, _) in type_counts.get("/") { + let val = facet.to_path_string().trim_start_matches('/').to_string(); + metadata.content_types.insert(val); + } + + Ok(metadata) + } + + pub async fn get_dashboard_stats( + &self, + accounts: &Option>, + ) -> BichonResult { + let searcher = self.create_searcher()?; + let now_ms = utc_now!(); + let week_ago_ms = (Utc::now() - Duration::from_secs(60 * 60 * 24 * 30)).timestamp_millis(); + + let aggregations: Aggregations = serde_json::from_value(json!({ + "total_size": { + "sum": { "field": F_SIZE } + }, + "recent_30d_histogram": { + "histogram": { + "field": F_DATE, + "interval": 86400000, + "hard_bounds": { + "min": week_ago_ms, + "max": now_ms + } + } + }, + "top_from_values": { + "terms": { + "field": F_FROM, + "size": 10 + } + }, + "top_account_values": { + "terms": { + "field": F_ACCOUNT_ID, + "size": 10 + } + }, + "attachment_stats": { + "range": { + "field": F_REGULAR_ATTACHMENT_COUNT, + "ranges": [ + { "to": 1, "key": "no_attachment" }, + { "from": 1, "key": "has_attachment" } + ] + } + } + })) + .unwrap(); + + let query: Box = match accounts { + Some(ref ids) if !ids.is_empty() => { + let mut subqueries = Vec::new(); + for &id in ids { + let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id); + subqueries.push(( + Occur::Should, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, + )); + } + Box::new(BooleanQuery::new(subqueries)) + } + Some(_) => Box::new(EmptyQuery), + None => Box::new(AllQuery), + }; + + let agg_collector = AggregationCollector::from_aggs(aggregations, Default::default()); + let agg_results = searcher + .search(&query, &agg_collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut stats = DashboardStats::default(); + let total_size = agg_results.0.get("total_size").ok_or_else(|| { + raise_error!( + "missing 'total_size' aggregation result".into(), + ErrorCode::InternalError + ) + })?; + + if let AggregationResult::MetricResult(MetricResult::Sum(v)) = total_size { + let total_size = v.value.map(|v| v as u64).ok_or_else(|| { + raise_error!( + "'total_size' sum metric has no value".into(), + ErrorCode::InternalError + ) + })?; + stats.total_size_bytes = total_size; + } + + let recent_30d_histogram = agg_results.0.get("recent_30d_histogram").ok_or_else(|| { + raise_error!( + "missing 'recent_30d_histogram' aggregation result".into(), + ErrorCode::InternalError + ) + })?; + + let mut recent_activity = Vec::with_capacity(31); + if let AggregationResult::BucketResult(BucketResult::Histogram { buckets, .. }) = + recent_30d_histogram + { + if let BucketEntries::Vec(bucket_list) = buckets { + for entry in bucket_list { + if let Key::F64(ms) = entry.key { + recent_activity.push(TimeBucket { + timestamp_ms: ms as i64, + count: entry.doc_count, + }); + } + } + } + } + stats.recent_activity = recent_activity; + let mut top_senders = Vec::with_capacity(11); + let top_from_values = agg_results.0.get("top_from_values").unwrap(); + if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = + top_from_values + { + for entry in buckets { + if let Key::Str(sender) = &entry.key { + top_senders.push(Group { + key: sender.clone(), + count: entry.doc_count, + }); + } + } + } + stats.top_senders = top_senders; + + let mut top_accounts = Vec::with_capacity(11); + let top_account_values = agg_results.0.get("top_account_values").unwrap(); + if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = + top_account_values + { + for entry in buckets { + if let Key::U64(account_id) = &entry.key { + match AccountModel::get(*account_id) { + Ok(account) => { + top_accounts.push(Group { + key: account.email, + count: entry.doc_count, + }); + } + Err(e) => { + warn!( + account_id = account_id, + error = %e, + "orphaned account index detected, scheduling cleanup" + ); + let account_id = *account_id; + tokio::spawn(async move { + if let Err(e) = + INDEX_MANAGER.delete_account_envelopes(account_id).await + { + tracing::error!( + account_id = account_id, + error = %e, + "failed to cleanup envelope index" + ); + } + //这里要删除eml和attachments, 要先得到content_hash, 似乎有点麻烦，因为id有点多 + // if let Err(e) = + // INDEX_MANAGER.delete_account_envelopes(account_id).await + // { + // tracing::error!( + // account_id = account_id, + // error = %e, + // "failed to cleanup eml index" + // ); + // } + }); + } + } + } + } + } + stats.top_accounts = top_accounts; + + let attachment_stats = agg_results.0.get("attachment_stats").unwrap(); + if let AggregationResult::BucketResult(BucketResult::Range { buckets, .. }) = + attachment_stats + { + match buckets { + BucketEntries::Vec(bucket_vec) => { + for entry in bucket_vec { + match entry.key.to_string().as_str() { + "no_attachment" => { + stats.without_attachment_count = entry.doc_count; + } + "has_attachment" => { + stats.with_attachment_count = entry.doc_count; + } + _ => {} + } + } + } + BucketEntries::HashMap(bucket_map) => { + if let Some(entry) = bucket_map.get("no_attachment") { + stats.without_attachment_count = entry.doc_count; + } + if let Some(entry) = bucket_map.get("has_attachment") { + stats.with_attachment_count = entry.doc_count; + } + } + } + } + Ok(stats) + } +} + +fn fatal_commit(writer: &mut IndexWriter) { + const MAX_RETRIES: usize = 3; + const RETRY_DELAY_MS: u64 = 1000; + + for attempt in 0..=MAX_RETRIES { + match writer.commit() { + Ok(_) => { + if attempt > 0 { + eprintln!("[INFO] Commit succeeded on attempt {}", attempt + 1); + } + return; + } + Err(e) => match &e { + tantivy::TantivyError::IoError(io_error) => { + if attempt < MAX_RETRIES { + eprintln!( + "[WARN] Commit failed (attempt {}/{}): {:?}. Retrying in {}ms...", + attempt + 1, + MAX_RETRIES + 1, + io_error, + RETRY_DELAY_MS * (attempt as u64 + 1) + ); + std::thread::sleep(std::time::Duration::from_millis( + RETRY_DELAY_MS * (attempt as u64 + 1), + )); + } else { + eprintln!( + "[FATAL] Tantivy commit failed after {} attempts: {:?}", + MAX_RETRIES + 1, + io_error + ); + std::process::exit(1); + } + } + _ => { + eprintln!("[FATAL] Tantivy commit failed with non-IO error: {e:?}"); + std::process::exit(1); + } + }, + } + } +} diff --git a/src/modules/duckdb/mod.rs b/src/modules/store/tantivy/mod.rs similarity index 91% rename from src/modules/duckdb/mod.rs rename to src/modules/store/tantivy/mod.rs index 61f48ac..e3cf3b2 100644 --- a/src/modules/duckdb/mod.rs +++ b/src/modules/store/tantivy/mod.rs @@ -16,8 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -pub mod build; -pub mod init; -pub mod refinery; -#[cfg(test)] -mod tests; +pub mod fields; +pub mod manager; +pub mod model; +pub mod schema; diff --git a/src/modules/store/tantivy/model.rs b/src/modules/store/tantivy/model.rs new file mode 100644 index 0000000..b077e33 --- /dev/null +++ b/src/modules/store/tantivy/model.rs @@ -0,0 +1,241 @@ +use std::collections::HashSet; + +use tantivy::{ + schema::{Facet, Value}, + TantivyDocument, +}; + +use crate::{ + modules::{ + account::migration::AccountModel, + cache::imap::mailbox::MailBox, + error::{code::ErrorCode, BichonResult}, + message::content::AttachmentInfo, + store::{envelope::Envelope, tantivy::schema::SchemaTools}, + }, + raise_error, +}; + +#[derive(Debug, Clone)] +pub struct EnvelopeWithAttachments { + pub envelope: Envelope, + pub attachments: Option>, +} + +impl EnvelopeWithAttachments { + pub fn to_document(&self, body_text: &str, shard_id: u64) -> BichonResult { + let fields = SchemaTools::email_fields(); + let mut doc = TantivyDocument::new(); + + doc.add_text(fields.f_id, &self.envelope.id); + doc.add_text(fields.f_message_id, &self.envelope.message_id); + doc.add_u64(fields.f_account_id, self.envelope.account_id); + doc.add_u64(fields.f_mailbox_id, self.envelope.mailbox_id); + doc.add_u64(fields.f_uid, self.envelope.uid as u64); + doc.add_text(fields.f_subject, &self.envelope.subject); + doc.add_text(fields.f_preview, &self.envelope.preview); + doc.add_text(fields.f_content_hash, &self.envelope.content_hash); + doc.add_text(fields.f_from, &self.envelope.from); + doc.add_text(fields.f_body, body_text); + + for to in &self.envelope.to { + doc.add_text(fields.f_to, to); + } + for cc in &self.envelope.cc { + doc.add_text(fields.f_cc, cc); + } + for bcc in &self.envelope.bcc { + doc.add_text(fields.f_bcc, bcc); + } + + doc.add_i64(fields.f_date, self.envelope.date); + doc.add_i64(fields.f_internal_date, self.envelope.internal_date); + doc.add_u64(fields.f_size, self.envelope.size as u64); + doc.add_i64(fields.f_ingest_at, self.envelope.ingest_at); + doc.add_text(fields.f_thread_id, &self.envelope.thread_id); + + if let Some(ref atts) = self.attachments { + let atts_json = serde_json::to_string(atts).unwrap_or_else(|_| "[]".to_string()); + doc.add_text(fields.f_attachments, atts_json); + let mut search_terms = Vec::new(); + + for att in atts { + if let Some(ref filename) = att.filename { + search_terms.push(filename.clone()); + } + + if let Some(ext) = att.get_extension() { + search_terms.push(ext.clone()); + doc.add_facet(fields.f_attachment_ext, Facet::from(&format!("/{}", ext))); + } + let category = att.get_category().to_string(); + search_terms.push(category.clone()); + let file_type = att.file_type.to_lowercase(); + search_terms.push(file_type.clone()); + + doc.add_facet( + fields.f_attachment_category, + Facet::from(&format!("/{}", category)), + ); + doc.add_facet( + fields.f_attachment_content_type, + Facet::from(&format!("/{}", file_type)), + ); + + doc.add_text(fields.f_attachment_content_hash, &att.content_hash); + } + doc.add_text(fields.f_attachment_glue, search_terms.join(" ")); + } + + doc.add_u64( + fields.f_attachment_count, + self.envelope.attachment_count as u64, + ); + doc.add_u64( + fields.f_regular_attachment_count, + self.envelope.regular_attachment_count as u64, + ); + doc.add_u64(fields.f_shard_id, shard_id); + Ok(doc) + } + + pub fn from_tantivy_doc(doc: &TantivyDocument) -> BichonResult { + let fields = SchemaTools::email_fields(); + + let attachments_raw = extract_string_field(doc, fields.f_attachments).ok(); + let attachments: Option> = + attachments_raw.and_then(|json| serde_json::from_str(&json).ok()); + + let tags: Vec = doc + .get_all(fields.f_tags) + .filter_map(|value| value.as_facet()) + .map(|f| f.to_string()) + .collect(); + + let account_id = extract_u64_field(doc, fields.f_account_id)?; + let mailbox_id = extract_u64_field(doc, fields.f_mailbox_id)?; + + let account = AccountModel::get(account_id)?; + let mailbox = MailBox::get(mailbox_id)?; + let envelope = Envelope { + id: extract_string_field(doc, fields.f_id)?, + message_id: extract_string_field(doc, fields.f_message_id)?, + account_id, + account_email: Some(account.email), + mailbox_id, + mailbox_name: Some(mailbox.name), + uid: extract_u64_field(doc, fields.f_uid)? as u32, + subject: extract_string_field(doc, fields.f_subject)?, + preview: extract_string_field(doc, fields.f_preview).unwrap_or_default(), + from: extract_string_field(doc, fields.f_from)?, + to: extract_vec_string_field(doc, fields.f_to)?, + cc: extract_vec_string_field(doc, fields.f_cc)?, + bcc: extract_vec_string_field(doc, fields.f_bcc)?, + date: extract_i64_field(doc, fields.f_date)?, + internal_date: extract_i64_field(doc, fields.f_internal_date)?, + size: extract_u64_field(doc, fields.f_size)? as u32, + thread_id: extract_string_field(doc, fields.f_thread_id)?, + attachment_count: extract_u64_field(doc, fields.f_attachment_count)? as usize, + regular_attachment_count: extract_u64_field(doc, fields.f_regular_attachment_count)? + as usize, + tags: if tags.is_empty() { None } else { Some(tags) }, + content_hash: extract_string_field(doc, fields.f_content_hash)?, + ingest_at: extract_i64_field(doc, fields.f_ingest_at)?, + }; + + Ok(EnvelopeWithAttachments { + envelope, + attachments, + }) + } +} + +fn extract_u64_field( + document: &TantivyDocument, + field: tantivy::schema::Field, +) -> BichonResult { + let value = document.get_first(field).ok_or_else(|| { + raise_error!( + format!("miss '{}' field in tantivy document", stringify!(field)), + ErrorCode::InternalError + ) + })?; + value.as_u64().ok_or_else(|| { + raise_error!( + format!("'{}' field is not a u64", stringify!(field)), + ErrorCode::InternalError + ) + }) +} + +fn extract_i64_field( + document: &TantivyDocument, + field: tantivy::schema::Field, +) -> BichonResult { + let value = document.get_first(field).ok_or_else(|| { + raise_error!( + format!("miss '{}' field in tantivy document", stringify!(field)), + ErrorCode::InternalError + ) + })?; + value.as_i64().ok_or_else(|| { + raise_error!( + format!("'{}' field is not a i64", stringify!(field)), + ErrorCode::InternalError + ) + }) +} + +fn extract_string_field( + document: &TantivyDocument, + field: tantivy::schema::Field, +) -> BichonResult { + let value = document.get_first(field).ok_or_else(|| { + raise_error!( + format!("'{}' field not found", stringify!(field)), + ErrorCode::InternalError + ) + })?; + value.as_str().map(|s| s.to_string()).ok_or_else(|| { + raise_error!( + format!("'{}' field is not a string", stringify!(field)), + ErrorCode::InternalError + ) + }) +} + +fn extract_vec_string_field( + document: &TantivyDocument, + field: tantivy::schema::Field, +) -> BichonResult> { + let value = document + .get_all(field) + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect(); + Ok(value) +} + +pub async fn extract_contacts(doc: &TantivyDocument) -> BichonResult> { + let fields = SchemaTools::email_fields(); + let mut all_contacts = HashSet::new(); + + if let Ok(from_val) = extract_string_field(doc, fields.f_from) { + if !from_val.is_empty() { + all_contacts.insert(from_val); + } + } + + let multi_fields = [fields.f_to, fields.f_cc, fields.f_bcc]; + + for field in multi_fields { + if let Ok(vals) = extract_vec_string_field(doc, field) { + for v in vals { + if !v.is_empty() { + all_contacts.insert(v); + } + } + } + } + + Ok(all_contacts) +} diff --git a/src/modules/store/tantivy/schema.rs b/src/modules/store/tantivy/schema.rs new file mode 100644 index 0000000..9e7117b --- /dev/null +++ b/src/modules/store/tantivy/schema.rs @@ -0,0 +1,130 @@ +// +// Copyright (c) 2025 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::sync::{Arc, LazyLock}; +use tantivy::schema::{FacetOptions, Field, INDEXED}; +use tantivy::schema::{Schema, FAST, STORED, STRING, TEXT}; + +use crate::modules::store::tantivy::fields::{ + EmailFields, F_ACCOUNT_ID, F_ATTACHMENTS, F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_HASH, + F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_COUNT, F_ATTACHMENT_EXT, F_ATTACHMENT_GLUE, F_BCC, + F_BODY, F_CC, F_CONTENT_HASH, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE, F_MAILBOX_ID, + F_MESSAGE_ID, F_PREVIEW, F_REGULAR_ATTACHMENT_COUNT, F_SHARD_ID, F_SIZE, F_SUBJECT, F_TAGS, + F_THREAD_ID, F_TO, F_UID, +}; + +static EMAIL_FIELDS: LazyLock> = LazyLock::new(|| { + let (_, fields) = SchemaTools::create_email_schema(); + Arc::new(fields) +}); + +pub struct SchemaTools; + +impl SchemaTools { + pub fn email_schema() -> Schema { + let (schema, _) = Self::create_email_schema(); + schema + } + + pub fn email_fields() -> &'static EmailFields { + &EMAIL_FIELDS + } + + pub fn email_default_fields() -> Vec { + let fields = Self::email_fields(); + vec![ + fields.f_subject, + fields.f_body, + fields.f_attachment_glue, + fields.f_from, + fields.f_to, + ] + } + + pub fn create_email_schema() -> (Schema, EmailFields) { + let mut builder = Schema::builder(); + + let f_id = builder.add_text_field(F_ID, STRING | STORED | FAST); + let f_message_id = builder.add_text_field(F_MESSAGE_ID, STRING | STORED); + let f_account_id = builder.add_u64_field(F_ACCOUNT_ID, INDEXED | STORED | FAST); + let f_mailbox_id = builder.add_u64_field(F_MAILBOX_ID, INDEXED | STORED | FAST); + let f_uid = builder.add_u64_field(F_UID, INDEXED | STORED | FAST); + let f_subject = builder.add_text_field(F_SUBJECT, TEXT | STORED); + let f_body = builder.add_text_field(F_BODY, TEXT); + let f_preview = builder.add_text_field(F_PREVIEW, STORED); + let f_content_hash = builder.add_text_field(F_CONTENT_HASH, STRING | FAST | STORED); + let f_from = builder.add_text_field(F_FROM, STRING | STORED | FAST); + let f_to = builder.add_text_field(F_TO, STRING | STORED); + let f_cc = builder.add_text_field(F_CC, STRING | STORED); + let f_bcc = builder.add_text_field(F_BCC, STRING | STORED); + let f_date = builder.add_i64_field(F_DATE, INDEXED | STORED | FAST); + let f_internal_date = builder.add_i64_field(F_INTERNAL_DATE, INDEXED | STORED | FAST); + let f_ingest_at = builder.add_i64_field(F_INGEST_AT, INDEXED | STORED | FAST); + let f_size = builder.add_u64_field(F_SIZE, INDEXED | STORED | FAST); + let f_thread_id = builder.add_text_field(F_THREAD_ID, STRING | STORED | FAST); + let f_attachment_count = builder.add_u64_field(F_ATTACHMENT_COUNT, INDEXED | STORED | FAST); + let f_regular_attachment_count = + builder.add_u64_field(F_REGULAR_ATTACHMENT_COUNT, INDEXED | STORED | FAST); + let f_attachment_glue = builder.add_text_field(F_ATTACHMENT_GLUE, TEXT); + let f_attachments = builder.add_text_field(F_ATTACHMENTS, STORED); + let f_attachment_content_hash = + builder.add_text_field(F_ATTACHMENT_CONTENT_HASH, STRING | FAST | STORED); + let f_attachment_ext = + builder.add_facet_field(F_ATTACHMENT_EXT, FacetOptions::default().set_stored()); + let f_attachment_category = + builder.add_facet_field(F_ATTACHMENT_CATEGORY, FacetOptions::default().set_stored()); + let f_attachment_content_type = builder.add_facet_field( + F_ATTACHMENT_CONTENT_TYPE, + FacetOptions::default().set_stored(), + ); + let f_tags = builder.add_facet_field(F_TAGS, FacetOptions::default().set_stored()); + let f_shard_id = builder.add_u64_field(F_SHARD_ID, INDEXED | STORED | FAST); + + let fields = EmailFields { + f_id, + f_message_id, + f_account_id, + f_mailbox_id, + f_uid, + f_subject, + f_body, + f_preview, + f_content_hash, + f_from, + f_to, + f_cc, + f_bcc, + f_date, + f_internal_date, + f_ingest_at, + f_size, + f_thread_id, + f_attachment_count, + f_regular_attachment_count, + f_attachments, + f_attachment_glue, + f_attachment_content_hash, + f_attachment_ext, + f_attachment_category, + f_attachment_content_type, + f_tags, + f_shard_id, + }; + (builder.build(), fields) + } +} diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 69ca136..031c125 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -35,7 +35,7 @@ export interface EmailEnvelope { mailbox_name?: string; uid: number; subject: string; - text: string; + preview: string; from: string; to: string[]; cc: string[]; diff --git a/web/src/features/dashboard/index.tsx b/web/src/features/dashboard/index.tsx index 21fa42a..386585c 100644 --- a/web/src/features/dashboard/index.tsx +++ b/web/src/features/dashboard/index.tsx @@ -445,7 +445,7 @@ export default function MailArchiveDashboard() { handleQuickSearch({ text: m.subject })} + onClick={() => handleQuickSearch({ subject: `"${m.subject}"` })} > - -
- © 2025-2026 rustmailer.com - Bichon Email Archiving Project -
+
+ © 2025-2026 rustmailer.com - Bichon Email Archiving Project +
); } \ No newline at end of file diff --git a/web/src/features/search/mail-list-table.tsx b/web/src/features/search/mail-list-table.tsx index 3ea11ee..e499789 100755 --- a/web/src/features/search/mail-list-table.tsx +++ b/web/src/features/search/mail-list-table.tsx @@ -239,9 +239,9 @@ export function MailListTable({ id: "text_preview", header: () => null, cell: ({ row }) => { - const text = row.original.text + const preview = row.original.preview - if (!text) return null + if (!preview) return null return ( @@ -260,7 +260,7 @@ export function MailListTable({ align="start" className="max-w-[520px] max-h-[420px] overflow-auto whitespace-pre-wrap text-xs leading-relaxed" > - {text} + {preview} ) diff --git a/web/src/features/search/thread-dialog.tsx b/web/src/features/search/thread-dialog.tsx index bc303d1..b7dadc2 100644 --- a/web/src/features/search/thread-dialog.tsx +++ b/web/src/features/search/thread-dialog.tsx @@ -116,9 +116,7 @@ export function MailThreadDialog({ open, onOpenChange }: MailThreadDialogProps) .sort((a, b) => a.date - b.date) .map((msg) => { const isExpanded = expandedIds.has(msg.id); - const preview = - msg.text?.slice(0, 120) + - (msg.text?.length > 120 ? '...' : ''); + const preview = msg.preview; const date = new Date(msg.date); const formattedDate = isNaN(date.getTime()) ? t('search.thread.invalidDate') diff --git a/web/tsconfig.app.json b/web/tsconfig.app.json index f8072bd..b69cae9 100644 --- a/web/tsconfig.app.json +++ b/web/tsconfig.app.json @@ -3,10 +3,13 @@ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", "target": "ES2020", "useDefineForClassFields": true, - "lib": ["ES2020", "DOM", "DOM.Iterable"], + "lib": [ + "ES2020", + "DOM", + "DOM.Iterable" + ], "module": "ESNext", "skipLibCheck": true, - /* Bundler mode */ "moduleResolution": "Bundler", "allowImportingTsExtensions": true, @@ -14,13 +17,13 @@ "moduleDetection": "force", "noEmit": true, "jsx": "react-jsx", - /* Alias */ "baseUrl": ".", "paths": { - "@/*": ["./src/*"] + "@/*": [ + "./src/*" + ] }, - /* Linting */ "strict": true, "noUnusedLocals": true, @@ -28,5 +31,7 @@ "noFallthroughCasesInSwitch": true, "noUncheckedSideEffectImports": true }, - "include": ["src"] -} + "include": [ + "src" + ] +} \ No newline at end of file