diff --git a/Cargo.lock b/Cargo.lock
index 41ef030..c238532 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -73,17 +73,6 @@ dependencies = [
"opaque-debug",
]
-[[package]]
-name = "ahash"
-version = "0.7.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
-dependencies = [
- "getrandom 0.2.16",
- "once_cell",
- "version_check",
-]
-
[[package]]
name = "ahash"
version = "0.8.12"
@@ -91,7 +80,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
- "const-random",
"getrandom 0.3.4",
"once_cell",
"version_check",
@@ -202,6 +190,15 @@ dependencies = [
"derive_arbitrary",
]
+[[package]]
+name = "arc-swap"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
+dependencies = [
+ "rustversion",
+]
+
[[package]]
name = "arrayref"
version = "0.3.9"
@@ -214,225 +211,6 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
-[[package]]
-name = "arrow"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c"
-dependencies = [
- "arrow-arith",
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-csv",
- "arrow-data",
- "arrow-ipc",
- "arrow-json",
- "arrow-ord",
- "arrow-row",
- "arrow-schema",
- "arrow-select",
- "arrow-string",
-]
-
-[[package]]
-name = "arrow-arith"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "chrono",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-array"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d"
-dependencies = [
- "ahash 0.8.12",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "chrono",
- "half",
- "hashbrown 0.16.1",
- "num-complex",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-buffer"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4"
-dependencies = [
- "bytes 1.11.1",
- "half",
- "num-bigint",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-cast"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-ord",
- "arrow-schema",
- "arrow-select",
- "atoi",
- "base64 0.22.1",
- "chrono",
- "comfy-table",
- "half",
- "lexical-core",
- "num-traits",
- "ryu",
-]
-
-[[package]]
-name = "arrow-csv"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe"
-dependencies = [
- "arrow-array",
- "arrow-cast",
- "arrow-schema",
- "chrono",
- "csv",
- "csv-core",
- "regex",
-]
-
-[[package]]
-name = "arrow-data"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4"
-dependencies = [
- "arrow-buffer",
- "arrow-schema",
- "half",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-ipc"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
- "flatbuffers",
-]
-
-[[package]]
-name = "arrow-json"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-cast",
- "arrow-data",
- "arrow-schema",
- "chrono",
- "half",
- "indexmap",
- "itoa",
- "lexical-core",
- "memchr",
- "num-traits",
- "ryu",
- "serde_core",
- "serde_json",
- "simdutf8",
-]
-
-[[package]]
-name = "arrow-ord"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
-]
-
-[[package]]
-name = "arrow-row"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "half",
-]
-
-[[package]]
-name = "arrow-schema"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "arrow-select"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe"
-dependencies = [
- "ahash 0.8.12",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "num-traits",
-]
-
-[[package]]
-name = "arrow-string"
-version = "58.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e"
-dependencies = [
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-schema",
- "arrow-select",
- "memchr",
- "num-traits",
- "regex",
- "regex-syntax",
-]
-
[[package]]
name = "asn1-rs"
version = "0.7.1"
@@ -632,15 +410,6 @@ dependencies = [
"syn 2.0.117",
]
-[[package]]
-name = "atoi"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
-dependencies = [
- "num-traits",
-]
-
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -713,8 +482,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
name = "bichon"
version = "1.0.0"
dependencies = [
- "ahash 0.8.12",
- "arrow",
+ "ahash",
"async-imap",
"autoconfig",
"base64 0.22.1",
@@ -727,7 +495,6 @@ dependencies = [
"console",
"dashmap",
"dialoguer",
- "duckdb",
"email_address",
"encoding_rs",
"fjall",
@@ -757,11 +524,8 @@ dependencies = [
"poem",
"poem-derive",
"poem-openapi",
- "r2d2",
"rand 0.10.0",
"rcgen",
- "refinery",
- "refinery-core",
"regex",
"reqwest",
"ring",
@@ -769,11 +533,12 @@ dependencies = [
"rustls",
"rustls-pemfile",
"rustls-pki-types",
- "semver 1.0.27",
+ "semver 1.0.28",
"serde",
"serde_json",
"snafu",
"sysinfo",
+ "tantivy",
"tempfile",
"time 0.3.47",
"timeago",
@@ -808,15 +573,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
[[package]]
-name = "bitvec"
-version = "1.0.1"
+name = "bitpacking"
+version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
dependencies = [
- "funty",
- "radium",
- "tap",
- "wyz",
+ "crunchy",
]
[[package]]
@@ -865,25 +627,27 @@ dependencies = [
]
[[package]]
-name = "borsh"
-version = "1.6.0"
+name = "bon"
+version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f"
+checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
dependencies = [
- "borsh-derive",
- "cfg_aliases",
+ "bon-macros",
+ "rustversion",
]
[[package]]
-name = "borsh-derive"
-version = "1.6.0"
+name = "bon-macros"
+version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c"
+checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
dependencies = [
- "once_cell",
- "proc-macro-crate",
+ "darling",
+ "ident_case",
+ "prettyplease",
"proc-macro2",
"quote",
+ "rustversion",
"syn 2.0.117",
]
@@ -914,28 +678,6 @@ version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
-[[package]]
-name = "bytecheck"
-version = "0.6.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2"
-dependencies = [
- "bytecheck_derive",
- "ptr_meta",
- "simdutf8",
-]
-
-[[package]]
-name = "bytecheck_derive"
-version = "0.6.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
[[package]]
name = "bytecount"
version = "0.6.9"
@@ -1007,17 +749,11 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
dependencies = [
"camino",
"cargo-platform",
- "semver 1.0.27",
+ "semver 1.0.28",
"serde",
"serde_json",
]
-[[package]]
-name = "cast"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
-
[[package]]
name = "cc"
version = "1.2.47"
@@ -1030,6 +766,12 @@ dependencies = [
"shlex",
]
+[[package]]
+name = "census"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
+
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -1161,17 +903,6 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
-[[package]]
-name = "comfy-table"
-version = "7.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56"
-dependencies = [
- "strum 0.26.3",
- "strum_macros 0.26.4",
- "unicode-width",
-]
-
[[package]]
name = "compare"
version = "0.0.6"
@@ -1229,26 +960,6 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "const-random"
-version = "0.1.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
-dependencies = [
- "const-random-macro",
-]
-
-[[package]]
-name = "const-random-macro"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
-dependencies = [
- "getrandom 0.2.16",
- "once_cell",
- "tiny-keccak",
-]
-
[[package]]
name = "const_fn"
version = "0.4.11"
@@ -1363,6 +1074,16 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
@@ -1414,27 +1135,6 @@ dependencies = [
"subtle",
]
-[[package]]
-name = "csv"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde_core",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "ctr"
version = "0.6.0"
@@ -1455,7 +1155,7 @@ dependencies = [
"openssl-probe 0.1.6",
"openssl-sys",
"schannel",
- "socket2 0.6.1",
+ "socket2 0.6.3",
"windows-sys 0.59.0",
]
@@ -1530,6 +1230,12 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
+[[package]]
+name = "datasketches"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
+
[[package]]
name = "deflate64"
version = "0.1.10"
@@ -1557,6 +1263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587"
dependencies = [
"powerfmt",
+ "serde_core",
]
[[package]]
@@ -1641,24 +1348,10 @@ dependencies = [
]
[[package]]
-name = "duckdb"
-version = "1.10501.0"
+name = "downcast-rs"
+version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f13bc6d6487032fc2825a62ef8b4924b2378a2eb3166e132e5f3141ae9dd633f"
-dependencies = [
- "arrow",
- "cast",
- "chrono",
- "fallible-iterator",
- "fallible-streaming-iterator",
- "hashlink",
- "libduckdb-sys",
- "num",
- "num-integer",
- "r2d2",
- "rust_decimal",
- "strum 0.27.2",
-]
+checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
[[package]]
name = "dunce"
@@ -1748,6 +1441,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+[[package]]
+name = "erased-serde"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec"
+dependencies = [
+ "serde",
+ "serde_core",
+ "typeid",
+]
+
[[package]]
name = "errno"
version = "0.3.14"
@@ -1795,16 +1499,10 @@ dependencies = [
]
[[package]]
-name = "fallible-iterator"
-version = "0.3.0"
+name = "fastdivide"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
-
-[[package]]
-name = "fallible-streaming-iterator"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
[[package]]
name = "fastrand"
@@ -1821,17 +1519,6 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
-[[package]]
-name = "filetime"
-version = "0.2.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db"
-dependencies = [
- "cfg-if",
- "libc",
- "libredox",
-]
-
[[package]]
name = "find-msvc-tools"
version = "0.1.5"
@@ -1855,16 +1542,6 @@ dependencies = [
"xxhash-rust",
]
-[[package]]
-name = "flatbuffers"
-version = "25.12.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
-dependencies = [
- "bitflags",
- "rustc_version 0.4.1",
-]
-
[[package]]
name = "flate2"
version = "1.1.5"
@@ -1938,18 +1615,22 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "fs4"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
+dependencies = [
+ "rustix",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
-[[package]]
-name = "funty"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
-
[[package]]
name = "futures"
version = "0.3.32"
@@ -2214,27 +1895,6 @@ dependencies = [
"tracing",
]
-[[package]]
-name = "half"
-version = "2.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
-dependencies = [
- "cfg-if",
- "crunchy",
- "num-traits",
- "zerocopy",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-dependencies = [
- "ahash 0.7.8",
-]
-
[[package]]
name = "hashbrown"
version = "0.14.5"
@@ -2272,15 +1932,6 @@ dependencies = [
"syn 2.0.117",
]
-[[package]]
-name = "hashlink"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
-dependencies = [
- "hashbrown 0.15.5",
-]
-
[[package]]
name = "headers"
version = "0.4.1"
@@ -2445,6 +2096,12 @@ dependencies = [
"markup5ever",
]
+[[package]]
+name = "htmlescape"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
+
[[package]]
name = "http"
version = "0.2.12"
@@ -2595,7 +2252,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2 0.6.1",
+ "socket2 0.6.3",
"tokio",
"tower-service",
"tracing",
@@ -2804,6 +2461,15 @@ dependencies = [
"compare",
]
+[[package]]
+name = "inventory"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
+dependencies = [
+ "rustversion",
+]
+
[[package]]
name = "ipconfig"
version = "0.3.2"
@@ -2928,9 +2594,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "lettre"
-version = "0.11.20"
+version = "0.11.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "471816f3e24b85e820dee02cde962379ea1a669e5242f19c61bcbcffedf4c4fb"
+checksum = "dabda5859ee7c06b995b9d1165aa52c39110e079ef609db97178d86aeb051fa7"
dependencies = [
"base64 0.22.1",
"email-encoding",
@@ -2945,67 +2611,16 @@ dependencies = [
"nom 8.0.0",
"percent-encoding",
"quoted_printable",
- "socket2 0.6.1",
+ "socket2 0.6.3",
"tokio",
"url",
]
[[package]]
-name = "lexical-core"
-version = "1.0.6"
+name = "levenshtein_automata"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594"
-dependencies = [
- "lexical-parse-float",
- "lexical-parse-integer",
- "lexical-util",
- "lexical-write-float",
- "lexical-write-integer",
-]
-
-[[package]]
-name = "lexical-parse-float"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56"
-dependencies = [
- "lexical-parse-integer",
- "lexical-util",
-]
-
-[[package]]
-name = "lexical-parse-integer"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34"
-dependencies = [
- "lexical-util",
-]
-
-[[package]]
-name = "lexical-util"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17"
-
-[[package]]
-name = "lexical-write-float"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361"
-dependencies = [
- "lexical-util",
- "lexical-write-integer",
-]
-
-[[package]]
-name = "lexical-write-integer"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df"
-dependencies = [
- "lexical-util",
-]
+checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
[[package]]
name = "libbz2-rs-sys"
@@ -3015,32 +2630,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
[[package]]
name = "libc"
-version = "0.2.182"
+version = "0.2.184"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
-
-[[package]]
-name = "libduckdb-sys"
-version = "1.10501.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12096c1694924782b3fe21e790630b77bacb4fcb7ad9d7ee0fec626f985bf248"
-dependencies = [
- "cc",
- "flate2",
- "pkg-config",
- "reqwest",
- "serde",
- "serde_json",
- "tar",
- "vcpkg",
- "zip",
-]
-
-[[package]]
-name = "libm"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
+checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
[[package]]
name = "libmimalloc-sys"
@@ -3062,17 +2654,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "libredox"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616"
-dependencies = [
- "bitflags",
- "libc",
- "redox_syscall 0.7.0",
-]
-
[[package]]
name = "libz-rs-sys"
version = "0.5.5"
@@ -3202,7 +2783,7 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b7da45f78cc525d3750b623c967ae21c0cd28b2e6a9a2ee4b536a7cce3b21ce"
dependencies = [
- "ahash 0.8.12",
+ "ahash",
"flate2",
"hashify",
"hickory-resolver",
@@ -3289,6 +2870,15 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
+[[package]]
+name = "measure_time"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
+dependencies = [
+ "log",
+]
+
[[package]]
name = "memchr"
version = "2.7.6"
@@ -3347,9 +2937,9 @@ dependencies = [
[[package]]
name = "mio"
-version = "1.1.0"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873"
+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
dependencies = [
"libc",
"wasi 0.11.1+wasi-snapshot-preview1",
@@ -3397,6 +2987,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b"
+[[package]]
+name = "murmurhash32"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+
[[package]]
name = "native-tls"
version = "0.2.18"
@@ -3424,7 +3020,7 @@ dependencies = [
"native_model",
"redb 1.5.1",
"redb 2.6.3",
- "semver 1.0.27",
+ "semver 1.0.28",
"serde",
"skeptic",
"thiserror 1.0.69",
@@ -3528,20 +3124,6 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "num"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
-dependencies = [
- "num-bigint",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational",
- "num-traits",
-]
-
[[package]]
name = "num-bigint"
version = "0.4.6"
@@ -3552,15 +3134,6 @@ dependencies = [
"num-traits",
]
-[[package]]
-name = "num-complex"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
-dependencies = [
- "num-traits",
-]
-
[[package]]
name = "num-conv"
version = "0.2.0"
@@ -3576,28 +3149,6 @@ dependencies = [
"num-traits",
]
-[[package]]
-name = "num-iter"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-rational"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
-dependencies = [
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -3605,7 +3156,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
- "libm",
]
[[package]]
@@ -3672,7 +3222,7 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a95602f5a6eec5b15394516448c0b6fc2f2ef4ca2a9ab2951a91a18cec516bf0"
dependencies = [
- "ahash 0.8.12",
+ "ahash",
"lazy_static",
]
@@ -3701,6 +3251,12 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+[[package]]
+name = "oneshot"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
+
[[package]]
name = "opaque-debug"
version = "0.3.1"
@@ -3767,6 +3323,15 @@ dependencies = [
"vcpkg",
]
+[[package]]
+name = "ordered-float"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "outlook-pst"
version = "1.1.0"
@@ -3777,6 +3342,15 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "ownedbytes"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
+dependencies = [
+ "stable_deref_trait",
+]
+
[[package]]
name = "parking"
version = "2.2.1"
@@ -3801,7 +3375,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.18",
+ "redox_syscall",
"smallvec",
"windows-link",
]
@@ -4140,26 +3714,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "ptr_meta"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
-dependencies = [
- "ptr_meta_derive",
-]
-
-[[package]]
-name = "ptr_meta_derive"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
[[package]]
name = "pulldown-cmark"
version = "0.9.6"
@@ -4211,7 +3765,7 @@ version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3"
dependencies = [
- "ahash 0.8.12",
+ "ahash",
"equivalent",
"hashbrown 0.16.1",
"parking_lot",
@@ -4230,7 +3784,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls",
- "socket2 0.6.1",
+ "socket2 0.6.3",
"thiserror 2.0.17",
"tokio",
"tracing",
@@ -4267,7 +3821,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2 0.6.1",
+ "socket2 0.6.3",
"tracing",
"windows-sys 0.60.2",
]
@@ -4293,23 +3847,6 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
-[[package]]
-name = "r2d2"
-version = "0.8.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
-dependencies = [
- "log",
- "parking_lot",
- "scheduled-thread-pool",
-]
-
-[[package]]
-name = "radium"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
-
[[package]]
name = "rand"
version = "0.7.3"
@@ -4436,6 +3973,26 @@ dependencies = [
"bitflags",
]
+[[package]]
+name = "rayon"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
[[package]]
name = "rcgen"
version = "0.14.7"
@@ -4477,55 +4034,6 @@ dependencies = [
"bitflags",
]
-[[package]]
-name = "redox_syscall"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "refinery"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52c427f2572afe5c6cbfa2b1bf40071c89bf1a8539e958ea582842f6f38dcfae"
-dependencies = [
- "refinery-core",
- "refinery-macros",
-]
-
-[[package]]
-name = "refinery-core"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702655abfc67f93a6f735e9fa4ace7d2e580633f8961f28acbfd7583ddce936c"
-dependencies = [
- "async-trait",
- "cfg-if",
- "log",
- "regex",
- "siphasher",
- "thiserror 2.0.17",
- "time 0.3.47",
- "url",
- "walkdir",
-]
-
-[[package]]
-name = "refinery-macros"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5145756cdf293b5089dc6b4f103f1a1229cc55d67082c866f8c8289531c4b983"
-dependencies = [
- "proc-macro2",
- "quote",
- "refinery-core",
- "regex",
- "syn 2.0.117",
-]
-
[[package]]
name = "regex"
version = "1.12.3"
@@ -4555,15 +4063,6 @@ version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
-[[package]]
-name = "rend"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c"
-dependencies = [
- "bytecheck",
-]
-
[[package]]
name = "reqwest"
version = "0.12.24"
@@ -4635,35 +4134,6 @@ dependencies = [
"windows-sys 0.52.0",
]
-[[package]]
-name = "rkyv"
-version = "0.7.46"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1"
-dependencies = [
- "bitvec",
- "bytecheck",
- "bytes 1.11.1",
- "hashbrown 0.12.3",
- "ptr_meta",
- "rend",
- "rkyv_derive",
- "seahash",
- "tinyvec",
- "uuid",
-]
-
-[[package]]
-name = "rkyv_derive"
-version = "0.7.46"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
[[package]]
name = "rust-embed"
version = "8.11.0"
@@ -4699,19 +4169,13 @@ dependencies = [
]
[[package]]
-name = "rust_decimal"
-version = "1.40.0"
+name = "rust-stemmers"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0"
+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
dependencies = [
- "arrayvec",
- "borsh",
- "bytes 1.11.1",
- "num-traits",
- "rand 0.8.5",
- "rkyv",
"serde",
- "serde_json",
+ "serde_derive",
]
[[package]]
@@ -4729,15 +4193,6 @@ dependencies = [
"semver 0.9.0",
]
-[[package]]
-name = "rustc_version"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
-dependencies = [
- "semver 1.0.27",
-]
-
[[package]]
name = "rusticata-macros"
version = "4.1.0"
@@ -4837,27 +4292,12 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "scheduled-thread-pool"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
-dependencies = [
- "parking_lot",
-]
-
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-[[package]]
-name = "seahash"
-version = "4.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
-
[[package]]
name = "security-framework"
version = "3.6.0"
@@ -4898,9 +4338,9 @@ dependencies = [
[[package]]
name = "semver"
-version = "1.0.27"
+version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
dependencies = [
"serde",
"serde_core",
@@ -5120,12 +4560,6 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
-[[package]]
-name = "simdutf8"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
-
[[package]]
name = "siphasher"
version = "1.0.1"
@@ -5147,6 +4581,15 @@ dependencies = [
"walkdir",
]
+[[package]]
+name = "sketches-ddsketch"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "slab"
version = "0.4.11"
@@ -5209,12 +4652,12 @@ dependencies = [
[[package]]
name = "socket2"
-version = "0.6.1"
+version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881"
+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -5266,7 +4709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
dependencies = [
"discard",
- "rustc_version 0.2.3",
+ "rustc_version",
"stdweb-derive",
"stdweb-internal-macros",
"stdweb-internal-runtime",
@@ -5350,46 +4793,6 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-[[package]]
-name = "strum"
-version = "0.26.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
-
-[[package]]
-name = "strum"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
-dependencies = [
- "strum_macros 0.27.2",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.26.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.117",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
-
[[package]]
name = "subtle"
version = "2.6.1"
@@ -5482,20 +4885,152 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
-name = "tap"
-version = "1.0.1"
+name = "tantivy"
+version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+checksum = "778da245841522199d512d19511b041425d8cff3a8f262b4e1516fceb050289a"
+dependencies = [
+ "aho-corasick",
+ "arc-swap",
+ "base64 0.22.1",
+ "bitpacking",
+ "bon",
+ "byteorder",
+ "census",
+ "crc32fast",
+ "crossbeam-channel",
+ "datasketches",
+ "downcast-rs",
+ "fastdivide",
+ "fnv",
+ "fs4",
+ "htmlescape",
+ "itertools",
+ "levenshtein_automata",
+ "log",
+ "lru",
+ "lz4_flex",
+ "measure_time",
+ "memmap2",
+ "once_cell",
+ "oneshot",
+ "rayon",
+ "regex",
+ "rust-stemmers",
+ "rustc-hash",
+ "serde",
+ "serde_json",
+ "sketches-ddsketch",
+ "smallvec",
+ "tantivy-bitpacker",
+ "tantivy-columnar",
+ "tantivy-common",
+ "tantivy-fst",
+ "tantivy-query-grammar",
+ "tantivy-stacker",
+ "tantivy-tokenizer-api",
+ "tempfile",
+ "thiserror 2.0.17",
+ "time 0.3.47",
+ "typetag",
+ "uuid",
+ "winapi",
+ "zstd",
+]
[[package]]
-name = "tar"
-version = "0.4.44"
+name = "tantivy-bitpacker"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
+checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4"
dependencies = [
- "filetime",
- "libc",
- "xattr",
+ "bitpacking",
+]
+
+[[package]]
+name = "tantivy-columnar"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
+dependencies = [
+ "downcast-rs",
+ "fastdivide",
+ "itertools",
+ "serde",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-sstable",
+ "tantivy-stacker",
+]
+
+[[package]]
+name = "tantivy-common"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "ownedbytes",
+ "serde",
+ "time 0.3.47",
+]
+
+[[package]]
+name = "tantivy-fst"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
+dependencies = [
+ "byteorder",
+ "regex-syntax",
+ "utf8-ranges",
+]
+
+[[package]]
+name = "tantivy-query-grammar"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82"
+dependencies = [
+ "fnv",
+ "nom 7.1.3",
+ "ordered-float",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "tantivy-sstable"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
+dependencies = [
+ "futures-util",
+ "itertools",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-fst",
+ "zstd",
+]
+
+[[package]]
+name = "tantivy-stacker"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951"
+dependencies = [
+ "murmurhash32",
+ "tantivy-common",
+]
+
+[[package]]
+name = "tantivy-tokenizer-api"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98"
+dependencies = [
+ "serde",
]
[[package]]
@@ -5651,15 +5186,6 @@ dependencies = [
"isolang",
]
-[[package]]
-name = "tiny-keccak"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
-dependencies = [
- "crunchy",
-]
-
[[package]]
name = "tinystr"
version = "0.8.2"
@@ -5687,9 +5213,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.50.0"
+version = "1.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
+checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd"
dependencies = [
"bytes 1.11.1",
"libc",
@@ -5697,7 +5223,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
- "socket2 0.6.1",
+ "socket2 0.6.3",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -5714,9 +5240,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
-version = "2.6.0"
+version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
@@ -6019,12 +5545,42 @@ version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+[[package]]
+name = "typeid"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
+
[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
+[[package]]
+name = "typetag"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf"
+dependencies = [
+ "erased-serde",
+ "inventory",
+ "once_cell",
+ "serde",
+ "typetag-impl",
+]
+
+[[package]]
+name = "typetag-impl"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "uncased"
version = "0.9.10"
@@ -6131,6 +5687,12 @@ dependencies = [
"regex",
]
+[[package]]
+name = "utf8-ranges"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
+
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -6342,7 +5904,7 @@ dependencies = [
"bitflags",
"hashbrown 0.15.5",
"indexmap",
- "semver 1.0.27",
+ "semver 1.0.28",
]
[[package]]
@@ -6875,7 +6437,7 @@ dependencies = [
"id-arena",
"indexmap",
"log",
- "semver 1.0.27",
+ "semver 1.0.28",
"serde",
"serde_derive",
"serde_json",
@@ -6889,15 +6451,6 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
-[[package]]
-name = "wyz"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
-dependencies = [
- "tap",
-]
-
[[package]]
name = "x509-parser"
version = "0.18.1"
@@ -6916,16 +6469,6 @@ dependencies = [
"time 0.3.47",
]
-[[package]]
-name = "xattr"
-version = "1.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
-dependencies = [
- "libc",
- "rustix",
-]
-
[[package]]
name = "xml-rs"
version = "0.8.28"
diff --git a/Cargo.toml b/Cargo.toml
index cd7ea22..38dee68 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -46,7 +46,7 @@ poem-openapi = { version = "5.1.16", features = [
ring = { version = "0.17.14", features = ["std"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.149"
-tokio = { version = "1.50.0", features = ["full"] }
+tokio = { version = "1.51.0", features = ["full"] }
tracing = "0.1.44"
tracing-appender = "0.2.3"
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }
@@ -89,7 +89,7 @@ webpki-roots = "1.0.6"
rustls = { version = "0.23.37", default-features = false, features = ["ring"] }
rustls-pki-types = "1.14.0"
tokio-io-timeout = "1.2.1"
-semver = "1.0.27"
+semver = "1.0.28"
governor = "0.10.4"
lru = "0.16.3"
mime_guess = "2.0.5"
@@ -118,23 +118,14 @@ outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch
compressed-rtf = "1.0.1"
codepage-strings = "1.0.2"
mail-send = "0.5.2"
-duckdb = { version = "1.10501.0", features = [
- "chrono",
- "bundled",
- "r2d2",
- "appender-arrow",
-] }
-arrow = { version = "58", features = ["ffi"] }
-r2d2 = { version = "0.8", default-features = false }
-refinery = { version = "0.9", default-features = false }
-refinery-core = { version = "0.9", default-features = false }
rcgen = "0.14.7"
rustls-pemfile = "2.2.0"
blake3 = "1.8.4"
uuid = { version = "1.23.0", features = ["v4", "serde"] }
fjall = { version = "3.1.2", features = ["lz4", "metrics", "bytes_1"] }
+tantivy = { version = "0.26.0", features = ["zstd-compression"] }
[dev-dependencies]
#bincode = "1.3.3"
#secret-lib = "1.0.0"
tempfile = "3.27.0"
-lettre = "0.11.20"
+lettre = "0.11.21"
diff --git a/src/main.rs b/src/main.rs
index 08f3119..934e0cc 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -19,16 +19,15 @@
use bichon::{
bichon_version,
modules::{
- blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::imap::task::SYNC_TASKS,
common::rustls::BichonTls,
context::{executors::BichonContext, Initialize},
- duckdb::init::DuckDBManager,
error::{code::ErrorCode, BichonResult},
logger,
rest::start_http_server,
settings::cli::SETTINGS,
smtp::{start_smtp_server, SmtpServer},
+ store::{storage::BLOB_MANAGER, tantivy::manager::INDEX_MANAGER},
tasks::PeriodicTasks,
},
raise_error,
@@ -95,7 +94,7 @@ async fn main() -> BichonResult<()> {
}
SYNC_TASKS.shutdown().await;
- ENVELOPE_INDEX_MANAGER.shutdown().await;
+ INDEX_MANAGER.shutdown().await;
BLOB_MANAGER.shutdown().await;
info!("Bichon server stopped.");
Ok(())
@@ -105,7 +104,6 @@ async fn main() -> BichonResult<()> {
async fn initialize() -> BichonResult<()> {
SignalManager::initialize().await?;
DataDirManager::initialize().await?;
- DuckDBManager::initialize().await?;
UserManager::initialize().await?;
BichonTls::initialize().await?;
BichonContext::initialize().await?;
diff --git a/src/modules/account/migration.rs b/src/modules/account/migration.rs
index 59891c6..d7ca695 100644
--- a/src/modules/account/migration.rs
+++ b/src/modules/account/migration.rs
@@ -32,10 +32,10 @@ use crate::{
since::{DateSince, RelativeDate},
state::AccountRunningState,
},
- blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::imap::mailbox::MailBox,
database::{list_all_impl, secondary_find_impl, with_transaction},
error::BichonResult,
+ store::tantivy::manager::INDEX_MANAGER,
users::{role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel, DEFAULT_ADMIN_USER_ID},
},
utc_now,
@@ -359,10 +359,7 @@ impl AccountV4 {
OAuth2AccessToken::try_delete(account.id).await?;
UserModel::cleanup_account(account.id).await?;
MailBox::clean(account.id).await?;
- let content_hashes = ENVELOPE_INDEX_MANAGER
- .delete_account_envelopes(account.id)
- .await?;
- BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
+ INDEX_MANAGER.delete_account_envelopes(account.id).await?;
Self::delete_account(account.id).await?;
info!("Sequential cleanup completed for account: {}", account.id);
Ok(())
diff --git a/src/modules/blob/envelope.rs b/src/modules/blob/envelope.rs
deleted file mode 100644
index 8b05f3b..0000000
--- a/src/modules/blob/envelope.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-//
-// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
-//
-// This file is part of the Bichon Email Archiving Project
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-use duckdb::types::Value;
-use poem_openapi::Object;
-use serde::{Deserialize, Serialize};
-
-use crate::modules::{account::migration::AccountModel, cache::imap::mailbox::MailBox};
-
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
-pub struct Envelope {
- pub id: String,
- pub message_id: String,
- pub account_id: u64,
- pub account_email: Option,
- pub mailbox_id: u64,
- pub mailbox_name: Option,
- pub uid: u32,
- pub subject: String,
- pub text: String,
- pub from: String,
- pub to: Vec,
- pub cc: Vec,
- pub bcc: Vec,
- pub date: i64,
- pub internal_date: i64,
- pub size: u32,
- pub thread_id: String,
- pub attachment_count: usize,
- pub regular_attachment_count: usize,
- pub tags: Option>,
- /// Hash of the content.
- pub content_hash: String,
-}
-
-impl Envelope {
- pub fn has_any_attachments(&self) -> bool {
- self.attachment_count > 0
- }
-
- pub fn from_row(row: &duckdb::Row) -> duckdb::Result {
- let get_list = |col_name: &str| -> Vec {
- row.get::<_, Value>(col_name)
- .map(|v| {
- if let Value::List(inner_list) = v {
- inner_list
- .into_iter()
- .filter_map(|item| {
- if let Value::Text(s) = item {
- Some(s)
- } else {
- None
- }
- })
- .collect()
- } else {
- vec![]
- }
- })
- .unwrap_or_default()
- };
- let account_id = row.get("account_id")?;
- let mailbox_id = row.get("mailbox_id")?;
- let email = match AccountModel::get(account_id) {
- Ok(account) => account.email,
- Err(_) => "unknown".to_string(),
- };
-
- let mailbox_name = MailBox::find_mailbox(account_id, mailbox_id)
- .ok()
- .and_then(|m| m)
- .map(|m| m.name)
- .unwrap_or_else(|| "unknown".to_string());
-
- Ok(Self {
- id: row.get("id")?,
- message_id: row.get("message_id").unwrap_or_default(),
- account_id,
- account_email: Some(email),
- mailbox_id,
- mailbox_name: Some(mailbox_name),
- uid: row.get::<_, u64>("uid")? as u32,
- subject: row.get("subject").unwrap_or_default(),
- text: row.get("body").unwrap_or_default(),
- from: row.get("sender").unwrap_or_default(),
- to: get_list("recipients"),
- cc: get_list("cc"),
- bcc: get_list("bcc"),
- date: row.get("sent_at").unwrap_or(0),
- internal_date: row.get("received_at").unwrap_or(0),
- size: row.get::<_, u64>("size_bytes")? as u32,
- thread_id: row.get("thread_id")?,
- attachment_count: row.get::<_, i32>("attachment_count")? as usize,
- regular_attachment_count: row.get::<_, i32>("regular_attachment_count")? as usize,
- tags: {
- let t = get_list("tags");
- if t.is_empty() {
- None
- } else {
- Some(t)
- }
- },
- content_hash: row.get("content_hash")?,
- })
- }
-}
diff --git a/src/modules/blob/manager.rs b/src/modules/blob/manager.rs
deleted file mode 100644
index f042ac9..0000000
--- a/src/modules/blob/manager.rs
+++ /dev/null
@@ -1,349 +0,0 @@
-//
-// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
-//
-// This file is part of the Bichon Email Archiving Project
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-use std::{
- collections::{HashMap, HashSet},
- sync::LazyLock,
- time::Duration,
-};
-
-use crate::modules::{
- duckdb::init::duckdb,
- message::{
- attachment::AttachmentMetadata,
- content::{AttachmentDetail, AttachmentInfo},
- search::SortBy,
- tags::{TagCount, TagsRequest},
- },
-};
-use crate::{
- modules::{
- blob::envelope::Envelope,
- common::signal::SIGNAL_MANAGER,
- dashboard::{DashboardStats, LargestEmail},
- error::{code::ErrorCode, BichonResult},
- message::search::SearchFilter,
- rest::response::DataPage,
- },
- raise_error,
-};
-
-use tokio::{
- sync::{mpsc, Mutex},
- task::{self, JoinHandle},
-};
-
-pub static ENVELOPE_INDEX_MANAGER: LazyLock =
- LazyLock::new(EnvelopeIndexManager::new);
-
-pub const ENVELOPE_BATCH_SIZE: usize = 100;
-
-const MAX_BUFFER_DURATION: Duration = Duration::from_secs(10);
-
-pub struct EnvelopeIndexManager {
- sender: mpsc::Sender<(Envelope, Vec)>,
- handle: Mutex>>,
-}
-
-impl EnvelopeIndexManager {
- pub async fn shutdown(&self) {
- let mut guard = self.handle.lock().await;
- if let Some(handle) = guard.take() {
- tracing::info!("Waiting for EnvelopeIndexManager to sync all data...");
- let _ = handle.await;
- tracing::info!("EnvelopeIndexManager synchronized and closed.");
- }
- }
-
- pub fn new() -> Self {
- let (sender, mut receiver) = mpsc::channel::<(Envelope, Vec)>(1000);
- let handle = task::spawn(async move {
- let mut buffer: Vec<(Envelope, Vec)> =
- Vec::with_capacity(ENVELOPE_BATCH_SIZE);
- let mut interval = tokio::time::interval(MAX_BUFFER_DURATION);
- let mut shutdown = SIGNAL_MANAGER.subscribe();
- loop {
- tokio::select! {
- maybe_msg = receiver.recv() => {
- match maybe_msg {
- Some(doc) => {
- buffer.push(doc);
- if buffer.len() >= ENVELOPE_BATCH_SIZE {
- ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await;
- }
- }
- None => {
- if !buffer.is_empty() {
- tracing::info!("Channel closed, flushing remaining {} items", buffer.len());
- ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await;
- }
- break;
- },
- }
- }
- _ = interval.tick() => {
- if !buffer.is_empty() {
- ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await;
- }
- }
- _ = shutdown.recv() => {
- ENVELOPE_INDEX_MANAGER.flush(&mut buffer).await;
- break;
- }
- }
- }
- });
- Self {
- sender,
- handle: Mutex::new(Some(handle)),
- }
- }
-
- pub async fn queue(&self, doc: (Envelope, Vec)) {
- let _ = self.sender.send(doc).await;
- }
-
- async fn flush(&self, buffer: &mut Vec<(Envelope, Vec)>) {
- if buffer.is_empty() {
- return;
- }
-
- let items: Vec<(Envelope, Vec)> = buffer.drain(..).collect();
-
- let result = (|| -> BichonResult<()> {
- duckdb()?.append_envelopes_with_attachments(&items)?;
- Ok(())
- })();
-
- if let Err(e) = result {
- tracing::error!("Failed to drain and commit envelopes to DuckDB: {:#?}", e);
- }
- }
-
- pub async fn total_emails(&self, accounts: Option>) -> BichonResult {
- tokio::task::spawn_blocking(move || duckdb()?.total_emails(accounts))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn delete_account_envelopes(&self, account_id: u64) -> BichonResult> {
- let content_hashes = tokio::task::spawn_blocking(move || {
- duckdb()?.delete_account_envelopes_with_orphans(account_id)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))??;
- Ok(content_hashes)
- }
-
- pub async fn delete_mailbox_envelopes(
- &self,
- account_id: u64,
- mailbox_ids: Vec,
- ) -> BichonResult> {
- if mailbox_ids.is_empty() {
- tracing::warn!("delete_mailbox_envelopes: mailbox_ids is empty, nothing to delete");
- return Ok(vec![]);
- }
-
- let content_hashes = tokio::task::spawn_blocking(move || {
- duckdb()?.delete_mailbox_envelopes_with_orphans(account_id, mailbox_ids)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))??;
- Ok(content_hashes)
- }
-
- pub async fn get_all_tags(
- &self,
- accounts: Option>,
- ) -> BichonResult> {
- tokio::task::spawn_blocking(move || duckdb()?.get_all_tags(accounts))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_all_contacts(
- &self,
- accounts: Option>,
- ) -> BichonResult> {
- tokio::task::spawn_blocking(move || duckdb()?.get_all_contacts(accounts))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_attachment_metadata(
- &self,
- accounts: Option>,
- ) -> BichonResult {
- tokio::task::spawn_blocking(move || duckdb()?.get_attachment_metadata(accounts))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_orphan_hashes_in_memory(
- &self,
- deletes: HashMap>,
- ) -> BichonResult> {
- tokio::task::spawn_blocking(move || duckdb()?.get_orphan_hashes_in_memory(deletes))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn delete_envelopes_multi_account(
- &self,
- deletes: HashMap>, // HashMap
- ) -> BichonResult<()> {
- if deletes.is_empty() {
- tracing::warn!("delete_envelopes_multi_account: deletes is empty, nothing to delete");
- return Ok(());
- }
- tokio::task::spawn_blocking(move || duckdb()?.delete_envelopes_multi_account(deletes))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn update_envelope_tags(&self, request: TagsRequest) -> BichonResult<()> {
- if request.updates.is_empty() {
- tracing::warn!("update_envelope_tags: request is empty, nothing to update");
- return Ok(());
- }
- tokio::task::spawn_blocking(move || duckdb()?.update_envelope_tags(request))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn search(
- &self,
- accounts: Option>,
- filter: SearchFilter,
- page: u64,
- page_size: u64,
- desc: bool,
- sort_by: SortBy,
- ) -> BichonResult> {
- assert!(page > 0, "Page number must be greater than 0");
- assert!(page_size > 0, "Page size must be greater than 0");
- tokio::task::spawn_blocking(move || {
- duckdb()?.search(accounts, filter, page, page_size, desc, sort_by)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn list_mailbox_envelopes(
- &self,
- account_id: u64,
- mailbox_id: u64,
- page: u64,
- page_size: u64,
- desc: bool,
- ) -> BichonResult> {
- assert!(page > 0, "Page number must be greater than 0");
- assert!(page_size > 0, "Page size must be greater than 0");
- tokio::task::spawn_blocking(move || {
- duckdb()?.list_mailbox_envelopes(account_id, mailbox_id, page, page_size, desc)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn list_thread_envelopes(
- &self,
- account_id: u64,
- thread_id: String,
- page: u64,
- page_size: u64,
- desc: bool,
- ) -> BichonResult> {
- assert!(page > 0, "Page number must be greater than 0");
- assert!(page_size > 0, "Page size must be greater than 0");
- tokio::task::spawn_blocking(move || {
- duckdb()?.list_thread_envelopes(account_id, thread_id, page, page_size, desc)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_envelope_by_id(
- &self,
- account_id: u64,
- envelope_id: String,
- ) -> BichonResult> {
- tokio::task::spawn_blocking(move || duckdb()?.get_envelope_by_id(account_id, envelope_id))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_attachments_by_envelope_id(
- &self,
- account_id: u64,
- envelope_id: String,
- ) -> BichonResult> {
- tokio::task::spawn_blocking(move || {
- duckdb()?.get_attachments_by_envelope_id(account_id, envelope_id)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn top_10_largest_emails(
- &self,
- accounts: Option>,
- ) -> BichonResult> {
- tokio::task::spawn_blocking(move || duckdb()?.top_10_largest_emails(accounts))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> {
- tokio::task::spawn_blocking(move || duckdb()?.get_max_uid(account_id, mailbox_id))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn num_messages_in_mailbox(
- &self,
- account_id: u64,
- mailbox_id: u64,
- ) -> BichonResult {
- tokio::task::spawn_blocking(move || {
- duckdb()?.num_messages_in_mailbox(account_id, mailbox_id)
- })
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn num_messages_in_thread(
- &self,
- account_id: u64,
- thread_id: String,
- ) -> BichonResult {
- tokio::task::spawn_blocking(move || duckdb()?.num_messages_in_thread(account_id, thread_id))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-
- pub async fn get_dashboard_stats(
- &self,
- accounts: Option>,
- ) -> BichonResult {
- tokio::task::spawn_blocking(move || duckdb()?.get_dashboard_stats(accounts))
- .await
- .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?
- }
-}
diff --git a/src/modules/cache/imap/mailbox.rs b/src/modules/cache/imap/mailbox.rs
index 7cd4a67..4ff48d7 100644
--- a/src/modules/cache/imap/mailbox.rs
+++ b/src/modules/cache/imap/mailbox.rs
@@ -22,7 +22,7 @@ use crate::{
database::{
async_filter_by_secondary_key_impl, async_find_impl, batch_delete_impl,
batch_insert_impl, batch_upsert_impl, delete_impl, filter_by_secondary_key_impl,
- manager::DB_MANAGER,
+ find_impl, manager::DB_MANAGER,
},
error::{code::ErrorCode, BichonResult},
},
@@ -72,7 +72,7 @@ impl MailBox {
encode_mailbox_name!(&self.name)
}
- pub async fn get(id: u64) -> BichonResult {
+ pub async fn async_get(id: u64) -> BichonResult {
let result = async_find_impl::(DB_MANAGER.envelope_db(), id).await?;
Ok(result.ok_or_else(|| {
raise_error!(
@@ -82,6 +82,16 @@ impl MailBox {
})?)
}
+ pub fn get(id: u64) -> BichonResult {
+ let result = find_impl::(DB_MANAGER.envelope_db(), id)?;
+ Ok(result.ok_or_else(|| {
+ raise_error!(
+ format!("mailbox {} not found", id),
+ ErrorCode::InternalError
+ )
+ })?)
+ }
+
pub async fn delete(id: u64) -> BichonResult<()> {
delete_impl(DB_MANAGER.envelope_db(), move |rw| {
rw.get()
diff --git a/src/modules/cache/imap/sync/flow.rs b/src/modules/cache/imap/sync/flow.rs
index 5e71338..eb02d0e 100644
--- a/src/modules/cache/imap/sync/flow.rs
+++ b/src/modules/cache/imap/sync/flow.rs
@@ -32,7 +32,7 @@ use crate::{
},
error::{code::ErrorCode, BichonError, BichonResult},
imap::executor::ImapExecutor,
- blob::manager::ENVELOPE_INDEX_MANAGER,
+ store::tantivy::manager::INDEX_MANAGER,
},
raise_error,
};
@@ -426,7 +426,7 @@ async fn perform_incremental_sync(
remote_mailbox: &MailBox,
) -> BichonResult<()> {
if remote_mailbox.exists > 0 {
- let local_max_uid = ENVELOPE_INDEX_MANAGER
+ let local_max_uid = INDEX_MANAGER
.get_max_uid(account.id, local_mailbox.id)
.await?;
match local_max_uid {
diff --git a/src/modules/cache/imap/sync/rebuild.rs b/src/modules/cache/imap/sync/rebuild.rs
index d1c5e97..e3320fb 100644
--- a/src/modules/cache/imap/sync/rebuild.rs
+++ b/src/modules/cache/imap/sync/rebuild.rs
@@ -19,7 +19,6 @@
use crate::{
modules::{
account::migration::AccountModel,
- blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
cache::{
imap::{
mailbox::MailBox,
@@ -28,6 +27,7 @@ use crate::{
SEMAPHORE,
},
error::{code::ErrorCode, BichonError, BichonResult},
+ store::tantivy::manager::INDEX_MANAGER,
},
raise_error,
};
@@ -194,13 +194,9 @@ pub async fn rebuild_mailbox_cache(
local_mailbox: &MailBox,
remote_mailbox: &MailBox,
) -> BichonResult<()> {
- let content_hashes = ENVELOPE_INDEX_MANAGER
+ INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
.await?;
- // todo Distinguish these hashes to avoid mixing them
- if !content_hashes.is_empty() {
- BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
- }
if remote_mailbox.exists == 0 {
info!(
@@ -227,14 +223,9 @@ pub async fn rebuild_mailbox_cache_by_date(
remote: &MailBox,
direction: FetchDirection,
) -> BichonResult<()> {
- let content_hashes = ENVELOPE_INDEX_MANAGER
+ INDEX_MANAGER
.delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
.await?;
-
- if !content_hashes.is_empty() {
- BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
- }
-
if remote.exists == 0 {
info!(
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
diff --git a/src/modules/dashboard/mod.rs b/src/modules/dashboard/mod.rs
index 80bc746..64fdc84 100644
--- a/src/modules/dashboard/mod.rs
+++ b/src/modules/dashboard/mod.rs
@@ -16,16 +16,19 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use crate::modules::users::permissions::Permission;
+use crate::modules::{
+ store::tantivy::{manager::INDEX_MANAGER, schema::SchemaTools},
+ users::permissions::Permission,
+};
use poem_openapi::Object;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
+use tantivy::{schema::Value, TantivyDocument};
use crate::{
bichon_version,
modules::{
account::migration::AccountModel,
- blob::manager::ENVELOPE_INDEX_MANAGER,
common::auth::ClientContext,
error::{code::ErrorCode, BichonResult},
settings::dir::DATA_DIR_MANAGER,
@@ -63,13 +66,9 @@ impl DashboardStats {
Some(context.user.account_access_map.keys().cloned().collect())
};
- let mut stat = ENVELOPE_INDEX_MANAGER
- .get_dashboard_stats(authorized_ids.clone())
- .await?;
+ let mut stat = INDEX_MANAGER.get_dashboard_stats(&authorized_ids).await?;
- stat.top_largest_emails = ENVELOPE_INDEX_MANAGER
- .top_10_largest_emails(authorized_ids.clone())
- .await?;
+ stat.top_largest_emails = INDEX_MANAGER.top_10_largest_emails(&authorized_ids).await?;
stat.account_count = if has_all_accounts {
AccountModel::count().await?
@@ -77,15 +76,14 @@ impl DashboardStats {
authorized_ids.as_ref().map(|ids| ids.len()).unwrap_or(0)
};
- stat.email_count = ENVELOPE_INDEX_MANAGER.total_emails(authorized_ids).await?;
+ stat.email_count = INDEX_MANAGER.total_emails(&authorized_ids)?;
if has_all_accounts {
stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.eml_dir)
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- stat.index_usage_bytes =
- get_total_size(&DATA_DIR_MANAGER.envelope_dir.join("envelopes.db"))
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.tantivy_dir)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
} else {
stat.storage_usage_bytes = 0;
stat.index_usage_bytes = 0;
@@ -116,3 +114,48 @@ pub struct LargestEmail {
pub size_bytes: u64, // Email size in bytes
pub id: String,
}
+
+impl LargestEmail {
+ pub fn from_tantivy_doc(document: &TantivyDocument) -> BichonResult {
+ let fields = SchemaTools::email_fields();
+ let value = document.get_first(fields.f_size).ok_or_else(|| {
+ raise_error!(
+ "miss 'size' field in tantivy document".into(),
+ ErrorCode::InternalError
+ )
+ })?;
+ let size_bytes = value.as_u64().ok_or_else(|| {
+ raise_error!("'size' field is not a u64".into(), ErrorCode::InternalError)
+ })?;
+ let value = document.get_first(fields.f_subject).ok_or_else(|| {
+ raise_error!("'subject' field not found".into(), ErrorCode::InternalError)
+ })?;
+ let subject = value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+ raise_error!(
+ "'subject' field is not a string".into(),
+ ErrorCode::InternalError
+ )
+ })?;
+
+ let value = document.get_first(fields.f_id).ok_or_else(|| {
+ raise_error!(
+ format!("'{}' field not found", stringify!(field)),
+ ErrorCode::InternalError
+ )
+ })?;
+ let id = value.as_str().map(|s| s.to_string()).ok_or_else(|| {
+ raise_error!(
+ format!("'{}' field is not a string", stringify!(field)),
+ ErrorCode::InternalError
+ )
+ })?;
+
+ let envelope = LargestEmail {
+ subject,
+ size_bytes,
+ id,
+ };
+
+ Ok(envelope)
+ }
+}
diff --git a/src/modules/database/mod.rs b/src/modules/database/mod.rs
index cd4325a..12b3b10 100644
--- a/src/modules/database/mod.rs
+++ b/src/modules/database/mod.rs
@@ -208,6 +208,21 @@ pub async fn async_find_impl(
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
}
+pub fn find_impl(
+ database: &Arc>,
+ key: impl ToKey + Send + 'static,
+) -> BichonResult> {
+ let db = database.clone();
+ let r_transaction = db
+ .r_transaction()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let entity: Option = r_transaction
+ .get()
+ .primary(key)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ Ok(entity)
+}
+
pub async fn delete_impl(
database: &Arc>,
delete: impl FnOnce(&RwTransaction) -> BichonResult + Send + 'static,
diff --git a/src/modules/duckdb/build.rs b/src/modules/duckdb/build.rs
deleted file mode 100644
index a31a289..0000000
--- a/src/modules/duckdb/build.rs
+++ /dev/null
@@ -1,152 +0,0 @@
-//
-// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
-//
-// This file is part of the Bichon Email Archiving Project
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-use arrow::array::{Int32Array, Int64Array, ListBuilder, StringBuilder, UInt64Array};
-use arrow::datatypes::{DataType, Field, Schema};
-use arrow::record_batch::RecordBatch;
-use std::sync::Arc;
-
-use crate::modules::blob::envelope::Envelope;
-
-pub const DEFAULT_SHARD_ID: u64 = 0;
-
-pub fn build_record_batch(items: &[Envelope]) -> RecordBatch {
- let capacity = items.len();
-
- let schema = Arc::new(Schema::new(vec![
- Field::new("id", DataType::Utf8, false),
- Field::new("account_id", DataType::UInt64, false),
- Field::new("mailbox_id", DataType::UInt64, false),
- Field::new("uid", DataType::UInt64, false),
- Field::new("content_hash", DataType::Utf8, false),
- Field::new("subject", DataType::Utf8, true),
- Field::new("body", DataType::Utf8, true),
- Field::new("sender", DataType::Utf8, true),
- Field::new(
- "recipients",
- DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
- true,
- ),
- Field::new(
- "cc",
- DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
- true,
- ),
- Field::new(
- "bcc",
- DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
- true,
- ),
- Field::new("sent_at", DataType::Int64, true),
- Field::new("received_at", DataType::Int64, true),
- Field::new("size_bytes", DataType::UInt64, true),
- Field::new("thread_id", DataType::Utf8, true),
- Field::new("message_id", DataType::Utf8, true),
- Field::new("attachment_count", DataType::Int32, false),
- Field::new("regular_attachment_count", DataType::Int32, false),
- Field::new(
- "tags",
- DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
- true,
- ),
- Field::new("shard_id", DataType::UInt64, false),
- ]));
-
- let mut id_b = StringBuilder::with_capacity(capacity, capacity * 20);
- let mut account_id_b = UInt64Array::builder(capacity);
- let mut mailbox_id_b = UInt64Array::builder(capacity);
- let mut uid_b = UInt64Array::builder(capacity);
-
- let mut content_hash_b = StringBuilder::with_capacity(capacity, capacity * 64);
- let mut subject_b = StringBuilder::with_capacity(capacity, capacity * 20);
- let mut body_b = StringBuilder::with_capacity(capacity, capacity * 100);
- let mut from_b = StringBuilder::with_capacity(capacity, capacity * 20);
-
- let mut to_b = ListBuilder::new(StringBuilder::new());
- let mut cc_b = ListBuilder::new(StringBuilder::new());
- let mut bcc_b = ListBuilder::new(StringBuilder::new());
-
- let mut date_b = Int64Array::builder(capacity);
- let mut internal_date_b = Int64Array::builder(capacity);
- let mut size_b = UInt64Array::builder(capacity);
- let mut thread_id_b = StringBuilder::with_capacity(capacity, capacity * 20);
- let mut msg_id_b = StringBuilder::with_capacity(capacity, capacity * 30);
- let mut att_count_b = Int32Array::builder(capacity);
- let mut regular_att_count_b = Int32Array::builder(capacity);
- let mut tags_b = ListBuilder::new(StringBuilder::new());
- let mut shard_id_b = UInt64Array::builder(capacity);
-
- for e in items {
- id_b.append_value(&e.id);
- account_id_b.append_value(e.account_id);
- mailbox_id_b.append_value(e.mailbox_id);
- uid_b.append_value(e.uid as u64);
- content_hash_b.append_value(&e.content_hash);
- subject_b.append_value(&e.subject);
- body_b.append_value(&e.text);
- from_b.append_value(&e.from);
- for addr in &e.to {
- to_b.values().append_value(addr);
- }
- to_b.append(true);
- for addr in &e.cc {
- cc_b.values().append_value(addr);
- }
- cc_b.append(true);
- for addr in &e.bcc {
- bcc_b.values().append_value(addr);
- }
- bcc_b.append(true);
- date_b.append_value(e.date);
- internal_date_b.append_value(e.internal_date);
- size_b.append_value(e.size as u64);
- thread_id_b.append_value(&e.thread_id);
- msg_id_b.append_value(&e.message_id);
- att_count_b.append_value(e.attachment_count as i32);
- regular_att_count_b.append_value(e.regular_attachment_count as i32);
- tags_b.append(true);
- shard_id_b.append_value(DEFAULT_SHARD_ID);
- }
-
- RecordBatch::try_new(
- schema,
- vec![
- Arc::new(id_b.finish()),
- Arc::new(account_id_b.finish()),
- Arc::new(mailbox_id_b.finish()),
- Arc::new(uid_b.finish()),
- Arc::new(content_hash_b.finish()),
- Arc::new(subject_b.finish()),
- Arc::new(body_b.finish()),
- Arc::new(from_b.finish()),
- Arc::new(to_b.finish()),
- Arc::new(cc_b.finish()),
- Arc::new(bcc_b.finish()),
- Arc::new(date_b.finish()),
- Arc::new(internal_date_b.finish()),
- Arc::new(size_b.finish()),
- Arc::new(thread_id_b.finish()),
- Arc::new(msg_id_b.finish()),
- Arc::new(att_count_b.finish()),
- Arc::new(regular_att_count_b.finish()),
- Arc::new(tags_b.finish()),
- Arc::new(shard_id_b.finish()),
- ],
- )
- .expect("Failed to build RecordBatch")
-}
diff --git a/src/modules/duckdb/init.rs b/src/modules/duckdb/init.rs
deleted file mode 100644
index a75ad34..0000000
--- a/src/modules/duckdb/init.rs
+++ /dev/null
@@ -1,1716 +0,0 @@
-//
-// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
-//
-// This file is part of the Bichon Email Archiving Project
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-use chrono::{NaiveDateTime, Utc};
-use duckdb::{params, types::Value, DuckdbConnectionManager};
-use refinery::Runner;
-use std::{
- collections::{HashMap, HashSet},
- path::PathBuf,
- sync::OnceLock,
-};
-
-use crate::{
- modules::{
- account::migration::AccountModel,
- blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
- context::Initialize,
- dashboard::{DashboardStats, Group, LargestEmail, TimeBucket},
- duckdb::{build::build_record_batch, refinery::DuckDBConnection},
- error::{code::ErrorCode, BichonResult},
- message::{
- attachment::AttachmentMetadata,
- content::{AttachmentDetail, AttachmentInfo},
- search::{SearchFilter, SortBy},
- tags::{TagAction, TagCount, TagsRequest},
- },
- rest::response::DataPage,
- settings::{cli::SETTINGS, dir::DATA_DIR_MANAGER},
- },
- raise_error,
-};
-
-pub type DuckDBConn = r2d2::PooledConnection;
-
-pub static DUCKDBMANAGER: OnceLock = OnceLock::new();
-
-pub mod duckdb_tables {
- refinery::embed_migrations!("src/modules/duckdb/migrations");
-}
-
-pub fn duckdb() -> BichonResult<&'static DuckDBManager> {
- DUCKDBMANAGER.get().ok_or_else(|| {
- raise_error!(
- "DuckDB manager is not initialized".into(),
- ErrorCode::InternalError
- )
- })
-}
-
-fn debug_sql(sql: &str, args: &[Value]) -> String {
- let mut result = String::new();
- let mut parts = sql.split('?');
-
- for (i, part) in parts.by_ref().enumerate() {
- result.push_str(part);
-
- if i < args.len() {
- result.push_str(&format!("{:?}", args[i]));
- }
- }
-
- result
-}
-
-pub struct DuckDBManager {
- pool: r2d2::Pool,
-}
-
-impl Initialize for DuckDBManager {
- async fn initialize() -> BichonResult<()> {
- tracing::debug!("Initializing duckdb");
-
- if !&DATA_DIR_MANAGER.envelope_dir.exists() {
- std::fs::create_dir_all(&DATA_DIR_MANAGER.envelope_dir)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
-
- let pool = init(
- &DATA_DIR_MANAGER.envelope_dir.join("envelopes.db"),
- duckdb_tables::migrations::runner(),
- )?;
- let _ = DUCKDBMANAGER.set(DuckDBManager { pool });
- Ok(())
- }
-}
-
-impl DuckDBManager {
- pub fn conn(&self) -> BichonResult {
- Ok(self
- .pool
- .get()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?)
- }
-
- pub fn shutdown(&self) -> BichonResult<()> {
- self.conn()?
- .execute("FORCE CHECKPOINT", [])
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- tracing::info!("Shutting down");
- Ok(())
- }
-
- pub fn validate_regex(&self, pattern: &str) -> BichonResult<()> {
- let conn = self.conn()?;
- let check_sql = "SELECT regexp_matches('', ?)";
- if let Err(e) = conn
- .prepare(check_sql)
- .and_then(|mut stmt| stmt.execute([pattern]))
- {
- return Err(raise_error!(
- format!("Invalid Regular Expression for DuckDB: {}", e).into(),
- ErrorCode::InvalidParameter
- ));
- }
- Ok(())
- }
-
- pub fn delete_account_envelopes_with_orphans(
- &self,
- account_id: u64,
- ) -> BichonResult> {
- let mut conn = self.conn()?;
- let tx = conn
- .transaction()
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let orphan_sql = r#"
- WITH target_hashes AS (
- SELECT content_hash FROM envelopes WHERE account_id = ?
- UNION
- SELECT content_hash FROM envelope_attachments WHERE account_id = ?
- ),
- active_hashes AS (
- SELECT content_hash FROM envelopes WHERE account_id != ?
- UNION
- SELECT content_hash FROM envelope_attachments WHERE account_id != ?
- )
- SELECT content_hash FROM target_hashes
- EXCEPT
- SELECT content_hash FROM active_hashes
- "#;
-
- let mut stmt = tx
- .prepare(orphan_sql)
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let orphan_hashes: Vec = stmt
- .query_map([account_id, account_id, account_id, account_id], |row| {
- row.get::<_, String>(0)
- })
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- tx.execute(
- "DELETE FROM envelope_attachments WHERE account_id = ?",
- [account_id],
- )
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let count = tx
- .execute("DELETE FROM envelopes WHERE account_id = ?", [account_id])
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- tx.commit()
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- if count > 0 {
- tracing::info!(
- "Account {} data cleared. Deleted {} envelopes, {} orphan hashes identified.",
- account_id,
- count,
- orphan_hashes.len()
- );
- }
-
- Ok(orphan_hashes)
- }
-
- pub fn delete_mailbox_envelopes_with_orphans(
- &self,
- account_id: u64,
- mailbox_ids: Vec,
- ) -> BichonResult> {
- if mailbox_ids.is_empty() {
- return Ok(vec![]);
- }
-
- let mut conn = self.conn()?;
- let tx = conn
- .transaction()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let placeholders = mailbox_ids
- .iter()
- .map(|_| "?")
- .collect::>()
- .join(", ");
- let mut sql_params: Vec = vec![account_id.into()];
- sql_params.extend(mailbox_ids.iter().map(|&id| duckdb::types::Value::from(id)));
- let param_iter = duckdb::params_from_iter(sql_params);
-
- let orphan_sql = format!(
- r#"
- WITH target_hashes AS (
- SELECT content_hash FROM envelopes WHERE account_id = ? AND mailbox_id IN ({0})
- UNION
- SELECT content_hash FROM envelope_attachments WHERE account_id = ? AND mailbox_id IN ({0})
- ),
- active_hashes AS (
- SELECT content_hash FROM envelopes WHERE NOT (account_id = ? AND mailbox_id IN ({0}))
- UNION
- SELECT content_hash FROM envelope_attachments WHERE NOT (account_id = ? AND mailbox_id IN ({0}))
- )
- SELECT content_hash FROM target_hashes
- EXCEPT
- SELECT content_hash FROM active_hashes
- "#,
- placeholders
- );
-
- let mut query_params: Vec = Vec::new();
- for _ in 0..4 {
- query_params.push(account_id.into());
- query_params.extend(mailbox_ids.iter().map(|&id| duckdb::types::Value::from(id)));
- }
-
- let mut stmt = tx
- .prepare(&orphan_sql)
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
- let orphan_hashes: Vec = stmt
- .query_map(duckdb::params_from_iter(query_params), |row| {
- row.get::<_, String>(0)
- })
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let delete_att_sql = format!(
- "DELETE FROM envelope_attachments WHERE account_id = ? AND mailbox_id IN ({})",
- placeholders
- );
- tx.execute(&delete_att_sql, param_iter.clone())
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let delete_env_sql = format!(
- "DELETE FROM envelopes WHERE account_id = ? AND mailbox_id IN ({})",
- placeholders
- );
- let count = tx
- .execute(&delete_env_sql, param_iter)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- tx.commit()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- if count > 0 {
- tracing::info!(
- "Deleted {} emails. Found {} orphan hashes to clean up.",
- count,
- orphan_hashes.len()
- );
- }
-
- Ok(orphan_hashes)
- }
-
- pub fn append_envelopes_with_attachments(
- &self,
- items: &[(Envelope, Vec)],
- ) -> BichonResult<()> {
- let mut conn = self.conn()?;
- let tx = conn
- .transaction()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- {
- let mut env_appender = tx
- .appender("envelopes")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- let mut att_appender = tx
- .appender("envelope_attachments")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- for (env, atts) in items {
- env_appender
- .append_record_batch(build_record_batch(std::slice::from_ref(env)))
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- for att in atts {
- att_appender
- .append_row(params![
- env.id,
- env.account_id,
- env.mailbox_id,
- att.filename,
- att.is_message,
- att.inline,
- att.content_id,
- att.get_extension(),
- att.get_category(),
- att.file_type.to_ascii_lowercase(),
- att.size as u64,
- att.content_hash,
- 0
- ])
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
- }
- env_appender
- .flush()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- att_appender
- .flush()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
- tx.commit()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- Ok(())
- }
-
- pub fn total_emails(
- &self,
- accounts: Option>,
- ) -> BichonResult {
- let conn = self.conn()?;
- let total: u64 = match accounts {
- None => conn
- .query_row("SELECT COUNT(*) FROM envelopes", [], |row| row.get(0))
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?,
- Some(accts) if !accts.is_empty() => {
- let placeholders = accts.iter().map(|_| "?").collect::>().join(", ");
- let sql = format!(
- "SELECT COUNT(*) FROM envelopes WHERE account_id IN ({})",
- placeholders
- );
-
- conn.query_row(&sql, duckdb::params_from_iter(accts), |row| row.get(0))
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- }
- _ => 0,
- };
-
- Ok(total)
- }
-
- pub fn num_messages_in_thread(&self, account_id: u64, thread_id: String) -> BichonResult {
- let conn = self.conn()?;
- let count: u64 = conn
- .query_row(
- "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND thread_id = ?;",
- params![account_id, thread_id],
- |row| row.get(0),
- )
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- Ok(count)
- }
-
- pub fn num_messages_in_mailbox(&self, account_id: u64, mailbox_id: u64) -> BichonResult {
- let conn = self.conn()?;
- let count: u64 = conn
- .query_row(
- "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND mailbox_id = ?;",
- params![account_id, mailbox_id],
- |row| row.get(0),
- )
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- Ok(count)
- }
-
- pub fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> {
- let conn = self.conn()?;
- let max_uid: Option = conn
- .query_row(
- "SELECT MAX(uid) FROM envelopes WHERE account_id = ? AND mailbox_id = ?;",
- params![account_id, mailbox_id],
- |row| row.get(0),
- )
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(max_uid)
- }
-
- pub fn get_attachments_by_envelope_id(
- &self,
- account_id: u64,
- envelope_id: String,
- ) -> BichonResult> {
- let conn = self.conn()?;
-
- let mut stmt = conn
- .prepare(
- "SELECT * FROM envelope_attachments
- WHERE account_id = ? AND envelope_id = ?;",
- )
- .map_err(|e| {
- raise_error!(
- format!("Prepare failed: {:#?}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let attachment_iter = stmt
- .query_map(params![account_id, envelope_id], |row| {
- AttachmentDetail::from_row(row)
- })
- .map_err(|e| {
- raise_error!(format!("Query failed: {:#?}", e), ErrorCode::InternalError)
- })?;
-
- let mut attachments = Vec::new();
- for att_res in attachment_iter {
- attachments.push(att_res.map_err(|e| {
- raise_error!(
- format!("Row mapping failed: {:#?}", e),
- ErrorCode::InternalError
- )
- })?);
- }
-
- Ok(attachments)
- }
-
- pub fn get_envelope_by_id(
- &self,
- account_id: u64,
- envelope_id: String,
- ) -> BichonResult> {
- let conn = self.conn()?;
- let mut stmt = conn
- .prepare("SELECT * FROM envelopes WHERE account_id = ? AND id = ? LIMIT 1;")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let envelope_res = stmt.query_row(params![account_id, envelope_id], |row| {
- Envelope::from_row(row)
- });
-
- match envelope_res {
- Ok(env) => Ok(Some(env)),
- Err(duckdb::Error::QueryReturnedNoRows) => Ok(None),
- Err(e) => Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)),
- }
- }
-
- pub fn get_envelopes_by_ids(
- &self,
- account_id: u64,
- envelope_ids: &[&str],
- ) -> BichonResult> {
- if envelope_ids.is_empty() {
- return Ok(vec![]);
- }
- let conn = self.conn()?;
- let placeholders = vec!["?"; envelope_ids.len()].join(", ");
-
- let query = format!(
- "SELECT * FROM envelopes WHERE account_id = ? AND id IN ({})",
- placeholders
- );
-
- let mut stmt = conn.prepare(&query).map_err(|e| {
- raise_error!(format!("Prepare error: {:#?}", e), ErrorCode::InternalError)
- })?;
-
- let mut params: Vec> = Vec::new();
- params.push(Box::new(account_id));
- for id in envelope_ids {
- params.push(Box::new(id.to_string()));
- }
-
- let param_refs: Vec<&dyn duckdb::ToSql> = params.iter().map(|p| p.as_ref()).collect();
-
- let rows = stmt
- .query_map(duckdb::params_from_iter(param_refs), |row| {
- Envelope::from_row(row)
- })
- .map_err(|e| {
- raise_error!(format!("Query error: {:#?}", e), ErrorCode::InternalError)
- })?;
-
- let mut result = Vec::new();
- for row in rows {
- result.push(row.map_err(|e| {
- raise_error!(format!("Row error: {:#?}", e), ErrorCode::InternalError)
- })?);
- }
-
- Ok(result)
- }
-
- pub fn get_all_tags(&self, accounts: Option>) -> BichonResult> {
- let conn = self.conn()?;
- let mut sql = "
- SELECT t.tag, COUNT(*) as count
- FROM (
- SELECT unnest(tags) as tag, account_id
- FROM envelopes
- WHERE len(tags) > 0
- ) t
- "
- .to_string();
-
- let mut params_vec: Vec = Vec::new();
- if let Some(ref acc_set) = accounts {
- if !acc_set.is_empty() {
- let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", ");
- sql.push_str(&format!(" WHERE t.account_id IN ({})", placeholders));
- for id in acc_set {
- params_vec.push((*id).into());
- }
- }
- }
-
- sql.push_str(" GROUP BY t.tag ORDER BY count DESC, t.tag ASC;");
-
- let mut stmt = conn
- .prepare(&sql)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let tag_counts = stmt
- .query_map(duckdb::params_from_iter(params_vec), |row| {
- Ok(TagCount {
- tag: row.get(0)?,
- count: row.get(1)?,
- })
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(tag_counts)
- }
-
- pub fn get_all_contacts(
- &self,
- accounts: Option>,
- ) -> BichonResult> {
- let conn = self.conn()?;
- let mut sql = r#"
- SELECT DISTINCT LOWER(contact) AS contact
- FROM (
- SELECT sender AS contact, account_id
- FROM envelopes
- WHERE sender IS NOT NULL AND sender != ''
-
- UNION ALL
-
- SELECT unnest(recipients) AS contact, account_id
- FROM envelopes
- WHERE length(recipients) > 0
-
- UNION ALL
-
- SELECT unnest(cc) AS contact, account_id
- FROM envelopes
- WHERE length(cc) > 0
-
- UNION ALL
-
- SELECT unnest(bcc) AS contact, account_id
- FROM envelopes
- WHERE length(bcc) > 0
- ) t
- WHERE contact IS NOT NULL AND contact != ''
- "#
- .to_string();
-
- let mut params_vec: Vec = Vec::new();
-
- if let Some(ref acc_set) = accounts {
- if !acc_set.is_empty() {
- let placeholders = vec!["?"; acc_set.len()].join(", ");
- sql.push_str(&format!(" AND account_id IN ({})", placeholders));
-
- for &id in acc_set {
- params_vec.push(id.into());
- }
- }
- }
-
- sql.push_str(" ORDER BY contact;");
-
- let mut stmt = conn
- .prepare(&sql)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let contacts: HashSet = stmt
- .query_map(duckdb::params_from_iter(params_vec), |row| {
- let email: String = row.get(0)?;
- Ok(email)
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(contacts)
- }
-
- pub fn update_envelope_tags(&self, request: TagsRequest) -> BichonResult<()> {
- let mut conn = self.conn()?;
- let tags_json = serde_json::to_string(&request.tags)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let set_clause = match &request.action {
- TagAction::Overwrite => "SET tags = ?::JSON::VARCHAR[]",
- TagAction::Add => "SET tags = list_distinct(list_concat(tags, ?::JSON::VARCHAR[]))",
- TagAction::Remove => {
- "SET tags = list_filter(tags, x -> NOT list_contains(?::JSON::VARCHAR[], x))"
- }
- };
-
- let tx = conn
- .transaction()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- for (account_id, ids) in request.updates {
- if ids.is_empty() {
- continue;
- }
-
- for chunk in ids.chunks(100) {
- let placeholders = vec!["?"; chunk.len()].join(", ");
- let query = format!(
- "UPDATE envelopes
- {}
- WHERE account_id = ?
- AND id IN ({})",
- set_clause, placeholders
- );
-
- let mut params: Vec> = Vec::with_capacity(chunk.len() + 2);
-
- params.push(Box::new(tags_json.clone()));
- params.push(Box::new(account_id));
-
- for id in chunk {
- params.push(Box::new(id.clone()));
- }
-
- let param_refs: Vec<&dyn duckdb::ToSql> =
- params.iter().map(|p| p.as_ref()).collect();
-
- tx.execute(&query, duckdb::params_from_iter(param_refs))
- .map_err(|e| {
- raise_error!(
- format!("Update failed. Account: {}, Error: {}", account_id, e),
- ErrorCode::InternalError
- )
- })?;
- }
- }
-
- tx.commit()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- Ok(())
- }
-
- pub fn get_attachment_metadata(
- &self,
- accounts: Option>,
- ) -> BichonResult {
- let conn = self.conn()?;
- let mut sql = r#"
- SELECT
- COALESCE(CAST(array_agg(DISTINCT extension) FILTER (WHERE extension IS NOT NULL) AS JSON), '[]') AS extensions,
- COALESCE(CAST(array_agg(DISTINCT ext_category) AS JSON), '[]') AS categories,
- COALESCE(CAST(array_agg(DISTINCT content_type) AS JSON), '[]') AS content_types
- FROM envelope_attachments
- "#
- .to_string();
-
- let mut params_vec: Vec = Vec::new();
- if let Some(ref acc_set) = accounts {
- if !acc_set.is_empty() {
- let placeholders = vec!["?"; acc_set.len()].join(", ");
- sql.push_str(&format!(" WHERE account_id IN ({})", placeholders));
-
- for &id in acc_set {
- params_vec.push(id.into());
- }
- }
- }
-
- let mut stmt = conn
- .prepare(&sql)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let result = stmt
- .query_row(duckdb::params_from_iter(params_vec), |row| {
- let exts_raw: String = row.get(0).unwrap_or_else(|_| "[]".to_string());
- let cats_raw: String = row.get(1).unwrap_or_else(|_| "[]".to_string());
- let ctypes_raw: String = row.get(2).unwrap_or_else(|_| "[]".to_string());
-
- let exts: Vec = serde_json::from_str(&exts_raw).unwrap_or_default();
- let cats: Vec = serde_json::from_str(&cats_raw).unwrap_or_default();
- let ctypes: Vec = serde_json::from_str(&ctypes_raw).unwrap_or_default();
-
- Ok(AttachmentMetadata {
- extensions: exts.into_iter().collect(),
- categories: cats.into_iter().collect(),
- content_types: ctypes.into_iter().collect(),
- })
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- Ok(result)
- }
-
- pub fn delete_envelopes_multi_account(
- &self,
- deletes: HashMap>,
- ) -> BichonResult<()> {
- let mut conn = self.conn()?;
- let tx = conn
- .transaction()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- for (account_id, ids) in deletes {
- if ids.is_empty() {
- continue;
- }
-
- for chunk in ids.chunks(100) {
- let placeholders = vec!["?"; chunk.len()].join(", ");
- let mut params: Vec> = Vec::new();
- params.push(Box::new(account_id));
- for id in chunk {
- params.push(Box::new(id.clone()));
- }
-
- let param_refs: Vec<&dyn duckdb::ToSql> =
- params.iter().map(|p| p.as_ref()).collect();
- let query_params = duckdb::params_from_iter(param_refs);
-
- let del_attachments_query = format!(
- "DELETE FROM envelope_attachments
- WHERE account_id = ?
- AND envelope_id IN ({})",
- placeholders
- );
-
- tx.execute(&del_attachments_query, query_params.clone())
- .map_err(|e| {
- raise_error!(
- format!("Delete attachments fail: {:#?}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let query = format!(
- "DELETE FROM envelopes
- WHERE account_id = ?
- AND id IN ({})",
- placeholders
- );
-
- tx.execute(&query, query_params)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
- }
-
- tx.commit()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(())
- }
-
- pub fn get_orphan_hashes_in_memory(
- &self,
- deletes: HashMap>,
- ) -> BichonResult> {
- let conn = self.conn()?;
- let all_delete_ids: Vec = deletes.values().flatten().cloned().collect();
- if all_delete_ids.is_empty() {
- return Ok(vec![]);
- }
-
- if all_delete_ids.len() > 100 {
- return Err(raise_error!(
- "Too many IDs for batch delete, please shrink the batch".into(),
- ErrorCode::InvalidParameter
- ));
- }
-
- let mut target_hashes = HashSet::new();
- let placeholders = vec!["?"; all_delete_ids.len()].join(", ");
- let params = duckdb::params_from_iter(&all_delete_ids);
-
- let mut stmt = conn
- .prepare(&format!(
- "SELECT content_hash FROM envelopes WHERE id IN ({})",
- placeholders
- ))
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let rows = stmt
- .query_map(params.clone(), |r| r.get::<_, String>(0))
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- for h in rows {
- target_hashes
- .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?);
- }
-
- let mut stmt = conn
- .prepare(&format!(
- "SELECT content_hash FROM envelope_attachments WHERE envelope_id IN ({})",
- placeholders
- ))
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let rows = stmt
- .query_map(params, |r| r.get::<_, String>(0))
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
- for h in rows {
- target_hashes
- .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?);
- }
-
- if target_hashes.is_empty() {
- return Ok(vec![]);
- }
-
- let hash_placeholders = vec!["?"; target_hashes.len()].join(", ");
- let mut still_used_hashes = HashSet::new();
-
- let mut check_params: Vec> = Vec::new();
- for id in &all_delete_ids {
- check_params.push(Box::new(id.clone()));
- }
- for hash in &target_hashes {
- check_params.push(Box::new(hash.clone()));
- }
- let check_params_refs: Vec<&dyn duckdb::ToSql> =
- check_params.iter().map(|p| p.as_ref()).collect();
-
- let mut stmt = conn
- .prepare(&format!(
- "SELECT DISTINCT content_hash FROM envelopes WHERE id NOT IN ({}) AND content_hash IN ({})",
- placeholders, hash_placeholders
- ))
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let rows = stmt
- .query_map(duckdb::params_from_iter(&check_params_refs), |r| {
- r.get::<_, String>(0)
- })
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
- for h in rows {
- still_used_hashes
- .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?);
- }
-
- let mut stmt = conn.prepare(&format!(
- "SELECT DISTINCT content_hash FROM envelope_attachments WHERE envelope_id NOT IN ({}) AND content_hash IN ({})",
- placeholders, hash_placeholders
- )).map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
-
- let rows = stmt
- .query_map(duckdb::params_from_iter(&check_params_refs), |r| {
- r.get::<_, String>(0)
- })
- .map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?;
- for h in rows {
- still_used_hashes
- .insert(h.map_err(|e| raise_error!(e.to_string(), ErrorCode::InternalError))?);
- }
-
- let orphans: Vec = target_hashes
- .into_iter()
- .filter(|h| !still_used_hashes.contains(h))
- .collect();
-
- Ok(orphans)
- }
-
- pub fn top_10_largest_emails(
- &self,
- accounts: Option>,
- ) -> BichonResult> {
- let conn = self.conn()?;
- let mut sql = r#"
- SELECT id, subject, size_bytes
- FROM envelopes
- "#
- .to_string();
-
- let mut params_vec: Vec = Vec::new();
- if let Some(acc_set) = accounts {
- if !acc_set.is_empty() {
- let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", ");
- sql.push_str(&format!(" WHERE account_id IN ({})", placeholders));
-
- for id in acc_set {
- params_vec.push(id.into());
- }
- }
- }
-
- sql.push_str(" ORDER BY size_bytes DESC LIMIT 10;");
- let mut stmt = conn
- .prepare(&sql)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- let results = stmt
- .query_map(duckdb::params_from_iter(params_vec), |row| {
- Ok(LargestEmail {
- id: row.get(0)?,
- subject: row.get(1)?,
- size_bytes: row.get(2)?,
- })
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(results)
- }
-
- pub fn get_dashboard_stats(
- &self,
- accounts: Option>,
- ) -> BichonResult {
- let conn = self.conn()?;
-
- let mut account_filter = String::new();
- let mut params: Vec = Vec::new();
-
- if let Some(acc_set) = accounts {
- if !acc_set.is_empty() {
- let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", ");
- account_filter = format!("WHERE account_id IN ({})", placeholders);
- for id in acc_set {
- params.push(id.into());
- }
- }
- }
-
- let mut stats = DashboardStats::default();
-
- {
- let sql = format!(
- r#"
- SELECT COALESCE(SUM(size_bytes), 0) AS total_size
- FROM envelopes
- {account_filter}
- "#
- );
-
- let mut stmt = conn.prepare(&sql).map_err(|e| {
- raise_error!(
- format!("Prepare total_size failed: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut rows = stmt
- .query(duckdb::params_from_iter(params.clone()))
- .map_err(|e| {
- raise_error!(
- format!("Failed to execute total_size query: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- if let Some(row) = rows.next().map_err(|e| {
- raise_error!(
- format!("Failed to fetch row from total_size query: {}", e),
- ErrorCode::InternalError
- )
- })? {
- stats.total_size_bytes = row
- .get(0)
- .map_err(|e| {
- raise_error!(
- format!("Failed to read total_size value: {}", e),
- ErrorCode::InternalError
- )
- })
- .unwrap_or(0);
- }
- }
-
- {
- let thirty_days_ago = Utc::now() - chrono::Duration::days(30);
- let thirty_days_ago_ms = thirty_days_ago.timestamp_millis();
-
- let sql = format!(
- r#"
- SELECT
- date_trunc('day', to_timestamp(sent_at / 1000)) AS day,
- COUNT(*) AS cnt
- FROM envelopes
- {account_filter}
- WHERE sent_at >= ?
- GROUP BY day
- ORDER BY day ASC
- "#
- );
-
- let mut stmt = conn.prepare(&sql).map_err(|e| {
- raise_error!(
- format!("Prepare recent_activity failed: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut query_params = params.clone();
- query_params.push(thirty_days_ago_ms.into());
-
- let mut rows = stmt
- .query(duckdb::params_from_iter(query_params))
- .map_err(|e| {
- raise_error!(
- format!("Failed to execute recent_activity query: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut recent_activity = Vec::new();
-
- while let Some(row) = rows.next().map_err(|e| {
- raise_error!(
- format!("Failed to fetch row from recent_activity query: {}", e),
- ErrorCode::InternalError
- )
- })? {
- let day: NaiveDateTime = row.get(0).map_err(|e| {
- raise_error!(
- format!("Failed to read recent_activity value: {}", e),
- ErrorCode::InternalError
- )
- })?;
- let timestamp_ms = day.and_utc().timestamp_millis();
-
- recent_activity.push(TimeBucket {
- timestamp_ms,
- count: row.get(1).map_err(|e| {
- raise_error!(
- format!("Failed to read recent_activity value: {}", e),
- ErrorCode::InternalError
- )
- })?,
- });
- }
-
- stats.recent_activity = recent_activity;
- }
-
- {
- let sql = format!(
- r#"
- SELECT sender, COUNT(*) AS cnt
- FROM envelopes
- {account_filter}
- WHERE sender IS NOT NULL AND sender != ''
- GROUP BY sender
- ORDER BY cnt DESC, sender ASC
- LIMIT 10
- "#
- );
-
- let mut stmt = conn.prepare(&sql).map_err(|e| {
- raise_error!(
- format!("Prepare top_senders failed: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut rows = stmt
- .query(duckdb::params_from_iter(params.clone()))
- .map_err(|e| {
- raise_error!(
- format!("Failed to execute top_senders query: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut top_senders = Vec::new();
-
- while let Some(row) = rows.next().map_err(|e| {
- raise_error!(
- format!("Failed to fetch row from top_senders query: {}", e),
- ErrorCode::InternalError
- )
- })? {
- top_senders.push(Group {
- key: row.get(0).map_err(|e| {
- raise_error!(
- format!("Failed to read top_senders value: {}", e),
- ErrorCode::InternalError
- )
- })?,
- count: row.get(1).map_err(|e| {
- raise_error!(
- format!("Failed to read top_senders value: {}", e),
- ErrorCode::InternalError
- )
- })?,
- });
- }
-
- stats.top_senders = top_senders;
- }
- {
- let sql = format!(
- r#"
- SELECT account_id, COUNT(*) AS cnt
- FROM envelopes
- {account_filter}
- GROUP BY account_id
- ORDER BY cnt DESC
- LIMIT 10
- "#
- );
-
- let mut stmt = conn.prepare(&sql).map_err(|e| {
- raise_error!(
- format!("Prepare top_accounts failed: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut rows = stmt
- .query(duckdb::params_from_iter(params.clone()))
- .map_err(|e| {
- raise_error!(
- format!("Failed to execute top_accounts query: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut top_accounts = Vec::new();
-
- while let Some(row) = rows.next().map_err(|e| {
- raise_error!(
- format!("Failed to fetch row from top_accounts query: {}", e),
- ErrorCode::InternalError
- )
- })? {
- let account_id: u64 = row.get(0).map_err(|e| {
- raise_error!(
- format!("Failed to read top_accounts value: {}", e),
- ErrorCode::InternalError
- )
- })?;
- let count: u64 = row.get(1).map_err(|e| {
- raise_error!(
- format!("Failed to read top_accounts value: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- if let Ok(account) = AccountModel::get(account_id) {
- top_accounts.push(Group {
- key: account.email,
- count,
- });
- } else {
- tracing::warn!(account_id, "account not found in top accounts query");
- tokio::spawn(async move {
- let content_hashes = match ENVELOPE_INDEX_MANAGER
- .delete_account_envelopes(account_id)
- .await
- {
- Ok(content_hashes) => content_hashes,
- Err(e) => {
- tracing::error!(
- account_id = account_id,
- error = %e,
- "failed to cleanup envelope index"
- );
- return;
- }
- };
-
- if let Err(e) = BLOB_MANAGER.delete(&content_hashes, &content_hashes) {
- tracing::error!(
- account_id = account_id,
- error = %e,
- "failed to cleanup eml"
- );
- }
- });
- }
- }
-
- stats.top_accounts = top_accounts;
- }
-
- {
- let sql = format!(
- r#"
- SELECT (regular_attachment_count > 0) AS has_attachment, COUNT(*) AS cnt
- FROM envelopes
- {account_filter}
- GROUP BY has_attachment
- "#
- );
-
- let mut stmt = conn.prepare(&sql).map_err(|e| {
- raise_error!(
- format!("Prepare attachment_stats failed: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- let mut rows = stmt.query(duckdb::params_from_iter(params)).map_err(|e| {
- raise_error!(
- format!("Failed to execute attachment_stats query: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- while let Some(row) = rows.next().map_err(|e| {
- raise_error!(
- format!("Failed to fetch row from attachment_stats query: {}", e),
- ErrorCode::InternalError
- )
- })? {
- let has_attachment: bool = row.get(0).map_err(|e| {
- raise_error!(
- format!("Failed to read attachment_stats value: {}", e),
- ErrorCode::InternalError
- )
- })?;
- let cnt: u64 = row.get(1).map_err(|e| {
- raise_error!(
- format!("Failed to read attachment_stats value: {}", e),
- ErrorCode::InternalError
- )
- })?;
-
- if has_attachment {
- stats.with_attachment_count = cnt;
- } else {
- stats.without_attachment_count = cnt;
- }
- }
- }
-
- Ok(stats)
- }
-
- pub fn list_mailbox_envelopes(
- &self,
- account_id: u64,
- mailbox_id: u64,
- page: u64,
- page_size: u64,
- desc: bool,
- ) -> BichonResult> {
- let conn = self.conn()?;
- let current_page = page.max(1);
-
- let total_items: u64 = conn
- .query_row(
- "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND mailbox_id = ?",
- params![account_id, mailbox_id],
- |row| row.get(0),
- )
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let total_pages = if total_items == 0 {
- 0
- } else {
- (total_items + page_size - 1) / page_size
- };
-
- let offset = (current_page - 1) * page_size;
- let order_direction = if desc { "DESC" } else { "ASC" };
- let query = format!(
- "SELECT * FROM envelopes
- WHERE account_id = ? AND mailbox_id = ?
- ORDER BY sent_at {}
- LIMIT ? OFFSET ?",
- order_direction
- );
-
- let mut stmt = conn
- .prepare(&query)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- let items = stmt
- .query_map(params![account_id, mailbox_id, page_size, offset], |row| {
- Envelope::from_row(row)
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(DataPage {
- current_page: Some(current_page),
- page_size: Some(page_size),
- total_items,
- items,
- total_pages: Some(total_pages),
- })
- }
-
- pub fn list_thread_envelopes(
- &self,
- account_id: u64,
- thread_id: String,
- page: u64,
- page_size: u64,
- desc: bool,
- ) -> BichonResult> {
- let conn = self.conn()?;
- let current_page = page.max(1);
-
- let total_items: u64 = conn
- .query_row(
- "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND thread_id = ?",
- params![account_id, thread_id],
- |row| row.get(0),
- )
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let total_pages = if total_items == 0 {
- 0
- } else {
- (total_items + page_size - 1) / page_size
- };
-
- let offset = (current_page - 1) * page_size;
-
- let order_direction = if desc { "DESC" } else { "ASC" };
- let query = format!(
- "SELECT * FROM envelopes
- WHERE account_id = ? AND thread_id = ?
- ORDER BY sent_at {}
- LIMIT ? OFFSET ?",
- order_direction
- );
-
- let mut stmt = conn
- .prepare(&query)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let items = stmt
- .query_map(params![account_id, thread_id, page_size, offset], |row| {
- Envelope::from_row(row)
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- Ok(DataPage {
- current_page: Some(current_page),
- page_size: Some(page_size),
- total_items,
- items,
- total_pages: Some(total_pages),
- })
- }
-
- pub fn search(
- &self,
- accounts: Option>,
- filter: SearchFilter,
- page: u64,
- page_size: u64,
- desc: bool,
- sort_by: SortBy,
- ) -> BichonResult> {
- let page = page.max(1);
- let page_size = page_size.clamp(1, 500);
- let offset = (page - 1) * page_size;
-
- let allowed_accounts: Option> = match (accounts, filter.account_ids.clone()) {
- (Some(user_set), Some(filter_set)) => {
- let intersection: HashSet =
- user_set.intersection(&filter_set).cloned().collect();
- Some(intersection)
- }
- (Some(user_set), None) => Some(user_set),
- (None, Some(filter_set)) => Some(filter_set),
- (None, None) => None,
- };
-
- if let Some(ref set) = allowed_accounts {
- if set.is_empty() {
- return Ok(DataPage {
- current_page: Some(page),
- page_size: Some(page_size),
- total_items: 0,
- items: Vec::new(),
- total_pages: Some(0),
- });
- }
- }
-
- let mut base_sql = String::from(" FROM envelopes e ");
-
- let need_join_attachment = filter.attachment_name.is_some()
- || filter.attachment_extension.is_some()
- || filter.attachment_category.is_some()
- || filter.attachment_content_type.is_some();
-
- if need_join_attachment {
- base_sql.push_str(
- "
- LEFT JOIN envelope_attachments a
- ON a.account_id = e.account_id
- AND a.mailbox_id = e.mailbox_id
- AND a.envelope_id = e.id
- ",
- );
- }
-
- base_sql.push_str(" WHERE 1=1 ");
- let mut args: Vec = Vec::new();
-
- if let Some(set) = allowed_accounts {
- base_sql.push_str(" AND e.account_id IN (");
- for (i, _) in set.iter().enumerate() {
- if i > 0 {
- base_sql.push_str(",");
- }
- base_sql.push_str("?");
- }
- base_sql.push_str(")");
- for id in set {
- args.push(id.into());
- }
- }
-
- if let Some(mailboxes) = filter.mailbox_ids {
- if !mailboxes.is_empty() {
- base_sql.push_str(" AND e.mailbox_id IN (");
- for (i, _) in mailboxes.iter().enumerate() {
- if i > 0 {
- base_sql.push_str(",");
- }
- base_sql.push_str("?");
- }
- base_sql.push_str(")");
- for id in mailboxes {
- args.push(id.into());
- }
- }
- }
-
- if let Some(subject) = filter.subject {
- let pattern = format!("(?i){}", subject);
- base_sql.push_str(" AND regexp_matches(coalesce(e.subject, ''), ?)");
- args.push(pattern.into());
- }
-
- if let Some(body) = filter.body {
- let pattern = format!("(?i){}", body);
- base_sql.push_str(" AND regexp_matches(coalesce(e.body, ''), ?)");
- args.push(pattern.into());
- }
-
- if let Some(text) = filter.text {
- let pattern = format!("(?i){}", text);
- base_sql.push_str(
- "
- AND (
- regexp_matches(coalesce(e.subject, ''), ?)
- OR regexp_matches(coalesce(e.body, ''), ?)
- OR regexp_matches(coalesce(e.sender, ''), ?)
- OR regexp_matches(array_to_string(e.recipients, ','), ?)
- )
- ",
- );
- args.push(pattern.clone().into());
- args.push(pattern.clone().into());
- args.push(pattern.clone().into());
- args.push(pattern.into());
- }
-
- if let Some(from) = filter.from {
- base_sql.push_str(" AND e.sender ILIKE ? ");
- args.push(format!("%{}%", from).into());
- }
-
- if let Some(to) = filter.to {
- base_sql.push_str(" AND array_to_string(e.recipients, ',') ILIKE ?");
- args.push(format!("%{}%", to).into());
- }
-
- if let Some(cc) = filter.cc {
- base_sql.push_str(" AND array_to_string(e.cc, ',') ILIKE ?");
- args.push(format!("%{}%", cc).into());
- }
-
- if let Some(bcc) = filter.bcc {
- base_sql.push_str(" AND array_to_string(e.bcc, ',') ILIKE ?");
- args.push(format!("%{}%", bcc).into());
- }
-
- if let Some(since) = filter.since {
- base_sql.push_str(" AND e.sent_at >= ? ");
- args.push(since.into());
- }
-
- if let Some(before) = filter.before {
- base_sql.push_str(" AND e.sent_at <= ? ");
- args.push(before.into());
- }
-
- if let Some(min) = filter.min_size {
- base_sql.push_str(" AND e.size_bytes >= ? ");
- args.push(min.into());
- }
-
- if let Some(max) = filter.max_size {
- base_sql.push_str(" AND e.size_bytes <= ? ");
- args.push(max.into());
- }
-
- if let Some(mid) = filter.message_id {
- base_sql.push_str(" AND e.message_id ILIKE ? ");
- args.push(format!("%{}%", mid).into());
- }
-
- if let Some(has) = filter.has_attachment {
- if has {
- base_sql.push_str(" AND e.regular_attachment_count > 0 ");
- } else {
- base_sql.push_str(" AND e.regular_attachment_count = 0 ");
- }
- }
-
- if let Some(tags) = filter.tags {
- if !tags.is_empty() {
- base_sql.push_str(" AND (");
- for i in 0..tags.len() {
- if i > 0 {
- base_sql.push_str(" OR ");
- }
- base_sql.push_str("list_contains(e.tags, ?)");
- }
- base_sql.push_str(") ");
- for tag in tags {
- args.push(tag.into());
- }
- }
- }
-
- if let Some(name) = filter.attachment_name {
- base_sql.push_str(" AND a.filename ILIKE ? ");
- args.push(format!("%{}%", name).into());
- }
- // Normalized to lowercase at write-time. LIKE is sufficient here instead of ILIKE.
- if let Some(ext) = filter.attachment_extension {
- base_sql.push_str(" AND a.extension LIKE ? ");
- args.push(format!("%{}%", ext).into());
- }
- // Normalized to lowercase at write-time. LIKE is sufficient here instead of ILIKE.
- if let Some(cat) = filter.attachment_category {
- base_sql.push_str(" AND a.ext_category LIKE ? ");
- args.push(format!("%{}%", cat).into());
- }
- // Normalized to lowercase at write-time. LIKE is sufficient here instead of ILIKE.
- if let Some(ctype) = filter.attachment_content_type {
- base_sql.push_str(" AND a.content_type LIKE ? ");
- args.push(format!("%{}%", ctype).into());
- }
-
- let count_sql = if need_join_attachment {
- format!("SELECT COUNT(DISTINCT e.id) {}", base_sql)
- } else {
- format!("SELECT COUNT(*) {}", base_sql)
- };
-
- println!("================ COUNT SQL ================");
- println!("FINAL: {}", debug_sql(&count_sql, &args));
- println!("===========================================");
-
- let total_items: u64 = self
- .conn()?
- .query_row(&count_sql, duckdb::params_from_iter(args.iter()), |row| {
- row.get(0)
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- if total_items == 0 {
- return Ok(DataPage {
- current_page: Some(page),
- page_size: Some(page_size),
- total_items: 0,
- items: Vec::new(),
- total_pages: Some(0),
- });
- }
-
- let sort_column = match sort_by {
- SortBy::DATE => "e.sent_at",
- SortBy::SIZE => "e.size_bytes",
- };
-
- let order = if desc { "DESC" } else { "ASC" };
-
- let select_prefix = if need_join_attachment {
- "SELECT DISTINCT e.*"
- } else {
- "SELECT e.*"
- };
-
- let data_sql = format!(
- "{}{}ORDER BY {} {}
- LIMIT ? OFFSET ?
- ",
- select_prefix, base_sql, sort_column, order
- );
-
- let mut data_args = args.clone();
- data_args.push(page_size.into());
- data_args.push(offset.into());
- let conn = self.conn()?;
- let mut stmt = conn
- .prepare(&data_sql)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let items = stmt
- .query_map(duckdb::params_from_iter(data_args.iter()), |row| {
- Envelope::from_row(row)
- })
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .collect::, _>>()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let total_pages = if total_items == 0 {
- Some(0)
- } else {
- Some((total_items + page_size - 1) / page_size)
- };
-
- Ok(DataPage {
- current_page: Some(page),
- page_size: Some(page_size),
- total_items,
- items,
- total_pages,
- })
- }
-}
-
-pub fn init(
- path: &PathBuf,
- mut migrations_runner: Runner,
-) -> BichonResult> {
- let mut flags = duckdb::Config::default()
- .enable_autoload_extension(true)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .access_mode(duckdb::AccessMode::ReadWrite)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .with("enable_fsst_vectors", "true")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
- .with("allocator_background_threads", "true")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- if let Some(memory_limit) = &SETTINGS.bichon_duckdb_max_memory {
- flags = flags
- .max_memory(memory_limit)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
-
- if let Some(threads) = &SETTINGS.bichon_duckdb_threads {
- flags = flags
- .threads(*threads)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
-
- let conn = DuckdbConnectionManager::file_with_flags(path, flags)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
-
- let pool = r2d2::Pool::new(conn)
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- {
- let conn = pool
- .get()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- conn.execute("PRAGMA enable_checkpoint_on_shutdown", [])
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- conn.pragma_update(None, "autoload_known_extensions", &"true")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- conn.pragma_update(None, "allow_community_extensions", &"false")
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- }
-
- {
- let conn = pool
- .get()
- .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
- migrations_runner.set_migration_table_name("migrations");
- for migration in migrations_runner.run_iter(&mut DuckDBConnection(conn)) {
- match migration {
- Ok(migration) => {
- tracing::info!("Applied migration: {}", migration);
- }
- Err(err) => {
- return Err(raise_error!(
- format!("Failed to apply migration: {}", err),
- ErrorCode::InternalError
- ));
- }
- }
- }
- }
-
- Ok(pool)
-}
diff --git a/src/modules/duckdb/migrations/V1__initial.sql b/src/modules/duckdb/migrations/V1__initial.sql
deleted file mode 100644
index 4dcd397..0000000
--- a/src/modules/duckdb/migrations/V1__initial.sql
+++ /dev/null
@@ -1,69 +0,0 @@
--- =========================
--- envelopes (main table)
--- =========================
-CREATE TABLE IF NOT EXISTS envelopes (
- -- internal id (tantivy f_id)
- id UUID PRIMARY KEY,
-
- -- account / mailbox / uid
- account_id UBIGINT NOT NULL,
- mailbox_id UBIGINT NOT NULL,
- uid UBIGINT NOT NULL,
-
- -- headers / content
- content_hash VARCHAR(64) NOT NULL,
- subject TEXT,
- body TEXT,
-
- sender TEXT,
- recipients VARCHAR[],
- cc VARCHAR[],
- bcc VARCHAR[],
-
- -- dates
- sent_at BIGINT,
- received_at BIGINT,
-
- -- size
- size_bytes UBIGINT,
-
- -- thread
- thread_id VARCHAR NOT NULL,
-
- -- message-id
- message_id TEXT,
-
- -- attachment summary
- attachment_count INTEGER NOT NULL CHECK (attachment_count >= 0),
- regular_attachment_count INTEGER NOT NULL CHECK (regular_attachment_count >= 0),
- tags VARCHAR[],
- shard_id UBIGINT NOT NULL
-);
-
--- =========================
--- envelope_attachments
---
--- Normalized attachment metadata for fast filtering and UI display
--- =========================
-CREATE TABLE IF NOT EXISTS envelope_attachments (
- -- Reference to envelopes.id
- envelope_id UUID NOT NULL,
- account_id UBIGINT NOT NULL,
- mailbox_id UBIGINT NOT NULL,
- -- Original attachment filename (for display)
- filename TEXT,
- is_message BOOLEAN NOT NULL DEFAULT FALSE,
- is_inline BOOLEAN NOT NULL DEFAULT FALSE,
- cid TEXT,
- -- Normalized file extension (lowercase, without dot)
- extension TEXT,
-
- -- Extension category (document / image / archive / ...)
- ext_category TEXT NOT NULL,
- content_type TEXT NOT NULL,
- -- Attachment size in bytes
- -- 0 if unknown
- size_bytes UBIGINT NOT NULL,
- content_hash VARCHAR(64) NOT NULL,
- shard_id UINTEGER NOT NULL
-);
\ No newline at end of file
diff --git a/src/modules/duckdb/refinery.rs b/src/modules/duckdb/refinery.rs
deleted file mode 100644
index 2a70aa8..0000000
--- a/src/modules/duckdb/refinery.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-//
-// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
-//
-// This file is part of the Bichon Email Archiving Project
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-
-use chrono::{DateTime, Utc};
-use refinery::{error::WrapMigrationError, Migration};
-use refinery_core::{
- traits::sync::{Query, Transaction},
- Migrate,
-};
-use std::{ops::DerefMut, time::SystemTime};
-
-pub struct DuckDBConnection>(pub T);
-
-impl> From for DuckDBConnection {
- fn from(conn: T) -> Self {
- Self(conn)
- }
-}
-
-impl> Transaction for DuckDBConnection {
- type Error = duckdb::Error;
- fn execute<'a, T: Iterator- >(
- &mut self,
- mut queries: T,
- ) -> std::result::Result {
- let transaction = self.0.transaction()?;
- let count = queries.try_fold(0, |count, query| {
- transaction.execute_batch(query)?;
- Ok::<_, Self::Error>(count + 1)
- })?;
- transaction.commit()?;
- Ok(count)
- }
-}
-
-impl> Query> for DuckDBConnection {
- fn query(&mut self, query: &str) -> Result, Self::Error> {
- let mut stmt = self.0.prepare(query)?;
- let applied: Vec = stmt
- .query_map([], |row| {
- let version = row.get(0)?;
- let name: String = row.get(1)?;
- let applied_on: DateTime = row.get(2)?;
- let checksum: u64 = row.get(3)?;
- let applied_on: SystemTime = applied_on.into();
-
- Ok(Migration::applied(
- version,
- name,
- applied_on.into(),
- checksum,
- ))
- })?
- .collect::, _>>()?;
- Ok(applied)
- }
-}
-
-impl> Migrate for DuckDBConnection {
- fn assert_migrations_table(
- &mut self,
- migration_table_name: &str,
- ) -> std::result::Result {
- let query = format!(
- "CREATE TABLE IF NOT EXISTS {migration_table_name} (
- version INT primary key,
- name TEXT not null,
- applied_on TIMESTAMP not null,
- checksum TEXT not null
- )"
- );
-
- self.execute(std::iter::once(query.as_str()))
- .migration_err("failed to create or verify DuckDB migrations table", None)?;
- Ok(0)
- }
-}
diff --git a/src/modules/duckdb/tests.rs b/src/modules/duckdb/tests.rs
deleted file mode 100644
index 84c0bcc..0000000
--- a/src/modules/duckdb/tests.rs
+++ /dev/null
@@ -1,28 +0,0 @@
-use duckdb::Connection;
-
-#[test]
-fn test_get_envelopes_with_params_iter() {
- let conn = Connection::open_in_memory().unwrap();
- conn.execute("CREATE TABLE envelopes (id VARCHAR, account_id UBIGINT)", []).unwrap();
-
- conn.execute("INSERT INTO envelopes VALUES ('mail_1', 1), ('mail_2', 1), ('mail_3', 2)", []).unwrap();
-
- let search_ids = vec!["mail_1", "mail_2"];
- let acc_id = 1u64;
-
- let placeholders = search_ids.iter().map(|_| "?").collect::>().join(",");
- let query = format!("SELECT id FROM envelopes WHERE account_id = ? AND id IN ({})", placeholders);
-
- let mut params: Vec<&dyn duckdb::ToSql> = Vec::new();
- params.push(&acc_id);
- for id in &search_ids {
- params.push(id);
- }
-
- let mut stmt = conn.prepare(&query).unwrap();
- let res: Vec = stmt.query_map(duckdb::params_from_iter(params), |r| r.get(0)).unwrap()
- .map(|r| r.unwrap()).collect();
- println!("{:#?}", &res);
- assert_eq!(res.len(), 2);
- assert!(res.contains(&"mail_1".to_string()));
-}
\ No newline at end of file
diff --git a/src/modules/envelope/extractor.rs b/src/modules/envelope/extractor.rs
index b0435ef..c452300 100644
--- a/src/modules/envelope/extractor.rs
+++ b/src/modules/envelope/extractor.rs
@@ -16,16 +16,17 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER;
-use crate::modules::blob::storage::{DetachedEmail, BLOB_MANAGER};
use crate::modules::common::AddrVec;
use crate::modules::envelope::utils::normalize_subject;
use crate::modules::error::code::ErrorCode;
use crate::modules::error::BichonResult;
use crate::modules::message::content::AttachmentInfo;
+use crate::modules::store::storage::{DetachedEmail, BLOB_MANAGER};
+use crate::modules::store::tantivy::manager::INDEX_MANAGER;
+use crate::modules::store::tantivy::model::EnvelopeWithAttachments;
use crate::modules::utils::html::extract_text;
use crate::modules::utils::{compute_content_hash, hex_hash};
-use crate::{id, modules::blob::envelope::Envelope};
+use crate::{id, modules::store::envelope::Envelope};
use crate::{raise_error, utc_now};
use async_imap::types::Fetch;
use bytes::Bytes;
@@ -90,6 +91,7 @@ async fn extract_envelope_core(
)
})?;
+ let preview_limit = 100;
let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
text
} else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
@@ -98,6 +100,16 @@ async fn extract_envelope_core(
String::new()
};
+ let text = text.split_whitespace().collect::>().join(" ");
+
+ let preview = if text.chars().count() > preview_limit {
+ text.chars().take(preview_limit).collect::() + "..."
+ } else {
+ text.clone()
+ };
+
+ let body_text = text;
+
let message_id = message
.message_id()
.map(String::from)
@@ -153,13 +165,14 @@ async fn extract_envelope_core(
mailbox_id,
uid,
subject,
- text: String::new(),//for test
+ preview,
from,
to,
cc,
bcc,
date,
internal_date,
+ ingest_at: utc_now!(),
size,
thread_id,
attachment_count,
@@ -169,7 +182,12 @@ async fn extract_envelope_core(
mailbox_name: None,
content_hash: email_content_hash,
};
- ENVELOPE_INDEX_MANAGER.queue((envelope, attachments)).await;
+ let ea = EnvelopeWithAttachments {
+ envelope,
+ attachments: Some(attachments),
+ };
+ let doc = ea.to_document(&body_text, 0)?;
+ INDEX_MANAGER.queue(doc).await;
Ok(())
}
@@ -230,13 +248,14 @@ pub fn extract_envelope_from_nested_message(
mailbox_id: Default::default(),
uid: Default::default(),
subject,
- text,
+ preview: text,
from,
to,
cc,
bcc,
date,
internal_date: Default::default(),
+ ingest_at: Default::default(),
size: Default::default(),
thread_id,
attachment_count: Default::default(),
@@ -349,8 +368,8 @@ pub async fn reattach_eml_content(
account_id: u64,
envelope_id: String,
) -> BichonResult<(Envelope, Bytes)> {
- let envelope = ENVELOPE_INDEX_MANAGER
- .get_envelope_by_id(account_id, envelope_id.clone())
+ let e = INDEX_MANAGER
+ .get_envelope_by_id(account_id, &envelope_id)
.await?
.ok_or_else(|| {
raise_error!(
@@ -363,38 +382,37 @@ pub async fn reattach_eml_content(
})?;
let restored_eml = BLOB_MANAGER
- .get_email(&envelope.content_hash)?
+ .get_email(&e.envelope.content_hash)?
.ok_or_else(|| {
raise_error!(
format!(
"Original email content not found: account_id={} envelope_id={} content_hash={}",
- account_id, &envelope_id, &envelope.content_hash
+ account_id, &envelope_id, &e.envelope.content_hash
),
ErrorCode::ResourceNotFound
)
})?;
- if !envelope.has_any_attachments() {
- return Ok((envelope, restored_eml));
+ if !e.envelope.has_any_attachments() {
+ return Ok((e.envelope, restored_eml));
}
let mut restored_eml = restored_eml.to_vec();
-
- let account_detail = ENVELOPE_INDEX_MANAGER
- .get_attachments_by_envelope_id(account_id, envelope_id)
- .await?;
-
- if envelope.attachment_count != account_detail.len() {
+ let actual_count = e.attachments.as_ref().map(|a| a.len()).unwrap_or(0);
+ if e.envelope.attachment_count != actual_count {
return Err(raise_error!(
- "Consistency check failed: attachment_count does not match account_detail length"
- .into(),
+ format!(
+ "Consistency check failed: envelope.attachment_count ({}) does not match attachments.len ({})",
+ e.envelope.attachment_count,
+ actual_count
+ ),
ErrorCode::InternalError
));
}
let mut tasks = Vec::new();
- for detail in account_detail {
- let placeholder_str = format!("<>", &detail.info.content_hash);
+ for detail in e.attachments.unwrap() {
+ let placeholder_str = format!("<>", &detail.content_hash);
let pattern = placeholder_str.as_bytes();
let pattern_len = pattern.len();
@@ -409,7 +427,7 @@ pub async fn reattach_eml_content(
tasks.push((
absolute_start,
absolute_end,
- detail.info.content_hash.clone(),
+ detail.content_hash.clone(),
));
search_cursor = absolute_end;
}
@@ -433,7 +451,7 @@ pub async fn reattach_eml_content(
}
}
- Ok((envelope, Bytes::from(restored_eml)))
+ Ok((e.envelope, Bytes::from(restored_eml)))
}
#[cfg(test)]
diff --git a/src/modules/mailbox/delete.rs b/src/modules/mailbox/delete.rs
index 5209b36..ab7c303 100644
--- a/src/modules/mailbox/delete.rs
+++ b/src/modules/mailbox/delete.rs
@@ -1,11 +1,27 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
use crate::modules::{
- blob::{manager::ENVELOPE_INDEX_MANAGER, storage::BLOB_MANAGER},
- cache::imap::mailbox::MailBox,
- error::BichonResult,
+ cache::imap::mailbox::MailBox, error::BichonResult, store::tantivy::manager::INDEX_MANAGER,
};
pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResult<()> {
- let mailbox = MailBox::get(mailbox_id).await?;
+ let mailbox = MailBox::async_get(mailbox_id).await?;
let name = mailbox.name;
let delimiter = mailbox.delimiter.unwrap_or("/".to_owned());
@@ -26,10 +42,8 @@ pub async fn delete_mailbox_impl(account_id: u64, mailbox_id: u64) -> BichonResu
MailBox::delete(*id).await?;
}
- let content_hashes = ENVELOPE_INDEX_MANAGER
+ INDEX_MANAGER
.delete_mailbox_envelopes(account_id, ids_to_delete.clone())
.await?;
-
- BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
Ok(())
}
diff --git a/src/modules/message/content.rs b/src/modules/message/content.rs
index 71f4594..4769fe8 100644
--- a/src/modules/message/content.rs
+++ b/src/modules/message/content.rs
@@ -22,7 +22,7 @@ use crate::modules::envelope::extractor::{
extract_envelope_from_nested_message, reattach_eml_content,
};
use crate::modules::error::code::ErrorCode;
-use crate::modules::blob::envelope::Envelope;
+use crate::modules::store::envelope::Envelope;
use crate::modules::utils::compute_content_hash;
use crate::{modules::error::BichonResult, raise_error};
use mail_parser::{MessageParser, MimeHeaders};
@@ -121,30 +121,6 @@ impl AttachmentInfo {
"other"
}
}
-
-#[derive(Clone, Debug, Default, Eq, PartialEq)]
-pub struct AttachmentDetail {
- pub shard_id: usize,
- pub info: AttachmentInfo,
-}
-
-impl AttachmentDetail {
- pub fn from_row(row: &duckdb::Row) -> duckdb::Result {
- Ok(Self {
- shard_id: row.get("shard_id")?,
- info: AttachmentInfo {
- file_type: row.get("content_type")?,
- inline: row.get("is_inline")?,
- filename: row.get("filename")?,
- size: row.get("size_bytes")?,
- content_id: row.get("cid")?,
- content_hash: row.get("content_hash")?,
- is_message: row.get("is_message")?,
- },
- })
- }
-}
-
/// Represents the content of an email message in both plain text and HTML formats.
///
/// This struct contains optional fields for plain text and HTML versions of
diff --git a/src/modules/message/delete.rs b/src/modules/message/delete.rs
index a01a67b..8dfbb24 100644
--- a/src/modules/message/delete.rs
+++ b/src/modules/message/delete.rs
@@ -16,19 +16,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER;
-use crate::modules::blob::storage::BLOB_MANAGER;
use crate::modules::error::BichonResult;
+use crate::modules::store::tantivy::manager::INDEX_MANAGER;
use std::collections::HashMap;
pub async fn delete_messages_impl(request: HashMap>) -> BichonResult<()> {
- let content_hashes = ENVELOPE_INDEX_MANAGER
- .get_orphan_hashes_in_memory(request.clone())
- .await?;
- if !content_hashes.is_empty() {
- BLOB_MANAGER.delete(&content_hashes, &content_hashes)?;
- }
- ENVELOPE_INDEX_MANAGER
- .delete_envelopes_multi_account(request)
- .await
+ INDEX_MANAGER.delete_envelopes_multi_account(request).await
}
diff --git a/src/modules/message/list.rs b/src/modules/message/list.rs
index d468854..5fba652 100644
--- a/src/modules/message/list.rs
+++ b/src/modules/message/list.rs
@@ -16,53 +16,21 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use crate::{
- modules::{
- account::migration::AccountModel,
- error::{code::ErrorCode, BichonResult},
- blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER},
- rest::response::DataPage,
- },
- raise_error,
+use crate::modules::{
+ account::migration::AccountModel,
+ error::BichonResult,
+ rest::response::DataPage,
+ store::{envelope::Envelope, tantivy::manager::INDEX_MANAGER},
};
-pub async fn list_messages_impl(
- account_id: u64,
- mailbox_id: u64,
- page: u64,
- page_size: u64,
-) -> BichonResult> {
- AccountModel::check_account_exists(account_id).await?;
- validate_pagination_params(page, page_size)?;
- ENVELOPE_INDEX_MANAGER
- .list_mailbox_envelopes(account_id, mailbox_id, page, page_size, true)
- .await
-}
-
-fn validate_pagination_params(page: u64, page_size: u64) -> BichonResult<()> {
- if page == 0 || page_size == 0 {
- return Err(raise_error!(
- "Both page and page_size must be greater than 0.".into(),
- ErrorCode::InvalidParameter
- ));
- }
- if page_size > 500 {
- return Err(raise_error!(
- "The page_size exceeds the maximum allowed limit of 500.".into(),
- ErrorCode::InvalidParameter
- ));
- }
- Ok(())
-}
-
pub async fn get_thread_messages(
account_id: u64,
- thread_id: String,
+ thread_id: &str,
page: u64,
page_size: u64,
) -> BichonResult> {
AccountModel::check_account_exists(account_id).await?;
- ENVELOPE_INDEX_MANAGER
+ INDEX_MANAGER
.list_thread_envelopes(account_id, thread_id, page, page_size, true)
.await
}
diff --git a/src/modules/message/search.rs b/src/modules/message/search.rs
index fd4cb37..995dcb9 100644
--- a/src/modules/message/search.rs
+++ b/src/modules/message/search.rs
@@ -16,17 +16,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use std::collections::HashSet;
-
use poem_openapi::{Enum, Object};
use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
use crate::{
modules::{
- duckdb::init::duckdb,
error::{code::ErrorCode, BichonResult},
- blob::{envelope::Envelope, manager::ENVELOPE_INDEX_MANAGER},
rest::response::DataPage,
+ store::{envelope::Envelope, tantivy::manager::INDEX_MANAGER},
},
raise_error,
};
@@ -85,24 +83,6 @@ impl SearchRequest {
));
}
- let validate = |pattern: &Option| -> BichonResult<()> {
- if let Some(ref p) = pattern {
- if duckdb()?.validate_regex(p).is_err() {
- return Err(raise_error!(
- "Invalid search pattern: The regular expression is not supported by DuckDB.".into(),
- ErrorCode::InvalidParameter
- ));
- }
- }
- Ok(())
- };
-
- validate(&self.filter.text)?;
- validate(&self.filter.subject)?;
- validate(&self.filter.body)?;
- validate(&self.filter.from)?;
- validate(&self.filter.to)?;
-
Ok(())
}
}
@@ -112,7 +92,7 @@ pub async fn search_messages_impl(
request: SearchRequest,
) -> BichonResult> {
request.validate()?;
- ENVELOPE_INDEX_MANAGER
+ INDEX_MANAGER
.search(
accounts,
request.filter,
diff --git a/src/modules/mod.rs b/src/modules/mod.rs
index 6d249de..938bbd0 100644
--- a/src/modules/mod.rs
+++ b/src/modules/mod.rs
@@ -24,12 +24,10 @@ pub mod common;
pub mod context;
pub mod dashboard;
pub mod database;
-pub mod duckdb;
pub mod envelope;
pub mod error;
pub mod imap;
pub mod import;
-pub mod blob;
pub mod logger;
pub mod mailbox;
pub mod message;
@@ -37,6 +35,7 @@ pub mod oauth2;
pub mod rest;
pub mod settings;
pub mod smtp;
+pub mod store;
pub mod tasks;
pub mod token;
pub mod users;
diff --git a/src/modules/rest/api/message.rs b/src/modules/rest/api/message.rs
index 1be717c..8dc6a0d 100644
--- a/src/modules/rest/api/message.rs
+++ b/src/modules/rest/api/message.rs
@@ -17,9 +17,6 @@
// along with this program. If not, see .
use crate::modules::account::migration::AccountModel;
-use crate::modules::blob::envelope::Envelope;
-use crate::modules::blob::manager::ENVELOPE_INDEX_MANAGER;
-use crate::modules::blob::storage::get_reader;
use crate::modules::common::auth::ClientContext;
use crate::modules::message::append::restore_emails;
use crate::modules::message::append::RestoreMessagesRequest;
@@ -30,7 +27,7 @@ use crate::modules::message::content::retrieve_nested_eml_content;
use crate::modules::message::content::FullNestedMessageContent;
use crate::modules::message::content::{retrieve_email_content, FullMessageContent};
use crate::modules::message::delete::delete_messages_impl;
-use crate::modules::message::list::{get_thread_messages, list_messages_impl};
+use crate::modules::message::list::get_thread_messages;
use crate::modules::message::search::{search_messages_impl, SearchRequest};
use crate::modules::message::tags::TagCount;
use crate::modules::message::tags::TagsRequest;
@@ -38,6 +35,9 @@ use crate::modules::rest::api::ApiTags;
use crate::modules::rest::response::DataPage;
use crate::modules::rest::ApiResult;
use crate::modules::rest::ErrorCode;
+use crate::modules::store::envelope::Envelope;
+use crate::modules::store::storage::get_reader;
+use crate::modules::store::tantivy::manager::INDEX_MANAGER;
use crate::modules::users::permissions::Permission;
use crate::modules::utils::validate_tag;
use crate::raise_error;
@@ -73,32 +73,6 @@ impl MessageApi {
Ok(delete_messages_impl(request).await?)
}
- /// Lists messages in a mailbox. Requires `mailbox_id`, `page`, and `page_size` query parameters.
- #[oai(
- path = "/list-messages/:account_id",
- method = "get",
- operation_id = "list_messages"
- )]
- async fn list_messages(
- &self,
- /// The ID of the account.
- account_id: Path,
- /// The ID of the mailbox to list messages from.
- mailbox_id: Query,
- page: Query,
- page_size: Query,
- context: ClientContext,
- ) -> ApiResult>> {
- let account_id = account_id.0;
- let mailbox_id = mailbox_id.0;
- context
- .require_permission(Some(account_id), Permission::DATA_READ)
- .await?;
- Ok(Json(
- list_messages_impl(account_id, mailbox_id, page.0, page_size.0).await?,
- ))
- }
-
/// Searches messages across all mailboxes using various filter criteria.
/// The search filters are provided in the request body.
#[oai(
@@ -141,7 +115,7 @@ impl MessageApi {
context: ClientContext,
) -> ApiResult>> {
let account_id = account_id.0;
- let thread_id = thread_id.0;
+ let thread_id = thread_id.0.trim();
context
.require_permission(Some(account_id), Permission::DATA_READ)
.await?;
@@ -217,8 +191,8 @@ impl MessageApi {
.require_permission(Some(account_id), Permission::DATA_READ)
.await?;
let envelope_id = envelope_id.0;
- let envelope = ENVELOPE_INDEX_MANAGER
- .get_envelope_by_id(account_id, envelope_id.clone())
+ let e = INDEX_MANAGER
+ .get_envelope_by_id(account_id, &envelope_id)
.await?
.ok_or_else(|| {
raise_error!(
@@ -229,7 +203,7 @@ impl MessageApi {
ErrorCode::ResourceNotFound
)
})?;
- Ok(Json(envelope))
+ Ok(Json(e.envelope))
}
/// Downloads the raw EML file of a specific email.
@@ -361,9 +335,7 @@ impl MessageApi {
} else {
Some(context.user.account_access_map.keys().cloned().collect())
};
- Ok(Json(
- ENVELOPE_INDEX_MANAGER.get_all_tags(authorized_ids).await?,
- ))
+ Ok(Json(INDEX_MANAGER.get_all_tags(authorized_ids).await?))
}
/// Adds or removes facet tags for multiple emails across accounts.
@@ -389,7 +361,7 @@ impl MessageApi {
.await?;
}
- ENVELOPE_INDEX_MANAGER.update_envelope_tags(req).await?;
+ INDEX_MANAGER.update_envelope_tags(req).await?;
Ok(())
}
@@ -408,11 +380,7 @@ impl MessageApi {
} else {
Some(context.user.account_access_map.keys().cloned().collect())
};
- Ok(Json(
- ENVELOPE_INDEX_MANAGER
- .get_all_contacts(authorized_ids)
- .await?,
- ))
+ Ok(Json(INDEX_MANAGER.get_all_contacts(authorized_ids).await?))
}
/// Retrieves unique metadata for all attachments across authorized accounts.
@@ -434,9 +402,7 @@ impl MessageApi {
Some(context.user.account_access_map.keys().cloned().collect())
};
Ok(Json(
- ENVELOPE_INDEX_MANAGER
- .get_attachment_metadata(authorized_ids)
- .await?,
+ INDEX_MANAGER.collect_attachment_metadata(authorized_ids)?,
))
}
}
diff --git a/src/modules/settings/dir.rs b/src/modules/settings/dir.rs
index 7fbff74..0217a00 100644
--- a/src/modules/settings/dir.rs
+++ b/src/modules/settings/dir.rs
@@ -31,6 +31,7 @@ const ENVELOPE_DIR: &str = "envelope";
const EML_DIR: &str = "bichon-emls";
const TMP_DIR: &str = "tmp";
const LOG_DIR: &str = "logs";
+const TANTIVY_DIR: &str = "tantivy";
const TLS_CERT: &str = "cert.pem";
const TLS_KEY: &str = "key.pem";
@@ -45,7 +46,7 @@ pub struct DataDirManager {
pub temp_dir: PathBuf,
pub tls_cert: PathBuf,
pub tls_key: PathBuf,
- pub envelope_dir: PathBuf,
+ pub tantivy_dir: PathBuf,
pub eml_dir: PathBuf,
pub log_dir: PathBuf,
}
@@ -66,7 +67,7 @@ impl Initialize for DataDirManager {
impl DataDirManager {
pub fn new(root_dir: PathBuf) -> Self {
- let envelope_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir {
+ let index_dir = if let Some(ref index_dir) = SETTINGS.bichon_index_dir {
PathBuf::from(index_dir)
} else {
root_dir.join(ENVELOPE_DIR)
@@ -85,7 +86,7 @@ impl DataDirManager {
tls_key: root_dir.join(TLS_KEY),
tls_cert: root_dir.join(TLS_CERT),
log_dir: root_dir.join(LOG_DIR),
- envelope_dir,
+ tantivy_dir: index_dir.join(TANTIVY_DIR),
temp_dir: root_dir.join(TMP_DIR),
eml_dir,
}
diff --git a/src/modules/store/envelope.rs b/src/modules/store/envelope.rs
new file mode 100644
index 0000000..ac02919
--- /dev/null
+++ b/src/modules/store/envelope.rs
@@ -0,0 +1,53 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+use poem_openapi::Object;
+use serde::{Deserialize, Serialize};
+use tantivy::doc;
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)]
+pub struct Envelope {
+ pub id: String,
+ pub message_id: String,
+ pub account_id: u64,
+ pub account_email: Option,
+ pub mailbox_id: u64,
+ pub mailbox_name: Option,
+ pub uid: u32,
+ pub subject: String,
+ pub preview: String,
+ pub from: String,
+ pub to: Vec,
+ pub cc: Vec,
+ pub bcc: Vec,
+ pub date: i64,
+ pub internal_date: i64,
+ pub ingest_at: i64,
+ pub size: u32,
+ pub thread_id: String,
+ pub attachment_count: usize,
+ pub regular_attachment_count: usize,
+ pub tags: Option>,
+ pub content_hash: String,
+}
+
+impl Envelope {
+ pub fn has_any_attachments(&self) -> bool {
+ self.attachment_count > 0
+ }
+}
diff --git a/src/modules/blob/mod.rs b/src/modules/store/mod.rs
similarity index 98%
rename from src/modules/blob/mod.rs
rename to src/modules/store/mod.rs
index 0aab906..89220a3 100644
--- a/src/modules/blob/mod.rs
+++ b/src/modules/store/mod.rs
@@ -17,5 +17,5 @@
// along with this program. If not, see .
pub mod envelope;
-pub mod manager;
pub mod storage;
+pub mod tantivy;
diff --git a/src/modules/blob/storage.rs b/src/modules/store/storage.rs
similarity index 67%
rename from src/modules/blob/storage.rs
rename to src/modules/store/storage.rs
index f943cb7..7708676 100644
--- a/src/modules/blob/storage.rs
+++ b/src/modules/store/storage.rs
@@ -1,3 +1,21 @@
+//
+// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
use crate::modules::{
common::signal::SIGNAL_MANAGER,
envelope::extractor::reattach_eml_content,
@@ -6,7 +24,7 @@ use crate::modules::{
};
use crate::raise_error;
use bytes::Bytes;
-use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions};
+use fjall::{CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions, config::{BlockSizePolicy, CompressionPolicy}};
use std::{io::Cursor, sync::LazyLock};
use tokio::{
@@ -78,35 +96,46 @@ impl BlobManager {
pub fn new() -> Self {
let db = Database::builder(&DATA_DIR_MANAGER.eml_dir)
+ .cache_size(64 * 1024 * 1024)
+ .max_cached_files(Some(400))
+ .max_journaling_size(256 * 1024 * 1024)
.open()
.expect("Failed to initialize Fjall database: Check if the directory exists and has write permissions.");
let email_keyspace = db
.keyspace("email", || {
KeyspaceCreateOptions::default()
- .with_kv_separation(Some(
- KvSeparationOptions::default()
- .separation_threshold(0)
- .compression(CompressionType::Lz4)
- .file_target_size(128 * 1024 * 1024)
- .staleness_threshold(0.5)
- .age_cutoff(0.6),
- ))
- .max_memtable_size(64 * 1024 * 1024)
+ .max_memtable_size(16 * 1024 * 1024)
+ .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+ .data_block_compression_policy(
+ CompressionPolicy::all(CompressionType::Lz4)
+ )
+ .with_kv_separation(Some(
+ KvSeparationOptions::default()
+ .separation_threshold(0)
+ .compression(CompressionType::Lz4)
+ .file_target_size(128 * 1024 * 1024)
+ .staleness_threshold(0.5)
+ .age_cutoff(0.6),
+ ))
})
.expect("Failed to open 'email' keyspace: The partition metadata might be corrupted or inaccessible.");
let attachments_keyspace = db
.keyspace("attachments", || {
KeyspaceCreateOptions::default()
- .with_kv_separation(Some(
- KvSeparationOptions::default()
- .separation_threshold(0)
- .compression(CompressionType::Lz4)
- .file_target_size(256 * 1024 * 1024)
- .staleness_threshold(0.5)
- .age_cutoff(0.6),
- ))
- .max_memtable_size(64 * 1024 * 1024)
+ .data_block_size_policy(BlockSizePolicy::all(4 * 1024))
+ .data_block_compression_policy(
+ CompressionPolicy::all(CompressionType::Lz4)
+ )
+ .with_kv_separation(Some(
+ KvSeparationOptions::default()
+ .separation_threshold(0)
+ .compression(CompressionType::Lz4)
+ .file_target_size(256 * 1024 * 1024)
+ .staleness_threshold(0.5)
+ .age_cutoff(0.6),
+ ))
+ .max_memtable_size(16 * 1024 * 1024)
})
.expect("Failed to open 'attachments' keyspace: Check disk space for blob storage initialization.");
@@ -123,6 +152,9 @@ impl BlobManager {
match res {
Some(eml) => {
Self::process_detached_email(eml, &store, &email_ks, &attach_ks);
+ while let Ok(next_eml) = receiver.try_recv() {
+ Self::process_detached_email(next_eml, &store, &email_ks, &attach_ks);
+ }
}
None => {
tracing::info!("BlobManager: All senders dropped, closing storage.");
@@ -176,17 +208,22 @@ impl BlobManager {
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
}
- pub fn delete(
+ pub fn delete(
&self,
- email_content_hashes: &[String],
- attachment_content_hashes: &[String],
- ) -> BichonResult<()> {
+ email_content_hashes: I1,
+ attachment_content_hashes: I2,
+ ) -> BichonResult<()>
+ where
+ I1: IntoIterator,
+ I1::Item: AsRef,
+ I2: IntoIterator,
+ I2::Item: AsRef {
let mut batch = self.db.batch();
for hash in email_content_hashes {
- batch.remove(&self.email_keyspace, hash);
+ batch.remove(&self.email_keyspace, hash.as_ref());
}
for hash in attachment_content_hashes {
- batch.remove(&self.attachments_keyspace, hash);
+ batch.remove(&self.attachments_keyspace, hash.as_ref());
}
batch
.commit()
diff --git a/src/modules/store/tantivy/fields.rs b/src/modules/store/tantivy/fields.rs
new file mode 100644
index 0000000..267a0b7
--- /dev/null
+++ b/src/modules/store/tantivy/fields.rs
@@ -0,0 +1,80 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+use tantivy::schema::Field;
+
+pub const F_ID: &str = "id";
+pub const F_MESSAGE_ID: &str = "message_id";
+pub const F_ACCOUNT_ID: &str = "account_id";
+pub const F_MAILBOX_ID: &str = "mailbox_id";
+pub const F_UID: &str = "uid";
+pub const F_SUBJECT: &str = "subject";
+pub const F_BODY: &str = "body";
+pub const F_PREVIEW: &str = "preview";
+pub const F_CONTENT_HASH: &str = "content_hash";
+pub const F_FROM: &str = "from";
+pub const F_TO: &str = "to";
+pub const F_CC: &str = "cc";
+pub const F_BCC: &str = "bcc";
+pub const F_DATE: &str = "date";
+pub const F_INTERNAL_DATE: &str = "internal_date";
+pub const F_INGEST_AT: &str = "ingest_at";
+pub const F_SIZE: &str = "size";
+pub const F_THREAD_ID: &str = "thread_id";
+pub const F_ATTACHMENT_COUNT: &str = "attachment_count";
+pub const F_REGULAR_ATTACHMENT_COUNT: &str = "regular_attachment_count";
+pub const F_ATTACHMENTS: &str = "attachments";
+pub const F_ATTACHMENT_GLUE: &str = "attachment_glue";
+pub const F_ATTACHMENT_CONTENT_HASH: &str = "attachment_content_hash";
+pub const F_ATTACHMENT_EXT: &str = "attachment_ext";
+pub const F_ATTACHMENT_CATEGORY: &str = "attachment_category";
+pub const F_ATTACHMENT_CONTENT_TYPE: &str = "attachment_content_type";
+
+pub const F_TAGS: &str = "tags";
+pub const F_SHARD_ID: &str = "shard_id";
+
+pub struct EmailFields {
+ pub f_id: Field,
+ pub f_message_id: Field,
+ pub f_account_id: Field,
+ pub f_mailbox_id: Field,
+ pub f_uid: Field,
+ pub f_subject: Field,
+ pub f_body: Field,
+ pub f_preview: Field,
+ pub f_content_hash: Field,
+ pub f_from: Field,
+ pub f_to: Field,
+ pub f_cc: Field,
+ pub f_bcc: Field,
+ pub f_date: Field,
+ pub f_internal_date: Field,
+ pub f_ingest_at: Field,
+ pub f_size: Field,
+ pub f_thread_id: Field,
+ pub f_attachment_count: Field,
+ pub f_regular_attachment_count: Field,
+ pub f_attachments: Field,
+ pub f_attachment_glue: Field,
+ pub f_attachment_content_hash: Field,
+ pub f_attachment_ext: Field,
+ pub f_attachment_category: Field,
+ pub f_attachment_content_type: Field,
+ pub f_tags: Field,
+ pub f_shard_id: Field,
+}
diff --git a/src/modules/store/tantivy/manager.rs b/src/modules/store/tantivy/manager.rs
new file mode 100644
index 0000000..5494b38
--- /dev/null
+++ b/src/modules/store/tantivy/manager.rs
@@ -0,0 +1,1479 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+use std::{
+ collections::{HashMap, HashSet},
+ ops::Bound,
+ path::PathBuf,
+ sync::{Arc, LazyLock},
+ time::Duration,
+};
+
+use crate::{
+ modules::{
+ account::migration::AccountModel,
+ common::signal::SIGNAL_MANAGER,
+ dashboard::{DashboardStats, Group, LargestEmail, TimeBucket},
+ error::{code::ErrorCode, BichonResult},
+ message::{
+ attachment::AttachmentMetadata,
+ search::{SearchFilter, SortBy},
+ tags::{TagAction, TagCount, TagsRequest},
+ },
+ rest::response::DataPage,
+ settings::dir::DATA_DIR_MANAGER,
+ store::{
+ envelope::Envelope,
+ storage::BLOB_MANAGER,
+ tantivy::{
+ fields::{
+ F_ACCOUNT_ID, F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE,
+ F_ATTACHMENT_EXT, F_DATE, F_FROM, F_REGULAR_ATTACHMENT_COUNT, F_SIZE, F_TAGS,
+ F_THREAD_ID, F_UID,
+ },
+ model::{extract_contacts, EnvelopeWithAttachments},
+ schema::SchemaTools,
+ },
+ },
+ },
+ raise_error, utc_now,
+};
+
+use chrono::Utc;
+use serde_json::json;
+use tantivy::{
+ aggregation::{
+ agg_req::Aggregations,
+ agg_result::{
+ AggregationResult, AggregationResults, BucketEntries, BucketResult, MetricResult,
+ },
+ AggregationCollector, Key,
+ },
+ collector::{Count, DocSetCollector, FacetCollector, TopDocs},
+ indexer::{LogMergePolicy, UserOperation},
+ query::{AllQuery, BooleanQuery, EmptyQuery, Occur, Query, QueryParser, RangeQuery, TermQuery},
+ schema::{IndexRecordOption, Value},
+ DocAddress, Index, IndexReader, IndexWriter, Order, TantivyDocument, Term,
+};
+use tantivy::{query::RegexQuery, schema::Facet, Searcher};
+use tokio::{
+ sync::{mpsc, Mutex},
+ task::{self, JoinHandle},
+};
+use tracing::{info, warn};
+
+pub static INDEX_MANAGER: LazyLock = LazyLock::new(IndexManager::new);
+
+pub struct IndexManager {
+ index_writer: Arc>,
+ sender: mpsc::Sender,
+ reader: IndexReader,
+ query_parser: QueryParser,
+ handle: Mutex>>,
+}
+
+impl IndexManager {
+ pub async fn shutdown(&self) {
+ let mut guard = self.handle.lock().await;
+ if let Some(handle) = guard.take() {
+ let _ = handle.await;
+ }
+ }
+ pub fn new() -> Self {
+ let index = Self::open_or_create_index(&DATA_DIR_MANAGER.tantivy_dir);
+ let mut merge_policy = LogMergePolicy::default();
+ merge_policy.set_min_num_segments(25);
+ merge_policy.set_min_layer_size(10_000);
+ merge_policy.set_max_docs_before_merge(100_000);
+
+ let index_writer = index
+ .writer_with_num_threads(4, 134_217_728)
+ .unwrap_or_else(|e| {
+ panic!(
+ "Failed to create IndexWriter with 4 threads and 128MB buffer for {:?}: {}",
+ &DATA_DIR_MANAGER.tantivy_dir, e
+ )
+ });
+ index_writer.set_merge_policy(Box::new(merge_policy));
+ let index_writer = Arc::new(Mutex::new(index_writer));
+ let reader = index.reader().unwrap_or_else(|e| {
+ panic!(
+ "Failed to create IndexReader for {:?}: {}",
+ &DATA_DIR_MANAGER.tantivy_dir, e
+ )
+ });
+ let mut query_parser = QueryParser::for_index(&index, SchemaTools::email_default_fields());
+ query_parser.set_conjunction_by_default();
+
+ let (sender, mut receiver) = mpsc::channel::(100);
+
+ let writer = index_writer.clone();
+ let handler = task::spawn(async move {
+ let mut shutdown = SIGNAL_MANAGER.subscribe();
+ let mut commit_interval = tokio::time::interval(Duration::from_secs(30));
+ loop {
+ tokio::select! {
+ maybe_msg = receiver.recv() => {
+ match maybe_msg {
+ Some(doc) => {
+ let writer = writer.lock().await;
+ if let Err(e) = writer.add_document(doc) {
+ eprintln!("[ERROR] Failed to add document: {e:?}");
+ tracing::error!("Tantivy: Failed to add document: {e:?}");
+ }
+ while let Ok(next_doc) = receiver.try_recv() {
+ let _ = writer.add_document(next_doc);
+ }
+ }
+ None => {
+ tracing::info!("Tantivy: Receiver closed. Finalizing...");
+ let mut writer = writer.lock().await;
+ fatal_commit(&mut writer);
+ break;
+ },
+ }
+ }
+ _ = commit_interval.tick() => {
+ let mut writer = writer.lock().await;
+ fatal_commit(&mut writer);
+ }
+ _ = shutdown.recv() => {
+ tracing::info!("Tantivy: Shutdown signal received. Performing final commit...");
+ let mut writer = writer.lock().await;
+ fatal_commit(&mut writer);
+ tracing::info!("Tantivy: Shutdown cleanup complete.");
+ break;
+ }
+ }
+ }
+ });
+ Self {
+ index_writer,
+ sender,
+ reader,
+ query_parser,
+ handle: Mutex::new(Some(handler)),
+ }
+ }
+
+ pub async fn queue(&self, doc: TantivyDocument) {
+ let _ = self.sender.send(doc).await;
+ }
+
+ fn open_or_create_index(index_dir: &PathBuf) -> Index {
+ let need_create = !index_dir.exists()
+ || index_dir
+ .read_dir()
+ .map(|mut d| d.next().is_none())
+ .unwrap_or(true);
+ if need_create {
+ info!(
+ "Email index not found or empty, creating new index at {}",
+ index_dir.display()
+ );
+ std::fs::create_dir_all(&index_dir).unwrap_or_else(|e| {
+ panic!("Failed to create index directory {:?}: {}", index_dir, e)
+ });
+ Index::create_in_dir(&index_dir, SchemaTools::email_schema())
+ .unwrap_or_else(|e| panic!("Failed to create index in {:?}: {}", index_dir, e))
+ } else {
+ info!("Opening existing email index at {}", index_dir.display());
+ Self::open(&index_dir)
+ }
+ }
+
+ fn open(index_dir: &PathBuf) -> Index {
+ Index::open_in_dir(index_dir)
+ .unwrap_or_else(|e| panic!("Failed to open index in {:?}: {}", index_dir, e))
+ }
+
+ fn account_query(&self, account_id: u64) -> Box {
+ let account_term =
+ Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id);
+ Box::new(TermQuery::new(account_term, IndexRecordOption::Basic))
+ }
+
+ fn mailbox_query(&self, account_id: u64, mailbox_id: u64) -> Box {
+ let account_query = TermQuery::new(
+ Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id),
+ IndexRecordOption::Basic,
+ );
+ let mailbox_query = TermQuery::new(
+ Term::from_field_u64(SchemaTools::email_fields().f_mailbox_id, mailbox_id),
+ IndexRecordOption::Basic,
+ );
+ let boolean_query = BooleanQuery::new(vec![
+ (Occur::Must, Box::new(account_query)),
+ (Occur::Must, Box::new(mailbox_query)),
+ ]);
+ Box::new(boolean_query)
+ }
+
+ fn envelope_query(&self, account_id: u64, eid: &str) -> Box {
+ let account_id_query = TermQuery::new(
+ Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id),
+ IndexRecordOption::Basic,
+ );
+ let envelope_id_query = TermQuery::new(
+ Term::from_field_text(SchemaTools::email_fields().f_id, eid),
+ IndexRecordOption::Basic,
+ );
+ let boolean_query = BooleanQuery::new(vec![
+ (Occur::Must, Box::new(account_id_query)),
+ (Occur::Must, Box::new(envelope_id_query)),
+ ]);
+ Box::new(boolean_query)
+ }
+
+ fn filter_query(
+ &self,
+ accounts: Option>,
+ filter: SearchFilter,
+ parser: QueryParser,
+ ) -> BichonResult> {
+ let f = SchemaTools::email_fields();
+ let mut subqueries: Vec<(Occur, Box)> = Vec::new();
+
+ if let Some(authorized_ids) = accounts {
+ if authorized_ids.is_empty() {
+ let term = Term::from_field_u64(f.f_account_id, u64::MAX);
+ subqueries.push((
+ Occur::Must,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
+ ));
+ } else {
+ let mut account_must_queries = Vec::new();
+ for id in authorized_ids {
+ let term = Term::from_field_u64(f.f_account_id, id);
+ account_must_queries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ subqueries.push((
+ Occur::Must,
+ Box::new(BooleanQuery::new(account_must_queries)),
+ ));
+ }
+ }
+
+ if let Some(ref text) = filter.text {
+ let query = parser
+ .parse_query(text)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?;
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+
+ if let Some(ref subject_val) = filter.subject {
+ let term = Term::from_field_text(f.f_subject, subject_val);
+ let query = TermQuery::new(term, IndexRecordOption::Basic);
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+
+ if let Some(ref body_val) = filter.body {
+ let term = Term::from_field_text(f.f_body, body_val);
+ let query = TermQuery::new(term, IndexRecordOption::Basic);
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+
+ if let Some(ref tags) = filter.tags {
+ if !tags.is_empty() {
+ let mut should_queries: Vec<(Occur, Box)> = Vec::new();
+
+ for tag in tags {
+ let facet = Facet::from_text(tag).map_err(|e| {
+ raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter)
+ })?;
+
+ let term = Term::from_facet(f.f_tags, &facet);
+
+ should_queries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
+ ));
+ }
+ subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries))));
+ }
+ }
+
+ for (field, opt_value) in [
+ (f.f_from, &filter.from),
+ (f.f_to, &filter.to),
+ (f.f_cc, &filter.cc),
+ (f.f_bcc, &filter.bcc),
+ ] {
+ if let Some(ref v) = opt_value {
+ if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) {
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+ }
+ }
+
+ if let Some(has) = filter.has_attachment {
+ let lower: Bound;
+ let upper: Bound;
+
+ if has {
+ lower = Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, 1));
+ upper =
+ Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, u64::MAX));
+ } else {
+ lower = Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, 0));
+ upper = Bound::Included(Term::from_field_u64(f.f_regular_attachment_count, 0));
+ }
+
+ subqueries.push((Occur::Must, Box::new(RangeQuery::new(lower, upper))));
+ }
+
+ if let Some(ref name) = filter.attachment_name {
+ if let Ok(query) = RegexQuery::from_pattern(name.as_str(), f.f_attachment_glue) {
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+ }
+
+ if let Some(ref extension) = filter.attachment_extension {
+ if let Ok(query) = RegexQuery::from_pattern(extension.as_str(), f.f_attachment_glue) {
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+ }
+
+ if let Some(ref category) = filter.attachment_category {
+ if let Ok(query) = RegexQuery::from_pattern(category.as_str(), f.f_attachment_glue) {
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+ }
+
+ if let Some(ref content_type) = filter.attachment_content_type {
+ if let Ok(query) = RegexQuery::from_pattern(content_type.as_str(), f.f_attachment_glue)
+ {
+ subqueries.push((Occur::Must, Box::new(query)));
+ }
+ }
+
+ let start_bound = if let Some(from) = filter.since {
+ Bound::Included(Term::from_field_i64(f.f_date, from))
+ } else {
+ Bound::Unbounded
+ };
+
+ let end_bound = if let Some(to) = filter.before {
+ Bound::Included(Term::from_field_i64(f.f_date, to))
+ } else {
+ Bound::Unbounded
+ };
+
+ if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded {
+ let q = RangeQuery::new(start_bound, end_bound);
+ subqueries.push((Occur::Must, Box::new(q)));
+ }
+
+ if let Some(account_ids) = filter.account_ids {
+ let mut should_queries: Vec<(Occur, Box)> = Vec::new();
+ for id in account_ids {
+ let term = Term::from_field_u64(f.f_account_id, id);
+ should_queries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
+ ));
+ }
+ subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries))));
+ }
+
+ if let Some(mailbox_ids) = filter.mailbox_ids {
+ let mut should_queries: Vec<(Occur, Box)> = Vec::new();
+ for id in mailbox_ids {
+ let term = Term::from_field_u64(f.f_mailbox_id, id);
+ should_queries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
+ ));
+ }
+ subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries))));
+ }
+
+ let start_bound = if let Some(from) = filter.min_size {
+ Bound::Included(Term::from_field_u64(f.f_size, from))
+ } else {
+ Bound::Unbounded
+ };
+
+ let end_bound = if let Some(to) = filter.max_size {
+ Bound::Included(Term::from_field_u64(f.f_size, to))
+ } else {
+ Bound::Unbounded
+ };
+
+ if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded {
+ let q = RangeQuery::new(start_bound, end_bound);
+ subqueries.push((Occur::Must, Box::new(q)));
+ }
+
+ if let Some(ref msg_id) = filter.message_id {
+ let term = Term::from_field_text(f.f_message_id, msg_id);
+ subqueries.push((
+ Occur::Must,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)),
+ ));
+ }
+
+ if subqueries.is_empty() {
+ return Ok(Box::new(AllQuery));
+ }
+
+ Ok(Box::new(BooleanQuery::new(subqueries)))
+ }
+
+ fn thread_query(&self, account_id: u64, thread_id: &str) -> Box {
+ let account_query = TermQuery::new(
+ Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id),
+ IndexRecordOption::Basic,
+ );
+ let thread_query = TermQuery::new(
+ Term::from_field_text(SchemaTools::email_fields().f_thread_id, thread_id),
+ IndexRecordOption::Basic,
+ );
+ let boolean_query = BooleanQuery::new(vec![
+ (Occur::Must, Box::new(account_query)),
+ (Occur::Must, Box::new(thread_query)),
+ ]);
+ Box::new(boolean_query)
+ }
+
+ pub async fn get_envelope_by_id(
+ &self,
+ account_id: u64,
+ envelope_id: &str,
+ ) -> BichonResult> {
+ let searcher = self.create_searcher()?;
+ let f = SchemaTools::email_fields();
+
+ let query = BooleanQuery::new(vec![
+ (
+ Occur::Must,
+ Box::new(TermQuery::new(
+ Term::from_field_u64(f.f_account_id, account_id),
+ IndexRecordOption::Basic,
+ )),
+ ),
+ (
+ Occur::Must,
+ Box::new(TermQuery::new(
+ Term::from_field_text(f.f_id, envelope_id),
+ IndexRecordOption::Basic,
+ )),
+ ),
+ ]);
+
+ let docs: Vec<(f32, DocAddress)> = searcher
+ .search(&query, &TopDocs::with_limit(1).order_by_score())
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ if let Some((_, doc_address)) = docs.first() {
+ let doc: TantivyDocument = searcher
+ .doc(*doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let envelope = EnvelopeWithAttachments::from_tantivy_doc(&doc)?;
+ Ok(Some(envelope))
+ } else {
+ Ok(None)
+ }
+ }
+
+ pub async fn top_10_largest_emails(
+ &self,
+ accounts: &Option>,
+ ) -> BichonResult> {
+ self.reader
+ .reload()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let searcher = self.reader.searcher();
+
+ let query: Box = match accounts {
+ Some(ref ids) if !ids.is_empty() => {
+ let mut subqueries = Vec::new();
+ for &id in ids {
+ let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id);
+ subqueries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ Box::new(BooleanQuery::new(subqueries))
+ }
+ Some(_) => Box::new(EmptyQuery),
+ None => Box::new(AllQuery),
+ };
+
+ let mailbox_docs: Vec<(Option, DocAddress)> = searcher
+ .search(
+ &query,
+ &TopDocs::with_limit(10).order_by_fast_field(F_SIZE, Order::Desc),
+ )
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let mut result = Vec::new();
+
+ for (_, doc_address) in mailbox_docs {
+ let doc: TantivyDocument = searcher
+ .doc(doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let envelope = LargestEmail::from_tantivy_doc(&doc)?;
+ result.push(envelope);
+ }
+ Ok(result)
+ }
+
+ pub fn total_emails(&self, accounts: &Option>) -> BichonResult {
+ let searcher = self.create_searcher()?;
+
+ match accounts {
+ Some(ref ids) if !ids.is_empty() => {
+ let mut subqueries = Vec::new();
+ for &id in ids {
+ let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id);
+ subqueries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ let query = Box::new(BooleanQuery::new(subqueries)) as Box;
+ let count = searcher
+ .search(&query, &Count)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ Ok(count as u64)
+ }
+ Some(_) => Ok(0),
+ None => Ok(searcher.num_docs()),
+ }
+ }
+
+ pub async fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> {
+ let searcher = self.create_searcher()?;
+
+ let query = self.mailbox_query(account_id, mailbox_id);
+ let agg_req: Aggregations = serde_json::from_value(json!({
+ "max_uid": {
+ "max": {
+ "field": F_UID
+ }
+ }
+ }))
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+
+ let agg_res = searcher
+ .search(query.as_ref(), &collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ Ok(Self::extract_max_uid(&agg_res))
+ }
+
+ fn extract_max_uid(agg_res: &AggregationResults) -> Option {
+ agg_res.0.get("max_uid").and_then(|result| match result {
+ AggregationResult::MetricResult(MetricResult::Max(max)) => {
+ max.value.and_then(|value| {
+ (value >= 0.0 && value <= u64::MAX as f64).then(|| value.trunc() as u64)
+ })
+ }
+ _ => None,
+ })
+ }
+
+ pub async fn delete_account_envelopes(&self, account_id: u64) -> BichonResult<()> {
+ let query = self.account_query(account_id);
+ let (eml_content_hashes, attachments_content_hashes) =
+ self.collect_content_hashes(query).await?;
+
+ let query = self.account_query(account_id);
+
+ let mut writer = self.index_writer.lock().await;
+ writer
+ .delete_query(query)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ writer
+ .commit()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
+ self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)
+ .await?;
+ }
+ Ok(())
+ }
+
+ pub async fn delete_mailbox_envelopes(
+ &self,
+ account_id: u64,
+ mailbox_ids: Vec,
+ ) -> BichonResult<()> {
+ if mailbox_ids.is_empty() {
+ return Ok(());
+ }
+
+ let mut eml_content_hashes: HashSet = HashSet::new();
+ let mut attachments_content_hashes: HashSet = HashSet::new();
+
+ for mailbox_id in &mailbox_ids {
+ let query = self.mailbox_query(account_id, *mailbox_id);
+ let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query).await?;
+ eml_content_hashes.extend(eml_hashes);
+ attachments_content_hashes.extend(attachment_hashes);
+ }
+
+ let mut queries: Vec> = Vec::with_capacity(mailbox_ids.len());
+ for mailbox_id in mailbox_ids {
+ queries.push(self.mailbox_query(account_id, mailbox_id));
+ }
+ let mut writer = self.index_writer.lock().await;
+ for query in queries {
+ writer
+ .delete_query(query)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ }
+ writer
+ .commit()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
+ self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)
+ .await?;
+ }
+ Ok(())
+ }
+
+ async fn collect_content_hashes(
+ &self,
+ query: Box,
+ ) -> BichonResult<(HashSet, HashSet)> {
+ let mut eml_content_hashes = HashSet::new();
+ let mut attachments_content_hashes = HashSet::new();
+
+ let fields = SchemaTools::email_fields();
+ let searcher = self.create_searcher()?;
+
+ let docs = searcher
+ .search(&query, &DocSetCollector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ for doc_address in docs {
+ let doc = searcher
+ .doc::(doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ // Extract content_hash
+ if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) {
+ if let Some(str) = content_hash_value.as_str() {
+ eml_content_hashes.insert(str.to_string());
+ }
+ }
+
+ // Extract attachment_content_hash
+ let attachment_hash_values = doc.get_all(fields.f_attachment_content_hash);
+ for hash_value in attachment_hash_values {
+ if let Some(str) = hash_value.as_str() {
+ attachments_content_hashes.insert(str.to_string());
+ }
+ }
+ }
+
+ Ok((eml_content_hashes, attachments_content_hashes))
+ }
+
+ async fn cleanup_unused_content(
+ &self,
+ eml_content_hashes: HashSet,
+ attachments_content_hashes: HashSet,
+ ) -> BichonResult<()> {
+ let searcher = self.create_searcher()?;
+ let fields = SchemaTools::email_fields();
+ let mut eml: HashSet = HashSet::new();
+ for content_hash in eml_content_hashes {
+ // Check if any other emails still reference this content hash
+ let hash_term = Term::from_field_text(fields.f_content_hash, &content_hash);
+ let hash_query = TermQuery::new(hash_term, IndexRecordOption::Basic);
+ let count = searcher
+ .search(&hash_query, &Count)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ // If no references found, delete from KV store
+ if count == 0 {
+ eml.insert(content_hash);
+ }
+ }
+
+ let mut attachments: HashSet = HashSet::new();
+ for content_hash in attachments_content_hashes {
+ // Check if any other emails still reference this content hash
+ let hash_term = Term::from_field_text(fields.f_content_hash, &content_hash);
+ let hash_query = TermQuery::new(hash_term, IndexRecordOption::Basic);
+ let count = searcher
+ .search(&hash_query, &Count)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ // If no references found, delete from KV store
+ if count == 0 {
+ attachments.insert(content_hash);
+ }
+ }
+
+ BLOB_MANAGER.delete(&eml, &attachments)
+ }
+
+ pub async fn delete_envelopes_multi_account(
+ &self,
+ deletes: HashMap>,
+ ) -> BichonResult<()> {
+ if deletes.is_empty() {
+ tracing::warn!("delete_envelopes_multi_account: deletes is empty, nothing to delete");
+ return Ok(());
+ }
+
+ let mut eml_content_hashes: HashSet = HashSet::new();
+ let mut attachments_content_hashes: HashSet = HashSet::new();
+
+ for (account_id, envelope_ids) in &deletes {
+ let unique_ids: HashSet<&String> = envelope_ids.iter().collect();
+ if unique_ids.is_empty() {
+ continue;
+ }
+
+ for eid in unique_ids {
+ let query = self.envelope_query(*account_id, eid);
+ let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query).await?;
+ eml_content_hashes.extend(eml_hashes);
+ attachments_content_hashes.extend(attachment_hashes);
+ }
+ }
+
+ let mut writer = self.index_writer.lock().await;
+
+ for (account_id, envelope_ids) in deletes {
+ let unique_ids: HashSet<&String> = envelope_ids.iter().collect();
+ if unique_ids.is_empty() {
+ continue;
+ }
+ for eid in unique_ids {
+ let query = self.envelope_query(account_id, eid);
+ writer
+ .delete_query(query)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ }
+ }
+ writer
+ .commit()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
+ self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)
+ .await?;
+ }
+
+ Ok(())
+ }
+
+ fn collect_facets_recursive(
+ query: &dyn Query,
+ searcher: &Searcher,
+ parent_facet: &str,
+ all_facets: &mut Vec,
+ ) -> BichonResult<()> {
+ let mut facet_collector = FacetCollector::for_field(F_TAGS);
+ facet_collector.add_facet(parent_facet);
+
+ let facet_counts = searcher
+ .search(query, &facet_collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ for (facet, count) in facet_counts.get(parent_facet) {
+ all_facets.push(TagCount {
+ tag: facet.to_string(),
+ count,
+ });
+ Self::collect_facets_recursive(query, searcher, &facet.to_string(), all_facets)?;
+ }
+
+ Ok(())
+ }
+
+ pub async fn get_all_tags(
+ &self,
+ accounts: Option>,
+ ) -> BichonResult> {
+ let searcher = self.reader.searcher();
+
+ let query: Box = match accounts {
+ Some(ref ids) if !ids.is_empty() => {
+ let mut subqueries = Vec::new();
+ for &id in ids {
+ let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id);
+ subqueries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ Box::new(BooleanQuery::new(subqueries))
+ }
+ Some(_) => Box::new(EmptyQuery),
+ None => Box::new(AllQuery),
+ };
+
+ let mut all_facets = Vec::new();
+ Self::collect_facets_recursive(&query, &searcher, "/", &mut all_facets)?;
+ Ok(all_facets)
+ }
+
+ pub async fn get_all_contacts(
+ &self,
+ accounts: Option>,
+ ) -> BichonResult> {
+ let searcher = self.create_searcher()?;
+
+ let query: Box = match accounts {
+ Some(ref ids) if !ids.is_empty() => {
+ let mut subqueries = Vec::new();
+ for &id in ids {
+ let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id);
+ subqueries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ Box::new(BooleanQuery::new(subqueries))
+ }
+ Some(_) => Box::new(EmptyQuery),
+ None => Box::new(AllQuery),
+ };
+
+ let mut contacts_set: HashSet = HashSet::new();
+
+ let top_docs = searcher
+ .search(&query, &TopDocs::with_limit(1_000_000).order_by_score())
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ for (_score, doc_address) in top_docs {
+ let doc: TantivyDocument = searcher
+ .doc(doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let contacts = extract_contacts(&doc).await?;
+ for value in contacts {
+ contacts_set.insert(value);
+ }
+ }
+ Ok(contacts_set)
+ }
+
+ pub async fn update_envelope_tags(&self, request: TagsRequest) -> BichonResult<()> {
+ if request.updates.is_empty() {
+ tracing::warn!("update_envelope_tags: request is empty, nothing to update");
+ return Ok(());
+ }
+ let searcher = self.create_searcher()?;
+ let mut writer = self.index_writer.lock().await;
+
+ let f_tags = SchemaTools::email_fields().f_tags;
+ let f_id = SchemaTools::email_fields().f_id;
+ let deduplicated_updates: HashMap> = request
+ .updates
+ .into_iter()
+ .map(|(account_id, envelope_ids)| (account_id, envelope_ids.into_iter().collect()))
+ .collect();
+
+ let mut operations = Vec::new();
+
+ for (account_id, envelope_ids) in &deduplicated_updates {
+ for eid in envelope_ids {
+ let query = self.envelope_query(*account_id, eid);
+ let docs = searcher
+ .search(query.as_ref(), &TopDocs::with_limit(1).order_by_score())
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ if let Some((_, doc_address)) = docs.first() {
+ let old_doc: TantivyDocument = searcher
+ .doc(*doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ let mut current_tags: HashSet = old_doc
+ .get_all(f_tags)
+ .filter_map(|val| val.as_facet())
+ .map(|facet| facet.to_string())
+ .collect();
+
+ match request.action {
+ TagAction::Add => {
+ for tag in &request.tags {
+ current_tags.insert(tag.clone());
+ }
+ }
+ TagAction::Remove => {
+ for tag in &request.tags {
+ current_tags.remove(tag);
+ }
+ }
+ TagAction::Overwrite => {
+ current_tags = request.tags.iter().cloned().collect();
+ }
+ }
+
+ let mut new_doc = TantivyDocument::new();
+
+ for (field, value) in old_doc.field_values() {
+ if field != f_tags {
+ new_doc.add_field_value(field, value);
+ }
+ }
+ for tag in current_tags {
+ new_doc.add_facet(f_tags, &tag);
+ }
+
+ let delete_term = Term::from_field_text(f_id, eid);
+ operations.push(UserOperation::Delete(delete_term));
+ operations.push(UserOperation::Add(new_doc));
+ }
+ }
+ }
+
+ writer
+ .run(operations)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ // commit
+ writer
+ .commit()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ Ok(())
+ }
+
+ pub async fn search(
+ &self,
+ accounts: Option>,
+ filter: SearchFilter,
+ page: u64,
+ page_size: u64,
+ desc: bool,
+ sort_by: SortBy,
+ ) -> BichonResult> {
+ assert!(page > 0, "Page number must be greater than 0");
+ assert!(page_size > 0, "Page size must be greater than 0");
+ let query = self.filter_query(accounts, filter, self.query_parser.clone())?;
+ let searcher = self.create_searcher()?;
+ let total = searcher
+ .search(&query, &Count)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+ as u64;
+
+ if total == 0 {
+ return Ok(DataPage {
+ current_page: Some(page),
+ page_size: Some(page_size),
+ total_items: 0,
+ items: vec![],
+ total_pages: Some(0),
+ });
+ }
+ let offset = (page - 1) * page_size;
+ let total_pages = total.div_ceil(page_size);
+ if offset > total {
+ return Ok(DataPage {
+ current_page: Some(page),
+ page_size: Some(page_size),
+ total_items: total,
+ items: vec![],
+ total_pages: Some(total_pages),
+ });
+ }
+
+ let order = if desc { Order::Desc } else { Order::Asc };
+ let mailbox_docs: Vec;
+
+ match sort_by {
+ SortBy::DATE => {
+ let date_docs: Vec<(Option, DocAddress)> = searcher
+ .search(
+ &query,
+ &TopDocs::with_limit(page_size as usize)
+ .and_offset(offset as usize)
+ .order_by_fast_field(F_DATE, order),
+ )
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ mailbox_docs = date_docs.into_iter().map(|(_, addr)| addr).collect();
+ }
+ SortBy::SIZE => {
+ let size_docs: Vec<(Option, DocAddress)> = searcher
+ .search(
+ &query,
+ &TopDocs::with_limit(page_size as usize)
+ .and_offset(offset as usize)
+ .order_by_fast_field(F_SIZE, order),
+ )
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ mailbox_docs = size_docs.into_iter().map(|(_, addr)| addr).collect();
+ }
+ }
+
+ let mut result = Vec::new();
+
+ for doc_address in mailbox_docs {
+ let doc: TantivyDocument = searcher
+ .doc(doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let envelope = EnvelopeWithAttachments::from_tantivy_doc(&doc)?.envelope;
+ result.push(envelope);
+ }
+ Ok(DataPage {
+ current_page: Some(page),
+ page_size: Some(page_size),
+ total_items: total,
+ items: result,
+ total_pages: Some(total_pages),
+ })
+ }
+
+ fn create_searcher(&self) -> BichonResult {
+ self.reader
+ .reload()
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ Ok(self.reader.searcher())
+ }
+
+ pub async fn num_messages_in_thread(
+ &self,
+ searcher: &Searcher,
+ account_id: u64,
+ thread_id: &str,
+ ) -> BichonResult {
+ let query = self.thread_query(account_id, thread_id);
+
+ let agg_req: Aggregations = serde_json::from_value(json!({
+ "thread_count": {
+ "value_count": {
+ "field": F_THREAD_ID
+ }
+ }
+ }))
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let collector = AggregationCollector::from_aggs(agg_req, Default::default());
+
+ let agg_res = searcher
+ .search(query.as_ref(), &collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ Self::extract_thread_count(&agg_res)
+ }
+
+ fn extract_thread_count(agg_res: &AggregationResults) -> BichonResult {
+ let Some(result) = agg_res.0.get("thread_count") else {
+ return Err(raise_error!(
+ "Missing aggregation result: thread_count".into(),
+ ErrorCode::InternalError
+ ));
+ };
+
+ match result {
+ AggregationResult::MetricResult(MetricResult::Count(count)) => {
+ Ok(count.value.map(|v| v as u64).ok_or_else(|| {
+ raise_error!(
+ "Failed to get count value from aggregation result: value is None".into(),
+ ErrorCode::InternalError
+ )
+ })?)
+ }
+ other => Err(raise_error!(
+ format!("Unexpected aggregation result type: {other:?}"),
+ ErrorCode::InternalError
+ )),
+ }
+ }
+
+ pub async fn list_thread_envelopes(
+ &self,
+ account_id: u64,
+ thread_id: &str,
+ page: u64,
+ page_size: u64,
+ desc: bool,
+ ) -> BichonResult> {
+ assert!(page > 0, "Page number must be greater than 0");
+ assert!(page_size > 0, "Page size must be greater than 0");
+ let searcher = self.create_searcher()?;
+ let total = self
+ .num_messages_in_thread(&searcher, account_id, thread_id)
+ .await?;
+ if total == 0 {
+ return Ok(DataPage {
+ current_page: Some(page),
+ page_size: Some(page_size),
+ total_items: 0,
+ items: vec![],
+ total_pages: Some(0),
+ });
+ }
+ let offset = (page - 1) * page_size;
+ let total_pages = total.div_ceil(page_size);
+ if offset > total {
+ return Ok(DataPage {
+ current_page: Some(page),
+ page_size: Some(page_size),
+ total_items: total,
+ items: vec![],
+ total_pages: Some(total_pages),
+ });
+ }
+
+ let query = self.thread_query(account_id, thread_id);
+
+ let order = if desc { Order::Desc } else { Order::Asc };
+ let thread_docs: Vec<(Option, DocAddress)> = searcher
+ .search(
+ query.as_ref(),
+ &TopDocs::with_limit(page_size as usize)
+ .and_offset(offset as usize)
+ .order_by_fast_field(F_DATE, order),
+ )
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let mut result = Vec::new();
+
+ for (_, doc_address) in thread_docs {
+ let doc: TantivyDocument = searcher
+ .doc(doc_address)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let envelope = EnvelopeWithAttachments::from_tantivy_doc(&doc)?.envelope;
+ result.push(envelope);
+ }
+ Ok(DataPage {
+ current_page: Some(page),
+ page_size: Some(page_size),
+ total_items: total,
+ items: result,
+ total_pages: Some(total_pages),
+ })
+ }
+
+ pub fn collect_attachment_metadata(
+ &self,
+ accounts: Option>,
+ ) -> BichonResult {
+ let searcher = self.create_searcher()?;
+
+ let query: Box = match accounts {
+ Some(ref ids) if !ids.is_empty() => {
+ let mut subqueries = Vec::new();
+ for &id in ids {
+ let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id);
+ subqueries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ Box::new(BooleanQuery::new(subqueries))
+ }
+ Some(_) => Box::new(EmptyQuery),
+ None => Box::new(AllQuery),
+ };
+
+ // Extension collector
+ let mut ext_collector = FacetCollector::for_field(F_ATTACHMENT_EXT);
+ ext_collector.add_facet("/");
+
+ // Category collector
+ let mut cat_collector = FacetCollector::for_field(F_ATTACHMENT_CATEGORY);
+ cat_collector.add_facet("/");
+
+ // Content-Type collector
+ let mut type_collector = FacetCollector::for_field(F_ATTACHMENT_CONTENT_TYPE);
+ type_collector.add_facet("/");
+
+ let ext_counts = searcher
+ .search(&query, &ext_collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let cat_counts = searcher
+ .search(&query, &cat_collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+ let type_counts = searcher
+ .search(&query, &type_collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ let mut metadata = AttachmentMetadata {
+ extensions: HashSet::new(),
+ categories: HashSet::new(),
+ content_types: HashSet::new(),
+ };
+
+ for (facet, _) in ext_counts.get("/") {
+ let val = facet.to_path_string().trim_start_matches('/').to_string();
+ metadata.extensions.insert(val);
+ }
+
+ for (facet, _) in cat_counts.get("/") {
+ let val = facet.to_path_string().trim_start_matches('/').to_string();
+ metadata.categories.insert(val);
+ }
+
+ for (facet, _) in type_counts.get("/") {
+ let val = facet.to_path_string().trim_start_matches('/').to_string();
+ metadata.content_types.insert(val);
+ }
+
+ Ok(metadata)
+ }
+
+ pub async fn get_dashboard_stats(
+ &self,
+ accounts: &Option>,
+ ) -> BichonResult {
+ let searcher = self.create_searcher()?;
+ let now_ms = utc_now!();
+ let week_ago_ms = (Utc::now() - Duration::from_secs(60 * 60 * 24 * 30)).timestamp_millis();
+
+ let aggregations: Aggregations = serde_json::from_value(json!({
+ "total_size": {
+ "sum": { "field": F_SIZE }
+ },
+ "recent_30d_histogram": {
+ "histogram": {
+ "field": F_DATE,
+ "interval": 86400000,
+ "hard_bounds": {
+ "min": week_ago_ms,
+ "max": now_ms
+ }
+ }
+ },
+ "top_from_values": {
+ "terms": {
+ "field": F_FROM,
+ "size": 10
+ }
+ },
+ "top_account_values": {
+ "terms": {
+ "field": F_ACCOUNT_ID,
+ "size": 10
+ }
+ },
+ "attachment_stats": {
+ "range": {
+ "field": F_REGULAR_ATTACHMENT_COUNT,
+ "ranges": [
+ { "to": 1, "key": "no_attachment" },
+ { "from": 1, "key": "has_attachment" }
+ ]
+ }
+ }
+ }))
+ .unwrap();
+
+ let query: Box = match accounts {
+ Some(ref ids) if !ids.is_empty() => {
+ let mut subqueries = Vec::new();
+ for &id in ids {
+ let term = Term::from_field_u64(SchemaTools::email_fields().f_account_id, id);
+ subqueries.push((
+ Occur::Should,
+ Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box,
+ ));
+ }
+ Box::new(BooleanQuery::new(subqueries))
+ }
+ Some(_) => Box::new(EmptyQuery),
+ None => Box::new(AllQuery),
+ };
+
+ let agg_collector = AggregationCollector::from_aggs(aggregations, Default::default());
+ let agg_results = searcher
+ .search(&query, &agg_collector)
+ .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
+
+ let mut stats = DashboardStats::default();
+ let total_size = agg_results.0.get("total_size").ok_or_else(|| {
+ raise_error!(
+ "missing 'total_size' aggregation result".into(),
+ ErrorCode::InternalError
+ )
+ })?;
+
+ if let AggregationResult::MetricResult(MetricResult::Sum(v)) = total_size {
+ let total_size = v.value.map(|v| v as u64).ok_or_else(|| {
+ raise_error!(
+ "'total_size' sum metric has no value".into(),
+ ErrorCode::InternalError
+ )
+ })?;
+ stats.total_size_bytes = total_size;
+ }
+
+ let recent_30d_histogram = agg_results.0.get("recent_30d_histogram").ok_or_else(|| {
+ raise_error!(
+ "missing 'recent_30d_histogram' aggregation result".into(),
+ ErrorCode::InternalError
+ )
+ })?;
+
+ let mut recent_activity = Vec::with_capacity(31);
+ if let AggregationResult::BucketResult(BucketResult::Histogram { buckets, .. }) =
+ recent_30d_histogram
+ {
+ if let BucketEntries::Vec(bucket_list) = buckets {
+ for entry in bucket_list {
+ if let Key::F64(ms) = entry.key {
+ recent_activity.push(TimeBucket {
+ timestamp_ms: ms as i64,
+ count: entry.doc_count,
+ });
+ }
+ }
+ }
+ }
+ stats.recent_activity = recent_activity;
+ let mut top_senders = Vec::with_capacity(11);
+ let top_from_values = agg_results.0.get("top_from_values").unwrap();
+ if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) =
+ top_from_values
+ {
+ for entry in buckets {
+ if let Key::Str(sender) = &entry.key {
+ top_senders.push(Group {
+ key: sender.clone(),
+ count: entry.doc_count,
+ });
+ }
+ }
+ }
+ stats.top_senders = top_senders;
+
+ let mut top_accounts = Vec::with_capacity(11);
+ let top_account_values = agg_results.0.get("top_account_values").unwrap();
+ if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) =
+ top_account_values
+ {
+ for entry in buckets {
+ if let Key::U64(account_id) = &entry.key {
+ match AccountModel::get(*account_id) {
+ Ok(account) => {
+ top_accounts.push(Group {
+ key: account.email,
+ count: entry.doc_count,
+ });
+ }
+ Err(e) => {
+ warn!(
+ account_id = account_id,
+ error = %e,
+ "orphaned account index detected, scheduling cleanup"
+ );
+ let account_id = *account_id;
+ tokio::spawn(async move {
+ if let Err(e) =
+ INDEX_MANAGER.delete_account_envelopes(account_id).await
+ {
+ tracing::error!(
+ account_id = account_id,
+ error = %e,
+ "failed to cleanup envelope index"
+ );
+ }
+ //这里要删除eml和attachments, 要先得到content_hash, 似乎有点麻烦，因为id有点多
+ // if let Err(e) =
+ // INDEX_MANAGER.delete_account_envelopes(account_id).await
+ // {
+ // tracing::error!(
+ // account_id = account_id,
+ // error = %e,
+ // "failed to cleanup eml index"
+ // );
+ // }
+ });
+ }
+ }
+ }
+ }
+ }
+ stats.top_accounts = top_accounts;
+
+ let attachment_stats = agg_results.0.get("attachment_stats").unwrap();
+ if let AggregationResult::BucketResult(BucketResult::Range { buckets, .. }) =
+ attachment_stats
+ {
+ match buckets {
+ BucketEntries::Vec(bucket_vec) => {
+ for entry in bucket_vec {
+ match entry.key.to_string().as_str() {
+ "no_attachment" => {
+ stats.without_attachment_count = entry.doc_count;
+ }
+ "has_attachment" => {
+ stats.with_attachment_count = entry.doc_count;
+ }
+ _ => {}
+ }
+ }
+ }
+ BucketEntries::HashMap(bucket_map) => {
+ if let Some(entry) = bucket_map.get("no_attachment") {
+ stats.without_attachment_count = entry.doc_count;
+ }
+ if let Some(entry) = bucket_map.get("has_attachment") {
+ stats.with_attachment_count = entry.doc_count;
+ }
+ }
+ }
+ }
+ Ok(stats)
+ }
+}
+
+fn fatal_commit(writer: &mut IndexWriter) {
+ const MAX_RETRIES: usize = 3;
+ const RETRY_DELAY_MS: u64 = 1000;
+
+ for attempt in 0..=MAX_RETRIES {
+ match writer.commit() {
+ Ok(_) => {
+ if attempt > 0 {
+ eprintln!("[INFO] Commit succeeded on attempt {}", attempt + 1);
+ }
+ return;
+ }
+ Err(e) => match &e {
+ tantivy::TantivyError::IoError(io_error) => {
+ if attempt < MAX_RETRIES {
+ eprintln!(
+ "[WARN] Commit failed (attempt {}/{}): {:?}. Retrying in {}ms...",
+ attempt + 1,
+ MAX_RETRIES + 1,
+ io_error,
+ RETRY_DELAY_MS * (attempt as u64 + 1)
+ );
+ std::thread::sleep(std::time::Duration::from_millis(
+ RETRY_DELAY_MS * (attempt as u64 + 1),
+ ));
+ } else {
+ eprintln!(
+ "[FATAL] Tantivy commit failed after {} attempts: {:?}",
+ MAX_RETRIES + 1,
+ io_error
+ );
+ std::process::exit(1);
+ }
+ }
+ _ => {
+ eprintln!("[FATAL] Tantivy commit failed with non-IO error: {e:?}");
+ std::process::exit(1);
+ }
+ },
+ }
+ }
+}
diff --git a/src/modules/duckdb/mod.rs b/src/modules/store/tantivy/mod.rs
similarity index 91%
rename from src/modules/duckdb/mod.rs
rename to src/modules/store/tantivy/mod.rs
index 61f48ac..e3cf3b2 100644
--- a/src/modules/duckdb/mod.rs
+++ b/src/modules/store/tantivy/mod.rs
@@ -16,8 +16,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-pub mod build;
-pub mod init;
-pub mod refinery;
-#[cfg(test)]
-mod tests;
+pub mod fields;
+pub mod manager;
+pub mod model;
+pub mod schema;
diff --git a/src/modules/store/tantivy/model.rs b/src/modules/store/tantivy/model.rs
new file mode 100644
index 0000000..b077e33
--- /dev/null
+++ b/src/modules/store/tantivy/model.rs
@@ -0,0 +1,241 @@
+use std::collections::HashSet;
+
+use tantivy::{
+ schema::{Facet, Value},
+ TantivyDocument,
+};
+
+use crate::{
+ modules::{
+ account::migration::AccountModel,
+ cache::imap::mailbox::MailBox,
+ error::{code::ErrorCode, BichonResult},
+ message::content::AttachmentInfo,
+ store::{envelope::Envelope, tantivy::schema::SchemaTools},
+ },
+ raise_error,
+};
+
+#[derive(Debug, Clone)]
+pub struct EnvelopeWithAttachments {
+ pub envelope: Envelope,
+ pub attachments: Option>,
+}
+
+impl EnvelopeWithAttachments {
+ pub fn to_document(&self, body_text: &str, shard_id: u64) -> BichonResult {
+ let fields = SchemaTools::email_fields();
+ let mut doc = TantivyDocument::new();
+
+ doc.add_text(fields.f_id, &self.envelope.id);
+ doc.add_text(fields.f_message_id, &self.envelope.message_id);
+ doc.add_u64(fields.f_account_id, self.envelope.account_id);
+ doc.add_u64(fields.f_mailbox_id, self.envelope.mailbox_id);
+ doc.add_u64(fields.f_uid, self.envelope.uid as u64);
+ doc.add_text(fields.f_subject, &self.envelope.subject);
+ doc.add_text(fields.f_preview, &self.envelope.preview);
+ doc.add_text(fields.f_content_hash, &self.envelope.content_hash);
+ doc.add_text(fields.f_from, &self.envelope.from);
+ doc.add_text(fields.f_body, body_text);
+
+ for to in &self.envelope.to {
+ doc.add_text(fields.f_to, to);
+ }
+ for cc in &self.envelope.cc {
+ doc.add_text(fields.f_cc, cc);
+ }
+ for bcc in &self.envelope.bcc {
+ doc.add_text(fields.f_bcc, bcc);
+ }
+
+ doc.add_i64(fields.f_date, self.envelope.date);
+ doc.add_i64(fields.f_internal_date, self.envelope.internal_date);
+ doc.add_u64(fields.f_size, self.envelope.size as u64);
+ doc.add_i64(fields.f_ingest_at, self.envelope.ingest_at);
+ doc.add_text(fields.f_thread_id, &self.envelope.thread_id);
+
+ if let Some(ref atts) = self.attachments {
+ let atts_json = serde_json::to_string(atts).unwrap_or_else(|_| "[]".to_string());
+ doc.add_text(fields.f_attachments, atts_json);
+ let mut search_terms = Vec::new();
+
+ for att in atts {
+ if let Some(ref filename) = att.filename {
+ search_terms.push(filename.clone());
+ }
+
+ if let Some(ext) = att.get_extension() {
+ search_terms.push(ext.clone());
+ doc.add_facet(fields.f_attachment_ext, Facet::from(&format!("/{}", ext)));
+ }
+ let category = att.get_category().to_string();
+ search_terms.push(category.clone());
+ let file_type = att.file_type.to_lowercase();
+ search_terms.push(file_type.clone());
+
+ doc.add_facet(
+ fields.f_attachment_category,
+ Facet::from(&format!("/{}", category)),
+ );
+ doc.add_facet(
+ fields.f_attachment_content_type,
+ Facet::from(&format!("/{}", file_type)),
+ );
+
+ doc.add_text(fields.f_attachment_content_hash, &att.content_hash);
+ }
+ doc.add_text(fields.f_attachment_glue, search_terms.join(" "));
+ }
+
+ doc.add_u64(
+ fields.f_attachment_count,
+ self.envelope.attachment_count as u64,
+ );
+ doc.add_u64(
+ fields.f_regular_attachment_count,
+ self.envelope.regular_attachment_count as u64,
+ );
+ doc.add_u64(fields.f_shard_id, shard_id);
+ Ok(doc)
+ }
+
+ pub fn from_tantivy_doc(doc: &TantivyDocument) -> BichonResult {
+ let fields = SchemaTools::email_fields();
+
+ let attachments_raw = extract_string_field(doc, fields.f_attachments).ok();
+ let attachments: Option> =
+ attachments_raw.and_then(|json| serde_json::from_str(&json).ok());
+
+ let tags: Vec = doc
+ .get_all(fields.f_tags)
+ .filter_map(|value| value.as_facet())
+ .map(|f| f.to_string())
+ .collect();
+
+ let account_id = extract_u64_field(doc, fields.f_account_id)?;
+ let mailbox_id = extract_u64_field(doc, fields.f_mailbox_id)?;
+
+ let account = AccountModel::get(account_id)?;
+ let mailbox = MailBox::get(mailbox_id)?;
+ let envelope = Envelope {
+ id: extract_string_field(doc, fields.f_id)?,
+ message_id: extract_string_field(doc, fields.f_message_id)?,
+ account_id,
+ account_email: Some(account.email),
+ mailbox_id,
+ mailbox_name: Some(mailbox.name),
+ uid: extract_u64_field(doc, fields.f_uid)? as u32,
+ subject: extract_string_field(doc, fields.f_subject)?,
+ preview: extract_string_field(doc, fields.f_preview).unwrap_or_default(),
+ from: extract_string_field(doc, fields.f_from)?,
+ to: extract_vec_string_field(doc, fields.f_to)?,
+ cc: extract_vec_string_field(doc, fields.f_cc)?,
+ bcc: extract_vec_string_field(doc, fields.f_bcc)?,
+ date: extract_i64_field(doc, fields.f_date)?,
+ internal_date: extract_i64_field(doc, fields.f_internal_date)?,
+ size: extract_u64_field(doc, fields.f_size)? as u32,
+ thread_id: extract_string_field(doc, fields.f_thread_id)?,
+ attachment_count: extract_u64_field(doc, fields.f_attachment_count)? as usize,
+ regular_attachment_count: extract_u64_field(doc, fields.f_regular_attachment_count)?
+ as usize,
+ tags: if tags.is_empty() { None } else { Some(tags) },
+ content_hash: extract_string_field(doc, fields.f_content_hash)?,
+ ingest_at: extract_i64_field(doc, fields.f_ingest_at)?,
+ };
+
+ Ok(EnvelopeWithAttachments {
+ envelope,
+ attachments,
+ })
+ }
+}
+
+fn extract_u64_field(
+ document: &TantivyDocument,
+ field: tantivy::schema::Field,
+) -> BichonResult