diff --git a/Cargo.lock b/Cargo.lock index 34db093..dce2627 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,7 +36,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher 0.4.4", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -73,6 +73,17 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -80,6 +91,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -196,6 +208,224 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arrow" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num", +] + +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.16.1", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes 1.11.1", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap", + "lexical-core", + "memchr", + "num", + "serde", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +dependencies = [ + "bitflags", +] + +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash 0.8.12", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -263,14 +493,14 @@ dependencies = [ [[package]] name = "async-imap" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da885da5980f3934831e6370445c0e0e44ef251d7792308b39e908915a41d09" +checksum = "a78dceaba06f029d8f4d7df20addd4b7370a30206e3926267ecda2915b0f3f66" dependencies = [ "async-channel 2.5.0", "async-compression", "base64 0.22.1", - "bytes 1.11.0", + "bytes 1.11.1", "chrono", "futures", "imap-proto", @@ -356,6 +586,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -374,7 +613,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3dc5ca11ac66da7ba331bd707c8d6bca90a559c107aada6ee6abf1d4d5dbc2c" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "futures", "regex", "serde", @@ -432,13 +671,14 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon" -version = "0.3.7" +version = "1.0.0" dependencies = [ - "ahash", + "ahash 0.8.12", + "arrow", "async-imap", "autoconfig", "base64 0.22.1", - "bytes 1.11.0", + "bytes 1.11.1", "cacache", "chrono", "clap", @@ -447,6 +687,7 @@ dependencies = [ "console", "dashmap", "dialoguer", + "duckdb", "email_address", "encoding_rs", "futures", @@ -461,7 +702,7 @@ dependencies = [ "lru 0.16.3", "mail-parser", "mail-send", - "memmap2 0.9.9", + "memmap2 0.9.10", "mimalloc", "mime_guess", "murmur3", @@ -474,7 +715,10 @@ dependencies = [ "poem", "poem-derive", "poem-openapi", - "rand 0.9.2", + "r2d2", + "rand 0.10.0", + "refinery", + "refinery-core", "regex", "reqwest", "ring", @@ -488,7 +732,7 @@ dependencies = [ "sysinfo", "tantivy", "tempfile", - "time 0.3.46", + "time 0.3.47", "timeago", "tokio", "tokio-io-timeout", @@ -528,6 +772,18 @@ dependencies = [ "crunchy", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -584,6 +840,29 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "borsh" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "brotli" version = "8.0.2" @@ -611,6 +890,28 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytecount" version = "0.6.9" @@ -631,9 +932,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bzip2" @@ -702,6 +1003,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.47" @@ -733,10 +1040,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] -name = "chrono" -version = "0.4.43" +name = "chacha20" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -767,9 +1085,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.56" +version = "4.5.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" dependencies = [ "clap_builder", "clap_derive", @@ -777,9 +1095,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.56" +version = "4.5.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" dependencies = [ "anstream", "anstyle", @@ -801,9 +1119,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" [[package]] name = "cmake" @@ -840,6 +1158,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "comfy-table" +version = "7.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +dependencies = [ + "strum 0.26.3", + "strum_macros 0.26.4", + "unicode-width 0.2.2", +] + [[package]] name = "compressed-rtf" version = "1.0.0" @@ -892,6 +1221,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + [[package]] name = "const_fn" version = "0.4.11" @@ -936,6 +1285,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "cpuid-bool" version = "0.2.0" @@ -1032,6 +1390,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.6.0" @@ -1265,6 +1644,26 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" +[[package]] +name = "duckdb" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8685352ce688883098b61a361e86e87df66fc8c444f4a2411e884c16d5243a65" +dependencies = [ + "arrow", + "cast", + "chrono", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libduckdb-sys", + "num", + "num-integer", + "r2d2", + "rust_decimal", + "strum 0.27.2", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1377,6 +1776,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastdivide" version = "0.4.2" @@ -1398,12 +1809,33 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version 0.4.1", +] + [[package]] name = "flate2" version = "1.1.5" @@ -1470,10 +1902,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] -name = "futures" -version = "0.3.31" +name = "funty" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1486,9 +1924,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1496,15 +1934,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1513,9 +1951,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-lite" @@ -1547,9 +1985,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", @@ -1558,15 +1996,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" @@ -1576,9 +2014,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1588,7 +2026,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -1650,6 +2087,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.0", + "wasip2", + "wasip3", +] + [[package]] name = "ghash" version = "0.3.1" @@ -1708,7 +2159,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" dependencies = [ "atomic-waker", - "bytes 1.11.0", + "bytes 1.11.1", "fnv", "futures-core", "futures-sink", @@ -1720,6 +2171,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1759,6 +2231,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "headers" version = "0.4.1" @@ -1766,7 +2247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ "base64 0.22.1", - "bytes 1.11.0", + "bytes 1.11.1", "headers-core", "http 1.4.0", "httpdate", @@ -1815,7 +2296,7 @@ checksum = "a62d7684f766b0f96344be88c023f9b6650039aea09d526b4974cce302eb61b1" dependencies = [ "async-trait", "bitflags", - "bytes 1.11.0", + "bytes 1.11.1", "cfg-if", "data-encoding", "enum-as-inner 0.6.1", @@ -1830,7 +2311,7 @@ dependencies = [ "rustls", "rustls-pki-types", "thiserror 2.0.17", - "time 0.3.46", + "time 0.3.47", "tinyvec", "tokio", "tokio-rustls", @@ -1924,7 +2405,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "fnv", "itoa", ] @@ -1935,7 +2416,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "itoa", ] @@ -1945,7 +2426,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "http 1.4.0", ] @@ -1955,7 +2436,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "futures-core", "http 1.4.0", "http-body", @@ -2017,7 +2498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", - "bytes 1.11.0", + "bytes 1.11.1", "futures-channel", "futures-core", "h2", @@ -2057,7 +2538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ "base64 0.22.1", - "bytes 1.11.0", + "bytes 1.11.1", "futures-channel", "futures-core", "futures-util", @@ -2188,6 +2669,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -2243,6 +2730,8 @@ checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -2385,12 +2874,75 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "levenshtein_automata" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libbz2-rs-sys" version = "0.2.2" @@ -2399,9 +2951,26 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" + +[[package]] +name = "libduckdb-sys" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78bacb8933586cee3b550c39b610d314f9b7a48701ac7a914a046165a4ad8da" +dependencies = [ + "cc", + "flate2", + "pkg-config", + "reqwest", + "serde", + "serde_json", + "tar", + "vcpkg", + "zip", +] [[package]] name = "libm" @@ -2429,6 +2998,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags", + "libc", + "redox_syscall 0.7.0", +] + [[package]] name = "libz-rs-sys" version = "0.5.5" @@ -2458,9 +3038,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -2541,7 +3121,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b7da45f78cc525d3750b623c967ae21c0cd28b2e6a9a2ee4b536a7cce3b21ce" dependencies = [ - "ahash", + "ahash 0.8.12", "flate2", "hashify", "hickory-resolver", @@ -2567,9 +3147,9 @@ dependencies = [ [[package]] name = "mail-parser" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf4390741c4e6fa330bdeccdfb580815dbb462952de91838b723357985119a3" +checksum = "f82a3d6522697593ba4c683e0a6ee5a40fee93bc1a525e3cc6eeb3da11fd8897" dependencies = [ "encoding_rs", "hashify", @@ -2654,9 +3234,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", ] @@ -2759,7 +3339,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "encoding_rs", "futures-util", "http 1.4.0", @@ -2888,12 +3468,76 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2945,18 +3589,18 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ "bitflags", ] [[package]] name = "objc2-io-kit" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" dependencies = [ "libc", "objc2-core-foundation", @@ -2968,7 +3612,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a95602f5a6eec5b15394516448c0b6fc2f2ef4ca2a9ab2951a91a18cec516bf0" dependencies = [ - "ahash", + "ahash 0.8.12", "lazy_static", ] @@ -3031,7 +3675,7 @@ dependencies = [ [[package]] name = "outlook-pst" version = "1.1.0" -source = "git+https://github.com/rustmailer/outlook-pst-rs.git?branch=main#e0f961daea45068d9431de951f51ffdb8932f2a4" +source = "git+https://github.com/rustmailer/outlook-pst-rs.git?branch=main#3d36200fa359161a127287f47ca031ea8b802e48" dependencies = [ "byteorder", "thiserror 2.0.17", @@ -3071,7 +3715,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -3205,7 +3849,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f977080932c87287147dca052951c3e2696f8759863f6b4e4c0c9ffe7a4cc8b" dependencies = [ "async-compression", - "bytes 1.11.0", + "bytes 1.11.1", "futures-util", "headers", "hex", @@ -3261,7 +3905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ccbcc395bf4dd03df1da32da351b6b6732e4074ce27ddec315650e52a2be44c" dependencies = [ "base64 0.22.1", - "bytes 1.11.0", + "bytes 1.11.1", "derive_more", "email_address", "futures-util", @@ -3400,6 +4044,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "pulldown-cmark" version = "0.9.6" @@ -3451,7 +4115,7 @@ version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" dependencies = [ - "ahash", + "ahash 0.8.12", "equivalent", "hashbrown 0.16.1", "parking_lot", @@ -3463,7 +4127,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "cfg_aliases", "pin-project-lite", "quinn-proto", @@ -3483,7 +4147,7 @@ version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "getrandom 0.3.4", "lru-slab", "rand 0.9.2", @@ -3527,6 +4191,23 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -3561,6 +4242,17 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom 0.4.1", + "rand_core 0.10.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -3618,6 +4310,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "rand_distr" version = "0.4.3" @@ -3693,6 +4391,55 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags", +] + +[[package]] +name = "refinery" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c427f2572afe5c6cbfa2b1bf40071c89bf1a8539e958ea582842f6f38dcfae" +dependencies = [ + "refinery-core", + "refinery-macros", +] + +[[package]] +name = "refinery-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702655abfc67f93a6f735e9fa4ace7d2e580633f8961f28acbfd7583ddce936c" +dependencies = [ + "async-trait", + "cfg-if", + "log", + "regex", + "siphasher", + "thiserror 2.0.17", + "time 0.3.47", + "url", + "walkdir", +] + +[[package]] +name = "refinery-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5145756cdf293b5089dc6b4f103f1a1229cc55d67082c866f8c8289531c4b983" +dependencies = [ + "proc-macro2", + "quote", + "refinery-core", + "regex", + "syn 2.0.111", +] + [[package]] name = "reflink-copy" version = "0.1.28" @@ -3707,9 +4454,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -3734,6 +4481,15 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "reqwest" version = "0.12.24" @@ -3741,7 +4497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64 0.22.1", - "bytes 1.11.0", + "bytes 1.11.1", "futures-channel", "futures-core", "futures-util", @@ -3805,6 +4561,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck", + "bytes 1.11.1", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rust-embed" version = "8.11.0" @@ -3849,6 +4634,22 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "rust_decimal" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" +dependencies = [ + "arrayvec", + "borsh", + "bytes 1.11.1", + "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -3865,10 +4666,19 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "1.1.3" +name = "rustc_version" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver 1.0.27", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", @@ -3879,9 +4689,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "log", @@ -3954,12 +4764,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "self_cell" version = "1.2.1" @@ -4109,7 +4934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4129,7 +4954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4147,7 +4972,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -4159,7 +4984,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -4199,6 +5024,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.1" @@ -4362,7 +5193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" dependencies = [ "discard", - "rustc_version", + "rustc_version 0.2.3", "stdweb-derive", "stdweb-internal-macros", "stdweb-internal-runtime", @@ -4446,6 +5277,46 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.111", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "subtle" version = "2.6.1" @@ -4519,9 +5390,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.38.0" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe840c5b1afe259a5657392a4dbb74473a14c8db999c3ec2f4ae812e028a94da" +checksum = "1efc19935b4b66baa6f654ac7924c192f55b175c00a7ab72410fc24284dacda8" dependencies = [ "libc", "memchr", @@ -4566,7 +5437,7 @@ dependencies = [ "lru 0.12.5", "lz4_flex", "measure_time", - "memmap2 0.9.9", + "memmap2 0.9.10", "once_cell", "oneshot", "rayon", @@ -4587,7 +5458,7 @@ dependencies = [ "tantivy-tokenizer-api", "tempfile", "thiserror 2.0.17", - "time 0.3.46", + "time 0.3.47", "uuid", "winapi", "zstd", @@ -4628,7 +5499,7 @@ dependencies = [ "byteorder", "ownedbytes", "serde", - "time 0.3.46", + "time 0.3.47", ] [[package]] @@ -4688,13 +5559,30 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.24.0" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand 2.3.0", - "getrandom 0.3.4", + "getrandom 0.4.1", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4776,9 +5664,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -4788,7 +5676,7 @@ dependencies = [ "powerfmt", "serde_core", "time-core", - "time-macros 0.2.26", + "time-macros 0.2.27", ] [[package]] @@ -4809,9 +5697,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.26" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -4832,14 +5720,23 @@ dependencies = [ [[package]] name = "timeago" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05030782ebd7d1295cce15a98c8805de6e70776c95f8e3468f84f7f198824f49" +checksum = "e86660b0935ca5912eb155f5b3794e2f8f1a09d8c00c9efd5c1075ca26403638" dependencies = [ "chrono", "isolang", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -4871,7 +5768,7 @@ version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "libc", "mio", "parking_lot", @@ -4942,7 +5839,7 @@ version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ - "bytes 1.11.0", + "bytes 1.11.1", "futures-core", "futures-sink", "pin-project-lite", @@ -5022,7 +5919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" dependencies = [ "bitflags", - "bytes 1.11.0", + "bytes 1.11.1", "futures-util", "http 1.4.0", "http-body", @@ -5065,7 +5962,7 @@ checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" dependencies = [ "crossbeam-channel", "thiserror 2.0.17", - "time 0.3.46", + "time 0.3.47", "tracing-subscriber", ] @@ -5408,7 +6305,16 @@ version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] @@ -5469,6 +6375,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -5482,6 +6410,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver 1.0.27", +] + [[package]] name = "web-sys" version = "0.3.82" @@ -5516,9 +6456,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -5932,12 +6872,119 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn 2.0.111", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.111", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver 1.0.27", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xml-rs" version = "0.8.28" @@ -6088,7 +7135,7 @@ dependencies = [ "pbkdf2", "ppmd-rust", "sha1 0.10.6", - "time 0.3.46", + "time 0.3.47", "zeroize", "zopfli", "zstd", diff --git a/Cargo.toml b/Cargo.toml index 2332a01..15a4c14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bichon" -version = "0.3.7" +version = "1.0.0" edition = "2021" [[bin]] @@ -27,8 +27,8 @@ opt-level = 3 codegen-units = 1 [dependencies] -chrono = "0.4.43" -clap = { version = "4.5.56", features = ["derive", "env"] } +chrono = "0.4.44" +clap = { version = "4.5.60", features = ["derive", "env"] } mimalloc = "0.1.48" native_db = "0.8.2" itertools = "0.14.0" @@ -62,35 +62,35 @@ reqwest = { version = "0.12.24", default-features = false, features = [ ] } tokio-socks = "0.5.2" http = "1.4.0" -regex = "1.12.2" +regex = "1.12.3" email_address = "0.2.9" -futures = "0.3.31" +futures = "0.3.32" utf7-imap = "0.3.2" imap-proto = "0.16.6" -mail-parser = { version = '0.11.1', features = ["serde"] } +mail-parser = { version = '0.11.2', features = ["serde"] } # mail-send = "0.5.2" tokio-rustls = { version = "0.26.4", default-features = false, features = [ "ring", "tls12", ] } -timeago = "0.5.0" +timeago = "0.6.0" ahash = "0.8.12" oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] } url = { version = "2.5.8", features = ["serde"] } -sysinfo = "0.38.0" +sysinfo = "0.38.2" num_cpus = "1.17.0" cacache = { version = "13.1.0", default-features = false, features = [ "tokio-runtime", "mmap", ] } -rand = "0.9.2" +rand = "0.10.0" encoding_rs = "0.8.35" -async-imap = { version = "0.11.1", default-features = false, features = [ +async-imap = { version = "0.11.2", default-features = false, features = [ "runtime-tokio", "compress", ] } -webpki-roots = "1.0.5" -rustls = { version = "0.23.36", default-features = false, features = ["ring"] } +webpki-roots = "1.0.6" +rustls = { version = "0.23.37", default-features = false, features = ["ring"] } rustls-pki-types = "1.14.0" tokio-io-timeout = "1.2.1" semver = "1.0.27" @@ -98,7 +98,7 @@ governor = "0.10.4" lru = "0.16.3" mime_guess = "2.0.5" hex = "0.4.3" -time = { version = "0.3.46", features = [ +time = { version = "0.3.47", features = [ "formatting", "parsing", "local-offset", @@ -114,16 +114,27 @@ gethostname = "1.1.0" tantivy = { version = "0.25.0", features = ["quickwit", "zstd-compression"] } itoa = "1.0.17" html2text = "0.16.7" -bytes = "1.11.0" +bytes = "1.11.1" dialoguer = "0.12.0" console = "0.16.2" toml = "0.9.8" -memmap2 = "0.9.9" +memmap2 = "0.9.10" outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" } compressed-rtf = "1.0.0" codepage-strings = "1.0.2" mail-send = "0.5.2" +duckdb = { version = "1.4.4", features = [ + "chrono", + "bundled", + "r2d2", + "appender-arrow", +] } +arrow = { version = "56.2.0", features = ["ffi"] } +r2d2 = { version = "0.8", default-features = false } +refinery = { version = "0.9", default-features = false } +refinery-core = { version = "0.9", default-features = false } + [dev-dependencies] #bincode = "1.3.3" #secret-lib = "1.0.0" -tempfile = "3.24.0" +tempfile = "3.26.0" diff --git a/build.rs b/build.rs index ef46b47..c7711ed 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,9 @@ use std::{io::Result, process::Command}; fn main() -> Result<()> { + if cfg!(target_os = "windows") { + println!("cargo:rustc-link-lib=Rstrtmgr"); + } let output = Command::new("git") .args(&["rev-parse", "--short", "HEAD"]) .output() diff --git a/src/main.rs b/src/main.rs index 1485018..de5ed53 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ use bichon::{ modules::{ common::rustls::RustMailerTls, context::{executors::BichonContext, Initialize}, + duckdb::init::DuckDBManager, error::BichonResult, logger, rest::start_http_server, @@ -69,6 +70,7 @@ async fn initialize() -> BichonResult<()> { // SETTINGS.validate()?; SignalManager::initialize().await?; DataDirManager::initialize().await?; + DuckDBManager::initialize().await?; UserManager::initialize().await?; RustMailerTls::initialize().await?; BichonContext::initialize().await?; diff --git a/src/modules/account/grant.rs b/src/modules/account/grant.rs index 8452d66..9f6b1e4 100644 --- a/src/modules/account/grant.rs +++ b/src/modules/account/grant.rs @@ -58,7 +58,7 @@ impl BatchAccountRoleRequest { } for id in &self.account_ids { - let exists = AccountModel::find(*id).await?; // Assuming an exists helper + let exists = AccountModel::async_find(*id).await?; // Assuming an exists helper if exists.is_none() { return Err(raise_error!( format!("Account ID {} not found", id), diff --git a/src/modules/account/migration.rs b/src/modules/account/migration.rs index 2693cc3..44ad6c8 100644 --- a/src/modules/account/migration.rs +++ b/src/modules/account/migration.rs @@ -33,7 +33,7 @@ use crate::{ state::AccountRunningState, }, cache::imap::mailbox::MailBox, - database::{list_all_impl, with_transaction}, + database::{list_all_impl, secondary_find_impl, with_transaction}, error::BichonResult, indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER}, users::{role::DEFAULT_ACCOUNT_MANAGER_ROLE_ID, UserModel, DEFAULT_ADMIN_USER_ID}, @@ -51,14 +51,14 @@ use crate::modules::database::count_by_unique_secondary_key_impl; use crate::modules::database::delete_impl; use crate::modules::database::manager::DB_MANAGER; use crate::modules::database::{ - paginate_query_primary_scan_all_impl, secondary_find_impl, update_impl, + async_secondary_find_impl, paginate_query_primary_scan_all_impl, update_impl, }; use crate::modules::error::code::ErrorCode; use crate::modules::oauth2::token::OAuth2AccessToken; use crate::modules::rest::response::DataPage; use crate::raise_error; -pub type AccountModel = AccountV3; +pub type AccountModel = AccountV4; #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Enum)] pub enum AccountType { @@ -158,6 +158,42 @@ impl AccountV3 { fn pk(&self) -> String { format!("{}_{}", self.created_at, self.id) } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)] +#[native_model(id = 4, version = 4, from = AccountV3)] +#[native_db(primary_key(pk -> String))] +pub struct AccountV4 { + #[secondary_key(unique)] + pub id: u64, + pub imap: Option, + pub enabled: bool, + #[oai(validator(custom = "crate::modules::common::validator::EmailValidator"))] + pub email: String, + pub name: Option, + pub capabilities: Option>, + pub date_since: Option, + pub date_before: Option, + pub folder_limit: Option, + pub sync_folders: Option>, + pub account_type: AccountType, + pub sync_interval_min: Option, + pub sync_batch_size: Option, + pub known_folders: Option>, + pub created_at: i64, + pub updated_at: i64, + pub created_by: u64, //user id + pub use_proxy: Option, + pub use_dangerous: bool, + pub pgp_key: Option, + pub imap_daily_quota_bytes: Option, + pub auto_sync_new_mailboxes: Option, +} + +impl AccountV4 { + fn pk(&self) -> String { + format!("{}_{}", self.created_at, self.id) + } pub fn new(user_id: u64, request: AccountCreateRequest) -> BichonResult { Ok(Self { @@ -181,25 +217,30 @@ impl AccountV3 { created_by: user_id, sync_batch_size: request.sync_batch_size, date_before: request.date_before, + imap_daily_quota_bytes: request.imap_daily_quota_bytes, + auto_sync_new_mailboxes: request.auto_sync_new_mailboxes, }) } pub async fn check_account_exists(account_id: u64) -> BichonResult { - let account = - secondary_find_impl::(DB_MANAGER.meta_db(), AccountV3Key::id, account_id) - .await? - .ok_or_else(|| { - raise_error!( - format!("Account id='{account_id}' not found"), - ErrorCode::ResourceNotFound - ) - })?; + let account = async_secondary_find_impl::( + DB_MANAGER.meta_db(), + AccountV4Key::id, + account_id, + ) + .await? + .ok_or_else(|| { + raise_error!( + format!("Account id='{account_id}' not found"), + ErrorCode::ResourceNotFound + ) + })?; Ok(account) } /// Fetches an `AccountEntity` by its `id`. - pub async fn get(account_id: u64) -> BichonResult { - let result: AccountModel = Self::find(account_id).await?.ok_or_else(|| { + pub async fn async_get(account_id: u64) -> BichonResult { + let result: AccountModel = Self::async_find(account_id).await?.ok_or_else(|| { raise_error!( format!("Account with ID '{account_id}' not found"), ErrorCode::ResourceNotFound @@ -208,9 +249,27 @@ impl AccountV3 { Ok(result) } - pub async fn find(account_id: u64) -> BichonResult> { - secondary_find_impl::(DB_MANAGER.meta_db(), AccountV3Key::id, account_id) - .await + pub fn get(account_id: u64) -> BichonResult { + let result: AccountModel = Self::find(account_id)?.ok_or_else(|| { + raise_error!( + format!("Account with ID '{account_id}' not found"), + ErrorCode::ResourceNotFound + ) + })?; + Ok(result) + } + + pub async fn async_find(account_id: u64) -> BichonResult> { + async_secondary_find_impl::( + DB_MANAGER.meta_db(), + AccountV4Key::id, + account_id, + ) + .await + } + + pub fn find(account_id: u64) -> BichonResult> { + secondary_find_impl::(DB_MANAGER.meta_db(), AccountV4Key::id, account_id) } pub async fn create_account( @@ -258,7 +317,7 @@ impl AccountV3 { request: AccountUpdateRequest, validate: bool, ) -> BichonResult<()> { - let account = AccountModel::get(account_id).await?; + let account = AccountModel::async_get(account_id).await?; if validate { request.validate_update_request(&account)?; } @@ -273,7 +332,7 @@ impl AccountV3 { } pub async fn delete(account_id: u64) -> BichonResult<()> { - let account = Self::get(account_id).await?; + let account = Self::async_get(account_id).await?; if let Err(error) = Self::cleanup_account_resources_sequential(&account).await { tracing::error!( "[CLEANUP_ACCOUNT_ERROR] Account {}: failed to cleanup resources: {:#?}", @@ -287,7 +346,7 @@ impl AccountV3 { async fn delete_account(account_id: u64) -> BichonResult<()> { delete_impl(DB_MANAGER.meta_db(), move|rw|{ - rw.get().secondary::(AccountV3Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .ok_or_else(||raise_error!(format!("The account entity with id={account_id} that you want to delete was not found."), ErrorCode::ResourceNotFound)) }).await } @@ -317,7 +376,7 @@ impl AccountV3 { sync_folders: Vec, ) -> BichonResult<()> { update_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().secondary::(AccountV3Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .ok_or_else(|| raise_error!(format!("When trying to update account sync_folders, the corresponding record was not found. account_id={}", account_id), ErrorCode::ResourceNotFound)) }, |current|{ let mut updated = current.clone(); @@ -332,7 +391,7 @@ impl AccountV3 { known_folders: BTreeSet, ) -> BichonResult<()> { update_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().secondary::(AccountV3Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .ok_or_else(|| raise_error!(format!("When trying to update account known_folders, the corresponding record was not found. account_id={}", account_id), ErrorCode::ResourceNotFound)) }, |current|{ let mut updated = current.clone(); @@ -347,7 +406,7 @@ impl AccountV3 { capabilities: Vec, ) -> BichonResult<()> { update_impl(DB_MANAGER.meta_db(), move |rw| { - rw.get().secondary::(AccountV3Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + rw.get().secondary::(AccountV4Key::id, account_id).map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? .ok_or_else(|| raise_error!(format!("When trying to update account capabilities, the corresponding record was not found. account_id={}", account_id), ErrorCode::ResourceNotFound)) }, |current|{ let mut updated = current.clone(); @@ -378,7 +437,7 @@ impl AccountV3 { } pub async fn count() -> BichonResult { - count_by_unique_secondary_key_impl::(DB_MANAGER.meta_db(), AccountV3Key::id) + count_by_unique_secondary_key_impl::(DB_MANAGER.meta_db(), AccountV4Key::id) .await } @@ -483,6 +542,13 @@ impl AccountV3 { new.pgp_key = Some(pgp_key); } + if let Some(imap_daily_quota_bytes) = request.imap_daily_quota_bytes { + new.imap_daily_quota_bytes = Some(imap_daily_quota_bytes); + } + + if let Some(auto_sync_new_mailboxes) = request.auto_sync_new_mailboxes { + new.auto_sync_new_mailboxes = Some(auto_sync_new_mailboxes); + } new.updated_at = utc_now!(); Ok(new) } @@ -584,3 +650,59 @@ impl From for AccountV3 { } } } + +impl From for AccountV3 { + fn from(value: AccountV4) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + date_before: value.date_before, + folder_limit: value.folder_limit, + sync_folders: value.sync_folders, + account_type: value.account_type, + sync_interval_min: value.sync_interval_min, + sync_batch_size: value.sync_batch_size, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + created_by: value.created_by, + use_proxy: value.use_proxy, + use_dangerous: value.use_dangerous, + pgp_key: value.pgp_key, + } + } +} + +impl From for AccountV4 { + fn from(value: AccountV3) -> Self { + Self { + id: value.id, + imap: value.imap, + enabled: value.enabled, + email: value.email, + name: value.name, + capabilities: value.capabilities, + date_since: value.date_since, + date_before: value.date_before, + folder_limit: value.folder_limit, + sync_folders: value.sync_folders, + account_type: value.account_type, + sync_interval_min: value.sync_interval_min, + sync_batch_size: value.sync_batch_size, + known_folders: value.known_folders, + created_at: value.created_at, + updated_at: value.updated_at, + created_by: value.created_by, + use_proxy: value.use_proxy, + use_dangerous: value.use_dangerous, + pgp_key: value.pgp_key, + imap_daily_quota_bytes: None, + auto_sync_new_mailboxes: None, + } + } +} diff --git a/src/modules/account/payload.rs b/src/modules/account/payload.rs index 398eb0f..6611db2 100644 --- a/src/modules/account/payload.rs +++ b/src/modules/account/payload.rs @@ -44,6 +44,8 @@ pub struct AccountCreateRequest { pub use_proxy: Option, pub use_dangerous: bool, pub pgp_key: Option, + pub imap_daily_quota_bytes: Option, + pub auto_sync_new_mailboxes: Option, } impl AccountCreateRequest { @@ -155,6 +157,8 @@ pub struct AccountUpdateRequest { pub use_dangerous: Option, pub pgp_key: Option, + pub imap_daily_quota_bytes: Option, + pub auto_sync_new_mailboxes: Option, } impl AccountUpdateRequest { diff --git a/src/modules/account/view.rs b/src/modules/account/view.rs index b967fb1..e875c49 100644 --- a/src/modules/account/view.rs +++ b/src/modules/account/view.rs @@ -54,6 +54,7 @@ pub struct AccountResp { pub use_proxy: Option, pub use_dangerous: bool, pub pgp_key: Option, + pub imap_daily_quota_bytes: Option, } impl AccountResp { @@ -86,6 +87,7 @@ impl AccountResp { use_proxy: account.use_proxy, use_dangerous: account.use_dangerous, pgp_key: account.pgp_key, + imap_daily_quota_bytes: account.imap_daily_quota_bytes, } } } diff --git a/src/modules/cache/imap/mailbox.rs b/src/modules/cache/imap/mailbox.rs index 32bfdfd..bc33a4d 100644 --- a/src/modules/cache/imap/mailbox.rs +++ b/src/modules/cache/imap/mailbox.rs @@ -20,8 +20,9 @@ use crate::{ decode_mailbox_name, encode_mailbox_name, modules::{ database::{ - async_find_impl, batch_delete_impl, batch_insert_impl, batch_upsert_impl, delete_impl, - filter_by_secondary_key_impl, manager::DB_MANAGER, + async_filter_by_secondary_key_impl, async_find_impl, batch_delete_impl, + batch_insert_impl, batch_upsert_impl, delete_impl, filter_by_secondary_key_impl, + manager::DB_MANAGER, }, error::{code::ErrorCode, BichonResult}, }, @@ -71,24 +72,6 @@ impl MailBox { encode_mailbox_name!(&self.name) } - // pub async fn batch_delete(mailboxes: Vec) -> BichonResult<()> { - // batch_delete_impl(DB_MANAGER.envelope_db(), move |rw| { - // let mut to_deleted = Vec::new(); - // for mailbox in mailboxes { - // let retrived = rw - // .get() - // .primary::(mailbox.id) - // .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - // if let Some(retrived) = retrived { - // to_deleted.push(retrived); - // } - // } - // Ok(to_deleted) - // }) - // .await?; - // Ok(()) - // } - pub async fn get(id: u64) -> BichonResult { let result = async_find_impl::(DB_MANAGER.envelope_db(), id).await?; Ok(result.ok_or_else(|| { @@ -110,8 +93,21 @@ impl MailBox { } pub async fn list_all(account_id: u64) -> BichonResult> { - filter_by_secondary_key_impl(DB_MANAGER.envelope_db(), MailBoxKey::account_id, account_id) - .await + async_filter_by_secondary_key_impl( + DB_MANAGER.envelope_db(), + MailBoxKey::account_id, + account_id, + ) + .await + } + + pub fn find_mailbox(account_id: u64, mailbox_id: u64) -> BichonResult> { + let all: Vec = filter_by_secondary_key_impl( + DB_MANAGER.envelope_db(), + MailBoxKey::account_id, + account_id, + )?; + Ok(all.into_iter().find(|m| m.id == mailbox_id)) } pub async fn batch_insert(mailboxes: &[MailBox]) -> BichonResult<()> { diff --git a/src/modules/cache/imap/sync/sync_folders.rs b/src/modules/cache/imap/sync/sync_folders.rs index b4fa28e..9a02c8f 100644 --- a/src/modules/cache/imap/sync/sync_folders.rs +++ b/src/modules/cache/imap/sync/sync_folders.rs @@ -62,7 +62,7 @@ pub async fn get_sync_folders( mailboxes.iter().map(|(m, _)| m.name.clone()).collect(), ) .await?; - let account = AccountModel::get(account.id).await?; + let account = AccountModel::async_get(account.id).await?; let subscribed = &account.sync_folders.unwrap_or_default(); let is_noselect = |mailbox: &MailBox| { mailbox diff --git a/src/modules/cache/imap/task.rs b/src/modules/cache/imap/task.rs index a30fae6..ab50bdd 100644 --- a/src/modules/cache/imap/task.rs +++ b/src/modules/cache/imap/task.rs @@ -54,7 +54,7 @@ impl AccountSyncTask { let task = move |param: Option| { let account_id = param.unwrap(); Box::pin(async move { - let account = AccountModel::get(account_id).await.ok(); + let account = AccountModel::async_get(account_id).await.ok(); match account { Some(account) => { if !account.enabled { diff --git a/src/modules/common/signal.rs b/src/modules/common/signal.rs index b68cb43..37cf79a 100644 --- a/src/modules/common/signal.rs +++ b/src/modules/common/signal.rs @@ -16,12 +16,9 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use std::sync::LazyLock; -use crate::modules::{ - context::Initialize, error::BichonResult, utils::shutdown::shutdown_signal, -}; +use crate::modules::{context::Initialize, error::BichonResult, utils::shutdown::shutdown_signal}; use tokio::sync::broadcast; pub static SIGNAL_MANAGER: LazyLock = LazyLock::new(SignalManager::new); diff --git a/src/modules/dashboard/mod.rs b/src/modules/dashboard/mod.rs index 63f9996..6d4780e 100644 --- a/src/modules/dashboard/mod.rs +++ b/src/modules/dashboard/mod.rs @@ -20,7 +20,6 @@ use crate::modules::users::permissions::Permission; use poem_openapi::Object; use serde::{Deserialize, Serialize}; use std::collections::HashSet; -use tantivy::{schema::Value, TantivyDocument}; use crate::{ bichon_version, @@ -28,7 +27,7 @@ use crate::{ account::migration::AccountModel, common::auth::ClientContext, error::{code::ErrorCode, BichonResult}, - indexer::{manager::ENVELOPE_INDEX_MANAGER, schema::SchemaTools}, + indexer::manager::ENVELOPE_INDEX_MANAGER, settings::dir::DATA_DIR_MANAGER, utils::get_total_size, }, @@ -65,11 +64,11 @@ impl DashboardStats { }; let mut stat = ENVELOPE_INDEX_MANAGER - .get_dashboard_stats(&authorized_ids) + .get_dashboard_stats(authorized_ids.clone()) .await?; stat.top_largest_emails = ENVELOPE_INDEX_MANAGER - .top_10_largest_emails(&authorized_ids) + .top_10_largest_emails(authorized_ids.clone()) .await?; stat.account_count = if has_all_accounts { @@ -78,7 +77,7 @@ impl DashboardStats { authorized_ids.as_ref().map(|ids| ids.len()).unwrap_or(0) }; - stat.email_count = ENVELOPE_INDEX_MANAGER.total_emails(&authorized_ids)?; + stat.email_count = ENVELOPE_INDEX_MANAGER.total_emails(authorized_ids).await?; if has_all_accounts { stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.eml_dir) @@ -114,33 +113,5 @@ pub struct Group { pub struct LargestEmail { pub subject: String, // Email subject pub size_bytes: u64, // Email size in bytes -} - -impl LargestEmail { - pub fn from_tantivy_doc(document: &TantivyDocument) -> BichonResult { - let fields = SchemaTools::envelope_fields(); - let value = document.get_first(fields.f_size).ok_or_else(|| { - raise_error!( - "miss 'size' field in tantivy document".into(), - ErrorCode::InternalError - ) - })?; - let size_bytes = value.as_u64().ok_or_else(|| { - raise_error!("'size' field is not a u64".into(), ErrorCode::InternalError) - })?; - let value = document.get_first(fields.f_subject).ok_or_else(|| { - raise_error!("'subject' field not found".into(), ErrorCode::InternalError) - })?; - let subject = value.as_str().map(|s| s.to_string()).ok_or_else(|| { - raise_error!( - "'subject' field is not a string".into(), - ErrorCode::InternalError - ) - })?; - let envelope = LargestEmail { - subject, - size_bytes, - }; - Ok(envelope) - } + pub id: u64, } diff --git a/src/modules/database/mod.rs b/src/modules/database/mod.rs index 02e2ed0..ddbfbaf 100644 --- a/src/modules/database/mod.rs +++ b/src/modules/database/mod.rs @@ -16,7 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use crate::modules::account::migration::{AccountV1, AccountV2, AccountV3}; +use crate::modules::account::migration::{AccountV1, AccountV2, AccountV3, AccountV4}; use crate::modules::autoconfig::CachedMailSettings; use crate::modules::error::code::ErrorCode; use crate::modules::error::BichonResult; @@ -67,6 +67,7 @@ impl ModelsAdapter { self.register_model::(); self.register_model::(); self.register_model::(); + self.register_model::(); self.register_model::(); self.register_model::(); self.register_model::(); @@ -188,30 +189,6 @@ pub async fn update_impl( .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? } -// pub async fn batch_update_impl( -// database: &Arc>, -// filter: impl FnOnce(&RwTransaction) -> RustMailerResult> + Send + 'static, -// updated: impl FnOnce(&Vec) -> RustMailerResult> + Send + 'static, -// ) -> RustMailerResult> { -// let db = database.clone(); -// tokio::task::spawn_blocking(move || { -// let rw = db -// .rw_transaction() -// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; -// let targets = filter(&rw)?; -// let tuples = updated(&targets)?; -// for (old, updated) in tuples { -// rw.update(old, updated) -// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; -// } -// rw.commit() -// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; -// Ok(targets) -// }) -// .await -// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? -// } - pub async fn async_find_impl( database: &Arc>, key: impl ToKey + Send + 'static, @@ -231,21 +208,6 @@ pub async fn async_find_impl( .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? } -// pub fn find_impl( -// database: &Arc>, -// key: &str, -// ) -> BichonResult> { -// let db = database.clone(); -// let r_transaction = db -// .r_transaction() -// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; -// let entity: Option = r_transaction -// .get() -// .primary(key) -// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; -// Ok(entity) -// } - pub async fn delete_impl( database: &Arc>, delete: impl FnOnce(&RwTransaction) -> BichonResult + Send + 'static, @@ -423,7 +385,7 @@ pub async fn paginate_query_primary_scan_all_impl< .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? } -pub async fn filter_by_secondary_key_impl( +pub async fn async_filter_by_secondary_key_impl( database: &Arc>, key_def: impl ToKeyDefinition + Send + 'static, start_with: impl ToKey + Send + 'static, @@ -447,6 +409,26 @@ pub async fn filter_by_secondary_key_impl( .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? } +pub fn filter_by_secondary_key_impl( + database: &Arc>, + key_def: impl ToKeyDefinition + Send + 'static, + start_with: impl ToKey + Send + 'static, +) -> BichonResult> { + let db = database.clone(); + let r_transaction = db + .r_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let entities: Vec = r_transaction + .scan() + .secondary(key_def) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .start_with(start_with) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .try_collect() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(entities) +} + pub async fn count_by_unique_secondary_key_impl( database: &Arc>, key_def: impl ToKeyDefinition + Send + 'static, @@ -469,7 +451,7 @@ pub async fn count_by_unique_secondary_key_impl( +pub async fn async_secondary_find_impl( database: &Arc>, key_def: impl ToKeyDefinition + Send + 'static, key: impl ToKey + Send + 'static, @@ -491,6 +473,22 @@ pub async fn secondary_find_impl( .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? } +pub fn secondary_find_impl( + database: &Arc>, + key_def: impl ToKeyDefinition + Send + 'static, + key: impl ToKey + Send + 'static, +) -> BichonResult> { + let db = database.clone(); + let r_transaction = db + .r_transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let entities: Option = r_transaction + .get() + .secondary(key_def, key) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(entities) +} + #[derive(Debug)] pub struct Paginated { pub page: Option, diff --git a/src/modules/duckdb/build.rs b/src/modules/duckdb/build.rs new file mode 100644 index 0000000..8d2fcc7 --- /dev/null +++ b/src/modules/duckdb/build.rs @@ -0,0 +1,240 @@ +use arrow::array::{BooleanArray, Int32Array, Int64Array, ListBuilder, StringBuilder, UInt64Array}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use std::sync::Arc; + +use crate::modules::indexer::envelope::Envelope; + +pub const DEFAULT_SHARD_ID: u64 = 0; + +pub fn build_record_batch(items: &[Envelope]) -> RecordBatch { + let capacity = items.len(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("account_id", DataType::UInt64, false), + Field::new("mailbox_id", DataType::UInt64, false), + Field::new("uid", DataType::UInt64, false), + Field::new("subject", DataType::Utf8, true), + Field::new("body", DataType::Utf8, true), + Field::new("sender", DataType::Utf8, true), + Field::new( + "recipients", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + Field::new( + "cc", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + Field::new( + "bcc", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + Field::new("sent_at", DataType::Int64, true), + Field::new("received_at", DataType::Int64, true), + Field::new("size_bytes", DataType::UInt64, true), + Field::new("thread_id", DataType::UInt64, true), + Field::new("message_id", DataType::Utf8, true), + Field::new("has_attachment", DataType::Boolean, false), + Field::new("attachment_count", DataType::Int32, false), + Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + Field::new("shard_id", DataType::UInt64, false), + ])); + + let mut id_b = UInt64Array::builder(capacity); + let mut account_id_b = UInt64Array::builder(capacity); + let mut mailbox_id_b = UInt64Array::builder(capacity); + let mut uid_b = UInt64Array::builder(capacity); + let mut subject_b = StringBuilder::with_capacity(capacity, capacity * 20); + let mut body_b = StringBuilder::with_capacity(capacity, capacity * 100); + let mut from_b = StringBuilder::with_capacity(capacity, capacity * 20); + + let mut to_b = ListBuilder::new(StringBuilder::new()); + let mut cc_b = ListBuilder::new(StringBuilder::new()); + let mut bcc_b = ListBuilder::new(StringBuilder::new()); + + let mut date_b = Int64Array::builder(capacity); + let mut internal_date_b = Int64Array::builder(capacity); + let mut size_b = UInt64Array::builder(capacity); + let mut thread_id_b = UInt64Array::builder(capacity); + let mut msg_id_b = StringBuilder::with_capacity(capacity, capacity * 30); + let mut has_att_b = BooleanArray::builder(capacity); + let mut att_count_b = Int32Array::builder(capacity); + let mut tags_b = ListBuilder::new(StringBuilder::new()); + let mut shard_id_b = UInt64Array::builder(capacity); + + for e in items { + id_b.append_value(e.id); + account_id_b.append_value(e.account_id); + mailbox_id_b.append_value(e.mailbox_id); + uid_b.append_value(e.uid as u64); + subject_b.append_value(&e.subject); + body_b.append_value(&e.text); + from_b.append_value(&e.from); + + for addr in &e.to { + to_b.values().append_value(addr); + } + to_b.append(true); + + for addr in &e.cc { + cc_b.values().append_value(addr); + } + cc_b.append(true); + + for addr in &e.bcc { + bcc_b.values().append_value(addr); + } + bcc_b.append(true); + + date_b.append_value(e.date); + internal_date_b.append_value(e.internal_date); + size_b.append_value(e.size as u64); + thread_id_b.append_value(e.thread_id); + msg_id_b.append_value(&e.message_id); + + has_att_b.append_value(e.attachment_count > 0); + att_count_b.append_value(e.attachment_count as i32); + tags_b.append(true); + shard_id_b.append_value(DEFAULT_SHARD_ID); + } + + RecordBatch::try_new( + schema, + vec![ + Arc::new(id_b.finish()), + Arc::new(account_id_b.finish()), + Arc::new(mailbox_id_b.finish()), + Arc::new(uid_b.finish()), + Arc::new(subject_b.finish()), + Arc::new(body_b.finish()), + Arc::new(from_b.finish()), + Arc::new(to_b.finish()), + Arc::new(cc_b.finish()), + Arc::new(bcc_b.finish()), + Arc::new(date_b.finish()), + Arc::new(internal_date_b.finish()), + Arc::new(size_b.finish()), + Arc::new(thread_id_b.finish()), + Arc::new(msg_id_b.finish()), + Arc::new(has_att_b.finish()), + Arc::new(att_count_b.finish()), + Arc::new(tags_b.finish()), + Arc::new(shard_id_b.finish()), + ], + ) + .expect("Failed to build RecordBatch") +} + +#[cfg(test)] +mod integration_tests { + use super::*; + use duckdb::{Connection, Result}; + + #[test] + fn test_envelope_ingestion_and_query() -> Result<()> { + let conn = Connection::open_in_memory()?; + conn.execute_batch( + r#" + CREATE TABLE IF NOT EXISTS envelopes ( + -- internal id (tantivy f_id) + id UBIGINT NOT NULL, + + -- account / mailbox / uid + account_id UBIGINT NOT NULL, + mailbox_id UBIGINT NOT NULL, + uid UBIGINT NOT NULL, + + -- headers / content + subject TEXT, + body TEXT, + + sender TEXT, + recipients VARCHAR[], + cc VARCHAR[], + bcc VARCHAR[], + + -- dates + sent_at BIGINT, + received_at BIGINT, + + -- size + size_bytes UBIGINT, + + -- thread + thread_id UBIGINT, + + -- message-id + message_id TEXT, + + -- attachment summary + has_attachment BOOLEAN NOT NULL, + attachment_count INTEGER NOT NULL CHECK (attachment_count >= 0), + tags VARCHAR[], + shard_id UBIGINT NOT NULL + ); + "#, + )?; + + let items = vec![Envelope { + id: 101, + account_id: 1, + mailbox_id: 1, + uid: 50, + subject: "Testing Arrow".to_string(), + text: "Content".to_string(), + from: "sender@test.com".to_string(), + to: vec!["user1@test.com".to_string(), "user2@test.com".to_string()], + cc: vec!["manager@test.com".to_string()], + bcc: vec![], + date: 1000, + internal_date: 1001, + size: 2048, + thread_id: 1, + message_id: "id123".to_string(), + attachment_count: 1, + tags: None, + account_email: None, + mailbox_name: None, + }]; + + { + let batch = build_record_batch(&items); + let mut appender = conn.appender("envelopes")?; + appender.append_record_batch(batch)?; + appender.flush()?; + } + + let mut stmt = conn.prepare("SELECT subject, size_bytes FROM envelopes WHERE id = 101")?; + let mut rows = stmt.query([])?; + if let Some(row) = rows.next()? { + let subject: String = row.get(0)?; + let size: u64 = row.get(1)?; + assert_eq!(subject, "Testing Arrow"); + assert_eq!(size, 2048); + } + + let mut stmt = conn.prepare( + "SELECT count(*) FROM envelopes WHERE list_contains(\"recipients\", 'user2@test.com')", + )?; + let count: i64 = stmt.query_row([], |r| r.get(0))?; + assert_eq!(count, 1); + + let has_att: bool = conn.query_row( + "SELECT has_attachment FROM envelopes WHERE id = 101", + [], + |r| r.get(0), + )?; + assert!(has_att); + + println!("Integration test for ingestion and query passed!"); + Ok(()) + } +} diff --git a/src/modules/duckdb/init.rs b/src/modules/duckdb/init.rs new file mode 100644 index 0000000..a56add2 --- /dev/null +++ b/src/modules/duckdb/init.rs @@ -0,0 +1,1302 @@ +use chrono::{NaiveDateTime, Utc}; +use duckdb::{params, types::Value, DuckdbConnectionManager}; +use refinery::Runner; +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + sync::OnceLock, +}; + +use crate::{ + modules::{ + account::migration::AccountModel, + context::Initialize, + dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, + duckdb::{build::build_record_batch, refinery::DuckDBConnection}, + error::{code::ErrorCode, BichonResult}, + indexer::{ + envelope::Envelope, + manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER}, + }, + message::{ + content::AttachmentInfo, + search::{SearchFilter, SortBy}, + tags::TagCount, + }, + rest::response::DataPage, + settings::{cli::SETTINGS, dir::DATA_DIR_MANAGER}, + }, + raise_error, +}; + +pub type DuckDBConn = r2d2::PooledConnection; + +pub static DUCKDBMANAGER: OnceLock = OnceLock::new(); + +pub mod duckdb_tables { + refinery::embed_migrations!("src/modules/duckdb/migrations"); +} + +pub fn duckdb() -> BichonResult<&'static DuckDBManager> { + DUCKDBMANAGER.get().ok_or_else(|| { + raise_error!( + "DuckDB manager is not initialized".into(), + ErrorCode::InternalError + ) + }) +} + +fn debug_sql(sql: &str, args: &[Value]) -> String { + let mut result = String::new(); + let mut parts = sql.split('?'); + + for (i, part) in parts.by_ref().enumerate() { + result.push_str(part); + + if i < args.len() { + result.push_str(&format!("{:?}", args[i])); + } + } + + result +} + +pub struct DuckDBManager { + pool: r2d2::Pool, +} + +impl Initialize for DuckDBManager { + async fn initialize() -> BichonResult<()> { + tracing::debug!("Initializing databases"); + + if !&DATA_DIR_MANAGER.envelope_dir.exists() { + std::fs::create_dir_all(&DATA_DIR_MANAGER.envelope_dir) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + + let pool = init( + &DATA_DIR_MANAGER.envelope_dir.join("envelopes.db"), + duckdb_tables::migrations::runner(), + )?; + let _ = DUCKDBMANAGER.set(DuckDBManager { pool }); + Ok(()) + } +} + +impl DuckDBManager { + pub fn conn(&self) -> BichonResult { + Ok(self + .pool + .get() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?) + } + + pub fn shutdown(&self) -> BichonResult<()> { + self.conn()? + .execute("FORCE CHECKPOINT", []) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + tracing::info!("Shutting down"); + Ok(()) + } + + pub fn delete_envelopes_by_account(&self, account_id: u64) -> BichonResult { + let mut conn = self.conn()?; + let tx = conn + .transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + tx.execute( + "DELETE FROM envelope_attachments WHERE account_id = ?;", + params![account_id], + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let rows_deleted = tx + .execute( + "DELETE FROM envelopes WHERE account_id = ?;", + params![account_id], + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + tx.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if rows_deleted > 0 { + tracing::info!( + "Account cleanup: removed {} emails and their attachments for account {}", + rows_deleted, + account_id + ); + } + + Ok(rows_deleted) + } + + pub fn delete_mailbox_envelopes( + &self, + account_id: u64, + mailbox_ids: Vec, + ) -> BichonResult<()> { + if mailbox_ids.is_empty() { + return Ok(()); + } + let mut conn = self.conn()?; + let tx = conn + .transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + { + let placeholders = mailbox_ids + .iter() + .map(|_| "?") + .collect::>() + .join(", "); + + let mut sql_params: Vec = vec![account_id.into()]; + sql_params.extend(mailbox_ids.iter().map(|&id| duckdb::types::Value::from(id))); + let param_iter = duckdb::params_from_iter(sql_params); + + let delete_att_sql = format!( + "DELETE FROM envelope_attachments WHERE account_id = ? AND mailbox_id IN ({})", + placeholders + ); + tx.execute(&delete_att_sql, param_iter.clone()) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let delete_env_sql = format!( + "DELETE FROM envelopes WHERE account_id = ? AND mailbox_id IN ({})", + placeholders + ); + let count = tx + .execute(&delete_env_sql, param_iter) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if count > 0 { + tracing::info!( + "Deleted {} emails and their associated attachments for account: {}, mailboxes: {:?}", + count, + account_id, + mailbox_ids + ); + } + } + + tx.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(()) + } + + pub fn append_envelopes_with_attachments( + &self, + items: &[(Envelope, Vec)], + ) -> BichonResult<()> { + let mut conn = self.conn()?; + let tx = conn + .transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + { + let mut env_appender = tx + .appender("envelopes") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut att_appender = tx + .appender("envelope_attachments") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + for (env, atts) in items { + env_appender + .append_record_batch(build_record_batch(std::slice::from_ref(env))) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + for att in atts { + att_appender + .append_row(params![ + env.id, + env.account_id, + env.mailbox_id, + att.filename, + att.get_extension(), + att.get_category(), + att.file_type, + att.size as u64, + 0, + ]) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + } + env_appender + .flush() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + att_appender + .flush() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + tx.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(()) + } + + pub fn total_emails( + &self, + accounts: Option>, + ) -> BichonResult { + let conn = self.conn()?; + let total: u64 = match accounts { + None => conn + .query_row("SELECT COUNT(*) FROM envelopes", [], |row| row.get(0)) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?, + Some(accts) if !accts.is_empty() => { + let placeholders = accts.iter().map(|_| "?").collect::>().join(", "); + let sql = format!( + "SELECT COUNT(*) FROM envelopes WHERE account_id IN ({})", + placeholders + ); + + conn.query_row(&sql, duckdb::params_from_iter(accts), |row| row.get(0)) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + } + _ => 0, + }; + + Ok(total) + } + + pub fn num_messages_in_thread(&self, account_id: u64, thread_id: u64) -> BichonResult { + let conn = self.conn()?; + let count: u64 = conn + .query_row( + "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND thread_id = ?;", + params![account_id, thread_id], + |row| row.get(0), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(count) + } + + pub fn num_messages_in_mailbox(&self, account_id: u64, mailbox_id: u64) -> BichonResult { + let conn = self.conn()?; + let count: u64 = conn + .query_row( + "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND mailbox_id = ?;", + params![account_id, mailbox_id], + |row| row.get(0), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(count) + } + + pub fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { + let conn = self.conn()?; + let max_uid: Option = conn + .query_row( + "SELECT MAX(uid) FROM envelopes WHERE account_id = ? AND mailbox_id = ?;", + params![account_id, mailbox_id], + |row| row.get(0), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(max_uid) + } + + pub fn get_envelope_by_id( + &self, + account_id: u64, + envelope_id: u64, + ) -> BichonResult> { + let conn = self.conn()?; + let mut stmt = conn + .prepare("SELECT * FROM envelopes WHERE account_id = ? AND id = ? LIMIT 1;") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let envelope_res = stmt.query_row(params![account_id, envelope_id], |row| { + Envelope::from_row(row) + }); + + match envelope_res { + Ok(env) => Ok(Some(env)), + Err(duckdb::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError)), + } + } + + pub fn get_all_tags(&self, accounts: Option>) -> BichonResult> { + let conn = self.conn()?; + let mut sql = " + SELECT t.tag, COUNT(*) as count + FROM ( + SELECT unnest(tags) as tag, account_id + FROM envelopes + WHERE len(tags) > 0 + ) t + " + .to_string(); + + let mut params_vec: Vec = Vec::new(); + if let Some(ref acc_set) = accounts { + if !acc_set.is_empty() { + let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", "); + sql.push_str(&format!(" WHERE t.account_id IN ({})", placeholders)); + for id in acc_set { + params_vec.push((*id).into()); + } + } + } + + sql.push_str(" GROUP BY t.tag ORDER BY count DESC, t.tag ASC;"); + + let mut stmt = conn + .prepare(&sql) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let tag_counts = stmt + .query_map(duckdb::params_from_iter(params_vec), |row| { + Ok(TagCount { + tag: row.get(0)?, + count: row.get(1)?, + }) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .collect::, _>>() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(tag_counts) + } + + pub fn get_all_contacts( + &self, + accounts: Option>, + ) -> BichonResult> { + let conn = self.conn()?; + let mut sql = r#" + SELECT DISTINCT LOWER(contact) AS contact + FROM ( + SELECT sender AS contact, account_id + FROM envelopes + WHERE sender IS NOT NULL AND sender != '' + + UNION ALL + + SELECT unnest(recipients) AS contact, account_id + FROM envelopes + WHERE length(recipients) > 0 + + UNION ALL + + SELECT unnest(cc) AS contact, account_id + FROM envelopes + WHERE length(cc) > 0 + + UNION ALL + + SELECT unnest(bcc) AS contact, account_id + FROM envelopes + WHERE length(bcc) > 0 + ) t + WHERE contact IS NOT NULL AND contact != '' + "# + .to_string(); + + let mut params_vec: Vec = Vec::new(); + + if let Some(ref acc_set) = accounts { + if !acc_set.is_empty() { + let placeholders = vec!["?"; acc_set.len()].join(", "); + sql.push_str(&format!(" AND account_id IN ({})", placeholders)); + + for &id in acc_set { + params_vec.push(id.into()); + } + } + } + + sql.push_str(" ORDER BY contact;"); + + let mut stmt = conn + .prepare(&sql) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let contacts: HashSet = stmt + .query_map(duckdb::params_from_iter(params_vec), |row| { + let email: String = row.get(0)?; + Ok(email) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .collect::, _>>() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(contacts) + } + + pub fn update_envelope_tags( + &self, + updates: HashMap>, // account_id -> [envelope_id1, ...] + tags: Vec, + ) -> BichonResult<()> { + let mut conn = self.conn()?; + let tags_json = serde_json::to_string(&tags) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let tx = conn + .transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + for (account_id, ids) in updates { + if ids.is_empty() { + continue; + } + let ids_str = ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(","); + let query = format!( + "UPDATE envelopes + SET tags = CAST(json(?) AS VARCHAR[]) + WHERE account_id = ? + AND id IN ({})", + ids_str + ); + tx.execute(&query, params![tags_json, account_id]) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + tx.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(()) + } + + pub fn delete_envelopes_multi_account( + &self, + deletes: HashMap>, // account_id -> [envelope_id1, ...] + ) -> BichonResult<()> { + let mut conn = self.conn()?; + let tx = conn + .transaction() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + for (account_id, ids) in deletes { + if ids.is_empty() { + continue; + } + for chunk in ids.chunks(100) { + let ids_str = chunk + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(","); + let query = format!( + "DELETE FROM envelopes + WHERE account_id = ? + AND id IN ({})", + ids_str + ); + tx.execute(&query, params![account_id]) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + } + tx.commit() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(()) + } + + pub fn top_10_largest_emails( + &self, + accounts: Option>, + ) -> BichonResult> { + let conn = self.conn()?; + let mut sql = r#" + SELECT id, subject, size_bytes + FROM envelopes + "# + .to_string(); + + let mut params_vec: Vec = Vec::new(); + if let Some(acc_set) = accounts { + if !acc_set.is_empty() { + let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", "); + sql.push_str(&format!(" WHERE account_id IN ({})", placeholders)); + + for id in acc_set { + params_vec.push(id.into()); + } + } + } + + sql.push_str(" ORDER BY size_bytes DESC LIMIT 10;"); + let mut stmt = conn + .prepare(&sql) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let results = stmt + .query_map(duckdb::params_from_iter(params_vec), |row| { + Ok(LargestEmail { + id: row.get(0)?, + subject: row.get(1)?, + size_bytes: row.get(2)?, + }) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .collect::, _>>() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(results) + } + + pub fn get_dashboard_stats( + &self, + accounts: Option>, + ) -> BichonResult { + let conn = self.conn()?; + + let mut account_filter = String::new(); + let mut params: Vec = Vec::new(); + + if let Some(acc_set) = accounts { + if !acc_set.is_empty() { + let placeholders = acc_set.iter().map(|_| "?").collect::>().join(", "); + account_filter = format!("WHERE account_id IN ({})", placeholders); + for id in acc_set { + params.push(id.into()); + } + } + } + + let mut stats = DashboardStats::default(); + + { + let sql = format!( + r#" + SELECT COALESCE(SUM(size_bytes), 0) AS total_size + FROM envelopes + {account_filter} + "# + ); + + let mut stmt = conn.prepare(&sql).map_err(|e| { + raise_error!( + format!("Prepare total_size failed: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut rows = stmt + .query(duckdb::params_from_iter(params.clone())) + .map_err(|e| { + raise_error!( + format!("Failed to execute total_size query: {}", e), + ErrorCode::InternalError + ) + })?; + + if let Some(row) = rows.next().map_err(|e| { + raise_error!( + format!("Failed to fetch row from total_size query: {}", e), + ErrorCode::InternalError + ) + })? { + stats.total_size_bytes = row + .get(0) + .map_err(|e| { + raise_error!( + format!("Failed to read total_size value: {}", e), + ErrorCode::InternalError + ) + }) + .unwrap_or(0); + } + } + + { + let thirty_days_ago = Utc::now() - chrono::Duration::days(30); + let thirty_days_ago_ms = thirty_days_ago.timestamp_millis(); + + let sql = format!( + r#" + SELECT + date_trunc('day', to_timestamp(sent_at / 1000)) AS day, + COUNT(*) AS cnt + FROM envelopes + {account_filter} + WHERE sent_at >= ? + GROUP BY day + ORDER BY day ASC + "# + ); + + let mut stmt = conn.prepare(&sql).map_err(|e| { + raise_error!( + format!("Prepare recent_activity failed: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut query_params = params.clone(); + query_params.push(thirty_days_ago_ms.into()); + + let mut rows = stmt + .query(duckdb::params_from_iter(query_params)) + .map_err(|e| { + raise_error!( + format!("Failed to execute recent_activity query: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut recent_activity = Vec::new(); + + while let Some(row) = rows.next().map_err(|e| { + raise_error!( + format!("Failed to fetch row from recent_activity query: {}", e), + ErrorCode::InternalError + ) + })? { + let day: NaiveDateTime = row.get(0).map_err(|e| { + raise_error!( + format!("Failed to read recent_activity value: {}", e), + ErrorCode::InternalError + ) + })?; + let timestamp_ms = day.and_utc().timestamp_millis(); + + recent_activity.push(TimeBucket { + timestamp_ms, + count: row.get(1).map_err(|e| { + raise_error!( + format!("Failed to read recent_activity value: {}", e), + ErrorCode::InternalError + ) + })?, + }); + } + + stats.recent_activity = recent_activity; + } + + { + let sql = format!( + r#" + SELECT sender, COUNT(*) AS cnt + FROM envelopes + {account_filter} + WHERE sender IS NOT NULL AND sender != '' + GROUP BY sender + ORDER BY cnt DESC, sender ASC + LIMIT 10 + "# + ); + + let mut stmt = conn.prepare(&sql).map_err(|e| { + raise_error!( + format!("Prepare top_senders failed: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut rows = stmt + .query(duckdb::params_from_iter(params.clone())) + .map_err(|e| { + raise_error!( + format!("Failed to execute top_senders query: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut top_senders = Vec::new(); + + while let Some(row) = rows.next().map_err(|e| { + raise_error!( + format!("Failed to fetch row from top_senders query: {}", e), + ErrorCode::InternalError + ) + })? { + top_senders.push(Group { + key: row.get(0).map_err(|e| { + raise_error!( + format!("Failed to read top_senders value: {}", e), + ErrorCode::InternalError + ) + })?, + count: row.get(1).map_err(|e| { + raise_error!( + format!("Failed to read top_senders value: {}", e), + ErrorCode::InternalError + ) + })?, + }); + } + + stats.top_senders = top_senders; + } + { + let sql = format!( + r#" + SELECT account_id, COUNT(*) AS cnt + FROM envelopes + {account_filter} + GROUP BY account_id + ORDER BY cnt DESC + LIMIT 10 + "# + ); + + let mut stmt = conn.prepare(&sql).map_err(|e| { + raise_error!( + format!("Prepare top_accounts failed: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut rows = stmt + .query(duckdb::params_from_iter(params.clone())) + .map_err(|e| { + raise_error!( + format!("Failed to execute top_accounts query: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut top_accounts = Vec::new(); + + while let Some(row) = rows.next().map_err(|e| { + raise_error!( + format!("Failed to fetch row from top_accounts query: {}", e), + ErrorCode::InternalError + ) + })? { + let account_id: u64 = row.get(0).map_err(|e| { + raise_error!( + format!("Failed to read top_accounts value: {}", e), + ErrorCode::InternalError + ) + })?; + let count: u64 = row.get(1).map_err(|e| { + raise_error!( + format!("Failed to read top_accounts value: {}", e), + ErrorCode::InternalError + ) + })?; + + if let Ok(account) = AccountModel::get(account_id) { + top_accounts.push(Group { + key: account.email, + count, + }); + } else { + tracing::warn!(account_id, "account not found in top accounts query"); + tokio::spawn(async move { + if let Err(e) = ENVELOPE_INDEX_MANAGER + .delete_account_envelopes(account_id) + .await + { + tracing::error!( + account_id = account_id, + error = %e, + "failed to cleanup envelope index" + ); + } + if let Err(e) = EML_INDEX_MANAGER.delete_account_envelopes(account_id).await + { + tracing::error!( + account_id = account_id, + error = %e, + "failed to cleanup eml index" + ); + } + }); + } + } + + stats.top_accounts = top_accounts; + } + + { + let sql = format!( + r#" + SELECT has_attachment, COUNT(*) AS cnt + FROM envelopes + {account_filter} + GROUP BY has_attachment + "# + ); + + let mut stmt = conn.prepare(&sql).map_err(|e| { + raise_error!( + format!("Prepare attachment_stats failed: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut rows = stmt.query(duckdb::params_from_iter(params)).map_err(|e| { + raise_error!( + format!("Failed to execute attachment_stats query: {}", e), + ErrorCode::InternalError + ) + })?; + + while let Some(row) = rows.next().map_err(|e| { + raise_error!( + format!("Failed to fetch row from attachment_stats query: {}", e), + ErrorCode::InternalError + ) + })? { + let has_attachment: bool = row.get(0).map_err(|e| { + raise_error!( + format!("Failed to read attachment_stats value: {}", e), + ErrorCode::InternalError + ) + })?; + let cnt: u64 = row.get(1).map_err(|e| { + raise_error!( + format!("Failed to read attachment_stats value: {}", e), + ErrorCode::InternalError + ) + })?; + + if has_attachment { + stats.with_attachment_count = cnt; + } else { + stats.without_attachment_count = cnt; + } + } + } + + Ok(stats) + } + + pub fn list_mailbox_envelopes( + &self, + account_id: u64, + mailbox_id: u64, + page: u64, + page_size: u64, + desc: bool, + ) -> BichonResult> { + let conn = self.conn()?; + let current_page = page.max(1); + + let total_items: u64 = conn + .query_row( + "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND mailbox_id = ?", + params![account_id, mailbox_id], + |row| row.get(0), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let total_pages = if total_items == 0 { + 0 + } else { + (total_items + page_size - 1) / page_size + }; + + let offset = (current_page - 1) * page_size; + let order_direction = if desc { "DESC" } else { "ASC" }; + let query = format!( + "SELECT * FROM envelopes + WHERE account_id = ? AND mailbox_id = ? + ORDER BY sent_at {} + LIMIT ? OFFSET ?", + order_direction + ); + + let mut stmt = conn + .prepare(&query) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let items = stmt + .query_map(params![account_id, mailbox_id, page_size, offset], |row| { + Envelope::from_row(row) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .collect::, _>>() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(DataPage { + current_page: Some(current_page), + page_size: Some(page_size), + total_items, + items, + total_pages: Some(total_pages), + }) + } + + pub fn list_thread_envelopes( + &self, + account_id: u64, + thread_id: u64, + page: u64, + page_size: u64, + desc: bool, + ) -> BichonResult> { + let conn = self.conn()?; + let current_page = page.max(1); + + let total_items: u64 = conn + .query_row( + "SELECT COUNT(*) FROM envelopes WHERE account_id = ? AND thread_id = ?", + params![account_id, thread_id], + |row| row.get(0), + ) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let total_pages = if total_items == 0 { + 0 + } else { + (total_items + page_size - 1) / page_size + }; + + let offset = (current_page - 1) * page_size; + + let order_direction = if desc { "DESC" } else { "ASC" }; + let query = format!( + "SELECT * FROM envelopes + WHERE account_id = ? AND thread_id = ? + ORDER BY sent_at {} + LIMIT ? OFFSET ?", + order_direction + ); + + let mut stmt = conn + .prepare(&query) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let items = stmt + .query_map(params![account_id, thread_id, page_size, offset], |row| { + Envelope::from_row(row) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .collect::, _>>() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + Ok(DataPage { + current_page: Some(current_page), + page_size: Some(page_size), + total_items, + items, + total_pages: Some(total_pages), + }) + } + + pub fn search( + &self, + accounts: Option>, + filter: SearchFilter, + page: u64, + page_size: u64, + desc: bool, + sort_by: SortBy, + ) -> BichonResult> { + let page = page.max(1); + let page_size = page_size.clamp(1, 500); + let offset = (page - 1) * page_size; + + let allowed_accounts: Option> = match (accounts, filter.account_ids.clone()) { + (Some(user_set), Some(filter_set)) => { + let intersection: HashSet = + user_set.intersection(&filter_set).cloned().collect(); + Some(intersection) + } + (Some(user_set), None) => Some(user_set), + (None, Some(filter_set)) => Some(filter_set), + (None, None) => None, + }; + + if let Some(ref set) = allowed_accounts { + if set.is_empty() { + return Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: 0, + items: Vec::new(), + total_pages: Some(0), + }); + } + } + + let mut base_sql = String::from(" FROM envelopes e "); + + let need_join_attachment = filter.attachment_name.is_some(); + + if need_join_attachment { + base_sql.push_str( + " + LEFT JOIN envelope_attachments a + ON a.account_id = e.account_id + AND a.mailbox_id = e.mailbox_id + AND a.envelope_id = e.id + ", + ); + } + + base_sql.push_str(" WHERE 1=1 "); + let mut args: Vec = Vec::new(); + + if let Some(set) = allowed_accounts { + base_sql.push_str(" AND e.account_id IN ("); + for (i, _) in set.iter().enumerate() { + if i > 0 { + base_sql.push_str(","); + } + base_sql.push_str("?"); + } + base_sql.push_str(")"); + for id in set { + args.push(id.into()); + } + } + + if let Some(mailboxes) = filter.mailbox_ids { + if !mailboxes.is_empty() { + base_sql.push_str(" AND e.mailbox_id IN ("); + for (i, _) in mailboxes.iter().enumerate() { + if i > 0 { + base_sql.push_str(","); + } + base_sql.push_str("?"); + } + base_sql.push_str(")"); + for id in mailboxes { + args.push(id.into()); + } + } + } + + if let Some(text) = filter.text { + let pattern = format!("(?i){}", text); + base_sql.push_str( + " + AND ( + regexp_matches(coalesce(e.subject, ''), ?) + OR regexp_matches(coalesce(e.body, ''), ?) + ) + ", + ); + args.push(pattern.clone().into()); + args.push(pattern.into()); + } + + if let Some(from) = filter.from { + base_sql.push_str(" AND e.sender ILIKE ? "); + args.push(format!("%{}%", from).into()); + } + + if let Some(to) = filter.to { + base_sql.push_str( + " + AND EXISTS ( + SELECT 1 FROM UNNEST(e.recipients) r + WHERE r::VARCHAR ILIKE ? + ) + ", + ); + args.push(format!("%{}%", to).into()); + } + + if let Some(cc) = filter.cc { + base_sql.push_str( + " + AND EXISTS ( + SELECT 1 FROM UNNEST(e.cc) r + WHERE r::VARCHAR ILIKE ? + ) + ", + ); + args.push(format!("%{}%", cc).into()); + } + + if let Some(bcc) = filter.bcc { + base_sql.push_str( + " + AND EXISTS ( + SELECT 1 FROM UNNEST(e.bcc) r + WHERE r::VARCHAR ILIKE ? + ) + ", + ); + args.push(format!("%{}%", bcc).into()); + } + + if let Some(since) = filter.since { + base_sql.push_str(" AND e.sent_at >= ? "); + args.push(since.into()); + } + + if let Some(before) = filter.before { + base_sql.push_str(" AND e.sent_at <= ? "); + args.push(before.into()); + } + + if let Some(min) = filter.min_size { + base_sql.push_str(" AND e.size_bytes >= ? "); + args.push(min.into()); + } + + if let Some(max) = filter.max_size { + base_sql.push_str(" AND e.size_bytes <= ? "); + args.push(max.into()); + } + + if let Some(mid) = filter.message_id { + base_sql.push_str(" AND e.message_id ILIKE ? "); + args.push(format!("%{}%", mid).into()); + } + + if let Some(has) = filter.has_attachment { + base_sql.push_str(" AND e.has_attachment = ? "); + args.push(has.into()); + } + + if let Some(tags) = filter.tags { + if !tags.is_empty() { + base_sql.push_str(" AND ("); + for i in 0..tags.len() { + if i > 0 { + base_sql.push_str(" OR "); + } + base_sql.push_str("list_contains(e.tags, ?)"); + } + base_sql.push_str(") "); + for tag in tags { + args.push(tag.into()); + } + } + } + + if let Some(name) = filter.attachment_name { + base_sql.push_str(" AND a.filename ILIKE ? "); + args.push(format!("%{}%", name).into()); + } + + let count_sql = if need_join_attachment { + format!("SELECT COUNT(DISTINCT e.id) {}", base_sql) + } else { + format!("SELECT COUNT(*) {}", base_sql) + }; + + println!("================ COUNT SQL ================"); + println!("FINAL: {}", debug_sql(&count_sql, &args)); + println!("==========================================="); + + let total_items: u64 = self + .conn()? + .query_row(&count_sql, duckdb::params_from_iter(args.iter()), |row| { + row.get(0) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if total_items == 0 { + return Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items: 0, + items: Vec::new(), + total_pages: Some(0), + }); + } + + let sort_column = match sort_by { + SortBy::DATE => "e.sent_at", + SortBy::SIZE => "e.size_bytes", + }; + + let order = if desc { "DESC" } else { "ASC" }; + + let select_prefix = if need_join_attachment { + "SELECT DISTINCT e.*" + } else { + "SELECT e.*" + }; + + let data_sql = format!( + "{}{}ORDER BY {} {} + LIMIT ? OFFSET ? + ", + select_prefix, base_sql, sort_column, order + ); + + let mut data_args = args.clone(); + data_args.push(page_size.into()); + data_args.push(offset.into()); + let conn = self.conn()?; + let mut stmt = conn + .prepare(&data_sql) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let items = stmt + .query_map(duckdb::params_from_iter(data_args.iter()), |row| { + Envelope::from_row(row) + }) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .collect::, _>>() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let total_pages = if total_items == 0 { + Some(0) + } else { + Some((total_items + page_size - 1) / page_size) + }; + + Ok(DataPage { + current_page: Some(page), + page_size: Some(page_size), + total_items, + items, + total_pages, + }) + } +} + +pub fn init( + path: &PathBuf, + mut migrations_runner: Runner, +) -> BichonResult> { + let mut flags = duckdb::Config::default() + .enable_autoload_extension(true) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .access_mode(duckdb::AccessMode::ReadWrite) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .with("enable_fsst_vectors", "true") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? + .with("allocator_background_threads", "true") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if let Some(memory_limit) = &SETTINGS.bichon_duckdb_max_memory { + flags = flags + .max_memory(memory_limit) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + + if let Some(threads) = &SETTINGS.bichon_duckdb_threads { + flags = flags + .threads(*threads) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + + let conn = DuckdbConnectionManager::file_with_flags(path, flags) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let pool = r2d2::Pool::new(conn) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + { + let conn = pool + .get() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + conn.execute("PRAGMA enable_checkpoint_on_shutdown", []) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + conn.pragma_update(None, "autoload_known_extensions", &"true") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + conn.pragma_update(None, "allow_community_extensions", &"false") + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + } + + { + let conn = pool + .get() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + migrations_runner.set_migration_table_name("migrations"); + for migration in migrations_runner.run_iter(&mut DuckDBConnection(conn)) { + match migration { + Ok(migration) => { + tracing::info!("Applied migration: {}", migration); + } + Err(err) => { + return Err(raise_error!( + format!("Failed to apply migration: {}", err), + ErrorCode::InternalError + )); + } + } + } + } + + Ok(pool) +} diff --git a/src/modules/duckdb/migrations/V1__initial.sql b/src/modules/duckdb/migrations/V1__initial.sql new file mode 100644 index 0000000..d90652c --- /dev/null +++ b/src/modules/duckdb/migrations/V1__initial.sql @@ -0,0 +1,70 @@ +-- ========================= +-- envelopes (main table) +-- ========================= +CREATE TABLE IF NOT EXISTS envelopes ( + -- internal id (tantivy f_id) + id UBIGINT NOT NULL, + + -- account / mailbox / uid + account_id UBIGINT NOT NULL, + mailbox_id UBIGINT NOT NULL, + uid UBIGINT NOT NULL, + + -- headers / content + subject TEXT, + body TEXT, + + sender TEXT, + recipients VARCHAR[], + cc VARCHAR[], + bcc VARCHAR[], + + -- dates + sent_at BIGINT, + received_at BIGINT, + + -- size + size_bytes UBIGINT, + + -- thread + thread_id UBIGINT, + + -- message-id + message_id TEXT, + + -- attachment summary + has_attachment BOOLEAN NOT NULL, + attachment_count INTEGER NOT NULL CHECK (attachment_count >= 0), + tags VARCHAR[], + shard_id UBIGINT NOT NULL +); + +CREATE INDEX idx_env_mailbox_sent ON envelopes(account_id, mailbox_id, sent_at); + +-- ========================= +-- envelope_attachments +-- +-- Normalized attachment metadata for fast filtering and UI display +-- ========================= +CREATE TABLE IF NOT EXISTS envelope_attachments ( + -- Reference to envelopes.id + envelope_id UBIGINT NOT NULL, + account_id UBIGINT NOT NULL, + mailbox_id UBIGINT NOT NULL, + -- Original attachment filename (for display) + filename TEXT NOT NULL, + + -- Normalized file extension (lowercase, without dot) + extension TEXT NOT NULL, + + -- Extension category (document / image / archive / ...) + ext_category TEXT NOT NULL, + content_type TEXT NOT NULL, + -- Attachment size in bytes + -- 0 if unknown + size_bytes UBIGINT NOT NULL, + shard_id UBIGINT NOT NULL +); + + +CREATE INDEX IF NOT EXISTS idx_attachments_mailbox_id ON envelope_attachments (mailbox_id); diff --git a/src/modules/duckdb/mod.rs b/src/modules/duckdb/mod.rs new file mode 100644 index 0000000..421f066 --- /dev/null +++ b/src/modules/duckdb/mod.rs @@ -0,0 +1,3 @@ +pub mod build; +pub mod init; +pub mod refinery; diff --git a/src/modules/duckdb/refinery.rs b/src/modules/duckdb/refinery.rs new file mode 100644 index 0000000..e8d0fe5 --- /dev/null +++ b/src/modules/duckdb/refinery.rs @@ -0,0 +1,74 @@ +use chrono::{DateTime, Utc}; +use refinery::{error::WrapMigrationError, Migration}; +use refinery_core::{ + traits::sync::{Query, Transaction}, + Migrate, +}; +use std::{ops::DerefMut, time::SystemTime}; + +pub struct DuckDBConnection>(pub T); + +impl> From for DuckDBConnection { + fn from(conn: T) -> Self { + Self(conn) + } +} + +impl> Transaction for DuckDBConnection { + type Error = duckdb::Error; + fn execute<'a, T: Iterator>( + &mut self, + mut queries: T, + ) -> std::result::Result { + let transaction = self.0.transaction()?; + let count = queries.try_fold(0, |count, query| { + transaction.execute_batch(query)?; + Ok::<_, Self::Error>(count + 1) + })?; + transaction.commit()?; + Ok(count) + } +} + +impl> Query> for DuckDBConnection { + fn query(&mut self, query: &str) -> Result, Self::Error> { + let mut stmt = self.0.prepare(query)?; + let applied: Vec = stmt + .query_map([], |row| { + let version = row.get(0)?; + let name: String = row.get(1)?; + let applied_on: DateTime = row.get(2)?; + let checksum: u64 = row.get(3)?; + let applied_on: SystemTime = applied_on.into(); + + Ok(Migration::applied( + version, + name, + applied_on.into(), + checksum, + )) + })? + .collect::, _>>()?; + Ok(applied) + } +} + +impl> Migrate for DuckDBConnection { + fn assert_migrations_table( + &mut self, + migration_table_name: &str, + ) -> std::result::Result { + let query = format!( + "CREATE TABLE IF NOT EXISTS {migration_table_name} ( + version INT primary key, + name TEXT not null, + applied_on TIMESTAMP not null, + checksum TEXT not null + )" + ); + + self.execute(std::iter::once(query.as_str())) + .migration_err("failed to create or verify DuckDB migrations table", None)?; + Ok(0) + } +} diff --git a/src/modules/envelope/extractor.rs b/src/modules/envelope/extractor.rs index 9a9d58e..d7e713d 100644 --- a/src/modules/envelope/extractor.rs +++ b/src/modules/envelope/extractor.rs @@ -20,124 +20,55 @@ use crate::modules::common::AddrVec; use crate::modules::envelope::utils::normalize_subject; use crate::modules::error::code::ErrorCode; use crate::modules::error::BichonResult; -use crate::modules::utils::create_hash; +use crate::modules::message::content::AttachmentInfo; +use crate::modules::utils::create_hash2; use crate::modules::utils::html::extract_text; use crate::{calculate_hash, raise_error, utc_now}; use crate::{id, modules::indexer::envelope::Envelope}; use async_imap::types::Fetch; -use mail_parser::{HeaderName, Message, MessageParser, MimeHeaders}; +use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders}; -pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> BichonResult { +pub fn extract_envelope( + fetch: &Fetch, + account_id: u64, + mailbox_id: u64, +) -> BichonResult<(Envelope, Vec)> { let internal_date = fetch .internal_date() .map(|d| d.timestamp_millis()) .unwrap_or(0); let uid = fetch.uid.unwrap_or(0); - let body = fetch .body() .ok_or_else(|| raise_error!("No body available".into(), ErrorCode::InternalError))?; - let size = fetch.size.unwrap_or(body.len() as u32); - let message = MessageParser::new().parse(body).ok_or_else(|| { - raise_error!( - "Email header parse result is not available".into(), - ErrorCode::InternalError - ) - })?; - let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) { - text - } else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) { - extract_text(html) - } else { - String::new() - }; - - let message_id = message - .message_id() - .map(String::from) - .unwrap_or(generate_message_id()); - let in_reply_to = message.in_reply_to().as_text().map(String::from); - let references = extract_references(&message); - let thread_id = compute_thread_id(in_reply_to, references, &message_id); - - let mut subject = message.subject().map(String::from).unwrap_or_default(); - - if subject.contains('\u{FFFD}') { - subject = normalize_subject(message.header_raw(HeaderName::Subject)); - } - - let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0); - let bcc: Option> = message.bcc().map(|addr| { - AddrVec::from(addr) - .0 - .into_iter() - .filter_map(|a| a.address) - .collect() - }); - let cc: Option> = message.cc().map(|addr| { - AddrVec::from(addr) - .0 - .into_iter() - .filter_map(|a| a.address) - .collect() - }); - let to: Option> = message.to().map(|addr| { - AddrVec::from(addr) - .0 - .into_iter() - .filter_map(|a| a.address) - .collect() - }); - let from = message - .from() - .and_then(|addr| AddrVec::from(addr).0.into_iter().next()) - .and_then(|add| add.address) - .unwrap_or_else(|| "unknown".to_string()); - - let attachments: Vec = message - .attachments() - .filter(|att| { - let disp = att.content_disposition(); - let is_inline = disp.map(|d| d.is_inline()).unwrap_or(false); - let has_filename = att.attachment_name().is_some(); - has_filename && !is_inline - }) - .filter_map(|att| att.attachment_name()) - .map(|name| name.to_string()) - .collect(); - let envelope = Envelope { - id: create_hash(account_id, &message_id), - message_id, - account_id, - mailbox_id, - uid, - subject, - text, - from, - to: to.unwrap_or_default(), - cc: cc.unwrap_or_default(), - bcc: bcc.unwrap_or_default(), - date, - internal_date, - size, - thread_id, - attachments, - tags: None, - account_email: None, - mailbox_name: None, - }; - Ok(envelope) + extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id) } pub fn extract_envelope_from_eml( body: &[u8], account_id: u64, mailbox_id: u64, -) -> BichonResult { - let uid = 0; - let size = body.len() as u32; +) -> BichonResult<(Envelope, Vec)> { + extract_envelope_core(body, 0, body.len() as u32, 0, account_id, mailbox_id).map( + |(mut env, att)| { + if env.internal_date == 0 { + env.internal_date = env.date; + } + (env, att) + }, + ) +} + +fn extract_envelope_core( + body: &[u8], + uid: u32, + size: u32, + internal_date: i64, + account_id: u64, + mailbox_id: u64, +) -> BichonResult<(Envelope, Vec)> { let message = MessageParser::new().parse(body).ok_or_else(|| { raise_error!( "Email header parse result is not available".into(), @@ -156,7 +87,8 @@ pub fn extract_envelope_from_eml( let message_id = message .message_id() .map(String::from) - .unwrap_or(generate_message_id()); + .unwrap_or_else(generate_message_id); + let in_reply_to = message.in_reply_to().as_text().map(String::from); let references = extract_references(&message); let thread_id = compute_thread_id(in_reply_to, references, &message_id); @@ -167,46 +99,66 @@ pub fn extract_envelope_from_eml( } let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0); - let bcc: Option> = message.bcc().map(|addr| { - AddrVec::from(addr) - .0 - .into_iter() - .filter_map(|a| a.address) - .collect() - }); - let cc: Option> = message.cc().map(|addr| { - AddrVec::from(addr) - .0 - .into_iter() - .filter_map(|a| a.address) - .collect() - }); - let to: Option> = message.to().map(|addr| { - AddrVec::from(addr) - .0 - .into_iter() - .filter_map(|a| a.address) - .collect() - }); + + let parse_addrs = |addrs: Option<&Address<'_>>| { + addrs + .map(|addr| { + AddrVec::from(addr) + .0 + .into_iter() + .filter_map(|a| a.address) + .collect() + }) + .unwrap_or_default() + }; + + let bcc = parse_addrs(message.bcc()); + let cc = parse_addrs(message.cc()); + let to = parse_addrs(message.to()); + let from = message .from() .and_then(|addr| AddrVec::from(addr).0.into_iter().next()) .and_then(|add| add.address) .unwrap_or_else(|| "unknown".to_string()); - - let attachments: Vec = message + let attachments: Vec = message .attachments() - .filter(|att| { - let disp = att.content_disposition(); - let is_inline = disp.map(|d| d.is_inline()).unwrap_or(false); - let has_filename = att.attachment_name().is_some(); - has_filename && !is_inline + .filter_map(|attachment| { + let content_id = attachment.content_id().map(Into::into); + let inline = attachment + .content_disposition() + .map(|d| d.is_inline()) + .unwrap_or(false); + if inline && content_id.is_some() { + return None; + } + + let file_type = attachment + .content_type() + .map(|ct| { + format!( + "{}/{}", + ct.c_type.as_ref(), + ct.c_subtype.as_deref().unwrap_or("") + ) + }) + .unwrap_or_else(|| "application/octet-stream".to_string()); + + Some(AttachmentInfo { + filename: attachment + .attachment_name() + .map(|name| name.to_string()) + .unwrap_or_default(), + size: attachment.contents().len(), + inline, + file_type, + content_id, + }) }) - .filter_map(|att| att.attachment_name()) - .map(|name| name.to_string()) .collect(); + let envelope = Envelope { - id: create_hash(account_id, &message_id), + id: create_hash2(account_id, mailbox_id, &message_id), message_id, account_id, mailbox_id, @@ -214,19 +166,20 @@ pub fn extract_envelope_from_eml( subject, text, from, - to: to.unwrap_or_default(), - cc: cc.unwrap_or_default(), - bcc: bcc.unwrap_or_default(), + to, + cc, + bcc, date, - internal_date: date, + internal_date, size, thread_id, - attachments, + attachment_count: attachments.len(), tags: None, account_email: None, mailbox_name: None, }; - Ok(envelope) + + Ok((envelope, attachments)) } pub fn compute_thread_id( diff --git a/src/modules/imap/executor.rs b/src/modules/imap/executor.rs index 1829818..177354d 100644 --- a/src/modules/imap/executor.rs +++ b/src/modules/imap/executor.rs @@ -25,6 +25,7 @@ use crate::modules::error::code::ErrorCode; use crate::modules::imap::session::SessionStream; use crate::modules::indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER}; use crate::modules::indexer::schema::SchemaTools; +use crate::modules::utils::create_hash; use crate::modules::{error::BichonResult, imap::manager::ImapConnectionManager}; use crate::raise_error; use async_imap::types::Name; @@ -207,13 +208,15 @@ impl ImapExecutor { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? { let envelope = extract_envelope(&fetch, account_id, mailbox_id)?; + let eml_id = create_hash(account_id, &envelope.0.message_id); + ENVELOPE_INDEX_MANAGER - .add_document(envelope.id, envelope.to_document(mailbox_id)?) + .add_document(envelope.0.id, envelope) .await; let body = fetch.body().ok_or_else(|| { raise_error!("missing a body".into(), ErrorCode::ImapUnexpectedResult) })?; - EML_INDEX_MANAGER.add_document( envelope.id, doc!(fields.f_id => envelope.id, fields.f_account_id => account_id, fields.f_mailbox_id => mailbox_id, fields.f_eml => body)).await; + EML_INDEX_MANAGER.add_document( eml_id, doc!(fields.f_id => eml_id, fields.f_account_id => account_id, fields.f_mailbox_id => mailbox_id, fields.f_eml => body)).await; count += 1; } Ok(count) @@ -242,13 +245,14 @@ impl ImapExecutor { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? { let envelope = extract_envelope(&fetch, account_id, mailbox_id)?; + let eml_id = create_hash(account_id, &envelope.0.message_id); ENVELOPE_INDEX_MANAGER - .add_document(envelope.id, envelope.to_document(mailbox_id)?) + .add_document(envelope.0.id, envelope) .await; let body = fetch.body().ok_or_else(|| { raise_error!("missing a body".into(), ErrorCode::ImapUnexpectedResult) })?; - EML_INDEX_MANAGER.add_document( envelope.id, doc!(fields.f_id => envelope.id, fields.f_account_id => account_id, fields.f_mailbox_id => mailbox_id, fields.f_eml => body)).await; + EML_INDEX_MANAGER.add_document( eml_id, doc!(fields.f_id => eml_id, fields.f_account_id => account_id, fields.f_mailbox_id => mailbox_id, fields.f_eml => body)).await; } Ok(()) } diff --git a/src/modules/imap/manager.rs b/src/modules/imap/manager.rs index 89b746c..7d554c7 100644 --- a/src/modules/imap/manager.rs +++ b/src/modules/imap/manager.rs @@ -94,7 +94,7 @@ impl ImapConnectionManager { } pub async fn build(account_id: u64) -> BichonResult>> { - let account = AccountModel::get(account_id).await?; + let account = AccountModel::async_get(account_id).await?; let client = match Self::create_client(&account).await { Ok(client) => client, Err(error) => { diff --git a/src/modules/imap/tests.rs b/src/modules/imap/tests.rs index 67b390d..7d29fa6 100644 --- a/src/modules/imap/tests.rs +++ b/src/modules/imap/tests.rs @@ -16,13 +16,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use mail_parser::{parsers::MessageStream, HeaderName, MessageParser}; +use mail_parser::{parsers::MessageStream, MessageParser, MimeHeaders}; use crate::{ base64_encode_url_safe, - modules::{ - account::entity::Encryption, envelope::utils::normalize_subject, imap::client::Client, - }, + modules::{account::entity::Encryption, imap::client::Client}, }; #[tokio::test] @@ -109,13 +107,29 @@ R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 #[tokio::test] async fn test44() { - let path = r"C:\Users\polly\Downloads\test222.eml"; + let path = r"C:\Users\polly\Downloads\test333.eml"; let input = std::fs::read(path).unwrap(); let message = MessageParser::default().parse(&input).unwrap(); - let subject = message.subject().unwrap(); - println!("Subject: {}", subject); - if subject.contains('\u{FFFD}') { - let subject = normalize_subject(message.header_raw(HeaderName::Subject)); - println!("Subject: {}", subject); + for attachment in message.attachments() { + let content_type = attachment.content_type().unwrap(); + let filename = attachment + .attachment_name() + .map(|name| name.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let disposition = attachment.content_disposition(); + + let file_type = format!( + "{}/{}", + content_type.c_type.as_ref(), + content_type.c_subtype.as_deref().unwrap_or("") + ); + + let inline = disposition.map(|d| d.is_inline()).unwrap_or(false); + + println!( + "filename: {}, file_type: {}, inline: {}", + filename, file_type, inline + ); } } diff --git a/src/modules/import/mod.rs b/src/modules/import/mod.rs index 4f6b4cb..b36afcc 100644 --- a/src/modules/import/mod.rs +++ b/src/modules/import/mod.rs @@ -129,16 +129,16 @@ impl ImportEmls { continue; } }; - + let eml_id = create_hash(account_id, &envelope.0.message_id); ENVELOPE_INDEX_MANAGER - .add_document(envelope.id, envelope.to_document(mailbox_id).unwrap()) + .add_document(envelope.0.id, envelope) .await; EML_INDEX_MANAGER .add_document( - envelope.id, + eml_id, doc!( - fields.f_id => envelope.id, + fields.f_id => eml_id, fields.f_account_id => account_id, fields.f_mailbox_id => mailbox_id, fields.f_eml => decoded diff --git a/src/modules/indexer/envelope.rs b/src/modules/indexer/envelope.rs index 24dd96c..31ea9f2 100644 --- a/src/modules/indexer/envelope.rs +++ b/src/modules/indexer/envelope.rs @@ -16,18 +16,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::collections::HashSet; - -use crate::modules::account::migration::AccountModel; -use crate::modules::cache::imap::mailbox::MailBox; -use crate::modules::error::code::ErrorCode; -use crate::modules::utils::create_hash; -use crate::modules::{error::BichonResult, indexer::schema::SchemaTools}; -use crate::raise_error; +use duckdb::types::Value; use poem_openapi::Object; use serde::{Deserialize, Serialize}; -use tantivy::schema::Facet; -use tantivy::{doc, schema::Value, TantivyDocument}; + +use crate::modules::{account::migration::AccountModel, cache::imap::mailbox::MailBox}; #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize, Object)] pub struct Envelope { @@ -48,186 +41,72 @@ pub struct Envelope { pub internal_date: i64, pub size: u32, pub thread_id: u64, - pub attachments: Vec, + pub attachment_count: usize, pub tags: Option>, } -fn extract_u64_field( - document: &TantivyDocument, - field: tantivy::schema::Field, -) -> BichonResult { - let value = document.get_first(field).ok_or_else(|| { - raise_error!( - format!("miss '{}' field in tantivy document", stringify!(field)), - ErrorCode::InternalError - ) - })?; - value.as_u64().ok_or_else(|| { - raise_error!( - format!("'{}' field is not a u64", stringify!(field)), - ErrorCode::InternalError - ) - }) -} - -fn extract_i64_field( - document: &TantivyDocument, - field: tantivy::schema::Field, -) -> BichonResult { - let value = document.get_first(field).ok_or_else(|| { - raise_error!( - format!("miss '{}' field in tantivy document", stringify!(field)), - ErrorCode::InternalError - ) - })?; - value.as_i64().ok_or_else(|| { - raise_error!( - format!("'{}' field is not a i64", stringify!(field)), - ErrorCode::InternalError - ) - }) -} - -fn extract_string_field( - document: &TantivyDocument, - field: tantivy::schema::Field, -) -> BichonResult { - let value = document.get_first(field).ok_or_else(|| { - raise_error!( - format!("'{}' field not found", stringify!(field)), - ErrorCode::InternalError - ) - })?; - value.as_str().map(|s| s.to_string()).ok_or_else(|| { - raise_error!( - format!("'{}' field is not a string", stringify!(field)), - ErrorCode::InternalError - ) - }) -} - -fn extract_vec_string_field( - document: &TantivyDocument, - field: tantivy::schema::Field, -) -> BichonResult> { - let value = document - .get_all(field) - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect(); - Ok(value) -} - impl Envelope { - pub fn to_document(&self, mailbox_id: u64) -> BichonResult { - let fields = SchemaTools::envelope_fields(); - let mut doc = doc!(); - doc.add_u64(fields.f_id, self.id); - doc.add_text(fields.f_message_id, &self.message_id); - doc.add_u64(fields.f_account_id, self.account_id); - doc.add_u64(fields.f_mailbox_id, mailbox_id); - doc.add_u64(fields.f_uid, self.uid as u64); - doc.add_text(fields.f_subject, &self.subject); - doc.add_text(fields.f_text, &self.text); - doc.add_text(fields.f_from, &self.from); - for to in &self.to { - doc.add_text(fields.f_to, to); - } - for cc in &self.cc { - doc.add_text(fields.f_cc, cc); - } - for bcc in &self.bcc { - doc.add_text(fields.f_bcc, bcc); - } - doc.add_i64(fields.f_date, self.date); - doc.add_i64(fields.f_internal_date, self.internal_date); - doc.add_u64(fields.f_size, self.size as u64); - doc.add_u64(fields.f_thread_id, self.thread_id); - for att in &self.attachments { - doc.add_text(fields.f_attachments, att); - } - doc.add_bool(fields.f_has_attachment, self.attachments.len() > 0); - Ok(doc) - } - - pub async fn from_tantivy_doc(doc: &TantivyDocument) -> BichonResult { - let fields = SchemaTools::envelope_fields(); - let account_id = extract_u64_field(doc, fields.f_account_id)?; - let message_id = extract_string_field(doc, fields.f_message_id)?; - let mailbox_id = extract_u64_field(doc, fields.f_mailbox_id)?; - let id = create_hash(account_id, &message_id); - let full_text = extract_string_field(doc, fields.f_text)?; - - // Take up to the first 500 characters as a preview; - let preview = if full_text.chars().count() > 500 { - full_text.chars().take(500).collect::() + "..." - } else { - full_text + pub fn from_row(row: &duckdb::Row) -> duckdb::Result { + let get_list = |col_name: &str| -> Vec { + row.get::<_, Value>(col_name) + .map(|v| { + if let Value::List(inner_list) = v { + inner_list + .into_iter() + .filter_map(|item| { + if let Value::Text(s) = item { + Some(s) + } else { + None + } + }) + .collect() + } else { + vec![] + } + }) + .unwrap_or_default() + }; + let account_id = row.get("account_id")?; + let mailbox_id = row.get("mailbox_id")?; + let email = match AccountModel::get(account_id) { + Ok(account) => account.email, + Err(_) => "unknown".to_string(), }; - let tags: Vec = doc - .get_all(fields.f_tags) - .filter_map(|value| value.as_facet()) - .map(|facet_encoded_str| { - Facet::from_encoded(facet_encoded_str.as_bytes().to_vec()) - .ok() - .map(|facet| facet.to_string()) - }) - .flatten() - .collect(); - let account_email = AccountModel::find(account_id).await?.map(|a| a.email); + let mailbox_name = MailBox::find_mailbox(account_id, mailbox_id) + .ok() + .and_then(|m| m) + .map(|m| m.name) + .unwrap_or_else(|| "unknown".to_string()); - let mailboxes = MailBox::list_all(account_id).await?; - let mailbox_name = mailboxes - .iter() - .find(|m| m.id == mailbox_id) - .map(|m| m.name.clone()); - - let envelope = Envelope { - id, + Ok(Self { + id: row.get("id")?, + message_id: row.get("message_id").unwrap_or_default(), account_id, - account_email, + account_email: Some(email), mailbox_id, - mailbox_name, - message_id: extract_string_field(doc, fields.f_message_id)?, - uid: extract_u64_field(doc, fields.f_uid)? as u32, - subject: extract_string_field(doc, fields.f_subject)?, - text: preview, - from: extract_string_field(doc, fields.f_from)?, - to: extract_vec_string_field(doc, fields.f_to)?, - cc: extract_vec_string_field(doc, fields.f_cc)?, - bcc: extract_vec_string_field(doc, fields.f_bcc)?, - date: extract_i64_field(doc, fields.f_date)?, - internal_date: extract_i64_field(doc, fields.f_internal_date)?, - size: extract_u64_field(doc, fields.f_size)? as u32, - thread_id: extract_u64_field(doc, fields.f_thread_id)?, - attachments: extract_vec_string_field(doc, fields.f_attachments)?, - tags: Some(tags), - }; - Ok(envelope) - } -} - -pub async fn extract_contacts(doc: &TantivyDocument) -> BichonResult> { - let fields = SchemaTools::envelope_fields(); - let mut all_contacts = HashSet::new(); - - if let Ok(from_val) = extract_string_field(doc, fields.f_from) { - if !from_val.is_empty() { - all_contacts.insert(from_val); - } - } - - let multi_fields = [fields.f_to, fields.f_cc, fields.f_bcc]; - - for field in multi_fields { - if let Ok(vals) = extract_vec_string_field(doc, field) { - for v in vals { - if !v.is_empty() { - all_contacts.insert(v); + mailbox_name: Some(mailbox_name), + uid: row.get::<_, u64>("uid")? as u32, + subject: row.get("subject").unwrap_or_default(), + text: row.get("body").unwrap_or_default(), + from: row.get("sender").unwrap_or_default(), + to: get_list("recipients"), + cc: get_list("cc"), + bcc: get_list("bcc"), + date: row.get("sent_at").unwrap_or(0), + internal_date: row.get("received_at").unwrap_or(0), + size: row.get::<_, u64>("size_bytes")? as u32, + thread_id: row.get("thread_id").unwrap_or(0), + attachment_count: row.get::<_, i32>("attachment_count")? as usize, + tags: { + let t = get_list("tags"); + if t.is_empty() { + None + } else { + Some(t) } - } - } + }, + }) } - - Ok(all_contacts) } diff --git a/src/modules/indexer/manager.rs b/src/modules/indexer/manager.rs index 4a74070..8c292c2 100644 --- a/src/modules/indexer/manager.rs +++ b/src/modules/indexer/manager.rs @@ -18,113 +18,74 @@ use std::{ collections::{HashMap, HashSet}, - ops::Bound, path::PathBuf, sync::{Arc, LazyLock}, time::Duration, }; use crate::modules::{ - indexer::envelope::extract_contacts, - message::{search::SortBy, tags::TagCount}, + duckdb::init::duckdb, + message::{content::AttachmentInfo, search::SortBy, tags::TagCount}, }; use crate::{ modules::{ - account::migration::AccountModel, common::signal::SIGNAL_MANAGER, - dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, + dashboard::{DashboardStats, LargestEmail}, error::{code::ErrorCode, BichonResult}, - indexer::{ - envelope::Envelope, - fields::{ - F_ACCOUNT_ID, F_DATE, F_FROM, F_HAS_ATTACHMENT, F_MAILBOX_ID, F_SIZE, F_TAGS, - F_THREAD_ID, F_UID, - }, - schema::SchemaTools, - }, + indexer::{envelope::Envelope, schema::SchemaTools}, message::search::SearchFilter, rest::response::DataPage, settings::dir::DATA_DIR_MANAGER, }, - raise_error, utc_now, + raise_error, }; -use chrono::Utc; + use mail_parser::{MessageParser, MimeHeaders}; -use serde_json::json; + +use tantivy::indexer::UserOperation; use tantivy::{ - aggregation::{ - agg_req::Aggregations, - agg_result::{ - AggregationResult, AggregationResults, BucketEntries, BucketResult, MetricResult, - }, - AggregationCollector, Key, - }, - collector::{Count, FacetCollector, TopDocs}, - query::{ - AllQuery, BooleanQuery, EmptyQuery, Occur, Query, QueryParser, RangeQuery, RegexQuery, - TermQuery, - }, - schema::{Facet, IndexRecordOption, Value}, + collector::TopDocs, + query::{BooleanQuery, Occur, Query, TermQuery}, + schema::{IndexRecordOption, Value}, store::{Compressor, ZstdCompressor}, - DocAddress, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, Order, - TantivyDocument, Term, + Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, TantivyDocument, Term, }; -use tantivy::{indexer::UserOperation, Searcher}; use tokio::{ fs::File, io::AsyncWriteExt, sync::{mpsc, Mutex}, task, }; -use tracing::{info, warn}; +use tracing::info; pub static ENVELOPE_INDEX_MANAGER: LazyLock = LazyLock::new(EnvelopeIndexManager::new); pub static EML_INDEX_MANAGER: LazyLock = LazyLock::new(EmlIndexManager::new); -pub const ENVELOPE_BATCH_SIZE: usize = 1000; -pub const EML_BATCH_SIZE: usize = 200; +pub const ENVELOPE_BATCH_SIZE: usize = 500; +pub const EML_BATCH_SIZE: usize = 100; -const MAX_BUFFER_DURATION: Duration = Duration::from_secs(30); +const MAX_BUFFER_DURATION: Duration = Duration::from_secs(10); -pub enum WriteMessage { +pub enum MetadataOp { + Record((u64, (Envelope, Vec))), + Shutdown, +} + +pub enum DocumentOp { Document((u64, TantivyDocument)), Shutdown, } pub struct EnvelopeIndexManager { - index_writer: Arc>, - sender: mpsc::Sender, - reader: IndexReader, - query_parser: QueryParser, + sender: mpsc::Sender, } impl EnvelopeIndexManager { pub fn new() -> Self { - let index = Self::open_or_create_index(&DATA_DIR_MANAGER.envelope_dir); - let index_writer = Arc::new(Mutex::new( - index - .writer_with_num_threads(8, 536_870_912) - .unwrap_or_else(|e| { - panic!( - "Failed to create IndexWriter with 8 threads and 512MB buffer for {:?}: {}", - DATA_DIR_MANAGER.envelope_dir, e - ) - }), - )); - let reader = index.reader().unwrap_or_else(|e| { - panic!( - "Failed to create IndexReader for {:?}: {}", - DATA_DIR_MANAGER.envelope_dir, e - ) - }); - let mut query_parser = - QueryParser::for_index(&index, SchemaTools::envelope_default_fields()); - query_parser.set_conjunction_by_default(); - - let (sender, mut receiver) = mpsc::channel::(1000); + let (sender, mut receiver) = mpsc::channel::(1000); task::spawn(async move { - let mut buffer: HashMap = + let mut buffer: HashMap)> = HashMap::with_capacity(ENVELOPE_BATCH_SIZE); let mut interval = tokio::time::interval(MAX_BUFFER_DURATION); let mut shutdown = SIGNAL_MANAGER.subscribe(); @@ -132,13 +93,13 @@ impl EnvelopeIndexManager { tokio::select! { maybe_msg = receiver.recv() => { match maybe_msg { - Some(WriteMessage::Document((eid, doc))) => { + Some(MetadataOp::Record((eid, doc))) => { buffer.insert(eid, doc); if buffer.len() >= ENVELOPE_BATCH_SIZE { ENVELOPE_INDEX_MANAGER.drain_and_commit(&mut buffer).await; } } - Some(WriteMessage::Shutdown) => { + Some(MetadataOp::Shutdown) => { ENVELOPE_INDEX_MANAGER.drain_and_commit(&mut buffer).await; break; } @@ -151,316 +112,46 @@ impl EnvelopeIndexManager { } } _ = shutdown.recv() => { - let _ = ENVELOPE_INDEX_MANAGER.sender.send(WriteMessage::Shutdown).await; + let _ = ENVELOPE_INDEX_MANAGER.sender.send(MetadataOp::Shutdown).await; } } } }); - Self { - index_writer, - sender, - reader, - query_parser, - } + Self { sender } } - pub async fn add_document(&self, eid: u64, doc: TantivyDocument) { - let _ = self.sender.send(WriteMessage::Document((eid, doc))).await; + pub async fn add_document(&self, eid: u64, doc: (Envelope, Vec)) { + let _ = self.sender.send(MetadataOp::Record((eid, doc))).await; } - async fn drain_and_commit(&self, buffer: &mut HashMap) { + async fn drain_and_commit(&self, buffer: &mut HashMap)>) { if buffer.is_empty() { return; } - let mut writer = self.index_writer.lock().await; - let mut operations = Vec::new(); - for (eid, doc) in buffer.drain() { - let delete_term = Term::from_field_u64(SchemaTools::envelope_fields().f_id, eid); - operations.push(UserOperation::Delete(delete_term)); - operations.push(UserOperation::Add(doc)); - } - if let Err(e) = writer.run(operations) { - eprintln!("[FATAL] Tantivy run failed: {e:?}"); - std::process::exit(1); - } + let items: Vec<(Envelope, Vec)> = buffer.drain().map(|(_, v)| v).collect(); - fatal_commit(&mut writer); - } + let result = (|| -> BichonResult<()> { + duckdb()?.append_envelopes_with_attachments(&items)?; + Ok(()) + })(); - fn open_or_create_index(index_dir: &PathBuf) -> Index { - let need_create = !index_dir.exists() - || index_dir - .read_dir() - .map(|mut d| d.next().is_none()) - .unwrap_or(true); - if need_create { - info!( - "Email index not found or empty, creating new index at {}", - index_dir.display() - ); - std::fs::create_dir_all(&index_dir).unwrap_or_else(|e| { - panic!("Failed to create index directory {:?}: {}", index_dir, e) - }); - Index::create_in_dir(&index_dir, SchemaTools::envelope_schema()) - .unwrap_or_else(|e| panic!("Failed to create index in {:?}: {}", index_dir, e)) - } else { - info!("Opening existing email index at {}", index_dir.display()); - open(&index_dir) + if let Err(e) = result { + tracing::error!("Failed to drain and commit envelopes to DuckDB: {:#?}", e); } } - pub fn total_emails(&self, accounts: &Option>) -> BichonResult { - let searcher = self.create_searcher()?; - - match accounts { - Some(ref ids) if !ids.is_empty() => { - let mut subqueries = Vec::new(); - for &id in ids { - let term = - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, id); - subqueries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, - )); - } - let query = Box::new(BooleanQuery::new(subqueries)) as Box; - let count = searcher - .search(&query, &Count) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(count as u64) - } - Some(_) => Ok(0), - None => Ok(searcher.num_docs()), - } - } - - fn account_query(&self, account_id: u64) -> Box { - let account_term = - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, account_id); - Box::new(TermQuery::new(account_term, IndexRecordOption::Basic)) - } - - fn mailbox_query(&self, account_id: u64, mailbox_id: u64) -> Box { - let account_query = TermQuery::new( - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, account_id), - IndexRecordOption::Basic, - ); - let mailbox_query = TermQuery::new( - Term::from_field_u64(SchemaTools::envelope_fields().f_mailbox_id, mailbox_id), - IndexRecordOption::Basic, - ); - let boolean_query = BooleanQuery::new(vec![ - (Occur::Must, Box::new(account_query)), - (Occur::Must, Box::new(mailbox_query)), - ]); - Box::new(boolean_query) - } - - fn filter_query( - &self, - accounts: Option>, - filter: SearchFilter, - parser: QueryParser, - ) -> BichonResult> { - let f = SchemaTools::envelope_fields(); - let mut subqueries: Vec<(Occur, Box)> = Vec::new(); - - if let Some(authorized_ids) = accounts { - if authorized_ids.is_empty() { - let term = Term::from_field_u64(f.f_account_id, u64::MAX); - subqueries.push(( - Occur::Must, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)), - )); - } else { - let mut account_must_queries = Vec::new(); - for id in authorized_ids { - let term = Term::from_field_u64(f.f_account_id, id); - account_must_queries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, - )); - } - subqueries.push(( - Occur::Must, - Box::new(BooleanQuery::new(account_must_queries)), - )); - } - } - - if let Some(ref text) = filter.text { - let query = parser - .parse_query(text) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; - subqueries.push((Occur::Must, Box::new(query))); - } - - if let Some(ref tags) = filter.tags { - if !tags.is_empty() { - let mut should_queries: Vec<(Occur, Box)> = Vec::new(); - - for tag in tags { - let facet = Facet::from_text(tag).map_err(|e| { - raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter) - })?; - - let term = Term::from_facet(f.f_tags, &facet); - - should_queries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)), - )); - } - subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries)))); - } - } - - for (field, opt_value) in [ - (f.f_from, &filter.from), - (f.f_to, &filter.to), - (f.f_cc, &filter.cc), - (f.f_bcc, &filter.bcc), - ] { - if let Some(ref v) = opt_value { - if let Ok(query) = RegexQuery::from_pattern(v.as_str(), field) { - subqueries.push((Occur::Must, Box::new(query))); - } - } - } - - if let Some(has) = filter.has_attachment { - if has { - subqueries.push(( - Occur::Must, - Box::new(TermQuery::new( - Term::from_field_bool(f.f_has_attachment, true.into()), - IndexRecordOption::Basic, - )), - )); - } - } - - if let Some(ref name) = filter.attachment_name { - if let Ok(query) = RegexQuery::from_pattern(name.as_str(), f.f_attachments) { - subqueries.push((Occur::Must, Box::new(query))); - } - } - - let start_bound = if let Some(from) = filter.since { - Bound::Included(Term::from_field_i64(f.f_date, from)) - } else { - Bound::Unbounded - }; - - let end_bound = if let Some(to) = filter.before { - Bound::Included(Term::from_field_i64(f.f_date, to)) - } else { - Bound::Unbounded - }; - - if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded { - let q = RangeQuery::new(start_bound, end_bound); - subqueries.push((Occur::Must, Box::new(q))); - } - - if let Some(account_ids) = filter.account_ids { - let mut should_queries: Vec<(Occur, Box)> = Vec::new(); - for id in account_ids { - let term = Term::from_field_u64(f.f_account_id, id); - should_queries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)), - )); - } - subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries)))); - } - - if let Some(mailbox_ids) = filter.mailbox_ids { - let mut should_queries: Vec<(Occur, Box)> = Vec::new(); - for id in mailbox_ids { - let term = Term::from_field_u64(f.f_mailbox_id, id); - should_queries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)), - )); - } - subqueries.push((Occur::Must, Box::new(BooleanQuery::new(should_queries)))); - } - - let start_bound = if let Some(from) = filter.min_size { - Bound::Included(Term::from_field_u64(f.f_size, from)) - } else { - Bound::Unbounded - }; - - let end_bound = if let Some(to) = filter.max_size { - Bound::Included(Term::from_field_u64(f.f_size, to)) - } else { - Bound::Unbounded - }; - - if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded { - let q = RangeQuery::new(start_bound, end_bound); - subqueries.push((Occur::Must, Box::new(q))); - } - - if let Some(ref msg_id) = filter.message_id { - let term = Term::from_field_text(f.f_message_id, msg_id); - subqueries.push(( - Occur::Must, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)), - )); - } - - if subqueries.is_empty() { - return Ok(Box::new(AllQuery)); - } - - Ok(Box::new(BooleanQuery::new(subqueries))) - } - - fn thread_query(&self, account_id: u64, thread_id: u64) -> Box { - let account_query = TermQuery::new( - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, account_id), - IndexRecordOption::Basic, - ); - let thread_query = TermQuery::new( - Term::from_field_u64(SchemaTools::envelope_fields().f_thread_id, thread_id), - IndexRecordOption::Basic, - ); - let boolean_query = BooleanQuery::new(vec![ - (Occur::Must, Box::new(account_query)), - (Occur::Must, Box::new(thread_query)), - ]); - Box::new(boolean_query) - } - - fn envelope_query(&self, account_id: u64, eid: u64) -> Box { - let account_id_query = TermQuery::new( - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, account_id), - IndexRecordOption::Basic, - ); - let envelope_id_query = TermQuery::new( - Term::from_field_u64(SchemaTools::envelope_fields().f_id, eid), - IndexRecordOption::Basic, - ); - let boolean_query = BooleanQuery::new(vec![ - (Occur::Must, Box::new(account_id_query)), - (Occur::Must, Box::new(envelope_id_query)), - ]); - Box::new(boolean_query) + pub async fn total_emails(&self, accounts: Option>) -> BichonResult { + tokio::task::spawn_blocking(move || duckdb()?.total_emails(accounts)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn delete_account_envelopes(&self, account_id: u64) -> BichonResult<()> { - let query = self.account_query(account_id); - let mut writer = self.index_writer.lock().await; - writer - .delete_query(query) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - writer - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let _ = + tokio::task::spawn_blocking(move || duckdb()?.delete_envelopes_by_account(account_id)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?; Ok(()) } @@ -473,43 +164,12 @@ impl EnvelopeIndexManager { tracing::warn!("delete_mailbox_envelopes: mailbox_ids is empty, nothing to delete"); return Ok(()); } - let mut queries: Vec> = Vec::with_capacity(mailbox_ids.len()); - for mailbox_id in mailbox_ids { - queries.push(self.mailbox_query(account_id, mailbox_id)); - } - let mut writer = self.index_writer.lock().await; - for query in queries { - writer - .delete_query(query) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - writer - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(()) - } - - fn collect_facets_recursive( - query: &dyn Query, - searcher: &Searcher, - parent_facet: &str, - all_facets: &mut Vec, - ) -> BichonResult<()> { - let mut facet_collector = FacetCollector::for_field(F_TAGS); - facet_collector.add_facet(parent_facet); - - let facet_counts = searcher - .search(query, &facet_collector) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - for (facet, count) in facet_counts.get(parent_facet) { - all_facets.push(TagCount { - tag: facet.to_string(), - count, - }); - Self::collect_facets_recursive(query, searcher, &facet.to_string(), all_facets)?; - } + let _ = tokio::task::spawn_blocking(move || { + duckdb()?.delete_mailbox_envelopes(account_id, mailbox_ids) + }) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))?; Ok(()) } @@ -517,100 +177,32 @@ impl EnvelopeIndexManager { &self, accounts: Option>, ) -> BichonResult> { - let searcher = self.reader.searcher(); - - let query: Box = match accounts { - Some(ref ids) if !ids.is_empty() => { - let mut subqueries = Vec::new(); - for &id in ids { - let term = - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, id); - subqueries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, - )); - } - Box::new(BooleanQuery::new(subqueries)) - } - Some(_) => Box::new(EmptyQuery), - None => Box::new(AllQuery), - }; - - let mut all_facets = Vec::new(); - Self::collect_facets_recursive(&query, &searcher, "/", &mut all_facets)?; - Ok(all_facets) + tokio::task::spawn_blocking(move || duckdb()?.get_all_tags(accounts)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn get_all_contacts( &self, accounts: Option>, ) -> BichonResult> { - let searcher = self.create_searcher()?; - - let query: Box = match accounts { - Some(ref ids) if !ids.is_empty() => { - let mut subqueries = Vec::new(); - for &id in ids { - let term = - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, id); - subqueries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, - )); - } - Box::new(BooleanQuery::new(subqueries)) - } - Some(_) => Box::new(EmptyQuery), - None => Box::new(AllQuery), - }; - - let mut contacts_set: HashSet = HashSet::new(); - - let top_docs = searcher - .search(&query, &TopDocs::with_limit(1_000_000)) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - for (_score, doc_address) in top_docs { - let doc: TantivyDocument = searcher - .doc_async(doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let contacts = extract_contacts(&doc).await?; - for value in contacts { - contacts_set.insert(value); - } - } - Ok(contacts_set) + tokio::task::spawn_blocking(move || duckdb()?.get_all_contacts(accounts)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn delete_envelopes_multi_account( &self, - deletes: &HashMap>, // HashMap + deletes: HashMap>, // HashMap ) -> BichonResult<()> { if deletes.is_empty() { tracing::warn!("delete_envelopes_multi_account: deletes is empty, nothing to delete"); return Ok(()); } - let mut writer = self.index_writer.lock().await; - - for (account_id, envelope_ids) in deletes { - let unique_ids: HashSet = envelope_ids.iter().copied().collect(); - if unique_ids.is_empty() { - continue; - } - for eid in unique_ids { - let query = self.envelope_query(*account_id, eid); - writer - .delete_query(query) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - } - } - writer - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(()) + tokio::task::spawn_blocking(move || duckdb()?.delete_envelopes_multi_account(deletes)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn update_envelope_tags( @@ -622,59 +214,9 @@ impl EnvelopeIndexManager { tracing::warn!("update_envelope_tags: updates is empty, nothing to update"); return Ok(()); } - - let searcher = self.create_searcher()?; - let mut writer = self.index_writer.lock().await; - - let f_tags = SchemaTools::envelope_fields().f_tags; - let f_id = SchemaTools::envelope_fields().f_id; - let deduplicated_updates: HashMap> = updates - .into_iter() - .map(|(account_id, envelope_ids)| (account_id, envelope_ids.into_iter().collect())) - .collect(); - - let mut operations = Vec::new(); - - for (account_id, envelope_ids) in &deduplicated_updates { - for eid in envelope_ids { - let query = self.envelope_query(*account_id, *eid); - let docs = searcher - .search(query.as_ref(), &TopDocs::with_limit(1)) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - if let Some((_, doc_address)) = docs.first() { - let old_doc: TantivyDocument = searcher - .doc_async(*doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let mut new_doc = TantivyDocument::new(); - for (field, value) in old_doc.field_values() { - if field != f_tags { - new_doc.add_field_value(field, value); - } - } - for tag in &tags { - new_doc.add_facet(f_tags, tag); - } - - let delete_term = Term::from_field_u64(f_id, *eid); - operations.push(UserOperation::Delete(delete_term)); - operations.push(UserOperation::Add(new_doc)); - } - } - } - - writer - .run(operations) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - // commit - writer - .commit() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - Ok(()) + tokio::task::spawn_blocking(move || duckdb()?.update_envelope_tags(updates, tags)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn search( @@ -688,79 +230,11 @@ impl EnvelopeIndexManager { ) -> BichonResult> { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let query = self.filter_query(accounts, filter, self.query_parser.clone())?; - let searcher = self.create_searcher()?; - let total = searcher - .search(&query, &Count) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))? - as u64; - - if total == 0 { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: 0, - items: vec![], - total_pages: Some(0), - }); - } - let offset = (page - 1) * page_size; - let total_pages = total.div_ceil(page_size); - if offset > total { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: total, - items: vec![], - total_pages: Some(total_pages), - }); - } - - let order = if desc { Order::Desc } else { Order::Asc }; - let mailbox_docs: Vec; - - match sort_by { - SortBy::DATE => { - let date_docs: Vec<(i64, DocAddress)> = searcher - .search( - &query, - &TopDocs::with_limit(page_size as usize) - .and_offset(offset as usize) - .order_by_fast_field(F_DATE, order), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - mailbox_docs = date_docs.into_iter().map(|(_, addr)| addr).collect(); - } - SortBy::SIZE => { - let size_docs: Vec<(u64, DocAddress)> = searcher - .search( - &query, - &TopDocs::with_limit(page_size as usize) - .and_offset(offset as usize) - .order_by_fast_field(F_SIZE, order), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - mailbox_docs = size_docs.into_iter().map(|(_, addr)| addr).collect(); - } - } - - let mut result = Vec::new(); - - for doc_address in mailbox_docs { - let doc: TantivyDocument = searcher - .doc_async(doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let envelope = Envelope::from_tantivy_doc(&doc).await?; - result.push(envelope); - } - Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: total, - items: result, - total_pages: Some(total_pages), + tokio::task::spawn_blocking(move || { + duckdb()?.search(accounts, filter, page, page_size, desc, sort_by) }) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn list_mailbox_envelopes( @@ -773,57 +247,11 @@ impl EnvelopeIndexManager { ) -> BichonResult> { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let searcher = self.create_searcher()?; - let total = self - .num_messages_in_mailbox(&searcher, account_id, mailbox_id) - .await?; - if total == 0 { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: 0, - items: vec![], - total_pages: Some(0), - }); - } - let offset = (page - 1) * page_size; - let total_pages = total.div_ceil(page_size); - if offset > total { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: total, - items: vec![], - total_pages: Some(total_pages), - }); - } - let query = self.mailbox_query(account_id, mailbox_id); - let order = if desc { Order::Desc } else { Order::Asc }; - let mailbox_docs: Vec<(i64, DocAddress)> = searcher - .search( - query.as_ref(), - &TopDocs::with_limit(page_size as usize) - .and_offset(offset as usize) - .order_by_fast_field(F_DATE, order), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let mut result = Vec::new(); - - for (_, doc_address) in mailbox_docs { - let doc: TantivyDocument = searcher - .doc_async(doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let envelope = Envelope::from_tantivy_doc(&doc).await?; - result.push(envelope); - } - Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: total, - items: result, - total_pages: Some(total_pages), + tokio::task::spawn_blocking(move || { + duckdb()?.list_mailbox_envelopes(account_id, mailbox_id, page, page_size, desc) }) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn list_thread_envelopes( @@ -836,478 +264,73 @@ impl EnvelopeIndexManager { ) -> BichonResult> { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let searcher = self.create_searcher()?; - let total = self - .num_messages_in_thread(&searcher, account_id, thread_id) - .await?; - if total == 0 { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: 0, - items: vec![], - total_pages: Some(0), - }); - } - let offset = (page - 1) * page_size; - let total_pages = total.div_ceil(page_size); - if offset > total { - return Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: total, - items: vec![], - total_pages: Some(total_pages), - }); - } - - let query = self.thread_query(account_id, thread_id); - - let order = if desc { Order::Desc } else { Order::Asc }; - let thread_docs: Vec<(i64, DocAddress)> = searcher - .search( - query.as_ref(), - &TopDocs::with_limit(page_size as usize) - .and_offset(offset as usize) - .order_by_fast_field(F_DATE, order), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let mut result = Vec::new(); - - for (_, doc_address) in thread_docs { - let doc: TantivyDocument = searcher - .doc_async(doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let envelope = Envelope::from_tantivy_doc(&doc).await?; - result.push(envelope); - } - Ok(DataPage { - current_page: Some(page), - page_size: Some(page_size), - total_items: total, - items: result, - total_pages: Some(total_pages), + tokio::task::spawn_blocking(move || { + duckdb()?.list_thread_envelopes(account_id, thread_id, page, page_size, desc) }) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn get_envelope_by_id( &self, account_id: u64, - message_id: u64, + envelope_id: u64, ) -> BichonResult> { - let searcher = self.create_searcher()?; - let f = SchemaTools::envelope_fields(); - - let query = BooleanQuery::new(vec![ - ( - Occur::Must, - Box::new(TermQuery::new( - Term::from_field_u64(f.f_account_id, account_id), - IndexRecordOption::Basic, - )), - ), - ( - Occur::Must, - Box::new(TermQuery::new( - Term::from_field_u64(f.f_id, message_id), - IndexRecordOption::Basic, - )), - ), - ]); - - let docs: Vec<(f32, DocAddress)> = searcher - .search(&query, &TopDocs::with_limit(1)) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - if let Some((_, doc_address)) = docs.first() { - let doc: TantivyDocument = searcher - .doc_async(*doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let envelope = Envelope::from_tantivy_doc(&doc).await?; - Ok(Some(envelope)) - } else { - Ok(None) - } + tokio::task::spawn_blocking(move || duckdb()?.get_envelope_by_id(account_id, envelope_id)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn top_10_largest_emails( &self, - accounts: &Option>, + accounts: Option>, ) -> BichonResult> { - self.reader - .reload() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let searcher = self.reader.searcher(); - - let query: Box = match accounts { - Some(ref ids) if !ids.is_empty() => { - let mut subqueries = Vec::new(); - for &id in ids { - let term = - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, id); - subqueries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, - )); - } - Box::new(BooleanQuery::new(subqueries)) - } - Some(_) => Box::new(EmptyQuery), - None => Box::new(AllQuery), - }; - - let mailbox_docs: Vec<(u64, DocAddress)> = searcher - .search( - &query, - &TopDocs::with_limit(10).order_by_fast_field(F_SIZE, Order::Desc), - ) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let mut result = Vec::new(); - - for (_, doc_address) in mailbox_docs { - let doc: TantivyDocument = searcher - .doc_async(doc_address) - .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let envelope = LargestEmail::from_tantivy_doc(&doc)?; - result.push(envelope); - } - Ok(result) + tokio::task::spawn_blocking(move || duckdb()?.top_10_largest_emails(accounts)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn get_max_uid(&self, account_id: u64, mailbox_id: u64) -> BichonResult> { - self.reader - .reload() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let searcher = self.reader.searcher(); - - let query = self.mailbox_query(account_id, mailbox_id); - let agg_req: Aggregations = serde_json::from_value(json!({ - "max_uid": { - "max": { - "field": F_UID - } - } - })) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let collector = AggregationCollector::from_aggs(agg_req, Default::default()); - - let agg_res = searcher - .search(query.as_ref(), &collector) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(Self::extract_max_uid(&agg_res)) - } - - fn extract_max_uid(agg_res: &AggregationResults) -> Option { - agg_res.0.get("max_uid").and_then(|result| match result { - AggregationResult::MetricResult(MetricResult::Max(max)) => { - max.value.and_then(|value| { - (value >= 0.0 && value <= u64::MAX as f64).then(|| value.trunc() as u64) - }) - } - _ => None, - }) + tokio::task::spawn_blocking(move || duckdb()?.get_max_uid(account_id, mailbox_id)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn num_messages_in_mailbox( &self, - searcher: &Searcher, account_id: u64, mailbox_id: u64, ) -> BichonResult { - let query = self.mailbox_query(account_id, mailbox_id); - - let agg_req: Aggregations = serde_json::from_value(json!({ - "mailbox_count": { - "value_count": { - "field": F_MAILBOX_ID - } - } - })) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let collector = AggregationCollector::from_aggs(agg_req, Default::default()); - - let agg_res = searcher - .search(query.as_ref(), &collector) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Self::extract_mailbox_count(&agg_res) - } - - fn extract_mailbox_count(agg_res: &AggregationResults) -> BichonResult { - let Some(result) = agg_res.0.get("mailbox_count") else { - return Err(raise_error!( - "Missing aggregation result: mailbox_count".into(), - ErrorCode::InternalError - )); - }; - - match result { - AggregationResult::MetricResult(MetricResult::Count(count)) => { - Ok(count.value.map(|v| v as u64).ok_or_else(|| { - raise_error!( - "Failed to get count value from aggregation result: value is None".into(), - ErrorCode::InternalError - ) - })?) - } - other => Err(raise_error!( - format!("Unexpected aggregation result type: {other:?}"), - ErrorCode::InternalError - )), - } + tokio::task::spawn_blocking(move || { + duckdb()?.num_messages_in_mailbox(account_id, mailbox_id) + }) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn num_messages_in_thread( &self, - searcher: &Searcher, account_id: u64, thread_id: u64, ) -> BichonResult { - let query = self.thread_query(account_id, thread_id); - - let agg_req: Aggregations = serde_json::from_value(json!({ - "thread_count": { - "value_count": { - "field": F_THREAD_ID - } - } - })) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - let collector = AggregationCollector::from_aggs(agg_req, Default::default()); - - let agg_res = searcher - .search(query.as_ref(), &collector) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Self::extract_thread_count(&agg_res) - } - - fn extract_thread_count(agg_res: &AggregationResults) -> BichonResult { - let Some(result) = agg_res.0.get("thread_count") else { - return Err(raise_error!( - "Missing aggregation result: thread_count".into(), - ErrorCode::InternalError - )); - }; - - match result { - AggregationResult::MetricResult(MetricResult::Count(count)) => { - Ok(count.value.map(|v| v as u64).ok_or_else(|| { - raise_error!( - "Failed to get count value from aggregation result: value is None".into(), - ErrorCode::InternalError - ) - })?) - } - other => Err(raise_error!( - format!("Unexpected aggregation result type: {other:?}"), - ErrorCode::InternalError - )), - } - } - - fn create_searcher(&self) -> BichonResult { - self.reader - .reload() - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Ok(self.reader.searcher()) + tokio::task::spawn_blocking(move || duckdb()?.num_messages_in_thread(account_id, thread_id)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } pub async fn get_dashboard_stats( &self, - accounts: &Option>, + accounts: Option>, ) -> BichonResult { - let searcher = self.create_searcher()?; - let now_ms = utc_now!(); - let week_ago_ms = (Utc::now() - Duration::from_secs(60 * 60 * 24 * 30)).timestamp_millis(); - - let aggregations: Aggregations = serde_json::from_value(json!({ - "total_size": { - "sum": { "field": F_SIZE } - }, - "recent_30d_histogram": { - "histogram": { - "field": F_DATE, - "interval": 86400000, - "hard_bounds": { - "min": week_ago_ms, - "max": now_ms - } - } - }, - "top_from_values": { - "terms": { - "field": F_FROM, - "size": 10 - } - }, - "top_account_values": { - "terms": { - "field": F_ACCOUNT_ID, - "size": 10 - } - }, - "attachment_stats": { - "terms": { - "field": F_HAS_ATTACHMENT - } - } - })) - .unwrap(); - - let query: Box = match accounts { - Some(ref ids) if !ids.is_empty() => { - let mut subqueries = Vec::new(); - for &id in ids { - let term = - Term::from_field_u64(SchemaTools::envelope_fields().f_account_id, id); - subqueries.push(( - Occur::Should, - Box::new(TermQuery::new(term, IndexRecordOption::Basic)) as Box, - )); - } - Box::new(BooleanQuery::new(subqueries)) - } - Some(_) => Box::new(EmptyQuery), - None => Box::new(AllQuery), - }; - - let agg_collector = AggregationCollector::from_aggs(aggregations, Default::default()); - let agg_results = searcher - .search(&query, &agg_collector) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - - let mut stats = DashboardStats::default(); - let total_size = agg_results.0.get("total_size").ok_or_else(|| { - raise_error!( - "missing 'total_size' aggregation result".into(), - ErrorCode::InternalError - ) - })?; - - if let AggregationResult::MetricResult(MetricResult::Sum(v)) = total_size { - let total_size = v.value.map(|v| v as u64).ok_or_else(|| { - raise_error!( - "'total_size' sum metric has no value".into(), - ErrorCode::InternalError - ) - })?; - stats.total_size_bytes = total_size; - } - - let recent_30d_histogram = agg_results.0.get("recent_30d_histogram").ok_or_else(|| { - raise_error!( - "missing 'recent_30d_histogram' aggregation result".into(), - ErrorCode::InternalError - ) - })?; - - let mut recent_activity = Vec::with_capacity(31); - if let AggregationResult::BucketResult(BucketResult::Histogram { buckets, .. }) = - recent_30d_histogram - { - if let BucketEntries::Vec(bucket_list) = buckets { - for entry in bucket_list { - if let Key::F64(ms) = entry.key { - recent_activity.push(TimeBucket { - timestamp_ms: ms as i64, - count: entry.doc_count, - }); - } - } - } - } - stats.recent_activity = recent_activity; - let mut top_senders = Vec::with_capacity(11); - let top_from_values = agg_results.0.get("top_from_values").unwrap(); - if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = - top_from_values - { - for entry in buckets { - if let Key::Str(sender) = &entry.key { - top_senders.push(Group { - key: sender.clone(), - count: entry.doc_count, - }); - } - } - } - stats.top_senders = top_senders; - - let mut top_accounts = Vec::with_capacity(11); - let top_account_values = agg_results.0.get("top_account_values").unwrap(); - if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = - top_account_values - { - for entry in buckets { - if let Key::U64(account_id) = &entry.key { - match AccountModel::get(*account_id).await { - Ok(account) => { - top_accounts.push(Group { - key: account.email, - count: entry.doc_count, - }); - } - Err(e) => { - warn!( - account_id = account_id, - error = %e, - "orphaned account index detected, scheduling cleanup" - ); - let account_id = *account_id; - tokio::spawn(async move { - if let Err(e) = ENVELOPE_INDEX_MANAGER - .delete_account_envelopes(account_id) - .await - { - tracing::error!( - account_id = account_id, - error = %e, - "failed to cleanup envelope index" - ); - } - if let Err(e) = - EML_INDEX_MANAGER.delete_account_envelopes(account_id).await - { - tracing::error!( - account_id = account_id, - error = %e, - "failed to cleanup eml index" - ); - } - }); - } - } - } - } - } - stats.top_accounts = top_accounts; - - let attachment_stats = agg_results.0.get("attachment_stats").unwrap(); - if let AggregationResult::BucketResult(BucketResult::Terms { buckets, .. }) = - attachment_stats - { - for entry in buckets { - if let Key::U64(key) = entry.key { - if key == 0 { - stats.without_attachment_count = entry.doc_count; - } - if key == 1 { - stats.with_attachment_count = entry.doc_count; - } - } - } - } - Ok(stats) + tokio::task::spawn_blocking(move || duckdb()?.get_dashboard_stats(accounts)) + .await + .map_err(|e| raise_error!(format!("{:?}", e), ErrorCode::InternalError))? } } pub struct EmlIndexManager { index_writer: Arc>, - sender: mpsc::Sender, + sender: mpsc::Sender, reader: IndexReader, } @@ -1330,7 +353,7 @@ impl EmlIndexManager { DATA_DIR_MANAGER.eml_dir, e ) }); - let (sender, mut receiver) = mpsc::channel::(100); + let (sender, mut receiver) = mpsc::channel::(100); task::spawn(async move { let mut buffer: HashMap = HashMap::with_capacity(EML_BATCH_SIZE); let mut interval = tokio::time::interval(MAX_BUFFER_DURATION); @@ -1339,13 +362,13 @@ impl EmlIndexManager { tokio::select! { maybe_msg = receiver.recv() => { match maybe_msg { - Some(WriteMessage::Document((eid, doc))) => { + Some(DocumentOp::Document((eid, doc))) => { buffer.insert(eid, doc); if buffer.len() >= EML_BATCH_SIZE { EML_INDEX_MANAGER.drain_and_commit(&mut buffer).await; } } - Some(WriteMessage::Shutdown) => { + Some(DocumentOp::Shutdown) => { EML_INDEX_MANAGER.drain_and_commit(&mut buffer).await; break; } @@ -1358,7 +381,7 @@ impl EmlIndexManager { } } _ = shutdown.recv() => { - let _ = EML_INDEX_MANAGER.sender.send(WriteMessage::Shutdown).await; + let _ = EML_INDEX_MANAGER.sender.send(DocumentOp::Shutdown).await; } } } @@ -1369,9 +392,21 @@ impl EmlIndexManager { reader, } } - + /// Adds a document to the indexer. + /// + /// # Parameters + /// - `eid`: A hash derived from **Account ID + Message ID**. + /// This acts as a unique identifier for the EML content itself. + /// + /// - `doc`: The `TantivyDocument` representing the mail body/content. + /// + /// # Logical Design + /// Unlike the `envelope_id` (which is a hash of Account + Folder + Message ID), + /// this `eid` ignores the folder context. This ensures that while metadata + /// (envelopes) can be duplicated across different folders, the physical + /// EML/document storage remains de-duplicated and unique. pub async fn add_document(&self, eid: u64, doc: TantivyDocument) { - let _ = self.sender.send(WriteMessage::Document((eid, doc))).await; + let _ = self.sender.send(DocumentOp::Document((eid, doc))).await; } fn open_or_create_index(index_dir: &PathBuf) -> Index { diff --git a/src/modules/message/content.rs b/src/modules/message/content.rs index e40a9a6..7f15fc9 100644 --- a/src/modules/message/content.rs +++ b/src/modules/message/content.rs @@ -16,11 +16,11 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::base64_encode; use crate::modules::account::migration::AccountModel; use crate::modules::error::code::ErrorCode; -use crate::modules::indexer::manager::EML_INDEX_MANAGER; +use crate::modules::indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER}; +use crate::modules::utils::create_hash; use crate::{modules::error::BichonResult, raise_error}; use mail_parser::{MessageParser, MimeHeaders}; @@ -45,6 +45,72 @@ pub struct AttachmentInfo { pub content_id: Option, } +impl AttachmentInfo { + pub fn get_extension(&self) -> String { + std::path::Path::new(&self.filename) + .extension() + .and_then(|ext| ext.to_str()) + .map(|ext| ext.to_lowercase()) + .unwrap_or_default() + } + + pub fn get_category(&self) -> &'static str { + let ext = self.get_extension(); + + let category = match ext.as_str() { + "doc" | "docx" | "pdf" | "rtf" | "odt" | "pages" | "pptx" | "ppt" => Some("document"), + "xls" | "xlsx" | "ods" | "numbers" | "csv" => Some("spreadsheet"), + "ical" | "ics" | "vcs" | "ifb" | "icalendar" => Some("event"), + "txt" | "log" | "md" => Some("text"), + "jpg" | "jpeg" | "png" | "gif" | "bmp" | "tiff" | "avif" | "heic" | "heif" | "webp" => { + Some("image") + } + "mp4" | "mkv" | "mov" | "avi" | "webm" => Some("video"), + "wav" | "mp3" | "aac" | "ogg" | "wma" | "flac" | "aiff" => Some("audio"), + "psd" | "eps" | "svg" | "cdr" | "ai" => Some("graphics_2d"), + "stl" | "obj" | "3mf" | "amf" | "f3d" | "sldprt" | "stp" | "step" | "dwg" | "x_t" + | "x_b" | "sat" | "ipt" => Some("graphics_3d"), + "c" | "h" | "html" | "css" | "js" | "ts" | "vue" | "tsx" | "svelte" | "py" | "java" + | "cs" | "go" | "rb" | "php" | "swift" | "rs" | "r" | "jl" | "lua" | "sql" => { + Some("code") + } + "tsv" | "xml" | "json" | "yml" | "yaml" | "toml" | "env" | "ini" => Some("data"), + "ps1" | "sh" | "bat" | "cmd" | "exe" | "msi" | "dmg" | "pkg" | "deb" | "rpm" => { + Some("executable") + } + "zip" | "gz" | "tgz" | "7z" | "rar" | "tar" | "bz2" | "zst" | "xz" | "iso" | "img" => { + Some("archive") + } + _ => None, + }; + + if let Some(cat) = category { + return cat; + } + + let mime = self.file_type.to_lowercase(); + if mime.starts_with("image/") { + return "image"; + } + if mime.starts_with("video/") { + return "video"; + } + if mime.starts_with("audio/") { + return "audio"; + } + if mime.starts_with("text/") { + return "text"; + } + if mime.contains("compressed") || mime.contains("zip") || mime.contains("archive") { + return "archive"; + } + if mime.contains("pdf") || mime.contains("msword") || mime.contains("officedocument") { + return "document"; + } + + "other" + } +} /// Represents the content of an email message in both plain text and HTML formats. /// /// This struct contains optional fields for plain text and HTML versions of @@ -64,13 +130,23 @@ pub struct FullMessageContent { pub attachments: Option>, } -pub async fn retrieve_email_content( - account_id: u64, - id: u64, -) -> BichonResult { +pub async fn retrieve_email_content(account_id: u64, id: u64) -> BichonResult { AccountModel::check_account_exists(account_id).await?; + let envelope = ENVELOPE_INDEX_MANAGER + .get_envelope_by_id(account_id, id) + .await? + .ok_or_else(|| { + raise_error!( + format!( + "Email record not found: account_id={} id={}", + account_id, id + ), + ErrorCode::ResourceNotFound + ) + })?; + let eml_id = create_hash(account_id, &envelope.message_id); let eml = EML_INDEX_MANAGER - .get(account_id, id) + .get(account_id, eml_id) .await? .ok_or_else(|| { raise_error!( @@ -133,7 +209,7 @@ pub async fn retrieve_email_content( attachments.push(AttachmentInfo { filename, - size: attachment.len(), + size: attachment.contents().len(), inline, file_type, content_id: attachment.content_id().map(Into::into), diff --git a/src/modules/message/delete.rs b/src/modules/message/delete.rs index c390c75..888f82c 100644 --- a/src/modules/message/delete.rs +++ b/src/modules/message/delete.rs @@ -16,7 +16,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use crate::modules::error::BichonResult; use crate::modules::indexer::manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER}; use std::collections::HashMap; @@ -26,6 +25,6 @@ pub async fn delete_messages_impl(request: HashMap>) -> BichonResu .delete_email_multi_account(&request) .await?; ENVELOPE_INDEX_MANAGER - .delete_envelopes_multi_account(&request) + .delete_envelopes_multi_account(request) .await } diff --git a/src/modules/message/search.rs b/src/modules/message/search.rs index ccac1d5..2e2fa8a 100644 --- a/src/modules/message/search.rs +++ b/src/modules/message/search.rs @@ -39,14 +39,14 @@ pub struct SearchFilter { pub bcc: Option, pub since: Option, pub before: Option, - pub account_ids: Option>, - pub mailbox_ids: Option>, + pub account_ids: Option>, + pub mailbox_ids: Option>, pub min_size: Option, pub max_size: Option, pub message_id: Option, pub has_attachment: Option, pub attachment_name: Option, - pub tags: Option>, + pub tags: Option>, } #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Enum)] diff --git a/src/modules/mod.rs b/src/modules/mod.rs index d6eedd1..ef52c75 100644 --- a/src/modules/mod.rs +++ b/src/modules/mod.rs @@ -24,6 +24,7 @@ pub mod common; pub mod context; pub mod dashboard; pub mod database; +pub mod duckdb; pub mod envelope; pub mod error; pub mod imap; diff --git a/src/modules/oauth2/entity.rs b/src/modules/oauth2/entity.rs index 697573d..e737530 100644 --- a/src/modules/oauth2/entity.rs +++ b/src/modules/oauth2/entity.rs @@ -21,7 +21,7 @@ use crate::{ modules::{ database::{ delete_impl, insert_impl, manager::DB_MANAGER, paginate_query_primary_scan_all_impl, - secondary_find_impl, update_impl, + async_secondary_find_impl, update_impl, }, error::{code::ErrorCode, BichonResult}, rest::response::DataPage, @@ -133,7 +133,7 @@ impl OAuth2 { } pub async fn get(id: u64) -> BichonResult> { - secondary_find_impl(DB_MANAGER.meta_db(), OAuth2Key::id, id).await + async_secondary_find_impl(DB_MANAGER.meta_db(), OAuth2Key::id, id).await } pub async fn delete(id: u64) -> BichonResult<()> { diff --git a/src/modules/rest/api/account.rs b/src/modules/rest/api/account.rs index 9583ccd..cdb75e9 100644 --- a/src/modules/rest/api/account.rs +++ b/src/modules/rest/api/account.rs @@ -58,7 +58,7 @@ impl AccountApi { context .require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS) .await?; - Ok(Json(AccountModel::get(account_id).await?)) + Ok(Json(AccountModel::async_get(account_id).await?)) } /// Delete an account by ID - WARNING: This permanently removes the account and all associated resources diff --git a/src/modules/rest/api/message.rs b/src/modules/rest/api/message.rs index bdc4c55..0f2517e 100644 --- a/src/modules/rest/api/message.rs +++ b/src/modules/rest/api/message.rs @@ -34,6 +34,7 @@ use crate::modules::rest::response::DataPage; use crate::modules::rest::ApiResult; use crate::modules::rest::ErrorCode; use crate::modules::users::permissions::Permission; +use crate::modules::utils::validate_tag; use crate::raise_error; use poem::Body; use poem_openapi::param::{Path, Query}; @@ -41,7 +42,6 @@ use poem_openapi::payload::{Attachment, AttachmentType, Json}; use poem_openapi::OpenApi; use std::collections::HashMap; use std::collections::HashSet; -use tantivy::schema::Facet; pub struct MessageApi; @@ -147,7 +147,7 @@ impl MessageApi { /// Fetches the content of a specific email. #[oai( - path = "/message-content/:account_id/:message_id", + path = "/message-content/:account_id/:envelope_id", method = "get", operation_id = "fetch_message_content" )] @@ -156,7 +156,7 @@ impl MessageApi { /// The ID of the account. account_id: Path, /// The ID of the message to fetch. - message_id: Path, + envelope_id: Path, context: ClientContext, ) -> ApiResult> { let account_id = account_id.0; @@ -164,13 +164,13 @@ impl MessageApi { .require_permission(Some(account_id), Permission::DATA_READ) .await?; Ok(Json( - retrieve_email_content(account_id, message_id.0).await?, + retrieve_email_content(account_id, envelope_id.0).await?, )) } /// Retrieves the envelope (metadata) of a specific message. #[oai( - path = "/envelope/:account_id/:message_id", + path = "/envelope/:account_id/:envelope_id", method = "get", operation_id = "get_envelope" )] @@ -179,7 +179,7 @@ impl MessageApi { /// The ID of the account. account_id: Path, /// The ID of the message. - message_id: Path, + envelope_id: Path, context: ClientContext, ) -> ApiResult> { let account_id = account_id.0; @@ -187,13 +187,13 @@ impl MessageApi { .require_permission(Some(account_id), Permission::DATA_READ) .await?; let envelope = ENVELOPE_INDEX_MANAGER - .get_envelope_by_id(account_id, message_id.0) + .get_envelope_by_id(account_id, envelope_id.0) .await? .ok_or_else(|| { raise_error!( format!( - "Envelope not found: account_id={} message_id={}", - account_id, message_id.0 + "Envelope not found: account_id={} envelope_id={}", + account_id, envelope_id.0 ), ErrorCode::ResourceNotFound ) @@ -308,8 +308,8 @@ impl MessageApi { ) -> ApiResult<()> { let req = req.0; for tag in &req.tags { - Facet::from_text(tag) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InvalidParameter))?; + validate_tag(tag) + .map_err(|e| raise_error!(format!("{}", e), ErrorCode::InvalidParameter))?; } for account_id in req.updates.keys() { diff --git a/src/modules/settings/cli.rs b/src/modules/settings/cli.rs index d1b6ca3..7e8147b 100644 --- a/src/modules/settings/cli.rs +++ b/src/modules/settings/cli.rs @@ -247,6 +247,21 @@ pub struct Settings { value_parser = clap::value_parser!(u16).range(1..) )] pub bichon_sync_concurrency: Option, + + #[clap( + long, + env, + help = "Number of DuckDB execution threads (default: auto)", + value_parser = clap::value_parser!(u16).range(1..) + )] + pub bichon_duckdb_threads: Option, + + #[clap( + long, + env, + help = "Maximum memory DuckDB may use (e.g. 512MB, 2GB, 80%)" + )] + pub bichon_duckdb_max_memory: Option, } impl Settings { diff --git a/src/modules/token/mod.rs b/src/modules/token/mod.rs index 7e0f3dd..d2429d2 100644 --- a/src/modules/token/mod.rs +++ b/src/modules/token/mod.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use super::error::code::ErrorCode; use crate::modules::database::manager::DB_MANAGER; use crate::modules::database::{ - async_find_impl, delete_impl, filter_by_secondary_key_impl, with_transaction, + async_find_impl, delete_impl, async_filter_by_secondary_key_impl, with_transaction, }; use crate::modules::database::{insert_impl, list_all_impl, update_impl}; use crate::modules::settings::cli::SETTINGS; @@ -154,7 +154,7 @@ impl AccessTokenModel { } pub async fn get_user_webui_token(user_id: u64) -> BichonResult> { - let tokens = filter_by_secondary_key_impl::( + let tokens = async_filter_by_secondary_key_impl::( DB_MANAGER.meta_db(), AccessTokenModelKey::user_id, user_id, @@ -167,7 +167,7 @@ impl AccessTokenModel { } pub async fn get_user_api_tokens(user_id: u64) -> BichonResult> { - let tokens = filter_by_secondary_key_impl::( + let tokens = async_filter_by_secondary_key_impl::( DB_MANAGER.meta_db(), AccessTokenModelKey::user_id, user_id, diff --git a/src/modules/users/mod.rs b/src/modules/users/mod.rs index 5df1b10..d434fd2 100644 --- a/src/modules/users/mod.rs +++ b/src/modules/users/mod.rs @@ -21,7 +21,7 @@ use crate::{ modules::{ database::{ async_find_impl, batch_delete_impl, delete_impl, list_all_impl, manager::DB_MANAGER, - secondary_find_impl, update_impl, with_transaction, + async_secondary_find_impl, update_impl, with_transaction, }, error::{code::ErrorCode, BichonResult}, token::{AccessTokenModel, AccessTokenModelKey, TokenType}, @@ -282,7 +282,7 @@ impl BichonUserV2 { username: String, password: String, ) -> BichonResult { - let user_option = secondary_find_impl::( + let user_option = async_secondary_find_impl::( DB_MANAGER.meta_db(), BichonUserV2Key::username, username.clone(), @@ -292,7 +292,7 @@ impl BichonUserV2 { let user = match user_option { Some(u) => u, None => { - match secondary_find_impl::( + match async_secondary_find_impl::( DB_MANAGER.meta_db(), BichonUserV2Key::email, username, @@ -366,7 +366,7 @@ impl BichonUserV2 { pub async fn check_username_conflict(username: &str) -> BichonResult<()> { // Check username duplicate - if secondary_find_impl::( + if async_secondary_find_impl::( DB_MANAGER.meta_db(), BichonUserV2Key::username, username.to_string(), @@ -385,7 +385,7 @@ impl BichonUserV2 { pub async fn check_email_conflict(email: &str) -> BichonResult<()> { // Check email duplicate - if secondary_find_impl::( + if async_secondary_find_impl::( DB_MANAGER.meta_db(), BichonUserV2Key::email, email.to_string(), @@ -520,7 +520,7 @@ impl BichonUserV2 { } if let Some(username) = &request.username { - let user_option = secondary_find_impl::( + let user_option = async_secondary_find_impl::( DB_MANAGER.meta_db(), BichonUserV2Key::username, username.to_string(), @@ -538,7 +538,7 @@ impl BichonUserV2 { } if let Some(email) = &request.email { - let user_option = secondary_find_impl::( + let user_option = async_secondary_find_impl::( DB_MANAGER.meta_db(), BichonUserV2Key::email, email.to_string(), diff --git a/src/modules/users/payload.rs b/src/modules/users/payload.rs index a7e430c..072eb34 100644 --- a/src/modules/users/payload.rs +++ b/src/modules/users/payload.rs @@ -284,7 +284,7 @@ impl UserCreateRequest { } for (aid, rid) in &self.account_access_map { - if AccountModel::find(*aid).await?.is_none() { + if AccountModel::async_find(*aid).await?.is_none() { return Err(raise_error!( format!("Account {} not found", aid), ErrorCode::InvalidParameter @@ -404,7 +404,7 @@ impl UserUpdateRequest { if let Some(account_access_map) = &self.account_access_map { for (aid, rid) in account_access_map { - if AccountModel::find(*aid).await?.is_none() { + if AccountModel::async_find(*aid).await?.is_none() { return Err(raise_error!( format!("Account {} not found", aid), ErrorCode::InvalidParameter diff --git a/src/modules/utils/mod.rs b/src/modules/utils/mod.rs index 57610b7..7c01f64 100644 --- a/src/modules/utils/mod.rs +++ b/src/modules/utils/mod.rs @@ -21,7 +21,7 @@ use std::{fs, io, path::PathBuf}; use crate::modules::error::BichonResult; use base64::engine::general_purpose::STANDARD; use base64::{engine::general_purpose, Engine}; -use rand::{rng, Rng}; +use rand::{rng, RngExt}; use super::error::code::ErrorCode; @@ -277,14 +277,23 @@ pub fn hash(s: &str) -> u64 { } pub fn create_hash(account_id: u64, field: &str) -> u64 { - // Construct a buffer of bytes from account_id and mailbox_name let mut buffer = Vec::new(); - buffer.extend_from_slice(&account_id.to_le_bytes()); // Convert u64 to bytes + buffer.extend_from_slice(&account_id.to_le_bytes()); buffer.push(b':'); // Separator - buffer.extend_from_slice(field.as_bytes()); // Add mailbox name - // Create a Cursor for the buffer + buffer.extend_from_slice(field.as_bytes()); + let mut cursor = std::io::Cursor::new(buffer); + let hash = murmur3::murmur3_x64_128(&mut cursor, 0).unwrap(); + (hash & 0x1F_FFFF_FFFF_FFFF) as u64 +} + +pub fn create_hash2(account_id: u64, field1: u64, field2: &str) -> u64 { + let mut buffer = Vec::new(); + buffer.extend_from_slice(&account_id.to_le_bytes()); + buffer.push(b':'); // Separator + buffer.extend_from_slice(&field1.to_le_bytes()); + buffer.push(b':'); // Separator + buffer.extend_from_slice(field2.as_bytes()); let mut cursor = std::io::Cursor::new(buffer); - // Compute the 128-bit Murmur3 hash and cast to u64 let hash = murmur3::murmur3_x64_128(&mut cursor, 0).unwrap(); (hash & 0x1F_FFFF_FFFF_FFFF) as u64 } @@ -335,3 +344,28 @@ pub fn decode_avatar_bytes(base64_str: &str) -> BichonResult> { Ok(bytes) } + +pub fn validate_tag(tag: &str) -> Result<(), String> { + if tag.is_empty() { + return Err("Tag cannot be empty".to_string()); + } + + const INVALID: &[char] = &[ + '\'', '"', '`', ';', ',', '(', ')', '[', ']', '{', '}', '<', '>', + ]; + + let mut found = Vec::new(); + + for c in tag.chars() { + if INVALID.contains(&c) && !found.contains(&c) { + found.push(c); + } + } + + if !found.is_empty() { + let chars: String = found.iter().collect(); + return Err(format!("Tag contains invalid characters: {}", chars)); + } + + Ok(()) +} diff --git a/src/modules/utils/net.rs b/src/modules/utils/net.rs index 0506030..d56dc66 100644 --- a/src/modules/utils/net.rs +++ b/src/modules/utils/net.rs @@ -38,13 +38,6 @@ pub(crate) async fn establish_tcp_connection_with_timeout( ) -> BichonResult>>> { // Establish the TCP connection with a timeout let tcp_stream = connect_with_optional_proxy(use_proxy, address).await?; - - // // Disable Nagle's algorithm for more efficient network communication - // tcp_stream - // .set_nodelay(true) - // .map_err(|e| raise_error!(e.to_string(), ErrorCode::NetworkError))?; - - // Wrap the TCP stream in a TimeoutStream for timeout management let mut timeout_stream = TimeoutStream::new(tcp_stream); // Set read and write timeouts @@ -140,7 +133,7 @@ async fn connect_with_optional_proxy( ) })? .map(|s| s.into_inner()) - .map_err(|e| raise_error!(e.to_string(), ErrorCode::NetworkError)); + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)); } // Fallback to direct TCP connection timeout(TIMEOUT, TcpStream::connect(address)) @@ -160,5 +153,5 @@ async fn connect_with_optional_proxy( ErrorCode::ConnectionTimeout ) })? - .map_err(|e| raise_error!(e.to_string(), ErrorCode::NetworkError)) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)) } diff --git a/src/modules/utils/tls.rs b/src/modules/utils/tls.rs index af91c30..a0f77d0 100644 --- a/src/modules/utils/tls.rs +++ b/src/modules/utils/tls.rs @@ -77,7 +77,7 @@ async fn establish_rustls_stream( let tls_stream = tls_connector .connect(server_name, stream) .await - .map_err(|e| raise_error!(e.to_string(), ErrorCode::NetworkError))?; + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?; Ok(tls_stream) } diff --git a/web/src/api/index.ts b/web/src/api/index.ts index b0e6a6c..8725b08 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -43,6 +43,6 @@ export interface EmailEnvelope { internal_date: number; size: number; thread_id: number, - attachments: string[]; + attachment_count: number; tags: string[]; } \ No newline at end of file diff --git a/web/src/features/auth/sign-in/auth-layout.tsx b/web/src/features/auth/auth-layout.tsx similarity index 100% rename from web/src/features/auth/sign-in/auth-layout.tsx rename to web/src/features/auth/auth-layout.tsx diff --git a/web/src/features/auth/sign-in/index.tsx b/web/src/features/auth/index.tsx similarity index 96% rename from web/src/features/auth/sign-in/index.tsx rename to web/src/features/auth/index.tsx index d1692cc..bda33c6 100644 --- a/web/src/features/auth/sign-in/index.tsx +++ b/web/src/features/auth/index.tsx @@ -23,7 +23,7 @@ import { CardHeader, CardTitle, } from '@/components/ui/card' -import { UserAuthForm } from './components/user-auth-form' +import { UserAuthForm } from './user-auth-form' import { useTranslation } from 'react-i18next' import { AuthLayout } from './auth-layout' diff --git a/web/src/features/auth/sign-in/components/logout.tsx b/web/src/features/auth/sign-in/components/logout.tsx deleted file mode 100644 index 9b71923..0000000 --- a/web/src/features/auth/sign-in/components/logout.tsx +++ /dev/null @@ -1,63 +0,0 @@ -// -// Copyright (c) 2025 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - - -import { IconLogout } from '@tabler/icons-react' -import { ConfirmDialog } from '@/components/confirm-dialog' -import { useTranslation } from 'react-i18next' - -interface Props { - open: boolean - onOpenChange: (open: boolean) => void - handleConfirm: () => void -} - -export function LogoutConfirmDialog({ open, onOpenChange, handleConfirm }: Props) { - const { t } = useTranslation() - const handleLogout = () => { - handleConfirm(); - onOpenChange(false); - }; - - return ( - - {' '} - {t('auth.logout')} - - } - desc={ -

- {t('auth.areYouSureYouWantToLogOut')} -
- {t('auth.youWillNeedToLogInAgain')} -

- } - confirmText={t('auth.logout')} - cancelBtnText={t('common.cancel')} - /> - ) -} \ No newline at end of file diff --git a/web/src/features/auth/sign-in/components/user-auth-form.tsx b/web/src/features/auth/user-auth-form.tsx similarity index 100% rename from web/src/features/auth/sign-in/components/user-auth-form.tsx rename to web/src/features/auth/user-auth-form.tsx diff --git a/web/src/features/dashboard/index.tsx b/web/src/features/dashboard/index.tsx index 90622c7..6bd460c 100644 --- a/web/src/features/dashboard/index.tsx +++ b/web/src/features/dashboard/index.tsx @@ -29,6 +29,7 @@ import { get_dashboard_stats, TimeBucket } from '@/api/system/api'; import { Main } from '@/components/layout/main'; import { FixedHeader } from '@/components/layout/fixed-header'; import { useTranslation } from 'react-i18next'; +import { getToken } from '@/stores/authStore'; interface DailyActivity { date: string; @@ -102,8 +103,10 @@ const EmptyTable = ({ title }: { title: string }) => ( ); export default function MailArchiveDashboard() { + const token = getToken(); const { data: stats, isLoading } = useQuery({ queryKey: ['dashboard-stats'], + enabled: !!token, queryFn: get_dashboard_stats, }); diff --git a/web/src/features/mailbox/components/mail-list.tsx b/web/src/features/mailbox/components/mail-list.tsx index d61d9d7..668baae 100644 --- a/web/src/features/mailbox/components/mail-list.tsx +++ b/web/src/features/mailbox/components/mail-list.tsx @@ -118,7 +118,7 @@ export function MailList({ )} {items.map((item, index) => { - const hasAttachments = item.attachments && item.attachments.length > 0 + const hasAttachments = item.attachment_count > 0 const isSelected = currentEnvelope?.id === item.id const isChecked = hasSelected(item.id) return ( @@ -171,7 +171,7 @@ export function MailList({ {hasAttachments && (
- {item.attachments?.length} + {item.attachment_count}
)} {formatBytes(item.size)} diff --git a/web/src/features/search/mail-list-table.tsx b/web/src/features/search/mail-list-table.tsx index 7f6226e..a577f5f 100755 --- a/web/src/features/search/mail-list-table.tsx +++ b/web/src/features/search/mail-list-table.tsx @@ -207,7 +207,7 @@ export function MailListTable({ { id: "attachment_count", header: () => , - cell: ({ row }) => {(row.original.attachments ?? []).length}, + cell: ({ row }) => {row.original.attachment_count}, meta: { className: 'text-left text-xs' }, minSize: 40, maxSize: 40 diff --git a/web/src/features/search/mail-list.tsx b/web/src/features/search/mail-list.tsx index ec5d1bd..87ef6df 100644 --- a/web/src/features/search/mail-list.tsx +++ b/web/src/features/search/mail-list.tsx @@ -154,7 +154,7 @@ export function MailList({ )} {items.map((item, index) => { - const hasAttachments = item.attachments && item.attachments.length > 0 + const hasAttachments = item.attachment_count > 0 const isSelectedRow = currentEnvelope?.id === item.id const isChecked = hasSelected(item.account_id, item.id) @@ -209,7 +209,7 @@ export function MailList({ {hasAttachments && (
- {item.attachments?.length} + {item.attachment_count}
)} diff --git a/web/src/hooks/use-current-user.ts b/web/src/hooks/use-current-user.ts index 145875a..d27355b 100644 --- a/web/src/hooks/use-current-user.ts +++ b/web/src/hooks/use-current-user.ts @@ -21,10 +21,14 @@ import { useQuery } from '@tanstack/react-query' import { AxiosError } from 'axios' import { get_current_user, User } from '@/api/users/api' import { useMemo } from 'react' +import { getToken } from '@/stores/authStore' export function useCurrentUser() { + const token = getToken(); + const query = useQuery({ queryKey: ['current-user'], + enabled: !!token, queryFn: get_current_user, retry: false, }) diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.ts index 5464615..4318009 100644 --- a/web/src/lib/utils.ts +++ b/web/src/lib/utils.ts @@ -88,28 +88,12 @@ export function validateTag(facetPath: string) { }; } - if (!facetPath.startsWith('/')) { + const invalidChars = /['"`;,()[\]{}<>]/; + + if (invalidChars.test(facetPath)) { return { valid: false, - error: "Tag path must start with '/'" - }; - } - - let escaped = false; - for (let i = 1; i < facetPath.length; i++) { - const char = facetPath[i]; - - if (escaped) { - escaped = false; - } else if (char === '\\') { - escaped = true; - } - } - - if (escaped) { - return { - valid: false, - error: "Tag path has unmatched escape character at the end" + error: "Tag path contains invalid characters" }; } diff --git a/web/src/routes/(auth)/sign-in.tsx b/web/src/routes/(auth)/sign-in.tsx index 2500a6a..bf8fa38 100644 --- a/web/src/routes/(auth)/sign-in.tsx +++ b/web/src/routes/(auth)/sign-in.tsx @@ -18,7 +18,7 @@ import { createFileRoute } from '@tanstack/react-router' -import SignIn from '@/features/auth/sign-in' +import SignIn from '@/features/auth' export const Route = createFileRoute('/(auth)/sign-in')({ component: SignIn,