mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-03 07:48:34 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
86869ac848 | ||
|
|
38453accc6 | ||
|
|
a60b2c7dc4 | ||
|
|
8e46c7a162 | ||
|
|
1a615e1c45 | ||
|
|
83dd9cdd6b | ||
|
|
f30cd66e00 | ||
|
|
4bd714a670 | ||
|
|
c0a63a1e3c | ||
|
|
04a022e850 | ||
|
|
f9c2fc77ff | ||
|
|
ec3e842bbb | ||
|
|
7311529908 | ||
|
|
0792bb546d | ||
|
|
0be2670600 | ||
|
|
575f851cfb | ||
|
|
61430b72b0 | ||
|
|
15c0cfc1d9 | ||
|
|
ea8c493374 | ||
|
|
005b1c2116 | ||
|
|
d8b78b8010 | ||
|
|
36f3f19cdc | ||
|
|
9f3097df32 | ||
|
|
c075b9ef12 | ||
|
|
21a7f7e9d5 | ||
|
|
26c14fcaaf | ||
|
|
6873841ba4 | ||
|
|
0dd81f599d | ||
|
|
b3afc52a82 | ||
|
|
171a40d70f | ||
|
|
895ea543a9 | ||
|
|
6eca351994 | ||
|
|
1f477eca65 | ||
|
|
a3cdc094e8 | ||
|
|
95147a7824 | ||
|
|
b22811f78c | ||
|
|
fd61d013a2 | ||
|
|
3a950e7591 | ||
|
|
f17820bfa8 | ||
|
|
178b25d27d | ||
|
|
d160ca75f5 | ||
|
|
04136a4ae2 | ||
|
|
1d6f5d9a22 | ||
|
|
105a6d9b15 | ||
|
|
df440c8441 | ||
|
|
4116a59b79 | ||
|
|
609eee1b84 | ||
|
|
79b9f07888 | ||
|
|
ba28369202 | ||
|
|
ff64b66f79 | ||
|
|
a4f8e674c3 | ||
|
|
dde6b990da | ||
|
|
6b1f843bd5 | ||
|
|
66fd50bc23 | ||
|
|
9daab241b0 | ||
|
|
85d5490834 | ||
|
|
6e984f376c | ||
|
|
7470125a23 | ||
|
|
469d254e2b | ||
|
|
7fde7ee19a | ||
|
|
d543508a23 | ||
|
|
f440069912 |
+3
-1
@@ -1,4 +1,6 @@
|
||||
/target
|
||||
.vscode
|
||||
.idea
|
||||
config.toml
|
||||
config.toml
|
||||
node_modules
|
||||
dedup_report.txt
|
||||
Generated
+34
-33
@@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bichon-admin"
|
||||
version = "1.0.1"
|
||||
version = "1.4.0"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"console",
|
||||
@@ -301,7 +301,6 @@ dependencies = [
|
||||
"indicatif",
|
||||
"itertools",
|
||||
"memdb",
|
||||
"mimalloc",
|
||||
"native_db",
|
||||
"native_model",
|
||||
"serde",
|
||||
@@ -312,7 +311,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-cli"
|
||||
version = "1.0.1"
|
||||
version = "1.4.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
@@ -338,7 +337,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-core"
|
||||
version = "1.0.1"
|
||||
version = "1.4.0"
|
||||
dependencies = [
|
||||
"async-imap",
|
||||
"base64 0.22.1",
|
||||
@@ -346,6 +345,7 @@ dependencies = [
|
||||
"bytes 1.11.1",
|
||||
"chrono",
|
||||
"clap",
|
||||
"cron",
|
||||
"dashmap",
|
||||
"deunicode",
|
||||
"email_address",
|
||||
@@ -396,7 +396,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-server"
|
||||
version = "1.0.1"
|
||||
version = "1.4.0"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"bichon-smtp",
|
||||
@@ -404,7 +404,6 @@ dependencies = [
|
||||
"email_address",
|
||||
"governor",
|
||||
"http",
|
||||
"mimalloc",
|
||||
"poem",
|
||||
"poem-derive",
|
||||
"poem-openapi",
|
||||
@@ -421,7 +420,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bichon-smtp"
|
||||
version = "1.0.1"
|
||||
version = "1.4.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bichon-core",
|
||||
@@ -875,6 +874,17 @@ version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
|
||||
|
||||
[[package]]
|
||||
name = "cron"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5877d3fbf742507b66bc2a1945106bd30dd8504019d596901ddd012a4dd01740"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"once_cell",
|
||||
"winnow 0.6.26",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.15"
|
||||
@@ -1006,9 +1016,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.1.0"
|
||||
version = "6.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
|
||||
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
@@ -1779,9 +1789,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
|
||||
checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
|
||||
dependencies = [
|
||||
"bytes 1.11.1",
|
||||
"itoa",
|
||||
@@ -2252,15 +2262,6 @@ version = "0.2.185"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
@@ -2465,15 +2466,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -3931,9 +3923,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
@@ -4286,9 +4278,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.39.1"
|
||||
version = "0.39.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4deba334e1190ba7cb498327affa11e5ece10d26a30ab2f27fcf09504b8d8b6"
|
||||
checksum = "14311e7e9a03114cd4b65eedd54e8fed2945e17f08586ae97ef53bc0669f9581"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
@@ -5611,6 +5603,15 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.6.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
|
||||
+6
-6
@@ -6,23 +6,23 @@ members = [
|
||||
"crates/server",
|
||||
"crates/cli",
|
||||
"crates/admin",
|
||||
"crates/smtp",
|
||||
]
|
||||
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "1.0.1"
|
||||
version = "1.4.0"
|
||||
edition = "2021"
|
||||
|
||||
[workspace.dependencies]
|
||||
chrono = "0.4.44"
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
mimalloc = "0.1.50"
|
||||
memdb = { path = "crates/memdb" }
|
||||
itertools = "0.14.0"
|
||||
ring = { version = "0.17.14", features = ["std"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.149"
|
||||
serde_json = "1.0.150"
|
||||
tokio = { version = "1.52.3", features = ["full"] }
|
||||
tracing = "0.1.44"
|
||||
tracing-appender = "0.2.3"
|
||||
@@ -38,7 +38,7 @@ reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
"socks",
|
||||
] }
|
||||
tokio-socks = "0.5.2"
|
||||
http = "1.4.0"
|
||||
http = "1.4.1"
|
||||
regex = "1.12.3"
|
||||
email_address = "0.2.9"
|
||||
futures = "0.3.32"
|
||||
@@ -52,7 +52,7 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
timeago = "0.6.0"
|
||||
oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] }
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
sysinfo = "0.39.1"
|
||||
sysinfo = "0.39.2"
|
||||
num_cpus = "1.17.0"
|
||||
rand = "0.10.1"
|
||||
encoding_rs = "0.8.35"
|
||||
@@ -73,7 +73,7 @@ time = { version = "0.3.47", features = [
|
||||
rust-embed = "8.11.0"
|
||||
murmur3 = "0.5.2"
|
||||
urlencoding = "2.1.3"
|
||||
dashmap = "6.1.0"
|
||||
dashmap = "6.2.1"
|
||||
gethostname = "1.1.0"
|
||||
itoa = "1.0.18"
|
||||
html2text = "0.17.1"
|
||||
|
||||
@@ -106,6 +106,10 @@
|
||||
- **Admin Tooling**: Password reset for locked-out admins. Non-destructive v0.3.7 to v1.0 data migration.
|
||||
- **API Token Management**: Create, list, and revoke long-lived API tokens for programmatic access.
|
||||
- **SOCKS5 Proxy Management**: Configure and manage proxy profiles for routing IMAP traffic per account.
|
||||
- **Scheduled Download**: Configure per-account download schedules using cron expressions. Run syncs at specific times or intervals — for example, nightly-only or business-hours-only archiving.
|
||||
- **Remote Content Blocking**: External images and tracking pixels embedded in emails are blocked by default. Users can selectively allow remote content to load on a per-message basis from the WebUI.
|
||||
- **Async Index Deduplication**: Duplicate detection in the search index is performed asynchronously, reducing write latency during high-throughput ingestion.
|
||||
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -271,6 +275,10 @@ All settings accept both CLI flags (`--bichon-http-port`) and environment variab
|
||||
> [!TIP]
|
||||
> Place `BICHON_INDEX_DIR` on fast SSD storage for responsive search, and `BICHON_DATA_DIR` on high-capacity HDD for cost-effective blob storage.
|
||||
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Bichon does NOT support writing data directly to a network file system (NFS, CIFS/SMB, etc.). All directories — `BICHON_ROOT_DIR`, `BICHON_DATA_DIR`, and `BICHON_INDEX_DIR` — must reside on a **local file system**; otherwise, data corruption may occur.
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
| Variable | Default | Description |
|
||||
@@ -576,8 +584,8 @@ No. Bichon is an **archiver**, not an email client. The optional SMTP server **r
|
||||
|
||||
### What hardware does Bichon need?
|
||||
|
||||
- **Minimal:** 1 CPU core, 512 MB RAM
|
||||
- **Recommended (100+ accounts, 200+ GB):** 4+ cores, 2+ GB RAM
|
||||
- **Recommended:** 4+ CPU cores, 2+ GB RAM (sufficient for 10+ accounts and 200+ GB of archived data)
|
||||
- Filesystem: use a mainstream Linux filesystem such as **ext4** or **XFS**; avoid network / virtual filesystems (NFS, VirtIO-FS) for all data directories
|
||||
- Indices benefit from SSD storage; blob storage can use HDD
|
||||
|
||||
### How do I reset the admin password?
|
||||
@@ -648,7 +656,6 @@ Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or joi
|
||||
| **Frontend** | React 18, TypeScript, Vite 6, ShadCN UI, TanStack Router/Query/Table |
|
||||
| **Charts** | Recharts |
|
||||
| **i18n** | i18next (18 languages) |
|
||||
| **Allocator** | mimalloc |
|
||||
| **Container** | Ubuntu 24.04, Docker |
|
||||
|
||||
## License
|
||||
|
||||
@@ -17,5 +17,4 @@ serde_json.workspace = true
|
||||
|
||||
itertools.workspace = true
|
||||
snafu.workspace = true
|
||||
memdb.workspace = true
|
||||
mimalloc = "0.1.50"
|
||||
memdb.workspace = true
|
||||
@@ -18,7 +18,6 @@
|
||||
|
||||
use console::style;
|
||||
use dialoguer::{theme::ColorfulTheme, Select};
|
||||
use mimalloc::MiMalloc;
|
||||
|
||||
use crate::{migrate::handle_migration, reset::handle_reset_password};
|
||||
|
||||
@@ -26,8 +25,6 @@ pub mod meta;
|
||||
pub mod migrate;
|
||||
pub mod reset;
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: MiMalloc = MiMalloc;
|
||||
|
||||
fn main() {
|
||||
run_interactive();
|
||||
@@ -43,7 +40,7 @@ async fn run_interactive() {
|
||||
|
||||
let main_options = vec![
|
||||
"Reset Admin Password",
|
||||
"Migrate Legacy v0.3.7 Storage to v1.0.0",
|
||||
"Migrate Legacy v0.3.7 Storage to v1.x",
|
||||
"Exit",
|
||||
];
|
||||
|
||||
|
||||
@@ -117,7 +117,6 @@ pub struct AccountV3 {
|
||||
pub capabilities: Option<Vec<String>>,
|
||||
pub date_since: Option<DateSince>,
|
||||
pub date_before: Option<RelativeDate>,
|
||||
pub folder_limit: Option<u32>,
|
||||
pub sync_folders: Option<Vec<String>>,
|
||||
pub account_type: AccountType,
|
||||
pub sync_interval_min: Option<i64>,
|
||||
@@ -193,7 +192,7 @@ impl From<AccountV3> for AccountV2 {
|
||||
name: value.name,
|
||||
capabilities: value.capabilities,
|
||||
date_since: value.date_since,
|
||||
folder_limit: value.folder_limit,
|
||||
folder_limit: None,
|
||||
sync_folders: value.sync_folders,
|
||||
account_type: value.account_type,
|
||||
sync_interval_min: value.sync_interval_min,
|
||||
@@ -217,7 +216,6 @@ impl From<AccountV2> for AccountV3 {
|
||||
name: value.name,
|
||||
capabilities: value.capabilities,
|
||||
date_since: value.date_since,
|
||||
folder_limit: value.folder_limit,
|
||||
sync_folders: value.sync_folders,
|
||||
account_type: value.account_type,
|
||||
sync_interval_min: value.sync_interval_min,
|
||||
@@ -246,7 +244,6 @@ impl From<AccountV3> for AccountModel {
|
||||
capabilities: value.capabilities,
|
||||
date_since: value.date_since,
|
||||
date_before: value.date_before,
|
||||
folder_limit: value.folder_limit,
|
||||
download_folders: value.sync_folders,
|
||||
account_type: value.account_type,
|
||||
download_interval_min: value.sync_interval_min,
|
||||
@@ -261,6 +258,7 @@ impl From<AccountV3> for AccountModel {
|
||||
imap_quota_window: None,
|
||||
imap_quota_bytes: None,
|
||||
auto_download_new_mailboxes: None,
|
||||
download_schedule: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -660,6 +658,7 @@ impl From<MailBox> for bichon_core::cache::imap::mailbox::MailBox {
|
||||
unseen: value.unseen,
|
||||
uid_next: value.uid_next,
|
||||
uid_validity: value.uid_validity,
|
||||
highest_uid: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use indicatif::{ProgressBar, ProgressStyle};
|
||||
pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"\n{}",
|
||||
style("MIGRATION: Bichon v0.3.7 Storage Architecture → v1.0.0")
|
||||
style("MIGRATION: Bichon v0.3.7 Storage Architecture → v1.x")
|
||||
.bold()
|
||||
.yellow()
|
||||
);
|
||||
@@ -20,7 +20,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
"{}",
|
||||
style(
|
||||
"This tool migrates data from the legacy v0.3.7 Tantivy-based storage \
|
||||
architecture to the new v1.0.0 \
|
||||
architecture to the new v1.x \
|
||||
separated index and Fjall-backed storage format."
|
||||
)
|
||||
.dim()
|
||||
@@ -32,7 +32,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
"Legacy v0.3.7 architecture:\n\
|
||||
• envelope metadata stored in Tantivy\n\
|
||||
• message data stored in Tantivy\n\n\
|
||||
New v1.0.0 architecture:\n\
|
||||
New v1.x architecture:\n\
|
||||
• mail indexes stored in Tantivy\n\
|
||||
• attachment indexes stored in Tantivy\n\
|
||||
• raw message data stored in Fjall\n\
|
||||
@@ -163,7 +163,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!("----------------------------------------");
|
||||
|
||||
println!(
|
||||
"\n{} Checking legacy v0.x storage layout...",
|
||||
"\n{} Checking legacy v0.3.7 storage layout...",
|
||||
style("⌛").yellow()
|
||||
);
|
||||
|
||||
@@ -172,7 +172,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"{} {}",
|
||||
style("✔").green(),
|
||||
style("Legacy v0.x Tantivy-based storage detected. Migration to v1.0 is required.")
|
||||
style("Legacy v0.3.7 Tantivy-based storage detected. Migration to v1.x is required.")
|
||||
.yellow()
|
||||
);
|
||||
}
|
||||
@@ -180,13 +180,13 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
println!(
|
||||
"{} {}",
|
||||
style("✔").green(),
|
||||
style("No legacy v0.x storage layout was detected at the specified paths.").green()
|
||||
style("No legacy v0.3.7 storage layout was detected at the specified paths.").green()
|
||||
);
|
||||
|
||||
println!(
|
||||
"{}",
|
||||
style(
|
||||
"The selected directories may already be using the v1.0 storage architecture."
|
||||
"The selected directories may already be using the v1.x storage architecture."
|
||||
)
|
||||
.dim()
|
||||
);
|
||||
@@ -288,12 +288,6 @@ pub fn handle_migration(theme: &ColorfulTheme) {
|
||||
style(batch_size).cyan().bold()
|
||||
);
|
||||
|
||||
println!(
|
||||
"{} Using batch size: {}\n",
|
||||
style("✓").green(),
|
||||
style(batch_size).cyan().bold()
|
||||
);
|
||||
|
||||
let legacy = LegacyDirs::new(index_path.clone(), data_path.clone());
|
||||
let total_segments = match count_eml_segments(&legacy) {
|
||||
Ok(n) => n,
|
||||
|
||||
+67
-42
@@ -27,19 +27,17 @@ use bichon_core::{
|
||||
users::{permissions::Permission, view::UserView},
|
||||
};
|
||||
|
||||
use crate::BichonCliConfig as BichonCliConfig;
|
||||
|
||||
pub async fn verify_user_and_get_account(
|
||||
config: &BichonCliConfig,
|
||||
theme: &ColorfulTheme,
|
||||
only_nosync: bool,
|
||||
) -> MinimalAccount {
|
||||
let client = Client::new();
|
||||
let url = format!("{}/api/v1/current-user", config.base_url);
|
||||
use crate::BichonCliConfig;
|
||||
|
||||
async fn fetch_json<T: serde::de::DeserializeOwned>(
|
||||
client: &Client,
|
||||
url: &str,
|
||||
token: &str,
|
||||
label: &str,
|
||||
) -> T {
|
||||
let response = match client
|
||||
.get(&url)
|
||||
.header("Authorization", format!("Bearer {}", config.api_token))
|
||||
.get(url)
|
||||
.header("Authorization", format!("Bearer {}", token))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -59,19 +57,15 @@ pub async fn verify_user_and_get_account(
|
||||
}
|
||||
};
|
||||
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let error_body = response
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "No error detail provided".to_string());
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_else(|_| String::new());
|
||||
|
||||
if !status.is_success() {
|
||||
eprintln!(
|
||||
"\n{} Server returned an error (Status: {})",
|
||||
style("✘ API Error:").red().bold(),
|
||||
style(status).yellow()
|
||||
);
|
||||
|
||||
if status == 401 {
|
||||
eprintln!(
|
||||
"{} Your API Token seems to be invalid or expired.",
|
||||
@@ -83,36 +77,66 @@ pub async fn verify_user_and_get_account(
|
||||
style("Context:").dim()
|
||||
);
|
||||
}
|
||||
|
||||
eprintln!("{} {}", style("Response:").dim(), error_body);
|
||||
eprintln!("{} {}", style("Response:").dim(), body);
|
||||
process::exit(1);
|
||||
}
|
||||
|
||||
let user: UserView = response.json().await.expect("Failed to parse user data");
|
||||
println!("Welcome, {}!", style(&user.username).cyan());
|
||||
|
||||
let account_list_url = format!(
|
||||
"{}/api/v1/minimal-account-list?only_nosync={only_nosync}",
|
||||
config.base_url
|
||||
);
|
||||
let acc_response = client
|
||||
.get(&account_list_url)
|
||||
.header("Authorization", format!("Bearer {}", config.api_token))
|
||||
.send()
|
||||
.await
|
||||
.expect("Failed to fetch account list");
|
||||
|
||||
if !acc_response.status().is_success() {
|
||||
panic!(
|
||||
"Failed to retrieve accounts. Status: {}",
|
||||
acc_response.status()
|
||||
if body.is_empty() {
|
||||
eprintln!(
|
||||
"\n{} Server returned an empty response for [{}] (Status: {})",
|
||||
style("✘ Empty Response:").red().bold(),
|
||||
label,
|
||||
status
|
||||
);
|
||||
eprintln!(
|
||||
"{} This may be caused by a reverse proxy or middleware issue.",
|
||||
style("Tip:").cyan()
|
||||
);
|
||||
process::exit(1);
|
||||
}
|
||||
|
||||
let accounts: Vec<MinimalAccount> = acc_response
|
||||
.json()
|
||||
.await
|
||||
.expect("Failed to parse minimal account list");
|
||||
match serde_json::from_str::<T>(&body) {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"\n{} Failed to parse response for [{}]: {}",
|
||||
style("✘ Parse Error:").red().bold(),
|
||||
label,
|
||||
e
|
||||
);
|
||||
eprintln!("{} Raw body: {}", style("Debug:").dim(), body);
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn verify_user_and_get_account(
|
||||
config: &BichonCliConfig,
|
||||
theme: &ColorfulTheme,
|
||||
only_nosync: bool,
|
||||
) -> MinimalAccount {
|
||||
let client = Client::new();
|
||||
|
||||
let user: UserView = fetch_json(
|
||||
&client,
|
||||
&format!("{}/api/v1/current-user", config.base_url),
|
||||
&config.api_token,
|
||||
"current-user",
|
||||
)
|
||||
.await;
|
||||
|
||||
println!("Welcome, {}!", style(&user.username).cyan());
|
||||
|
||||
let accounts: Vec<MinimalAccount> = fetch_json(
|
||||
&client,
|
||||
&format!(
|
||||
"{}/api/v1/minimal-account-list?only_nosync={only_nosync}",
|
||||
config.base_url
|
||||
),
|
||||
&config.api_token,
|
||||
"minimal-account-list",
|
||||
)
|
||||
.await;
|
||||
|
||||
if accounts.is_empty() {
|
||||
println!(
|
||||
@@ -129,6 +153,7 @@ pub async fn verify_user_and_get_account(
|
||||
);
|
||||
process::exit(1);
|
||||
}
|
||||
|
||||
let required_permission = Permission::DATA_IMPORT_BATCH;
|
||||
let mut selectable_accounts = Vec::new();
|
||||
let mut options = Vec::new();
|
||||
|
||||
@@ -60,3 +60,75 @@ pub fn determine_folder(labels_raw: &str) -> String {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use mail_parser::{HeaderValue, MessageParser};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn parse_x_gmail_labels(raw_message: &[u8]) -> Option<String> {
|
||||
// MessageParser::new() has an empty header_map so the hardcoded match at
|
||||
// parsers/header.rs:76 treats ALL unknown headers as raw (no RFC 2047
|
||||
// decoding). We need three things to get decoding:
|
||||
// 1. A non-empty header_map (so the else branch runs)
|
||||
// 2. default_header_text() so the fallback fn is parse_unstructured
|
||||
// 3. OR register X-Gmail-Labels explicitly via header_text()
|
||||
let message = MessageParser::new()
|
||||
.with_minimal_headers()
|
||||
.default_header_text()
|
||||
.parse(raw_message)?;
|
||||
let value: &HeaderValue<'_> = message.header("X-Gmail-Labels")?;
|
||||
value.as_text().map(|s| s.to_string())
|
||||
}
|
||||
|
||||
/// Construct a raw MIME message with RFC 2047 encoded X-Gmail-Labels,
|
||||
/// parse it, and verify the header is correctly decoded.
|
||||
fn build_email(x_gmail_labels: &str) -> Vec<u8> {
|
||||
format!(
|
||||
"From: sender@example.com\r\n\
|
||||
To: recipient@example.com\r\n\
|
||||
Subject: Test\r\n\
|
||||
X-Gmail-Labels: {}\r\n\
|
||||
\r\n\
|
||||
Body text here.\r\n",
|
||||
x_gmail_labels
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rfc2047_encoded_labels_are_decoded() {
|
||||
// Exactly the format the user reported: French Gmail labels
|
||||
let raw = build_email("=?UTF-8?Q?Corbeille?=, =?UTF-8?Q?Messages_archiv=C3=A9s?=");
|
||||
let labels = parse_x_gmail_labels(&raw).expect("failed to parse X-Gmail-Labels");
|
||||
|
||||
// mail-parser decodes RFC 2047 header values during initial parsing.
|
||||
// The decoded text should NOT contain raw =?UTF-8?Q?... sequences.
|
||||
assert!(!labels.contains("=?UTF-8"), "labels still encoded: {labels:?}");
|
||||
assert!(labels.contains("Corbeille"), "missing 'Corbeille': {labels:?}");
|
||||
assert!(
|
||||
labels.contains("archivés"),
|
||||
"missing decoded 'archivés': {labels:?}",
|
||||
);
|
||||
|
||||
// Full pipeline: decoded labels → determine_folder
|
||||
let folder = determine_folder(&labels);
|
||||
assert_eq!(folder, "Corbeille");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_ascii_labels_passthrough() {
|
||||
let raw = build_email("Inbox, Important");
|
||||
let labels = parse_x_gmail_labels(&raw).expect("failed to parse X-Gmail-Labels");
|
||||
assert_eq!(labels, "Inbox, Important");
|
||||
assert_eq!(determine_folder(&labels), "Important");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_x_gmail_labels_header() {
|
||||
let raw = b"From: sender@example.com\r\nTo: r@example.com\r\n\r\nBody.\r\n";
|
||||
let message = MessageParser::new().parse(raw.as_slice()).unwrap();
|
||||
assert!(message.header("X-Gmail-Labels").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
+232
-13
@@ -28,10 +28,14 @@ use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata};
|
||||
use console::style;
|
||||
use dialoguer::{theme::ColorfulTheme, Input};
|
||||
use dialoguer::{Confirm, Select};
|
||||
use mail_parser::parsers::MessageStream;
|
||||
use mail_parser::MessageParser;
|
||||
use reqwest::Client;
|
||||
|
||||
/// Skip emails larger than this with a warning (100 MB).
|
||||
const MAX_EMAIL_BYTES: usize = 100 * 1024 * 1024;
|
||||
/// Flush a folder buffer when accumulated base64 bytes exceed this (200 MB).
|
||||
const MAX_BUFFER_BYTES: usize = 200 * 1024 * 1024;
|
||||
|
||||
pub mod gmail;
|
||||
pub mod reader;
|
||||
|
||||
@@ -133,14 +137,34 @@ pub async fn run_import(
|
||||
};
|
||||
|
||||
let mut folder_buffers: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut total_buffered_bytes: usize = 0;
|
||||
let batch_limit = 50;
|
||||
let mut skipped_count: u64 = 0;
|
||||
|
||||
println!("Starting import process...");
|
||||
|
||||
for (index, e) in mbox.iter().enumerate() {
|
||||
let msg_num = index + 1;
|
||||
let body = e.data;
|
||||
let message = match MessageParser::new().parse(body) {
|
||||
|
||||
if body.len() > MAX_EMAIL_BYTES {
|
||||
let size_mb = body.len() as f64 / 1024.0 / 1024.0;
|
||||
eprintln!(
|
||||
"{} {}: email #{} is {:.1} MB (limit 100 MB). Skipping...",
|
||||
style("Warning").yellow().bold(),
|
||||
style(format!("oversized")).dim(),
|
||||
msg_num,
|
||||
size_mb,
|
||||
);
|
||||
skipped_count += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let message = match MessageParser::new()
|
||||
.with_minimal_headers()
|
||||
.default_header_text()
|
||||
.parse(body)
|
||||
{
|
||||
Some(msg) => msg,
|
||||
None => {
|
||||
eprintln!(
|
||||
@@ -149,6 +173,7 @@ pub async fn run_import(
|
||||
style(format!("at message #{}", msg_num)).dim(),
|
||||
"Failed to parse email structure. Skipping..."
|
||||
);
|
||||
skipped_count += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -159,15 +184,12 @@ pub async fn run_import(
|
||||
}
|
||||
|
||||
let get_default_folder = || {
|
||||
let gmail_labels = message.header_raw("X-Gmail-Labels").unwrap_or("INBOX");
|
||||
let text_cow = MessageStream::new(gmail_labels.as_bytes())
|
||||
.parse_unstructured()
|
||||
.into_text();
|
||||
let data: &str = match &text_cow {
|
||||
Some(c) => c.as_ref(),
|
||||
None => "INBOX",
|
||||
};
|
||||
determine_folder(data)
|
||||
let labels = message
|
||||
.header("X-Gmail-Labels")
|
||||
.and_then(|h| h.as_text())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| "INBOX".to_string());
|
||||
determine_folder(&labels)
|
||||
};
|
||||
|
||||
let folder_name = if let Some(ref folder) = target_folder {
|
||||
@@ -178,14 +200,22 @@ pub async fn run_import(
|
||||
get_default_folder()
|
||||
};
|
||||
|
||||
// Drop message before base64-encoding to free MIME parse memory.
|
||||
drop(message);
|
||||
|
||||
let b64_eml = base64_encode_url_safe!(&body);
|
||||
let encoded_len = b64_eml.len();
|
||||
|
||||
let buffer = folder_buffers
|
||||
.entry(folder_name.clone())
|
||||
.or_insert_with(|| Vec::new());
|
||||
.or_insert_with(Vec::new);
|
||||
buffer.push(b64_eml);
|
||||
total_buffered_bytes += encoded_len;
|
||||
|
||||
if buffer.len() >= batch_limit {
|
||||
if buffer.len() >= batch_limit || total_buffered_bytes >= MAX_BUFFER_BYTES {
|
||||
let emls_to_send = folder_buffers.remove(&folder_name).unwrap();
|
||||
let freed: usize = emls_to_send.iter().map(|s| s.len()).sum();
|
||||
total_buffered_bytes = total_buffered_bytes.saturating_sub(freed);
|
||||
send_batch_request(&client, config, account_id, &folder_name, emls_to_send).await;
|
||||
}
|
||||
}
|
||||
@@ -196,5 +226,194 @@ pub async fn run_import(
|
||||
}
|
||||
}
|
||||
|
||||
if skipped_count > 0 {
|
||||
println!(
|
||||
"{}",
|
||||
style(format!(
|
||||
"Skipped {} email(s) (oversized or unparseable).",
|
||||
skipped_count
|
||||
))
|
||||
.yellow()
|
||||
.bold()
|
||||
);
|
||||
}
|
||||
|
||||
println!("{}", style("Import completed successfully!").green().bold());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Fake sender: records every flushed batch as (folder_name, email_count, total_bytes).
|
||||
struct FakeSender {
|
||||
batches: Vec<(String, usize, usize)>,
|
||||
}
|
||||
|
||||
impl FakeSender {
|
||||
fn new() -> Self {
|
||||
Self { batches: vec![] }
|
||||
}
|
||||
fn send(&mut self, folder: &str, emls: Vec<String>) {
|
||||
let count = emls.len();
|
||||
let bytes: usize = emls.iter().map(|s| s.len()).sum();
|
||||
self.batches.push((folder.to_string(), count, bytes));
|
||||
// emls is dropped here, simulating real send
|
||||
}
|
||||
}
|
||||
|
||||
fn fake_encode(size: usize) -> String {
|
||||
// base64 expands ~1.33x, so the encoded string is roughly this long.
|
||||
// We just need a predictable byte size, so use a repeated character.
|
||||
"x".repeat(size)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_on_global_byte_threshold() {
|
||||
let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut total_bytes: usize = 0;
|
||||
let batch_limit = 50;
|
||||
let mut sender = FakeSender::new();
|
||||
|
||||
// Simulate 3 emails, each 80 MB encoded, spread across 3 folders.
|
||||
// After each email, global total goes up by 80 MB.
|
||||
// After the 3rd email: 240 MB > 200 MB → flush the folder that got the 3rd email.
|
||||
let emails = vec![
|
||||
("Inbox", 80_000_000),
|
||||
("Sent", 80_000_000),
|
||||
("Archive", 80_000_000),
|
||||
];
|
||||
|
||||
for (folder, eml_size) in emails {
|
||||
let encoded = fake_encode(eml_size);
|
||||
let len = encoded.len();
|
||||
let buffer = buffers.entry(folder.to_string()).or_insert_with(Vec::new);
|
||||
buffer.push(encoded);
|
||||
total_bytes += len;
|
||||
|
||||
if buffer.len() >= batch_limit || total_bytes >= MAX_BUFFER_BYTES {
|
||||
let sent = buffers.remove(folder).unwrap();
|
||||
let freed: usize = sent.iter().map(|s| s.len()).sum();
|
||||
total_bytes = total_bytes.saturating_sub(freed);
|
||||
sender.send(folder, sent);
|
||||
}
|
||||
}
|
||||
|
||||
// The 3rd email should trigger a global flush of "Archive".
|
||||
assert_eq!(sender.batches.len(), 1);
|
||||
assert_eq!(sender.batches[0].0, "Archive");
|
||||
assert_eq!(sender.batches[0].1, 1);
|
||||
// "Inbox" and "Sent" are still buffered (160 MB total).
|
||||
assert_eq!(buffers.len(), 2);
|
||||
assert!(buffers.contains_key("Inbox"));
|
||||
assert!(buffers.contains_key("Sent"));
|
||||
assert_eq!(total_bytes, 160_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_on_count_threshold() {
|
||||
let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut total_bytes: usize = 0;
|
||||
let batch_limit = 3;
|
||||
let mut sender = FakeSender::new();
|
||||
|
||||
// 4 small emails all to Inbox, well under byte threshold.
|
||||
for _ in 0..4 {
|
||||
let encoded = fake_encode(100); // tiny
|
||||
let len = encoded.len();
|
||||
let buffer = buffers
|
||||
.entry("Inbox".to_string())
|
||||
.or_insert_with(Vec::new);
|
||||
buffer.push(encoded);
|
||||
total_bytes += len;
|
||||
|
||||
if buffer.len() >= batch_limit || total_bytes >= MAX_BUFFER_BYTES {
|
||||
let sent = buffers.remove("Inbox").unwrap();
|
||||
let freed: usize = sent.iter().map(|s| s.len()).sum();
|
||||
total_bytes = total_bytes.saturating_sub(freed);
|
||||
sender.send("Inbox", sent);
|
||||
}
|
||||
}
|
||||
|
||||
// Count=3 should trigger flush once; the 4th email stays buffered.
|
||||
assert_eq!(sender.batches.len(), 1);
|
||||
assert_eq!(sender.batches[0].1, 3); // 3 emails flushed
|
||||
let remaining = buffers.get("Inbox").unwrap();
|
||||
assert_eq!(remaining.len(), 1); // 1 still buffered
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn global_bytes_exact_boundary() {
|
||||
let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut total_bytes: usize = 0;
|
||||
let mut sender = FakeSender::new();
|
||||
|
||||
// Push one email that puts us right at 200 MB.
|
||||
let encoded = fake_encode(MAX_BUFFER_BYTES);
|
||||
let len = encoded.len();
|
||||
buffers
|
||||
.entry("Inbox".to_string())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(encoded);
|
||||
total_bytes += len;
|
||||
|
||||
if total_bytes >= MAX_BUFFER_BYTES {
|
||||
let sent = buffers.remove("Inbox").unwrap();
|
||||
let freed: usize = sent.iter().map(|s| s.len()).sum();
|
||||
total_bytes = total_bytes.saturating_sub(freed);
|
||||
sender.send("Inbox", sent);
|
||||
}
|
||||
|
||||
// Should have flushed on the boundary.
|
||||
assert_eq!(sender.batches.len(), 1);
|
||||
assert_eq!(total_bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_one_folder_does_not_lose_others() {
|
||||
let mut buffers: HashMap<String, Vec<String>> = HashMap::new();
|
||||
let mut total_bytes: usize = 0;
|
||||
let batch_limit = 50;
|
||||
let mut sender = FakeSender::new();
|
||||
|
||||
// Build up A to 150 MB, B to 100 MB (total 250 MB > 200 MB).
|
||||
// A should trigger flush; B should stay buffered.
|
||||
let folder_a = "A".to_string();
|
||||
let folder_b = "B".to_string();
|
||||
|
||||
// Folder A: 150 MB
|
||||
let encoded = fake_encode(150_000_000);
|
||||
let len = encoded.len();
|
||||
buffers.entry(folder_a.clone()).or_insert_with(Vec::new).push(encoded);
|
||||
total_bytes += len;
|
||||
|
||||
// Folder B: 100 MB → total 250 MB → trigger flush on B
|
||||
let encoded = fake_encode(100_000_000);
|
||||
let len = encoded.len();
|
||||
buffers.entry(folder_b.clone()).or_insert_with(Vec::new).push(encoded);
|
||||
total_bytes += len;
|
||||
|
||||
// Check trigger on B
|
||||
let b_buffer = buffers.get(&folder_b).unwrap();
|
||||
if b_buffer.len() >= batch_limit || total_bytes >= MAX_BUFFER_BYTES {
|
||||
let sent = buffers.remove(&folder_b).unwrap();
|
||||
let freed: usize = sent.iter().map(|s| s.len()).sum();
|
||||
total_bytes = total_bytes.saturating_sub(freed);
|
||||
sender.send(&folder_b, sent);
|
||||
}
|
||||
|
||||
assert_eq!(sender.batches.len(), 1);
|
||||
assert_eq!(sender.batches[0].0, "B"); // B flushed
|
||||
assert!(buffers.contains_key("A")); // A still there
|
||||
assert_eq!(total_bytes, 150_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skip_oversized_email() {
|
||||
assert!(100 <= MAX_EMAIL_BYTES);
|
||||
// Use vec! so the 100 MB array lives on the heap, not the stack.
|
||||
let huge = vec![0u8; MAX_EMAIL_BYTES + 1];
|
||||
assert!(huge.len() > MAX_EMAIL_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,5 +69,6 @@ tokio-util.workspace = true
|
||||
whichlang = "0.1.1"
|
||||
deunicode = "1.6.2"
|
||||
scopeguard = "1.2.0"
|
||||
cron = "0.15"
|
||||
quick-xml = { version = "0.40.0", features = ["serialize"] }
|
||||
hickory-resolver = "0.26.0-alpha.1"
|
||||
|
||||
@@ -80,7 +80,6 @@ pub struct Account {
|
||||
pub capabilities: Option<Vec<String>>,
|
||||
pub date_since: Option<DateSince>,
|
||||
pub date_before: Option<RelativeDate>,
|
||||
pub folder_limit: Option<u32>,
|
||||
pub download_folders: Option<Vec<String>>,
|
||||
pub account_type: AccountType,
|
||||
pub download_interval_min: Option<i64>,
|
||||
@@ -95,6 +94,7 @@ pub struct Account {
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
}
|
||||
|
||||
impl MemDbModel for Account {
|
||||
@@ -124,7 +124,6 @@ impl Account {
|
||||
created_at: utc_now!(),
|
||||
updated_at: utc_now!(),
|
||||
use_proxy: request.use_proxy,
|
||||
folder_limit: request.folder_limit,
|
||||
use_dangerous: request.use_dangerous,
|
||||
pgp_key: request.pgp_key,
|
||||
created_by: user_id,
|
||||
@@ -133,6 +132,7 @@ impl Account {
|
||||
auto_download_new_mailboxes: request.auto_download_new_mailboxes,
|
||||
imap_quota_bytes: request.imap_quota_bytes,
|
||||
imap_quota_window: request.imap_quota_window,
|
||||
download_schedule: request.download_schedule,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -365,14 +365,8 @@ impl Account {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(folder_limit) = request.folder_limit {
|
||||
new.folder_limit = Some(folder_limit);
|
||||
}
|
||||
|
||||
if let Some(clear_folder_limit) = request.clear_folder_limit {
|
||||
if clear_folder_limit {
|
||||
new.folder_limit = None;
|
||||
}
|
||||
if let Some(account_name) = request.account_name {
|
||||
new.account_name = Some(account_name);
|
||||
}
|
||||
|
||||
if matches!(old.account_type, AccountType::IMAP) {
|
||||
@@ -435,6 +429,12 @@ impl Account {
|
||||
if let Some(auto_download_new_mailboxes) = request.auto_download_new_mailboxes {
|
||||
new.auto_download_new_mailboxes = Some(auto_download_new_mailboxes);
|
||||
}
|
||||
if let Some(download_schedule) = request.download_schedule {
|
||||
new.download_schedule = Some(download_schedule);
|
||||
}
|
||||
if request.clear_download_schedule == Some(true) {
|
||||
new.download_schedule = None;
|
||||
}
|
||||
new.updated_at = utc_now!();
|
||||
Ok(new)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use crate::account::entity::ImapConfig;
|
||||
use crate::account::migration::{AccountModel, AccountType, QuotaWindow};
|
||||
use crate::account::since::{DateSince, RelativeDate};
|
||||
@@ -39,8 +41,6 @@ pub struct AccountCreateRequest {
|
||||
pub date_since: Option<DateSince>,
|
||||
pub date_before: Option<RelativeDate>,
|
||||
pub account_type: AccountType,
|
||||
#[cfg_attr(feature = "web-api", oai(validator(minimum(value = "100"))))]
|
||||
pub folder_limit: Option<u32>,
|
||||
#[cfg_attr(feature = "web-api", oai(validator(minimum(value = "10"))))]
|
||||
pub download_interval_min: Option<i64>,
|
||||
#[cfg_attr(
|
||||
@@ -54,6 +54,7 @@ pub struct AccountCreateRequest {
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
}
|
||||
|
||||
impl AccountCreateRequest {
|
||||
@@ -92,12 +93,15 @@ impl AccountCreateRequest {
|
||||
))
|
||||
}
|
||||
}
|
||||
if self.download_interval_min.is_none() {
|
||||
if self.download_interval_min.is_none() && self.download_schedule.is_none() {
|
||||
return Err(raise_error!(
|
||||
"`sync_interval_min` is required for IMAP account type".into(),
|
||||
"`sync_interval_min` or `download_schedule` is required for IMAP account type".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if let Some(ref schedule) = self.download_schedule {
|
||||
validate_cron_expression(schedule)?;
|
||||
}
|
||||
}
|
||||
AccountType::NoSync => {}
|
||||
}
|
||||
@@ -139,12 +143,6 @@ pub struct AccountUpdateRequest {
|
||||
pub date_since: Option<DateSince>,
|
||||
pub date_before: Option<RelativeDate>,
|
||||
pub clear_date_range: Option<bool>,
|
||||
/// Max emails to sync for this folder.
|
||||
/// If not set, sync all emails.
|
||||
/// otherwise sync up to `n` most recent emails (min 10).
|
||||
#[cfg_attr(feature = "web-api", oai(validator(minimum(value = "100"))))]
|
||||
pub folder_limit: Option<u32>,
|
||||
pub clear_folder_limit: Option<bool>,
|
||||
/// Configuration for selective folder (mailbox/label) synchronization
|
||||
///
|
||||
/// - For IMAP/SMTP accounts:
|
||||
@@ -178,6 +176,8 @@ pub struct AccountUpdateRequest {
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
pub clear_download_schedule: Option<bool>,
|
||||
}
|
||||
|
||||
impl AccountUpdateRequest {
|
||||
@@ -197,13 +197,6 @@ impl AccountUpdateRequest {
|
||||
));
|
||||
}
|
||||
|
||||
if self.clear_folder_limit == Some(true) && self.folder_limit.is_some() {
|
||||
return Err(raise_error!(
|
||||
"clear_folder_limit cannot be combined with folder_limit".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
|
||||
if self.clear_date_range == Some(true)
|
||||
&& (self.date_since.is_some() || self.date_before.is_some())
|
||||
{
|
||||
@@ -230,11 +223,36 @@ impl AccountUpdateRequest {
|
||||
));
|
||||
}
|
||||
}
|
||||
if self.clear_download_schedule == Some(true) && self.download_schedule.is_some() {
|
||||
return Err(raise_error!(
|
||||
"clear_download_schedule cannot be combined with download_schedule".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
if let Some(ref schedule) = self.download_schedule {
|
||||
validate_cron_expression(schedule)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_cron_expression(expr: &str) -> BichonResult<()> {
|
||||
if expr.trim().is_empty() {
|
||||
return Err(raise_error!(
|
||||
"download_schedule must not be empty".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
));
|
||||
}
|
||||
cron::Schedule::from_str(expr).map_err(|e| {
|
||||
raise_error!(
|
||||
format!("Invalid cron expression '{}': {}", expr, e),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
|
||||
@@ -253,3 +271,34 @@ pub fn filter_accessible_accounts<'a>(
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::validate_cron_expression;
|
||||
|
||||
#[test]
|
||||
fn valid_cron_expressions() {
|
||||
assert!(validate_cron_expression("0 0 0 * * *").is_ok()); // daily at midnight
|
||||
assert!(validate_cron_expression("0 */5 * * * *").is_ok()); // every 5 minutes
|
||||
assert!(validate_cron_expression("0 0 12 * * 1-5").is_ok()); // weekdays at noon
|
||||
assert!(validate_cron_expression("0 30 4 1 * *").is_ok()); // 1st of month at 04:30
|
||||
assert!(validate_cron_expression("0 0 * * * *").is_ok()); // every hour
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_cron_expression_too_few_fields() {
|
||||
assert!(validate_cron_expression("0 0 * *").is_err());
|
||||
assert!(validate_cron_expression("* * * * *").is_err()); // 5 fields, needs seconds
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_cron_expression_empty() {
|
||||
assert!(validate_cron_expression("").is_err());
|
||||
assert!(validate_cron_expression(" ").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_cron_expression_garbage() {
|
||||
assert!(validate_cron_expression("not a cron").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,4 +364,43 @@ mod test {
|
||||
};
|
||||
assert!(e.validate().is_err());
|
||||
}
|
||||
|
||||
// ── Sliding window tests ──────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn relative_date_calculate_returns_valid_format() {
|
||||
let r = RelativeDate {
|
||||
unit: Unit::Years,
|
||||
value: 1,
|
||||
};
|
||||
let date_str = r.calculate_date().unwrap();
|
||||
// Expect format like "26-May-2025"
|
||||
assert!(date_str.len() > 5);
|
||||
assert!(date_str.contains('-'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_date_one_year_ago_is_before_now() {
|
||||
let r = RelativeDate {
|
||||
unit: Unit::Years,
|
||||
value: 1,
|
||||
};
|
||||
let date_str = r.calculate_date().unwrap();
|
||||
let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap();
|
||||
let today = chrono::Local::now().date_naive();
|
||||
assert!(parsed < today, "1 year ago ({parsed}) should be before today ({today})");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_date_one_day_ago_is_yesterday() {
|
||||
let r = RelativeDate {
|
||||
unit: Unit::Days,
|
||||
value: 1,
|
||||
};
|
||||
let date_str = r.calculate_date().unwrap();
|
||||
let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap();
|
||||
let today = chrono::Local::now().date_naive();
|
||||
let yesterday = today - chrono::Duration::days(1);
|
||||
assert_eq!(parsed, yesterday, "1 day ago should be yesterday");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,7 +40,6 @@ pub struct AccountResp {
|
||||
pub capabilities: Option<Vec<String>>,
|
||||
pub date_since: Option<DateSince>,
|
||||
pub date_before: Option<RelativeDate>,
|
||||
pub folder_limit: Option<u32>,
|
||||
pub download_folders: Option<Vec<String>>,
|
||||
pub account_type: AccountType,
|
||||
pub download_interval_min: Option<i64>,
|
||||
@@ -57,6 +56,7 @@ pub struct AccountResp {
|
||||
pub imap_quota_bytes: Option<u64>,
|
||||
pub imap_quota_window: Option<QuotaWindow>,
|
||||
pub auto_download_new_mailboxes: Option<bool>,
|
||||
pub download_schedule: Option<String>,
|
||||
}
|
||||
|
||||
impl AccountResp {
|
||||
@@ -72,7 +72,6 @@ impl AccountResp {
|
||||
capabilities: account.capabilities,
|
||||
date_since: account.date_since,
|
||||
date_before: account.date_before,
|
||||
folder_limit: account.folder_limit,
|
||||
download_folders: account.download_folders,
|
||||
account_type: account.account_type,
|
||||
download_interval_min: account.download_interval_min,
|
||||
@@ -93,6 +92,7 @@ impl AccountResp {
|
||||
imap_quota_bytes: account.imap_quota_bytes,
|
||||
imap_quota_window: account.imap_quota_window,
|
||||
auto_download_new_mailboxes: account.auto_download_new_mailboxes,
|
||||
download_schedule: account.download_schedule,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +45,10 @@ pub struct IncomingServer {
|
||||
#[serde(rename = "socketType")]
|
||||
pub socket_type: String,
|
||||
pub username: String,
|
||||
/// Authentication method from the XML, e.g. "OAuth2", "password-cleartext",
|
||||
/// "password-encrypted", "GSSAPI", "NTLM". Absent in DNS SRV fallback.
|
||||
#[serde(default)]
|
||||
pub authentication: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
|
||||
@@ -137,6 +141,7 @@ async fn lookup_srv(domain: &str) -> Option<MailConfig> {
|
||||
port: imap_port,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
}],
|
||||
outgoing: vec![OutgoingServer {
|
||||
protocol: "smtp".to_string(),
|
||||
@@ -153,30 +158,37 @@ async fn lookup_srv(domain: &str) -> Option<MailConfig> {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Discover mail server configuration for a domain using the Thunderbird
|
||||
/// autoconfig protocol (ISPDB) and DNS SRV fallback.
|
||||
/// autoconfig protocol (ISPDB), DNS SRV, MX fallback, and finally guessing.
|
||||
///
|
||||
/// Probe order:
|
||||
/// 1. `https://autoconfig.{domain}/mail/config-v1.1.xml`
|
||||
/// 2. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml`
|
||||
/// 3. DNS SRV records (`_imaps._tcp` / `_submission._tcp`)
|
||||
/// 4. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`)
|
||||
/// 2. `http://autoconfig.{domain}/mail/config-v1.1.xml`
|
||||
/// 3. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml`
|
||||
/// 4. `http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml`
|
||||
/// 5. DNS SRV records (`_imaps._tcp` / `_submission._tcp`)
|
||||
/// 6. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`)
|
||||
/// 7. MX lookup → ISPDB for MX domain
|
||||
/// 8. MX lookup → ISP autoconfig for MX domain
|
||||
/// 9. GuessConfig — probe common hostnames + ports
|
||||
pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
|
||||
let client = Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
// 1. Try autoconfig subdomain
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("https://autoconfig.{domain}/mail/config-v1.1.xml"),
|
||||
)
|
||||
.await
|
||||
// ── ISP autoconfig (HTTPS, then HTTP) ──────────────────────────
|
||||
if let Some(config) =
|
||||
fetch_xml(&client, &format!("https://autoconfig.{domain}/mail/config-v1.1.xml")).await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
if let Some(config) =
|
||||
fetch_xml(&client, &format!("http://autoconfig.{domain}/mail/config-v1.1.xml")).await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
// 2. Try well-known path
|
||||
// ── Well-known path (HTTPS, then HTTP) ─────────────────────────
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"),
|
||||
@@ -185,28 +197,100 @@ pub async fn fetch(domain: &str) -> BichonResult<MailConfig> {
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
// 3. Try DNS SRV records
|
||||
// ── DNS SRV records ────────────────────────────────────────────
|
||||
if let Some(config) = lookup_srv(domain).await {
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
// 4. Fall back to Thunderbird central database
|
||||
if let Some(config) = fetch_xml(
|
||||
&client,
|
||||
&format!("https://autoconfig.thunderbird.net/v1.1/{domain}"),
|
||||
)
|
||||
.await
|
||||
// ── Thunderbird central ISPDB ──────────────────────────────────
|
||||
if let Some(config) =
|
||||
fetch_xml(&client, &format!("https://autoconfig.thunderbird.net/v1.1/{domain}")).await
|
||||
{
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
// ── MX fallback ────────────────────────────────────────────────
|
||||
if let Some(config) = fetch_for_mx(&client, domain).await {
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
// ── GuessConfig ────────────────────────────────────────────────
|
||||
if let Some(config) = crate::autoconfig::guess::guess_config(domain).await {
|
||||
return Ok(config);
|
||||
}
|
||||
|
||||
Err(raise_error!(
|
||||
format!("No autoconfig found for domain: {domain}"),
|
||||
ErrorCode::InternalError
|
||||
))
|
||||
}
|
||||
|
||||
/// DNS MX lookup → retry ISPDB and ISP autoconfig for the MX domain.
|
||||
///
|
||||
/// Many self-hosted domains have their MX pointed at Google, Microsoft, etc.
|
||||
/// The MX domain's ISPDB entry covers the original domain.
|
||||
async fn fetch_for_mx(client: &Client, domain: &str) -> Option<MailConfig> {
|
||||
let mx_domain = lookup_mx_domain(domain).await?;
|
||||
if mx_domain == domain.to_ascii_lowercase() {
|
||||
return None; // same domain, already tried above
|
||||
}
|
||||
|
||||
// Try ISPDB for the MX domain
|
||||
if let Some(config) =
|
||||
fetch_xml(client, &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}")).await
|
||||
{
|
||||
return Some(config);
|
||||
}
|
||||
|
||||
// Try ISP autoconfig for the MX domain (HTTPS then HTTP)
|
||||
if let Some(config) =
|
||||
fetch_xml(client, &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await
|
||||
{
|
||||
return Some(config);
|
||||
}
|
||||
if let Some(config) =
|
||||
fetch_xml(client, &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await
|
||||
{
|
||||
return Some(config);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// DNS MX lookup → extract the second-level domain of the first MX hostname.
|
||||
async fn lookup_mx_domain(domain: &str) -> Option<String> {
|
||||
let resolver = TokioResolver::builder(TokioConnectionProvider::default())
|
||||
.ok()?
|
||||
.build();
|
||||
let lookup = resolver.mx_lookup(domain).await.ok()?;
|
||||
let record = lookup.iter().next()?;
|
||||
let mx_host = record.to_string().trim_end_matches('.').to_string();
|
||||
|
||||
// Extract a reasonable base domain from the MX hostname.
|
||||
// E.g., "aspmx.l.google.com" → "google.com"
|
||||
// "company.mail.protection.outlook.com" → "outlook.com"
|
||||
extract_base_domain(&mx_host)
|
||||
}
|
||||
|
||||
/// Extract the top two labels from a hostname as a rough base domain.
|
||||
fn extract_base_domain(host: &str) -> Option<String> {
|
||||
let parts: Vec<&str> = host.split('.').collect();
|
||||
if parts.len() >= 2 {
|
||||
Some(parts[parts.len() - 2..].join("."))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::account::entity::Encryption;
|
||||
use crate::autoconfig::client::{IncomingServer, MailConfig};
|
||||
use crate::imap::client::Client;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// A single host:port:encryption combination to probe.
|
||||
struct Guess {
|
||||
hostname: String,
|
||||
port: u16,
|
||||
encryption: Encryption,
|
||||
socket_type: &'static str,
|
||||
}
|
||||
|
||||
/// Generate candidates in the same order Thunderbird uses:
|
||||
/// 1. imap.{domain} — most common
|
||||
/// 2. mail.{domain} — fallback
|
||||
/// 3. {domain} — bare domain (rare)
|
||||
fn make_guesses(domain: &str) -> Vec<Guess> {
|
||||
let hosts = [
|
||||
format!("imap.{domain}"),
|
||||
format!("mail.{domain}"),
|
||||
domain.to_string(),
|
||||
];
|
||||
|
||||
let mut guesses = Vec::with_capacity(hosts.len() * 2);
|
||||
for host in &hosts {
|
||||
guesses.push(Guess {
|
||||
hostname: host.clone(),
|
||||
port: 993,
|
||||
encryption: Encryption::Ssl,
|
||||
socket_type: "SSL",
|
||||
});
|
||||
guesses.push(Guess {
|
||||
hostname: host.clone(),
|
||||
port: 143,
|
||||
encryption: Encryption::StartTls,
|
||||
socket_type: "STARTTLS",
|
||||
});
|
||||
}
|
||||
guesses
|
||||
}
|
||||
|
||||
/// Try to open a connection, read the IMAP banner, and close.
|
||||
/// Returns `true` if the server responds with an IMAP greeting.
|
||||
async fn probe(hostname: &str, port: u16, encryption: &Encryption) -> bool {
|
||||
match Client::connection(hostname, encryption, port, None, true).await {
|
||||
Ok(_) => {
|
||||
debug!("GuessConfig probe succeeded: {hostname}:{port} ({encryption:?})");
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("GuessConfig probe failed for {hostname}:{port}: {e:?}");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Thunderbird-style guessing: try common hostnames and ports, probing
|
||||
/// each with a real TCP connection.
|
||||
///
|
||||
/// Returns the first working `MailConfig`, or `None` if nothing works.
|
||||
pub async fn guess_config(domain: &str) -> Option<MailConfig> {
|
||||
let guesses = make_guesses(domain);
|
||||
info!("GuessConfig: trying {} candidates for {domain}", guesses.len());
|
||||
|
||||
for g in &guesses {
|
||||
if probe(&g.hostname, g.port, &g.encryption).await {
|
||||
info!(
|
||||
"GuessConfig: found working IMAP at {}:{} ({})",
|
||||
g.hostname, g.port, g.socket_type
|
||||
);
|
||||
return Some(MailConfig {
|
||||
incoming: vec![IncomingServer {
|
||||
protocol: "imap".to_string(),
|
||||
hostname: g.hostname.clone(),
|
||||
port: g.port,
|
||||
socket_type: g.socket_type.to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
@@ -19,6 +19,7 @@
|
||||
use crate::account::entity::Encryption;
|
||||
use crate::autoconfig::client::{self, MailConfig};
|
||||
use crate::autoconfig::entity::{MailServerConfig, ServerConfig};
|
||||
use crate::autoconfig::oauth2_providers::lookup_oauth2;
|
||||
use crate::autoconfig::CachedMailSettings;
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
@@ -54,9 +55,17 @@ pub(crate) fn mail_config_to_server_config(config: &MailConfig) -> Option<MailSe
|
||||
}
|
||||
};
|
||||
|
||||
// Detect OAuth2 support: the XML <authentication> field and a known
|
||||
// hostname → issuer mapping determine whether the provider supports OAuth2.
|
||||
let oauth2 = if imap.authentication.eq_ignore_ascii_case("OAuth2") {
|
||||
lookup_oauth2(&imap.hostname)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Some(MailServerConfig {
|
||||
imap: ServerConfig::new(imap.hostname.clone(), port, encryption),
|
||||
oauth2: None,
|
||||
oauth2,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,9 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod client;
|
||||
pub mod entity;
|
||||
pub mod guess;
|
||||
pub mod load;
|
||||
mod oauth2_providers;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::autoconfig::entity::OAuth2Config;
|
||||
|
||||
/// Per-provider OAuth2 metadata, mirroring Thunderbird's `OAuth2Providers.sys.mjs`.
|
||||
///
|
||||
/// Each entry maps one or more IMAP hostname suffixes to a well-known OIDC issuer
|
||||
/// and the IMAP-specific OAuth2 scopes.
|
||||
struct Provider {
|
||||
/// Suffixes matched case-insensitively against the end of the IMAP hostname.
|
||||
host_suffixes: &'static [&'static str],
|
||||
/// The OIDC issuer URL used by the provider.
|
||||
issuer: &'static str,
|
||||
/// OAuth2 scope(s) required for IMAP access.
|
||||
scopes: &'static [&'static str],
|
||||
}
|
||||
|
||||
const PROVIDERS: &[Provider] = &[
|
||||
// Google
|
||||
Provider {
|
||||
host_suffixes: &["imap.gmail.com", ".gmail.com", ".googlemail.com"],
|
||||
issuer: "https://accounts.google.com",
|
||||
scopes: &["https://mail.google.com/"],
|
||||
},
|
||||
// Microsoft (Outlook / Office 365 / Hotmail / Live)
|
||||
Provider {
|
||||
host_suffixes: &[
|
||||
"outlook.office365.com",
|
||||
".outlook.com",
|
||||
".hotmail.com",
|
||||
".live.com",
|
||||
".office365.com",
|
||||
],
|
||||
issuer: "https://login.microsoftonline.com/common/v2.0",
|
||||
scopes: &[
|
||||
"https://outlook.office365.com/IMAP.AccessAsUser.All",
|
||||
"offline_access",
|
||||
],
|
||||
},
|
||||
// Yahoo / AOL / ATT / Verizon
|
||||
Provider {
|
||||
host_suffixes: &[
|
||||
"imap.mail.yahoo.com",
|
||||
".yahoo.com",
|
||||
".yahoodns.net",
|
||||
".aol.com",
|
||||
"imap.aol.com",
|
||||
],
|
||||
issuer: "https://login.yahoo.com",
|
||||
scopes: &["mail-w"],
|
||||
},
|
||||
// Yandex
|
||||
Provider {
|
||||
host_suffixes: &["imap.yandex.ru", "imap.yandex.com", ".yandex.ru"],
|
||||
issuer: "https://oauth.yandex.com",
|
||||
scopes: &["imap:all"],
|
||||
},
|
||||
// Mail.ru
|
||||
Provider {
|
||||
host_suffixes: &["imap.mail.ru", ".mail.ru", ".bk.ru", ".list.ru", ".inbox.ru"],
|
||||
issuer: "https://o2.mail.ru",
|
||||
scopes: &["imap"],
|
||||
},
|
||||
// Fastmail
|
||||
Provider {
|
||||
host_suffixes: &["imap.fastmail.com", ".fastmail.com"],
|
||||
issuer: "https://www.fastmail.com",
|
||||
scopes: &[
|
||||
"https://www.fastmail.com/dev/imap",
|
||||
"offline_access",
|
||||
],
|
||||
},
|
||||
// Comcast
|
||||
Provider {
|
||||
host_suffixes: &["imap.comcast.net", ".comcast.net"],
|
||||
issuer: "https://oauth.xfinity.com",
|
||||
scopes: &["https://email.comcast.net/"],
|
||||
},
|
||||
];
|
||||
|
||||
/// Try to find an OAuth2 provider that matches the given IMAP hostname.
|
||||
///
|
||||
/// Matching is case-insensitive and done by suffix: a hostname "imap.gmail.com"
|
||||
/// matches the suffix ".gmail.com".
|
||||
pub fn lookup_oauth2(hostname: &str) -> Option<OAuth2Config> {
|
||||
let host = hostname.to_ascii_lowercase();
|
||||
for provider in PROVIDERS {
|
||||
if provider
|
||||
.host_suffixes
|
||||
.iter()
|
||||
.any(|suffix| host.ends_with(&suffix.to_ascii_lowercase()))
|
||||
{
|
||||
return Some(OAuth2Config {
|
||||
issuer: provider.issuer.to_string(),
|
||||
scope: provider.scopes.iter().map(|s| s.to_string()).collect(),
|
||||
auth_url: String::new(),
|
||||
token_url: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_known_providers() {
|
||||
let cases = [
|
||||
("imap.gmail.com", Some("https://accounts.google.com")),
|
||||
("imap.gmail.com", Some("https://accounts.google.com")),
|
||||
("outlook.office365.com", Some("https://login.microsoftonline.com/common/v2.0")),
|
||||
("imap.mail.yahoo.com", Some("https://login.yahoo.com")),
|
||||
("imap.aol.com", Some("https://login.yahoo.com")),
|
||||
("imap.yandex.ru", Some("https://oauth.yandex.com")),
|
||||
("imap.mail.ru", Some("https://o2.mail.ru")),
|
||||
("imap.fastmail.com", Some("https://www.fastmail.com")),
|
||||
("imap.comcast.net", Some("https://oauth.xfinity.com")),
|
||||
];
|
||||
for (hostname, expected_issuer) in &cases {
|
||||
let result = lookup_oauth2(hostname);
|
||||
assert_eq!(
|
||||
result.map(|c| c.issuer),
|
||||
expected_issuer.map(|s| s.to_string()),
|
||||
"failed for hostname: {hostname}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_provider() {
|
||||
assert!(lookup_oauth2("mail.my-company.example").is_none());
|
||||
}
|
||||
}
|
||||
@@ -195,6 +195,7 @@ fn make_imap_server(host: &str, port: u16, socket_type: &str) -> IncomingServer
|
||||
port,
|
||||
socket_type: socket_type.to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +242,7 @@ fn convert_no_imap_only_pop3() {
|
||||
port: 995,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
};
|
||||
@@ -266,6 +268,7 @@ fn convert_picks_imap_over_pop3() {
|
||||
port: 995,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
},
|
||||
make_imap_server("imap.example.com", 993, "SSL"),
|
||||
],
|
||||
@@ -284,6 +287,7 @@ fn convert_imaps_protocol_variant() {
|
||||
port: 993,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
};
|
||||
@@ -300,9 +304,65 @@ fn convert_case_insensitive_protocol() {
|
||||
port: 143,
|
||||
socket_type: "STARTTLS".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: String::new(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
};
|
||||
let result = mail_config_to_server_config(&config).expect("should recognize 'IMAP'");
|
||||
assert_eq!(result.imap.host, "imap.example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convert_gmail_oauth2() {
|
||||
let config = MailConfig {
|
||||
incoming: vec![IncomingServer {
|
||||
protocol: "imap".to_string(),
|
||||
hostname: "imap.gmail.com".to_string(),
|
||||
port: 993,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: "OAuth2".to_string(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
};
|
||||
let result = mail_config_to_server_config(&config).expect("should convert");
|
||||
let oauth2 = result.oauth2.expect("Gmail should have OAuth2");
|
||||
assert_eq!(oauth2.issuer, "https://accounts.google.com");
|
||||
assert!(oauth2.scope.contains(&"https://mail.google.com/".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convert_outlook_oauth2() {
|
||||
let config = MailConfig {
|
||||
incoming: vec![IncomingServer {
|
||||
protocol: "imap".to_string(),
|
||||
hostname: "outlook.office365.com".to_string(),
|
||||
port: 993,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: "OAuth2".to_string(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
};
|
||||
let result = mail_config_to_server_config(&config).expect("should convert");
|
||||
let oauth2 = result.oauth2.expect("Outlook should have OAuth2");
|
||||
assert!(oauth2.issuer.contains("microsoftonline"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convert_unknown_host_no_oauth2() {
|
||||
// OAuth2 auth flag on an unknown hostname → no OAuth2 returned
|
||||
let config = MailConfig {
|
||||
incoming: vec![IncomingServer {
|
||||
protocol: "imap".to_string(),
|
||||
hostname: "mail.random-isp.example".to_string(),
|
||||
port: 993,
|
||||
socket_type: "SSL".to_string(),
|
||||
username: "%EMAILADDRESS%".to_string(),
|
||||
authentication: "OAuth2".to_string(),
|
||||
}],
|
||||
outgoing: vec![],
|
||||
};
|
||||
let result = mail_config_to_server_config(&config).expect("should convert");
|
||||
assert!(result.oauth2.is_none(), "unknown hostname → no OAuth2 mapping");
|
||||
}
|
||||
|
||||
+94
-11
@@ -16,6 +16,11 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use chrono::{DateTime, Local, TimeZone, Utc};
|
||||
use cron::Schedule;
|
||||
|
||||
use crate::{
|
||||
utc_now,
|
||||
{
|
||||
@@ -48,11 +53,20 @@ pub async fn decide_next_download_task(
|
||||
|
||||
let should_start = match trigger_type {
|
||||
TriggerType::Manual => true,
|
||||
TriggerType::Scheduled => should_trigger_next_download(
|
||||
state.last_trigger_at,
|
||||
state.last_finished_at.unwrap_or(0),
|
||||
account.download_interval_min.unwrap_or(60),
|
||||
),
|
||||
TriggerType::Scheduled => {
|
||||
let now = utc_now!();
|
||||
let cooldown_ok = now - state.last_finished_at.unwrap_or(0) > 60 * 1000;
|
||||
if !cooldown_ok {
|
||||
false
|
||||
} else if let Some(ref schedule) = account.download_schedule {
|
||||
should_trigger_scheduled(schedule, state.last_trigger_at)
|
||||
} else {
|
||||
should_trigger_next_download(
|
||||
state.last_trigger_at,
|
||||
account.download_interval_min.unwrap_or(60),
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if should_start {
|
||||
@@ -63,11 +77,80 @@ pub async fn decide_next_download_task(
|
||||
}
|
||||
}
|
||||
|
||||
fn should_trigger_next_download(
|
||||
last_trigger_at: i64,
|
||||
last_finished_at: i64,
|
||||
sync_interval_min: i64,
|
||||
) -> bool {
|
||||
fn should_trigger_next_download(last_trigger_at: i64, sync_interval_min: i64) -> bool {
|
||||
let now = utc_now!();
|
||||
now - last_trigger_at > (sync_interval_min * 60 * 1000) && now - last_finished_at > 60 * 1000
|
||||
now - last_trigger_at > (sync_interval_min * 60 * 1000)
|
||||
}
|
||||
|
||||
fn should_trigger_scheduled(schedule_str: &str, last_trigger_at: i64) -> bool {
|
||||
let schedule = match Schedule::from_str(schedule_str) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Invalid cron expression '{}', falling back to no trigger: {}",
|
||||
schedule_str,
|
||||
e
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
// last_trigger_at is a UTC millis timestamp; convert to server local time
|
||||
let last_utc = match Utc.timestamp_millis_opt(last_trigger_at) {
|
||||
chrono::LocalResult::Single(dt) => dt,
|
||||
_ => {
|
||||
tracing::warn!("Invalid last_trigger_at timestamp: {}", last_trigger_at);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let last_dt: DateTime<Local> = last_utc.with_timezone(&Local);
|
||||
let now = Local::now();
|
||||
schedule
|
||||
.after(&last_dt)
|
||||
.next()
|
||||
.map_or(false, |next| next <= now)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cron_every_minute_triggers_after_60s() {
|
||||
// "0 * * * * *" = every minute at second 0. last_trigger 90s ago → should trigger
|
||||
let now = Local::now();
|
||||
let last_trigger = now.timestamp_millis() - 90_000;
|
||||
assert!(should_trigger_scheduled("0 * * * * *", last_trigger));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_daily_midnight_triggers_when_missed() {
|
||||
// "0 0 0 * * *" = daily at midnight
|
||||
// last_trigger was 25 hours ago → should trigger (we missed midnight)
|
||||
let now = Local::now();
|
||||
let last_trigger = now.timestamp_millis() - 25 * 60 * 60 * 1000;
|
||||
assert!(should_trigger_scheduled("0 0 0 * * *", last_trigger));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_daily_midnight_no_trigger_if_already_fired() {
|
||||
// "0 0 0 * * *" = daily at midnight
|
||||
// last_trigger was 1 minute ago → should NOT trigger
|
||||
let now = Local::now();
|
||||
let last_trigger = now.timestamp_millis() - 60_000;
|
||||
assert!(!should_trigger_scheduled("0 0 0 * * *", last_trigger));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_cron_returns_false() {
|
||||
assert!(!should_trigger_scheduled("invalid cron expression", 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cron_every_hour_triggers() {
|
||||
// "0 0 * * * *" = every hour at minute 0, second 0
|
||||
// last_trigger was 61 minutes ago → should trigger
|
||||
let now = Local::now();
|
||||
let last_trigger = now.timestamp_millis() - 61 * 60 * 1000;
|
||||
assert!(should_trigger_scheduled("0 0 * * * *", last_trigger));
|
||||
}
|
||||
}
|
||||
|
||||
+119
-152
@@ -32,7 +32,9 @@ use crate::{
|
||||
SEMAPHORE,
|
||||
},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
imap::executor::ImapExecutor,
|
||||
imap::executor::{
|
||||
generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE,
|
||||
},
|
||||
store::tantivy::envelope::ENVELOPE_MANAGER,
|
||||
},
|
||||
};
|
||||
@@ -40,7 +42,6 @@ use std::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
pub const DEFAULT_BATCH_SIZE: u32 = 30;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum FetchDirection {
|
||||
@@ -54,7 +55,7 @@ pub async fn fetch_and_save_by_date(
|
||||
mailbox: &MailBox,
|
||||
direction: FetchDirection,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<()> {
|
||||
) -> BichonResult<Option<u32>> {
|
||||
let account_id = account.id;
|
||||
let mut session = match ImapExecutor::create_connection(account_id).await {
|
||||
Ok(session) => session,
|
||||
@@ -108,32 +109,18 @@ pub async fn fetch_and_save_by_date(
|
||||
FolderStatus::Success,
|
||||
None,
|
||||
)?;
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let folder_limit = account.folder_limit;
|
||||
// sort small -> bigger
|
||||
let mut uid_vec: Vec<u32> = uid_list.into_iter().collect();
|
||||
uid_vec.sort();
|
||||
|
||||
if let Some(limit) = folder_limit {
|
||||
let limit = limit.max(100) as usize;
|
||||
if len > limit {
|
||||
uid_vec = match direction {
|
||||
FetchDirection::Since => uid_vec.split_off(len - limit),
|
||||
FetchDirection::Before => {
|
||||
uid_vec.truncate(limit);
|
||||
uid_vec
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let max_uid = uid_vec.last().copied();
|
||||
let planned = uid_vec.len() as u64;
|
||||
let uid_batches = generate_uid_sequence_hashset(
|
||||
uid_vec,
|
||||
account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize,
|
||||
false,
|
||||
);
|
||||
DownloadState::update_folder_progress(
|
||||
account_id,
|
||||
@@ -212,14 +199,16 @@ pub async fn fetch_and_save_by_date(
|
||||
)?;
|
||||
}
|
||||
session.logout().await.ok();
|
||||
Ok(())
|
||||
Ok(max_uid)
|
||||
}
|
||||
|
||||
/// Fetches all messages from a mailbox.
|
||||
/// Returns `Ok(Some(max_uid))` with the highest UID stored, or `Ok(None)` if empty.
|
||||
pub async fn fetch_and_save_full_mailbox(
|
||||
account: &AccountModel,
|
||||
mailbox: &MailBox,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<()> {
|
||||
) -> BichonResult<Option<u32>> {
|
||||
let mailbox_id = mailbox.id;
|
||||
let account_id = account.id;
|
||||
|
||||
@@ -262,33 +251,17 @@ pub async fn fetch_and_save_full_mailbox(
|
||||
}
|
||||
};
|
||||
|
||||
let folder_limit = account.folder_limit;
|
||||
let total_to_fetch = match folder_limit {
|
||||
Some(limit) if (limit as u64) < total => {
|
||||
let limit64 = limit as u64;
|
||||
total.min(limit64.max(100))
|
||||
}
|
||||
_ => total,
|
||||
};
|
||||
|
||||
let page_size = if let Some(limit) = folder_limit {
|
||||
limit
|
||||
.max(100)
|
||||
.min(account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE))
|
||||
} else {
|
||||
account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE)
|
||||
};
|
||||
|
||||
let total_batches = total_to_fetch.div_ceil(page_size as u64);
|
||||
let desc = folder_limit.is_some();
|
||||
let page_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE);
|
||||
let total_batches = total.div_ceil(page_size as u64);
|
||||
|
||||
info!(
|
||||
"Starting full mailbox download for '{}', total={}, limit={:?}, batches={}, desc={}",
|
||||
mailbox.name, total, folder_limit, total_batches, desc
|
||||
"Starting full mailbox download for '{}', total={}, batches={}",
|
||||
mailbox.name, total, total_batches
|
||||
);
|
||||
|
||||
let mut current_processed = 0u64;
|
||||
let mut has_error_or_cancel = false;
|
||||
let mut max_uid: Option<u32> = None;
|
||||
|
||||
for page in 1..=total_batches {
|
||||
if token.is_cancelled() {
|
||||
@@ -300,7 +273,7 @@ pub async fn fetch_and_save_full_mailbox(
|
||||
DownloadState::update_folder_progress(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
total_to_fetch,
|
||||
total,
|
||||
current_processed,
|
||||
FolderStatus::Cancelled,
|
||||
None,
|
||||
@@ -313,12 +286,12 @@ pub async fn fetch_and_save_full_mailbox(
|
||||
&mut session,
|
||||
account_id,
|
||||
mailbox_id,
|
||||
total_to_fetch,
|
||||
total,
|
||||
page as u64,
|
||||
page_size as u64,
|
||||
&mailbox.encoded_name(),
|
||||
desc,
|
||||
token.clone(),
|
||||
&mut max_uid,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -327,7 +300,7 @@ pub async fn fetch_and_save_full_mailbox(
|
||||
DownloadState::update_folder_progress(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
total_to_fetch,
|
||||
total,
|
||||
current_processed,
|
||||
FolderStatus::Downloading,
|
||||
None,
|
||||
@@ -339,7 +312,7 @@ pub async fn fetch_and_save_full_mailbox(
|
||||
DownloadState::update_folder_progress(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
total_to_fetch,
|
||||
total,
|
||||
current_processed,
|
||||
FolderStatus::Failed,
|
||||
Some(err_msg),
|
||||
@@ -354,71 +327,14 @@ pub async fn fetch_and_save_full_mailbox(
|
||||
DownloadState::update_folder_progress(
|
||||
account_id,
|
||||
mailbox.name.clone(),
|
||||
total_to_fetch,
|
||||
total,
|
||||
current_processed,
|
||||
FolderStatus::Success,
|
||||
None,
|
||||
)?;
|
||||
}
|
||||
session.logout().await.ok();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn generate_uid_sequence_hashset(
|
||||
unique_nums: Vec<u32>,
|
||||
chunk_size: usize,
|
||||
desc: bool,
|
||||
) -> Vec<(String, u64)> {
|
||||
assert!(!unique_nums.is_empty());
|
||||
let mut nums = unique_nums;
|
||||
if desc {
|
||||
nums.reverse();
|
||||
}
|
||||
|
||||
let mut result = Vec::new();
|
||||
|
||||
for chunk in nums.chunks(chunk_size) {
|
||||
let size = chunk.len() as u64;
|
||||
let compressed = compress_uid_list(chunk.to_vec());
|
||||
result.push((compressed, size));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn compress_uid_list(nums: Vec<u32>) -> String {
|
||||
if nums.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut sorted_nums = nums;
|
||||
sorted_nums.sort();
|
||||
|
||||
let mut result = Vec::new();
|
||||
let mut current_range_start = sorted_nums[0];
|
||||
let mut current_range_end = sorted_nums[0];
|
||||
|
||||
for &n in sorted_nums.iter().skip(1) {
|
||||
if n == current_range_end + 1 {
|
||||
current_range_end = n;
|
||||
} else {
|
||||
if current_range_start == current_range_end {
|
||||
result.push(current_range_start.to_string());
|
||||
} else {
|
||||
result.push(format!("{}:{}", current_range_start, current_range_end));
|
||||
}
|
||||
current_range_start = n;
|
||||
current_range_end = n;
|
||||
}
|
||||
}
|
||||
|
||||
if current_range_start == current_range_end {
|
||||
result.push(current_range_start.to_string());
|
||||
} else {
|
||||
result.push(format!("{}:{}", current_range_start, current_range_end));
|
||||
}
|
||||
|
||||
result.join(",")
|
||||
Ok(max_uid)
|
||||
}
|
||||
|
||||
pub async fn reconcile_mailboxes(
|
||||
@@ -448,7 +364,7 @@ pub async fn reconcile_mailboxes(
|
||||
break;
|
||||
}
|
||||
|
||||
if local_mailbox.uid_validity != remote_mailbox.uid_validity {
|
||||
let new_highest_uid = if local_mailbox.uid_validity != remote_mailbox.uid_validity {
|
||||
if remote_mailbox.uid_validity.is_none() {
|
||||
let err_msg = format!(
|
||||
"Mailbox '{}' logic error: Server did not provide UIDVALIDITY.",
|
||||
@@ -493,7 +409,7 @@ pub async fn reconcile_mailboxes(
|
||||
FetchDirection::Since,
|
||||
token.clone(),
|
||||
)
|
||||
.await?;
|
||||
.await?
|
||||
}
|
||||
None => match &account.date_before {
|
||||
Some(r) => {
|
||||
@@ -505,7 +421,7 @@ pub async fn reconcile_mailboxes(
|
||||
FetchDirection::Before,
|
||||
token.clone(),
|
||||
)
|
||||
.await?;
|
||||
.await?
|
||||
}
|
||||
None => {
|
||||
rebuild_mailbox_cache(
|
||||
@@ -520,10 +436,12 @@ pub async fn reconcile_mailboxes(
|
||||
}
|
||||
} else {
|
||||
perform_incremental_sync(account, local_mailbox, remote_mailbox, token.clone())
|
||||
.await?;
|
||||
}
|
||||
.await?
|
||||
};
|
||||
|
||||
mailboxes_to_update.push(remote_mailbox.clone());
|
||||
let mut updated = remote_mailbox.clone();
|
||||
updated.highest_uid = new_highest_uid;
|
||||
mailboxes_to_update.push(updated);
|
||||
}
|
||||
//The metadata of this mailbox must only be updated after a successful synchronization;
|
||||
//otherwise, it may cause synchronization errors and result in missing emails in the local sync results.
|
||||
@@ -598,7 +516,11 @@ pub async fn reconcile_mailboxes(
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(_) => {}
|
||||
Ok(new_highest_uid) => {
|
||||
let mut updated = mailbox.clone();
|
||||
updated.highest_uid = new_highest_uid;
|
||||
MailBox::batch_upsert(&[updated])?;
|
||||
}
|
||||
Err(err) => {
|
||||
has_error = true;
|
||||
tracing::error!("Folder sync task failed: {:#?}", err);
|
||||
@@ -622,57 +544,102 @@ pub async fn reconcile_mailboxes(
|
||||
}
|
||||
|
||||
//only check new emails and sync
|
||||
/// Incrementally syncs a mailbox.
|
||||
/// Returns the new highest UID after sync, or `None` if nothing changed.
|
||||
async fn perform_incremental_sync(
|
||||
account: &AccountModel,
|
||||
local_mailbox: &MailBox,
|
||||
remote_mailbox: &MailBox,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<()> {
|
||||
) -> BichonResult<Option<u32>> {
|
||||
if remote_mailbox.exists > 0 {
|
||||
let local_max_uid = ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?;
|
||||
match local_max_uid {
|
||||
Some(max_uid) => {
|
||||
let mut session = ImapExecutor::create_connection(account.id).await?;
|
||||
let before_date = account
|
||||
.date_before
|
||||
.as_ref()
|
||||
.map(|r| r.calculate_date())
|
||||
.transpose()?;
|
||||
|
||||
ImapExecutor::fetch_new_mail(
|
||||
&mut session,
|
||||
account,
|
||||
local_mailbox,
|
||||
max_uid + 1,
|
||||
before_date.as_deref(),
|
||||
token,
|
||||
)
|
||||
.await?;
|
||||
session.logout().await.ok();
|
||||
// Use stored highest_uid if available; otherwise fall back to Tantivy
|
||||
// query once (backward compatibility with pre-existing databases).
|
||||
let start_uid = match local_mailbox.highest_uid {
|
||||
Some(uid) => {
|
||||
tracing::info!(
|
||||
"[account {}][mailbox {}] perform_incremental_sync: stored highest_uid={}, remote.exists={}",
|
||||
account.id,
|
||||
local_mailbox.name,
|
||||
uid,
|
||||
remote_mailbox.exists
|
||||
);
|
||||
uid as u64 + 1
|
||||
}
|
||||
None => {
|
||||
info!(
|
||||
"No maximum UID found in index for mailbox, assuming local cache is missing."
|
||||
let local_max_uid =
|
||||
ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?;
|
||||
tracing::info!(
|
||||
"[account {}][mailbox {}] perform_incremental_sync: highest_uid unset, Tantivy max_uid={:?}, remote.exists={}",
|
||||
account.id,
|
||||
local_mailbox.name,
|
||||
local_max_uid,
|
||||
remote_mailbox.exists
|
||||
);
|
||||
|
||||
match &account.date_since {
|
||||
Some(date_since) => {
|
||||
fetch_and_save_by_date(
|
||||
account,
|
||||
date_since.since_date()?.as_str(),
|
||||
remote_mailbox,
|
||||
FetchDirection::Since,
|
||||
token,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
match local_max_uid {
|
||||
Some(uid) => uid + 1,
|
||||
None => {
|
||||
fetch_and_save_full_mailbox(account, remote_mailbox, token).await?;
|
||||
info!(
|
||||
"No maximum UID found in index for mailbox, assuming local storage is missing."
|
||||
);
|
||||
|
||||
let result = match &account.date_since {
|
||||
Some(date_since) => {
|
||||
fetch_and_save_by_date(
|
||||
account,
|
||||
date_since.since_date()?.as_str(),
|
||||
remote_mailbox,
|
||||
FetchDirection::Since,
|
||||
token,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
None => match &account.date_before {
|
||||
Some(r) => {
|
||||
fetch_and_save_by_date(
|
||||
account,
|
||||
&r.calculate_date()?,
|
||||
remote_mailbox,
|
||||
FetchDirection::Before,
|
||||
token,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
None => {
|
||||
fetch_and_save_full_mailbox(
|
||||
account, remote_mailbox, token,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
},
|
||||
};
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
let mut session = ImapExecutor::create_connection(account.id).await?;
|
||||
let before_date = account
|
||||
.date_before
|
||||
.as_ref()
|
||||
.map(|r| r.calculate_date())
|
||||
.transpose()?;
|
||||
|
||||
let new_max_uid = ImapExecutor::fetch_new_mail(
|
||||
&mut session,
|
||||
account,
|
||||
local_mailbox,
|
||||
start_uid,
|
||||
before_date.as_deref(),
|
||||
token,
|
||||
)
|
||||
.await?;
|
||||
session.logout().await.ok();
|
||||
|
||||
// Keep existing highest_uid if no new mail was fetched.
|
||||
Ok(new_max_uid.or(local_mailbox.highest_uid))
|
||||
} else {
|
||||
Ok(local_mailbox.highest_uid)
|
||||
}
|
||||
}
|
||||
|
||||
+38
-29
@@ -17,24 +17,20 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::{
|
||||
raise_error,
|
||||
{
|
||||
account::{
|
||||
migration::AccountModel,
|
||||
state::{DownloadState, DownloadStatus, FolderStatus},
|
||||
},
|
||||
cache::{
|
||||
imap::{
|
||||
download::flow::{
|
||||
fetch_and_save_by_date, fetch_and_save_full_mailbox, FetchDirection,
|
||||
},
|
||||
mailbox::MailBox,
|
||||
},
|
||||
SEMAPHORE,
|
||||
},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
store::tantivy::envelope::ENVELOPE_MANAGER,
|
||||
account::{
|
||||
migration::AccountModel,
|
||||
state::{DownloadState, DownloadStatus, FolderStatus},
|
||||
},
|
||||
cache::{
|
||||
imap::{
|
||||
download::flow::{fetch_and_save_by_date, fetch_and_save_full_mailbox, FetchDirection},
|
||||
mailbox::MailBox,
|
||||
},
|
||||
SEMAPHORE,
|
||||
},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
raise_error,
|
||||
store::tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
|
||||
};
|
||||
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -91,9 +87,13 @@ pub async fn rebuild_cache(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
match fetch_and_save_full_mailbox(&account, &mailbox, token.clone()).await {
|
||||
Ok(_) => {}
|
||||
Ok(new_highest_uid) => {
|
||||
let mut updated = mailbox.clone();
|
||||
updated.highest_uid = new_highest_uid;
|
||||
MailBox::batch_upsert(&[updated])?;
|
||||
}
|
||||
Err(err) => {
|
||||
has_error = true;
|
||||
tracing::error!("Folder sync task failed: {:#?}", err);
|
||||
@@ -173,7 +173,11 @@ pub async fn rebuild_cache_by_date(
|
||||
match fetch_and_save_by_date(&account, date.as_str(), &mailbox, direction, token.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
Ok(new_highest_uid) => {
|
||||
let mut updated = mailbox.clone();
|
||||
updated.highest_uid = new_highest_uid;
|
||||
MailBox::batch_upsert(&[updated])?;
|
||||
}
|
||||
Err(err) => {
|
||||
has_error = true;
|
||||
tracing::error!("Folder sync task failed: {:#?}", err);
|
||||
@@ -200,11 +204,13 @@ pub async fn rebuild_mailbox_cache(
|
||||
local_mailbox: &MailBox,
|
||||
remote_mailbox: &MailBox,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<()> {
|
||||
) -> BichonResult<Option<u32>> {
|
||||
ENVELOPE_MANAGER
|
||||
.delete_mailbox_envelopes(account.id, vec![local_mailbox.id])
|
||||
.await?;
|
||||
|
||||
ATTACHMENT_MANAGER
|
||||
.delete_mailbox_attachments(account.id, vec![local_mailbox.id])
|
||||
.await?;
|
||||
if remote_mailbox.exists == 0 {
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
|
||||
@@ -219,11 +225,11 @@ pub async fn rebuild_mailbox_cache(
|
||||
FolderStatus::Success,
|
||||
None,
|
||||
)?;
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
fetch_and_save_full_mailbox(account, remote_mailbox, token).await?;
|
||||
Ok(())
|
||||
let result = fetch_and_save_full_mailbox(account, remote_mailbox, token).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn rebuild_mailbox_cache_by_date(
|
||||
@@ -233,10 +239,13 @@ pub async fn rebuild_mailbox_cache_by_date(
|
||||
remote: &MailBox,
|
||||
direction: FetchDirection,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<()> {
|
||||
) -> BichonResult<Option<u32>> {
|
||||
ENVELOPE_MANAGER
|
||||
.delete_mailbox_envelopes(account.id, vec![local_mailbox_id])
|
||||
.await?;
|
||||
ATTACHMENT_MANAGER
|
||||
.delete_mailbox_attachments(account.id, vec![local_mailbox_id])
|
||||
.await?;
|
||||
if remote.exists == 0 {
|
||||
info!(
|
||||
"Account {}: Mailbox '{}' has no emails on the remote server. The mailbox is empty, no envelopes to fetch.",
|
||||
@@ -251,9 +260,9 @@ pub async fn rebuild_mailbox_cache_by_date(
|
||||
FolderStatus::Success,
|
||||
None,
|
||||
)?;
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
fetch_and_save_by_date(account, date, remote, direction, token).await?;
|
||||
Ok(())
|
||||
let result = fetch_and_save_by_date(account, date, remote, direction, token).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
+4
@@ -56,6 +56,10 @@ pub struct MailBox {
|
||||
/// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions.
|
||||
/// If `None`, the IMAP server has not provided this information.
|
||||
pub uid_validity: Option<u32>,
|
||||
/// The highest UID that has been successfully downloaded and stored locally.
|
||||
/// Used for incremental sync: next fetch starts from `highest_uid + 1`.
|
||||
/// If `None`, a fallback query against the Tantivy index will be performed once.
|
||||
pub highest_uid: Option<u32>,
|
||||
}
|
||||
|
||||
impl MemDbModel for MailBox {
|
||||
|
||||
@@ -83,16 +83,11 @@ impl DashboardStats {
|
||||
|
||||
stat.email_count = ENVELOPE_MANAGER.total_emails(&authorized_ids)?;
|
||||
stat.attachment_count = ATTACHMENT_MANAGER.total_attachments(&authorized_ids)?;
|
||||
if has_all_accounts {
|
||||
stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.storage_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
stat.storage_usage_bytes = get_total_size(&DATA_DIR_MANAGER.storage_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.envelope_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
} else {
|
||||
stat.storage_usage_bytes = 0;
|
||||
stat.index_usage_bytes = 0;
|
||||
}
|
||||
stat.index_usage_bytes = get_total_size(&&DATA_DIR_MANAGER.envelope_dir)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
stat.system_version = bichon_version!().to_string();
|
||||
|
||||
|
||||
@@ -32,12 +32,16 @@ impl DatabaseManager {
|
||||
let db_path = &DATA_DIR_MANAGER.memdb_dir;
|
||||
std::fs::create_dir_all(db_path).expect("Failed to create memdb data directory");
|
||||
|
||||
let db = MemDb::open_with(db_path, Durability::Full)
|
||||
let db = MemDb::open_with(db_path, Durability::Batch { max_ops: 100 })
|
||||
.expect("Failed to open memdb database");
|
||||
|
||||
// Start periodic snapshot worker (every 5 minutes)
|
||||
db.start_snapshot_worker(Duration::from_secs(300));
|
||||
|
||||
// Start periodic flush worker (every 10 seconds) so buffered writes
|
||||
// are flushed regularly and not only at the batch threshold.
|
||||
db.start_flush_worker(Duration::from_secs(10));
|
||||
|
||||
DatabaseManager { db }
|
||||
}
|
||||
|
||||
@@ -45,4 +49,12 @@ impl DatabaseManager {
|
||||
pub fn db(&self) -> &MemDb {
|
||||
&self.db
|
||||
}
|
||||
|
||||
/// Flush any buffered WAL entries to disk. Must be called before shutdown
|
||||
/// to avoid losing writes that haven't hit the batch threshold yet.
|
||||
pub fn flush(&self) {
|
||||
if let Err(e) = self.db.flush() {
|
||||
eprintln!("[memdb] flush error on shutdown: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,11 +16,13 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::cache::imap::mailbox::MailBox;
|
||||
use crate::common::AddrVec;
|
||||
use crate::envelope::meta::parse_bichon_metadata;
|
||||
use crate::envelope::utils::normalize_subject;
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::imap::executor::ImapExecutor;
|
||||
use crate::message::content::AttachmentInfo;
|
||||
use crate::store::blob::{DetachedEmail, BLOB_MANAGER};
|
||||
use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
|
||||
@@ -87,6 +89,7 @@ async fn extract_envelope_core(
|
||||
account_id: u64,
|
||||
mailbox_id: u64,
|
||||
) -> BichonResult<()> {
|
||||
//The content hash of the original raw EML
|
||||
let email_content_hash = compute_content_hash(body);
|
||||
let message: Message<'_> = MessageParser::new().parse(body).ok_or_else(|| {
|
||||
raise_error!(
|
||||
@@ -194,33 +197,37 @@ async fn extract_envelope_core(
|
||||
let attachment_docs: Vec<TantivyDocument> = attachments
|
||||
.iter()
|
||||
.filter(|a| !a.inline || a.content_id.is_none())
|
||||
.map(|a| AttachmentModel {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
envelope_id: envelope_id.clone(),
|
||||
account_id,
|
||||
account_email: None,
|
||||
mailbox_id,
|
||||
mailbox_name: None,
|
||||
subject: subject.clone(),
|
||||
content_hash: a.content_hash.clone(),
|
||||
from: from.clone(),
|
||||
date,
|
||||
ingest_at: now,
|
||||
size: a.size as u64,
|
||||
ext: a.get_extension(),
|
||||
category: a.get_category().to_string(),
|
||||
content_type: a.file_type.clone(),
|
||||
shard_id: 0,
|
||||
text: None,
|
||||
has_text: false,
|
||||
is_ocr: false,
|
||||
page_count: None,
|
||||
is_indexed: false,
|
||||
is_message: a.is_message,
|
||||
name: a.filename.clone(),
|
||||
tags: None,
|
||||
auto_tags: None,
|
||||
}).map(|a|a.into_document())
|
||||
.map(|a| {
|
||||
let has_text = a.extracted_text.is_some();
|
||||
AttachmentModel {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
envelope_id: envelope_id.clone(),
|
||||
account_id,
|
||||
account_email: None,
|
||||
mailbox_id,
|
||||
mailbox_name: None,
|
||||
subject: subject.clone(),
|
||||
content_hash: a.content_hash.clone(),
|
||||
from: from.clone(),
|
||||
date,
|
||||
ingest_at: now,
|
||||
size: a.size as u64,
|
||||
ext: a.get_extension(),
|
||||
category: a.get_category().to_string(),
|
||||
content_type: a.file_type.clone(),
|
||||
shard_id: 0,
|
||||
text: a.extracted_text.clone(),
|
||||
has_text,
|
||||
is_ocr: a.extracted_is_ocr,
|
||||
page_count: a.extracted_page_count.map(|n| n as u64),
|
||||
is_indexed: has_text,
|
||||
is_message: a.is_message,
|
||||
name: a.filename.clone(),
|
||||
tags: None,
|
||||
auto_tags: None,
|
||||
}
|
||||
})
|
||||
.map(|a| a.into_document())
|
||||
.collect();
|
||||
|
||||
let envelope = Envelope {
|
||||
@@ -253,6 +260,14 @@ async fn extract_envelope_core(
|
||||
attachments: Some(attachments),
|
||||
};
|
||||
let doc = ea.to_document(&body_text, 0)?;
|
||||
tracing::debug!(
|
||||
"[account {}][mailbox {}] extract: uid={} msg_id={} content_hash={}",
|
||||
account_id,
|
||||
mailbox_id,
|
||||
uid,
|
||||
&ea.envelope.message_id,
|
||||
&ea.envelope.content_hash,
|
||||
);
|
||||
ENVELOPE_MANAGER.queue(doc).await;
|
||||
for doc in attachment_docs {
|
||||
ATTACHMENT_MANAGER.queue(doc).await;
|
||||
@@ -386,11 +401,23 @@ pub async fn detach_and_store_attachments(
|
||||
|
||||
ranges.sort_by(|a, b| b.0.cmp(&a.0));
|
||||
let mut attachments = Vec::with_capacity(ranges.len());
|
||||
|
||||
// Collect candidates for text extraction (non-inline, known document types).
|
||||
struct TextCandidate {
|
||||
content_hash: String,
|
||||
file_type: String,
|
||||
ext: String,
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
let mut text_candidates: Vec<TextCandidate> = Vec::new();
|
||||
|
||||
for (raw_start, raw_end, att) in ranges {
|
||||
// Step 2: Extract raw bytes and store them as standalone documents
|
||||
let raw_bytes = &original_body[raw_start..raw_end];
|
||||
//This is the content hash of the decoded attachment, not the undecoded one.
|
||||
let content_hash = compute_content_hash(att.contents());
|
||||
|
||||
//"The actual content stored in the blob is the raw undecoded data, to avoid the reconstructed EML differing from the original due to decoding and re-encoding.
|
||||
attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes)));//
|
||||
|
||||
// Step 3: Replace raw attachment content with a hash-based placeholder
|
||||
@@ -398,30 +425,88 @@ pub async fn detach_and_store_attachments(
|
||||
let p_bytes = placeholder.as_bytes();
|
||||
stripped_eml.splice(raw_start..raw_end, p_bytes.iter().cloned());
|
||||
|
||||
let inline = att
|
||||
.content_disposition()
|
||||
.map(|d| d.is_inline())
|
||||
.unwrap_or(false);
|
||||
let file_type = att
|
||||
.content_type()
|
||||
.map(|ct| {
|
||||
format!(
|
||||
"{}/{}",
|
||||
ct.c_type.as_ref(),
|
||||
ct.c_subtype.as_deref().unwrap_or("")
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||
let has_cid = att.content_id().is_some();
|
||||
let ext = att
|
||||
.attachment_name()
|
||||
.and_then(|n| {
|
||||
std::path::Path::new(&n)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|s| s.to_ascii_lowercase())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
if !inline || !has_cid {
|
||||
let decoded_len = att.contents().len();
|
||||
if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
|
||||
&& crate::ext::text_extractor::should_try_extract(&file_type, &ext)
|
||||
{
|
||||
text_candidates.push(TextCandidate {
|
||||
content_hash: content_hash.clone(),
|
||||
file_type: file_type.clone(),
|
||||
ext: ext.clone(),
|
||||
bytes: att.contents().to_vec(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let info = AttachmentInfo {
|
||||
filename: att.attachment_name().map(|n| n.to_string()),
|
||||
size: att.contents().len(),
|
||||
inline: att
|
||||
.content_disposition()
|
||||
.map(|d| d.is_inline())
|
||||
.unwrap_or(false),
|
||||
file_type: att
|
||||
.content_type()
|
||||
.map(|ct| {
|
||||
format!(
|
||||
"{}/{}",
|
||||
ct.c_type.as_ref(),
|
||||
ct.c_subtype.as_deref().unwrap_or("")
|
||||
)
|
||||
})
|
||||
.unwrap_or_else(|| "application/octet-stream".to_string()),
|
||||
inline,
|
||||
file_type,
|
||||
content_id: att.content_id().map(|id| id.to_string()),
|
||||
content_hash: content_hash.clone(),
|
||||
is_message: att.is_message(),
|
||||
extracted_text: None,
|
||||
extracted_page_count: None,
|
||||
extracted_is_ocr: false,
|
||||
};
|
||||
|
||||
attachment_infos.push(info);
|
||||
}
|
||||
|
||||
// Run text extraction in a single spawn_blocking batch.
|
||||
if !text_candidates.is_empty() {
|
||||
if let Ok(mut extracted_map) = tokio::task::spawn_blocking(move || {
|
||||
let mut map: std::collections::HashMap<
|
||||
String,
|
||||
(String, Option<u32>, bool),
|
||||
> = std::collections::HashMap::new();
|
||||
for c in text_candidates {
|
||||
if let Some(r) =
|
||||
crate::ext::text_extractor::extract_text(&c.file_type, &c.ext, &c.bytes)
|
||||
{
|
||||
map.insert(c.content_hash, (r.text, r.page_count, r.is_ocr));
|
||||
}
|
||||
}
|
||||
map
|
||||
})
|
||||
.await
|
||||
{
|
||||
for info in &mut attachment_infos {
|
||||
if let Some((text, pages, is_ocr)) = extracted_map.remove(&info.content_hash) {
|
||||
info.extracted_text = Some(text);
|
||||
info.extracted_page_count = pages;
|
||||
info.extracted_is_ocr = is_ocr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Step 4: Store the final stripped EML content
|
||||
BLOB_MANAGER
|
||||
.queue(DetachedEmail {
|
||||
@@ -515,6 +600,114 @@ pub fn reattach_eml_content(
|
||||
Ok((e.envelope, Bytes::from(restored_eml)))
|
||||
}
|
||||
|
||||
/// Returns the raw EML for an indexed message, self-healing a missing content blob.
|
||||
///
|
||||
/// Behaves like [`reattach_eml_content`], but when the message's content blob is
|
||||
/// absent from the blob store it fetches that single message on demand from the
|
||||
/// IMAP server (`UID FETCH <uid> (BODY.PEEK[])`), persists it for future requests,
|
||||
/// and returns it. If the on-demand fetch itself fails, the original "content not
|
||||
/// found" error from [`reattach_eml_content`] is surfaced unchanged so the caller
|
||||
/// still produces its 404.
|
||||
pub async fn reattach_eml_content_self_healing(
|
||||
account_id: u64,
|
||||
envelope_id: String,
|
||||
) -> BichonResult<(Envelope, Bytes)> {
|
||||
let envelope = ENVELOPE_MANAGER
|
||||
.get_envelope_by_id(account_id, &envelope_id)?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"Envelope not found: account_id={} envelope_id={}",
|
||||
account_id, &envelope_id
|
||||
),
|
||||
ErrorCode::ResourceNotFound
|
||||
)
|
||||
})?
|
||||
.envelope;
|
||||
|
||||
// Fast path: the content blob is present, reuse the regular reattach logic.
|
||||
if BLOB_MANAGER.get_email(&envelope.content_hash)?.is_some() {
|
||||
return reattach_eml_content(account_id, envelope_id);
|
||||
}
|
||||
|
||||
// The blob is missing. Try to recover it directly from the IMAP server.
|
||||
match recover_message_blob(&envelope).await {
|
||||
Ok(raw_body) => {
|
||||
tracing::info!(
|
||||
account_id,
|
||||
envelope_id = %envelope_id,
|
||||
uid = envelope.uid,
|
||||
"Self-healed missing email content blob via on-demand IMAP fetch"
|
||||
);
|
||||
Ok((envelope, raw_body))
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
account_id,
|
||||
envelope_id = %envelope_id,
|
||||
uid = envelope.uid,
|
||||
error = %e,
|
||||
"On-demand IMAP fetch for missing content blob failed; returning not-found"
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches one message from IMAP and re-stores its detached blob.
|
||||
///
|
||||
/// On success the freshly fetched raw RFC822 body is returned; it is also queued
|
||||
/// (in detached form) into the blob store so subsequent requests hit the cache.
|
||||
/// Fails if the message cannot be fetched, or if the fetched bytes do not match
|
||||
/// the archived `content_hash` (the server-side message no longer matches what
|
||||
/// Bichon archived, so it cannot be treated as a recovery of that blob).
|
||||
async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
|
||||
let mailbox = MailBox::find_mailbox(envelope.account_id, envelope.mailbox_id)?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!(
|
||||
"Mailbox not found: account_id={} mailbox_id={}",
|
||||
envelope.account_id, envelope.mailbox_id
|
||||
),
|
||||
ErrorCode::ResourceNotFound
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut session = ImapExecutor::create_connection(envelope.account_id).await?;
|
||||
let result = ImapExecutor::fetch_single_message_body(
|
||||
&mut session,
|
||||
&mailbox.encoded_name(),
|
||||
envelope.uid,
|
||||
)
|
||||
.await;
|
||||
session.logout().await.ok();
|
||||
let raw_body = result?;
|
||||
|
||||
let fetched_hash = compute_content_hash(&raw_body);
|
||||
if fetched_hash != envelope.content_hash {
|
||||
return Err(raise_error!(
|
||||
format!(
|
||||
"Fetched message does not match archived content: expected content_hash={} got={}",
|
||||
envelope.content_hash, fetched_hash
|
||||
),
|
||||
ErrorCode::ImapUnexpectedResult
|
||||
));
|
||||
}
|
||||
|
||||
// Re-create the detached blob (stripped EML + attachments) so the missing
|
||||
// blob is repopulated for future requests. The detached EML is queued under
|
||||
// `fetched_hash`, which equals `envelope.content_hash`.
|
||||
let message = MessageParser::new().parse(raw_body.as_slice()).ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Failed to parse fetched email content".into(),
|
||||
ErrorCode::InternalError
|
||||
)
|
||||
})?;
|
||||
detach_and_store_attachments(&raw_body, &message, &fetched_hash).await;
|
||||
|
||||
Ok(Bytes::from(raw_body))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use html2text::config;
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Event bus extension point.
|
||||
//
|
||||
// Community edition: NoopEventBus — all events are discarded.
|
||||
// Pro edition: AuditEventBus — events are persisted to audit database.
|
||||
// Enterprise edition: adds SIEM webhook to the same trait impl.
|
||||
//
|
||||
// The open-source server emits events at key points (login, view, delete, search).
|
||||
// It never reads from the event bus — events are fire-and-forget.
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::sync::{LazyLock, RwLock};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Event {
|
||||
EmailViewed {
|
||||
email_id: String,
|
||||
user: String,
|
||||
ip: IpAddr,
|
||||
},
|
||||
EmailDeleted {
|
||||
email_id: String,
|
||||
user: String,
|
||||
},
|
||||
UserLoggedIn {
|
||||
user: String,
|
||||
ip: IpAddr,
|
||||
},
|
||||
UserCreated {
|
||||
created_by: String,
|
||||
new_user: String,
|
||||
},
|
||||
SearchPerformed {
|
||||
query: String,
|
||||
user: String,
|
||||
},
|
||||
SettingsChanged {
|
||||
key: String,
|
||||
user: String,
|
||||
},
|
||||
AttachmentDownloaded {
|
||||
email_id: String,
|
||||
content_hash: String,
|
||||
user: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub trait EventBus: Send + Sync {
|
||||
fn emit(&self, event: Event);
|
||||
}
|
||||
|
||||
/// Default — all events are discarded.
|
||||
struct NoopEventBus;
|
||||
impl EventBus for NoopEventBus {
|
||||
fn emit(&self, _event: Event) {}
|
||||
}
|
||||
|
||||
static EVENT_BUS: LazyLock<RwLock<Box<dyn EventBus>>> =
|
||||
LazyLock::new(|| RwLock::new(Box::new(NoopEventBus)));
|
||||
|
||||
/// Called by Pro/Enterprise at startup to replace the noop default.
|
||||
pub fn set_event_bus(bus: Box<dyn EventBus>) {
|
||||
*EVENT_BUS.write().unwrap() = bus;
|
||||
}
|
||||
|
||||
/// Fire-and-forget. Called by the server at key points.
|
||||
pub fn emit(event: Event) {
|
||||
EVENT_BUS.read().unwrap().emit(event);
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Event bus extension point.
|
||||
//
|
||||
// Community edition: NoopEventBus — all events are discarded.
|
||||
// Pro edition: AuditEventBus — events are persisted to audit database.
|
||||
// Enterprise edition: adds SIEM webhook to the same trait impl.
|
||||
//
|
||||
// The open-source server emits events at key points (login, view, delete, search).
|
||||
// It never reads from the event bus — events are fire-and-forget.
|
||||
|
||||
pub mod event_bus;
|
||||
pub mod text_extractor;
|
||||
@@ -0,0 +1,75 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Attachment text extraction extension point.
|
||||
//
|
||||
// Community edition: NoopExtractor — no attachments are text-indexed.
|
||||
// Pro edition: PdfExtractor — extracts text from PDF, Word, etc.
|
||||
//
|
||||
// Used in: crates/core/src/envelope/extractor.rs
|
||||
|
||||
use std::sync::{LazyLock, RwLock};
|
||||
|
||||
pub struct ExtractedText {
|
||||
pub text: String,
|
||||
pub page_count: Option<u32>,
|
||||
pub is_ocr: bool,
|
||||
}
|
||||
|
||||
pub trait AttachmentTextExtractor: Send + Sync {
|
||||
/// Returns None if this extractor doesn't handle the file type.
|
||||
/// Returns Some(ExtractedText) if text was successfully extracted.
|
||||
fn extract(&self, content_type: &str, ext: &str, bytes: &[u8]) -> Option<ExtractedText>;
|
||||
}
|
||||
|
||||
/// Default — all attachments are skipped.
|
||||
struct NoopExtractor;
|
||||
impl AttachmentTextExtractor for NoopExtractor {
|
||||
fn extract(&self, _ct: &str, _ext: &str, _bytes: &[u8]) -> Option<ExtractedText> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
static EXTRACTOR: LazyLock<RwLock<Box<dyn AttachmentTextExtractor>>> =
|
||||
LazyLock::new(|| RwLock::new(Box::new(NoopExtractor)));
|
||||
|
||||
/// Called by Pro/Enterprise at startup to replace the noop default.
|
||||
pub fn set_extractor(extractor: Box<dyn AttachmentTextExtractor>) {
|
||||
*EXTRACTOR.write().unwrap() = extractor;
|
||||
}
|
||||
|
||||
/// Attachments larger than this are skipped (10 MiB). Avoids excessive memory
|
||||
/// and CPU cost for huge files whose text is rarely useful for search.
|
||||
pub const MAX_EXTRACT_BYTES: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// Quick pre-filter: returns true for file types where text extraction may
|
||||
/// produce useful results. Avoids cloning attachment bytes for images, videos,
|
||||
/// archives, etc. when no registered extractor would handle them.
|
||||
pub fn should_try_extract(content_type: &str, ext: &str) -> bool {
|
||||
matches!(
|
||||
ext,
|
||||
"pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx"
|
||||
| "txt" | "rtf" | "odt" | "ods" | "odp"
|
||||
) || content_type.starts_with("text/")
|
||||
}
|
||||
|
||||
/// Called by the attachment pipeline during IMAP sync.
|
||||
/// The caller should wrap this in spawn_blocking for CPU-bound extraction.
|
||||
pub fn extract_text(content_type: &str, ext: &str, bytes: &[u8]) -> Option<ExtractedText> {
|
||||
EXTRACTOR.read().unwrap().extract(content_type, ext, bytes)
|
||||
}
|
||||
+351
-101
@@ -17,8 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::account::migration::AccountModel;
|
||||
use crate::account::state::{DownloadState, FolderStatus};
|
||||
use crate::cache::imap::download::flow::{generate_uid_sequence_hashset, DEFAULT_BATCH_SIZE};
|
||||
use crate::account::state::{DownloadState, DownloadStatus, FolderStatus};
|
||||
use crate::cache::imap::mailbox::MailBox;
|
||||
use crate::envelope::extractor::extract_envelope_and_store_it;
|
||||
use crate::error::code::ErrorCode;
|
||||
@@ -80,6 +79,15 @@ impl ImapExecutor {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))
|
||||
}
|
||||
|
||||
/// Fetches new mail for a mailbox.
|
||||
///
|
||||
/// When `before` is `Some(date)`, a two-step approach is used:
|
||||
/// `UID SEARCH` to find matching UIDs (standard IMAP), then batch `UID FETCH`
|
||||
/// for the specific UIDs. When `before` is `None`, a direct ranged
|
||||
/// `UID FETCH {start}:*` is issued and results are streamed.
|
||||
///
|
||||
/// Returns `Ok(Some(max_uid))` with the highest UID fetched, or `Ok(None)`
|
||||
/// if no new mail was found.
|
||||
pub async fn fetch_new_mail(
|
||||
session: &mut Session<Box<dyn SessionStream>>,
|
||||
account: &AccountModel,
|
||||
@@ -87,114 +95,197 @@ impl ImapExecutor {
|
||||
start_uid: u64,
|
||||
before: Option<&str>,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<()> {
|
||||
) -> BichonResult<Option<u32>> {
|
||||
assert!(start_uid > 0, "start_uid must be greater than 0");
|
||||
|
||||
let query = match before {
|
||||
Some(date) => format!("UID {start_uid}:* BEFORE {date}"),
|
||||
None => format!("UID {start_uid}:*"),
|
||||
};
|
||||
session
|
||||
.examine(&mailbox.encoded_name())
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?;
|
||||
|
||||
let uid_list = match Self::uid_search(session, &mailbox.encoded_name(), &query).await {
|
||||
Ok(uid_list) => uid_list,
|
||||
Err(e) => {
|
||||
let err_msg = format!("UID search failed in [{}]: {:#?}", mailbox.name, e);
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
0,
|
||||
0,
|
||||
FolderStatus::Failed,
|
||||
Some(err_msg.clone()),
|
||||
)?;
|
||||
DownloadState::append_session_error(account.id, err_msg)?;
|
||||
return Err(e);
|
||||
match before {
|
||||
Some(date) => {
|
||||
Self::fetch_new_mail_with_before(session, account, mailbox, start_uid, date, token)
|
||||
.await
|
||||
}
|
||||
};
|
||||
None => Self::fetch_new_mail_range(session, account, mailbox, start_uid, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
let len = uid_list.len();
|
||||
if len == 0 {
|
||||
let msg = match before {
|
||||
Some(date) => format!("No emails found before {}.", date),
|
||||
None => "No new emails found.".into(),
|
||||
};
|
||||
/// Two-step approach for date-filtered incremental fetch: UID SEARCH first,
|
||||
/// then batch UID FETCH for matching UIDs. Uses standard IMAP syntax that
|
||||
/// works across all compliant servers.
|
||||
async fn fetch_new_mail_with_before(
|
||||
session: &mut Session<Box<dyn SessionStream>>,
|
||||
account: &AccountModel,
|
||||
mailbox: &MailBox,
|
||||
start_uid: u64,
|
||||
date: &str,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<Option<u32>> {
|
||||
let query = format!("UID {start_uid}:* BEFORE {date}");
|
||||
info!(
|
||||
"[account {}][mailbox {}] fetch_new_mail: UID SEARCH {}",
|
||||
account.id, mailbox.name, query
|
||||
);
|
||||
let results = session.uid_search(&query).await.map_err(|e| {
|
||||
let err_msg = format!("UID SEARCH failed in [{}]: {:#?}", mailbox.name, e);
|
||||
let _ = DownloadState::append_session_error(account.id, err_msg);
|
||||
raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed)
|
||||
})?;
|
||||
|
||||
if results.is_empty() {
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
0,
|
||||
0,
|
||||
FolderStatus::Success,
|
||||
Some(msg),
|
||||
Some("No new emails found.".into()),
|
||||
)?;
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
}
|
||||
info!(
|
||||
"[account {}][mailbox {}] {} envelopes need to be fetched",
|
||||
account.id, mailbox.name, len
|
||||
);
|
||||
|
||||
let mut uid_vec: Vec<u32> = uid_list.into_iter().collect();
|
||||
let mut uid_vec: Vec<u32> = results.into_iter().collect();
|
||||
uid_vec.sort();
|
||||
let uid_batches = generate_uid_sequence_hashset(
|
||||
uid_vec,
|
||||
account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize,
|
||||
false,
|
||||
);
|
||||
let mut current_processed = 0u64;
|
||||
let mut has_error_or_cancel = false;
|
||||
for (index, batch) in uid_batches.into_iter().enumerate() {
|
||||
let max_uid = uid_vec.last().copied();
|
||||
let planned = uid_vec.len() as u64;
|
||||
let batch_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize;
|
||||
let uid_batches = generate_uid_sequence_hashset(uid_vec, batch_size);
|
||||
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
planned,
|
||||
0,
|
||||
FolderStatus::Pending,
|
||||
None,
|
||||
)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
for batch in uid_batches {
|
||||
if token.is_cancelled() {
|
||||
break;
|
||||
DownloadState::update_session_status(
|
||||
account.id,
|
||||
DownloadStatus::Cancelled,
|
||||
Some("User stopped or system shutdown".to_string()),
|
||||
)?;
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
planned,
|
||||
count,
|
||||
FolderStatus::Cancelled,
|
||||
None,
|
||||
)?;
|
||||
return Err(raise_error!(
|
||||
"Stream cancelled".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
match Self::uid_batch_retrieve_emails(
|
||||
Self::uid_batch_retrieve_emails(
|
||||
session,
|
||||
account.id,
|
||||
mailbox.id,
|
||||
&batch.0,
|
||||
token.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
current_processed += batch.1;
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
len as u64,
|
||||
current_processed,
|
||||
FolderStatus::Downloading,
|
||||
None,
|
||||
)?;
|
||||
}
|
||||
Err(e) => {
|
||||
let err_msg = format!("Batch {} failed: {:#?}", index, e);
|
||||
DownloadState::append_session_error(account.id, err_msg.clone())?;
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
len as u64,
|
||||
current_processed,
|
||||
FolderStatus::Failed,
|
||||
Some(err_msg),
|
||||
)?;
|
||||
has_error_or_cancel = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !has_error_or_cancel {
|
||||
.await?;
|
||||
count += batch.1;
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
len as u64,
|
||||
current_processed,
|
||||
planned,
|
||||
count,
|
||||
FolderStatus::Downloading,
|
||||
None,
|
||||
)?;
|
||||
}
|
||||
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
count,
|
||||
count,
|
||||
FolderStatus::Success,
|
||||
None,
|
||||
)?;
|
||||
|
||||
Ok(max_uid)
|
||||
}
|
||||
|
||||
/// Direct ranged UID FETCH without date filtering. Streams results from
|
||||
/// the server in a single IMAP round-trip.
|
||||
async fn fetch_new_mail_range(
|
||||
session: &mut Session<Box<dyn SessionStream>>,
|
||||
account: &AccountModel,
|
||||
mailbox: &MailBox,
|
||||
start_uid: u64,
|
||||
token: CancellationToken,
|
||||
) -> BichonResult<Option<u32>> {
|
||||
let uid_range = format!("{start_uid}:*");
|
||||
info!(
|
||||
"[account {}][mailbox {}] fetch_new_mail: direct UID FETCH {}",
|
||||
account.id, mailbox.name, uid_range
|
||||
);
|
||||
|
||||
let mut stream = session
|
||||
.uid_fetch(&uid_range, BODY_FETCH_COMMAND)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
let err_msg = format!("UID FETCH failed in [{}]: {:#?}", mailbox.name, e);
|
||||
let _ = DownloadState::append_session_error(account.id, err_msg);
|
||||
raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed)
|
||||
})?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut max_uid: Option<u32> = None;
|
||||
while let Some(fetch) = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
{
|
||||
if token.is_cancelled() {
|
||||
tracing::info!("Account {}: fetch_new_mail stream interrupted.", account.id);
|
||||
DownloadState::update_session_status(
|
||||
account.id,
|
||||
DownloadStatus::Cancelled,
|
||||
Some("User stopped or system shutdown".to_string()),
|
||||
)?;
|
||||
return Err(raise_error!(
|
||||
"Stream cancelled".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(uid) = fetch.uid {
|
||||
max_uid = Some(max_uid.unwrap_or(0).max(uid));
|
||||
}
|
||||
extract_envelope_and_store_it(fetch, account.id, mailbox.id).await?;
|
||||
count += 1;
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
0,
|
||||
0,
|
||||
FolderStatus::Success,
|
||||
Some("No new emails found.".into()),
|
||||
)?;
|
||||
} else {
|
||||
DownloadState::update_folder_progress(
|
||||
account.id,
|
||||
mailbox.name.clone(),
|
||||
count,
|
||||
count,
|
||||
FolderStatus::Success,
|
||||
None,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(max_uid)
|
||||
}
|
||||
|
||||
pub async fn batch_retrieve_emails(
|
||||
@@ -205,36 +296,23 @@ impl ImapExecutor {
|
||||
page: u64,
|
||||
page_size: u64,
|
||||
encoded_mailbox_name: &str,
|
||||
desc: bool,
|
||||
token: CancellationToken,
|
||||
max_uid: &mut Option<u32>,
|
||||
) -> BichonResult<usize> {
|
||||
assert!(page > 0, "Page number must be greater than 0");
|
||||
assert!(page_size > 0, "Page size must be greater than 0");
|
||||
|
||||
let (start, end) = if desc {
|
||||
// Fetch messages starting from the newest (descending order)
|
||||
let end = total.saturating_sub((page - 1) * page_size);
|
||||
if end == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
// Calculate start as end - page_size + 1 to avoid off-by-one errors
|
||||
let start = end.saturating_sub(page_size - 1).max(1);
|
||||
(start, end)
|
||||
} else {
|
||||
// Fetch messages starting from the oldest (ascending order)
|
||||
let start = (page - 1) * page_size + 1;
|
||||
if start > total {
|
||||
return Ok(0);
|
||||
}
|
||||
// Calculate end, capped by the total number of messages
|
||||
let end = (start + page_size - 1).min(total);
|
||||
(start, end)
|
||||
};
|
||||
// Fetch messages starting from the oldest (ascending order).
|
||||
let start = (page - 1) * page_size + 1;
|
||||
if start > total {
|
||||
return Ok(0);
|
||||
}
|
||||
let end = (start + page_size - 1).min(total);
|
||||
|
||||
let sequence_set = format!("{}:{}", start, end);
|
||||
info!(
|
||||
"Fetching mailbox '{}' messages: sequence {} (page {}, page_size {}, desc={})",
|
||||
encoded_mailbox_name, sequence_set, page, page_size, desc
|
||||
"Fetching mailbox '{}' messages: sequence {} (page {}, page_size {})",
|
||||
encoded_mailbox_name, sequence_set, page, page_size
|
||||
);
|
||||
|
||||
let mut stream = session
|
||||
@@ -255,6 +333,9 @@ impl ImapExecutor {
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
if let Some(uid) = fetch.uid {
|
||||
*max_uid = Some((*max_uid).unwrap_or(0).max(uid));
|
||||
}
|
||||
extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?;
|
||||
count += 1;
|
||||
}
|
||||
@@ -289,9 +370,178 @@ impl ImapExecutor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetches the raw RFC822 body of a single message by UID.
|
||||
///
|
||||
/// Selects (read-only) the given mailbox and issues `UID FETCH <uid> (BODY.PEEK[])`.
|
||||
/// Used for on-demand self-healing when an indexed message's content blob is missing.
|
||||
/// Returns the raw bytes, or an error if the message cannot be retrieved.
|
||||
pub async fn fetch_single_message_body(
|
||||
session: &mut Session<Box<dyn SessionStream>>,
|
||||
encoded_mailbox_name: &str,
|
||||
uid: u32,
|
||||
) -> BichonResult<Vec<u8>> {
|
||||
session
|
||||
.examine(encoded_mailbox_name)
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?;
|
||||
|
||||
let mut stream = session
|
||||
.uid_fetch(uid.to_string(), BODY_FETCH_COMMAND)
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?;
|
||||
|
||||
let fetch = stream
|
||||
.try_next()
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("UID {uid} not found on IMAP server"),
|
||||
ErrorCode::ResourceNotFound
|
||||
)
|
||||
})?;
|
||||
|
||||
let body = fetch
|
||||
.body()
|
||||
.ok_or_else(|| {
|
||||
raise_error!(
|
||||
format!("No body returned for UID {uid}"),
|
||||
ErrorCode::ImapUnexpectedResult
|
||||
)
|
||||
})?
|
||||
.to_vec();
|
||||
|
||||
// // Drain any remaining items so the stream is fully consumed before reuse.
|
||||
// while stream
|
||||
// .try_next()
|
||||
// .await
|
||||
// .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?
|
||||
// .is_some()
|
||||
// {}
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub async fn create_connection(
|
||||
account_id: u64,
|
||||
) -> BichonResult<Session<Box<dyn SessionStream>>> {
|
||||
ImapConnectionManager::build(account_id).await
|
||||
}
|
||||
}
|
||||
|
||||
pub const DEFAULT_BATCH_SIZE: u32 = 30;
|
||||
|
||||
/// Compresses a sorted list of UIDs into an IMAP sequence-set string.
|
||||
/// Consecutive UIDs become ranges (e.g. `1:5`), non-consecutive are
|
||||
/// comma-separated (e.g. `1:5,10,12:15`).
|
||||
pub fn compress_uid_list(nums: Vec<u32>) -> String {
|
||||
if nums.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let mut sorted_nums = nums;
|
||||
sorted_nums.sort();
|
||||
|
||||
let mut result = Vec::new();
|
||||
let mut current_range_start = sorted_nums[0];
|
||||
let mut current_range_end = sorted_nums[0];
|
||||
|
||||
for &n in sorted_nums.iter().skip(1) {
|
||||
if n == current_range_end + 1 {
|
||||
current_range_end = n;
|
||||
} else {
|
||||
if current_range_start == current_range_end {
|
||||
result.push(current_range_start.to_string());
|
||||
} else {
|
||||
result.push(format!("{}:{}", current_range_start, current_range_end));
|
||||
}
|
||||
current_range_start = n;
|
||||
current_range_end = n;
|
||||
}
|
||||
}
|
||||
|
||||
if current_range_start == current_range_end {
|
||||
result.push(current_range_start.to_string());
|
||||
} else {
|
||||
result.push(format!("{}:{}", current_range_start, current_range_end));
|
||||
}
|
||||
|
||||
result.join(",")
|
||||
}
|
||||
|
||||
/// Splits a sorted list of unique UIDs into compressed sequence-set batches.
|
||||
/// Returns `Vec<(sequence_set_string, batch_count)>`.
|
||||
pub fn generate_uid_sequence_hashset(
|
||||
unique_nums: Vec<u32>,
|
||||
chunk_size: usize,
|
||||
) -> Vec<(String, u64)> {
|
||||
assert!(!unique_nums.is_empty());
|
||||
|
||||
let mut result = Vec::new();
|
||||
let nums = unique_nums;
|
||||
|
||||
for chunk in nums.chunks(chunk_size) {
|
||||
let size = chunk.len() as u64;
|
||||
let compressed = compress_uid_list(chunk.to_vec());
|
||||
result.push((compressed, size));
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
// ── compress_uid_list ──────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn compress_empty() {
|
||||
assert_eq!(compress_uid_list(vec![]), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_single_uid() {
|
||||
assert_eq!(compress_uid_list(vec![42]), "42");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_consecutive_range() {
|
||||
assert_eq!(compress_uid_list(vec![1, 2, 3, 4, 5]), "1:5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_mixed_ranges() {
|
||||
assert_eq!(
|
||||
compress_uid_list(vec![1, 2, 3, 5, 7, 8, 9, 10]),
|
||||
"1:3,5,7:10"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_gap_at_boundary() {
|
||||
assert_eq!(compress_uid_list(vec![1, 2, 4, 5]), "1:2,4:5");
|
||||
}
|
||||
|
||||
// ── generate_uid_sequence_hashset ──────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn batch_single_chunk() {
|
||||
let batches = generate_uid_sequence_hashset(vec![1, 2, 3], 10);
|
||||
assert_eq!(batches.len(), 1);
|
||||
assert_eq!(batches[0].0, "1:3");
|
||||
assert_eq!(batches[0].1, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_multiple_chunks() {
|
||||
let batches = generate_uid_sequence_hashset(vec![1, 2, 3, 4, 5], 2);
|
||||
assert_eq!(batches.len(), 3);
|
||||
assert_eq!(batches[0].0, "1:2");
|
||||
assert_eq!(batches[0].1, 2);
|
||||
assert_eq!(batches[1].0, "3:4");
|
||||
assert_eq!(batches[1].1, 2);
|
||||
assert_eq!(batches[2].0, "5");
|
||||
assert_eq!(batches[2].1, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,9 @@ use crate::{
|
||||
raise_error,
|
||||
};
|
||||
|
||||
/// Skip individual emails larger than this after decoding (100 MB).
|
||||
const MAX_SINGLE_EML_BYTES: usize = 100 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
|
||||
pub struct BatchEmlRequest {
|
||||
@@ -66,7 +69,7 @@ pub struct BatchEmlResult {
|
||||
pub struct ImportEmls;
|
||||
|
||||
impl ImportEmls {
|
||||
pub async fn do_import(request: BatchEmlRequest) -> BichonResult<BatchEmlResult> {
|
||||
pub async fn do_import(mut request: BatchEmlRequest) -> BichonResult<BatchEmlResult> {
|
||||
let account = AccountModel::check_account_exists(request.account_id)?;
|
||||
|
||||
if !account.enabled {
|
||||
@@ -102,6 +105,7 @@ impl ImportEmls {
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
let mailbox_id = mailbox.id;
|
||||
// Upsert the mailbox, creating it if it doesn't exist
|
||||
@@ -115,7 +119,8 @@ impl ImportEmls {
|
||||
let mut failed_details: Vec<FailedEmlDetail> = Vec::new(); // Store failure details
|
||||
|
||||
let total = request.emls.len();
|
||||
for (index, eml_base64) in request.emls.into_iter().enumerate() {
|
||||
let mut index: usize = 0;
|
||||
while let Some(eml_base64) = request.emls.pop() {
|
||||
let decoded = match base64_decode_url_safe!(eml_base64.as_bytes()) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
@@ -126,9 +131,26 @@ impl ImportEmls {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
// eml_base64 string dropped here — frees base64 memory before parsing
|
||||
|
||||
if decoded.len() > MAX_SINGLE_EML_BYTES {
|
||||
let size_mb = decoded.len() as f64 / 1024.0 / 1024.0;
|
||||
let error_msg = format!(
|
||||
"Email at index {} is {:.1} MB (limit 50 MB). Skipping.",
|
||||
index, size_mb,
|
||||
);
|
||||
tracing::warn!("{}", error_msg);
|
||||
failed_details.push(FailedEmlDetail {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
match extract_envelope_from_eml(&decoded, account_id, mailbox_id).await {
|
||||
Ok(_) => {
|
||||
@@ -144,9 +166,11 @@ impl ImportEmls {
|
||||
index,
|
||||
error_message: error_msg,
|
||||
});
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
index += 1;
|
||||
}
|
||||
|
||||
let failed_count = failed_details.len();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod account;
|
||||
pub mod ext;
|
||||
pub mod admin;
|
||||
pub mod autoconfig;
|
||||
pub mod cache;
|
||||
|
||||
@@ -157,8 +157,13 @@ async fn fetch_remote_with_progress(account_id: u64) -> BichonResult<Vec<MailBox
|
||||
|
||||
mailbox.account_id = account_id;
|
||||
mailbox.id = create_hash(account_id, &mailbox.name);
|
||||
// Use STATUS instead of EXAMINE: gets MESSAGES/UNSEEN/UIDNEXT/UIDVALIDITY
|
||||
// without selecting the mailbox, avoiding context switches.
|
||||
let mx = session
|
||||
.examine(mailbox_name.as_str())
|
||||
.status(
|
||||
mailbox_name.as_str(),
|
||||
"(MESSAGES UNSEEN UIDNEXT UIDVALIDITY)",
|
||||
)
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?;
|
||||
mailbox.exists = mx.exists;
|
||||
@@ -205,8 +210,13 @@ pub async fn convert_names_to_mailboxes(
|
||||
|
||||
mailbox.account_id = account_id;
|
||||
mailbox.id = create_hash(account_id, &mailbox.name);
|
||||
// Use STATUS instead of EXAMINE: gets MESSAGES/UNSEEN/UIDNEXT/UIDVALIDITY
|
||||
// without selecting the mailbox, avoiding context switches.
|
||||
let mx = session
|
||||
.examine(mailbox_name.as_str())
|
||||
.status(
|
||||
mailbox_name.as_str(),
|
||||
"(MESSAGES UNSEEN UIDNEXT UIDVALIDITY)",
|
||||
)
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?;
|
||||
mailbox.exists = mx.exists;
|
||||
|
||||
@@ -22,6 +22,7 @@ use crate::envelope::extractor::{extract_envelope_from_nested_message, reattach_
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::store::envelope::Envelope;
|
||||
use crate::utils::compute_content_hash;
|
||||
use crate::utils::html::block_remote_content;
|
||||
use crate::{error::BichonResult, raise_error};
|
||||
use mail_parser::{MessageParser, MimeHeaders};
|
||||
//use poem_openapi::Object;
|
||||
@@ -46,6 +47,13 @@ pub struct AttachmentInfo {
|
||||
/// Hash of the content.
|
||||
pub content_hash: String,
|
||||
pub is_message: bool,
|
||||
/// Text extracted from the attachment body (Pro/Enterprise feature).
|
||||
/// Populated during IMAP sync; None for inline attachments and unsupported file types.
|
||||
pub extracted_text: Option<String>,
|
||||
/// Page count reported by the extractor, if any.
|
||||
pub extracted_page_count: Option<u32>,
|
||||
/// Whether the extracted text came from OCR.
|
||||
pub extracted_is_ocr: bool,
|
||||
}
|
||||
|
||||
impl AttachmentInfo {
|
||||
@@ -142,6 +150,9 @@ pub struct FullMessageContent {
|
||||
pub html: Option<String>,
|
||||
// all Attachments include inline attachments
|
||||
pub attachments: Option<Vec<AttachmentInfo>>,
|
||||
/// True when remote content (http/https URLs) was detected and stripped from html.
|
||||
#[serde(default)]
|
||||
pub has_remote_content: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
@@ -155,11 +166,15 @@ pub struct FullNestedMessageContent {
|
||||
pub attachments: Option<Vec<AttachmentInfo>>,
|
||||
/// Metadata for the email envelope.
|
||||
pub envelope: Envelope,
|
||||
/// True when remote content (http/https URLs) was detected and stripped from html.
|
||||
#[serde(default)]
|
||||
pub has_remote_content: bool,
|
||||
}
|
||||
|
||||
pub fn retrieve_email_content(
|
||||
account_id: u64,
|
||||
envelope_id: String,
|
||||
block_remote: bool,
|
||||
) -> BichonResult<FullMessageContent> {
|
||||
AccountModel::check_account_exists(account_id)?;
|
||||
let (envelope, eml) = reattach_eml_content(account_id, envelope_id)?;
|
||||
@@ -214,19 +229,31 @@ pub fn retrieve_email_content(
|
||||
let is_message = attachment.is_message();
|
||||
let content_hash = compute_content_hash(attachment.contents());
|
||||
attachments.push(AttachmentInfo {
|
||||
filename: filename.or(Some(content_hash.clone())), // Fallback to content_hash as the default filename if it is not provided.
|
||||
filename: filename.or(Some(content_hash.clone())),
|
||||
size: attachment.contents().len(),
|
||||
inline,
|
||||
file_type,
|
||||
is_message,
|
||||
content_hash,
|
||||
content_id: attachment.content_id().map(Into::into),
|
||||
extracted_text: None,
|
||||
extracted_page_count: None,
|
||||
extracted_is_ocr: false,
|
||||
});
|
||||
}
|
||||
let mut has_remote_content = false;
|
||||
if let Some(ref html_body) = html {
|
||||
let filtered = block_remote_content(html_body);
|
||||
has_remote_content = *html_body != filtered;
|
||||
if block_remote {
|
||||
html = Some(filtered);
|
||||
}
|
||||
}
|
||||
Ok(FullMessageContent {
|
||||
text,
|
||||
html,
|
||||
attachments: Some(attachments),
|
||||
has_remote_content,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -234,6 +261,7 @@ pub fn retrieve_nested_eml_content(
|
||||
account_id: u64,
|
||||
envelope_id: String,
|
||||
content_hash: &str,
|
||||
block_remote: bool,
|
||||
) -> BichonResult<FullNestedMessageContent> {
|
||||
let (_, eml) = reattach_eml_content(account_id, envelope_id)?;
|
||||
let parent_message = MessageParser::default().parse(&eml).ok_or_else(|| {
|
||||
@@ -302,22 +330,34 @@ pub fn retrieve_nested_eml_content(
|
||||
filename: attachment
|
||||
.attachment_name()
|
||||
.map(|n| n.to_string())
|
||||
.or(Some(content_hash.clone())), // Fallback to content_hash as the default filename if it is not provided.
|
||||
.or(Some(content_hash.clone())),
|
||||
size: attachment.contents().len(),
|
||||
inline: is_inline,
|
||||
file_type,
|
||||
content_hash,
|
||||
is_message: attachment.is_message(),
|
||||
content_id: cid.map(Into::into),
|
||||
extracted_text: None,
|
||||
extracted_page_count: None,
|
||||
extracted_is_ocr: false,
|
||||
});
|
||||
}
|
||||
|
||||
let envelope = extract_envelope_from_nested_message(nested_message, account_id)?;
|
||||
|
||||
let mut has_remote_content = false;
|
||||
if let Some(ref html_body) = html {
|
||||
let filtered = block_remote_content(html_body);
|
||||
has_remote_content = *html_body != filtered;
|
||||
if block_remote {
|
||||
html = Some(filtered);
|
||||
}
|
||||
}
|
||||
Ok(FullNestedMessageContent {
|
||||
text,
|
||||
html,
|
||||
attachments: Some(attachments),
|
||||
envelope,
|
||||
has_remote_content,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -26,6 +26,6 @@ pub async fn delete_messages_impl(request: HashMap<u64, Vec<String>>) -> BichonR
|
||||
.delete_envelopes_multi_account(request.clone())
|
||||
.await?;
|
||||
ATTACHMENT_MANAGER
|
||||
.delete_envelopes_multi_account(request)
|
||||
.delete_attachments_multi_account(request)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -45,6 +45,14 @@ pub struct EmailSearchFilter {
|
||||
pub bcc: Option<String>,
|
||||
pub since: Option<i64>,
|
||||
pub before: Option<i64>,
|
||||
/// Lower bound (inclusive) on the IMAP server INTERNALDATE timestamp.
|
||||
pub internal_date_since: Option<i64>,
|
||||
/// Upper bound (inclusive) on the IMAP server INTERNALDATE timestamp.
|
||||
pub internal_date_before: Option<i64>,
|
||||
/// Lower bound (inclusive) on Bichon's archival (ingest) timestamp.
|
||||
pub ingest_since: Option<i64>,
|
||||
/// Upper bound (inclusive) on Bichon's archival (ingest) timestamp.
|
||||
pub ingest_before: Option<i64>,
|
||||
pub account_ids: Option<HashSet<u64>>,
|
||||
pub mailbox_ids: Option<HashSet<u64>>,
|
||||
pub min_size: Option<u64>,
|
||||
@@ -64,6 +72,14 @@ pub enum SortBy {
|
||||
#[default]
|
||||
DATE,
|
||||
SIZE,
|
||||
/// Sort by the IMAP server INTERNALDATE timestamp.
|
||||
#[serde(rename = "INTERNAL_DATE")]
|
||||
#[cfg_attr(feature = "web-api", oai(rename = "INTERNAL_DATE"))]
|
||||
InternalDate,
|
||||
/// Sort by Bichon's archival (ingest) timestamp.
|
||||
#[serde(rename = "INGEST_AT")]
|
||||
#[cfg_attr(feature = "web-api", oai(rename = "INGEST_AT"))]
|
||||
IngestAt,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
|
||||
|
||||
@@ -13,10 +13,7 @@ use fjall::{
|
||||
CompressionType, Database, Keyspace, KeyspaceCreateOptions, KvSeparationOptions,
|
||||
};
|
||||
use mail_parser::MessageParser;
|
||||
use tantivy::{
|
||||
indexer::{LogMergePolicy, NoMergePolicy},
|
||||
Index, IndexWriter, TantivyDocument,
|
||||
};
|
||||
use tantivy::{indexer::NoMergePolicy, Index, IndexWriter, TantivyDocument};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
@@ -117,6 +114,9 @@ pub fn detach_attachments_standalone(
|
||||
content_id: att.content_id().map(|id| id.to_string()),
|
||||
content_hash,
|
||||
is_message: att.is_message(),
|
||||
extracted_text: None,
|
||||
extracted_page_count: None,
|
||||
extracted_is_ocr: false,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -262,6 +262,12 @@ impl NewIndexWriter {
|
||||
.parse(eml_bytes)
|
||||
.ok_or_else(|| raise_error!("failed to parse eml".into(), ErrorCode::InternalError))?;
|
||||
|
||||
if message.parts.is_empty() {
|
||||
return Err(raise_error!(
|
||||
"Malformed or completely empty EML (no parts found)".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
// ── text / preview ────────────────────────────────────────────────
|
||||
let text = message
|
||||
.body_text(0)
|
||||
@@ -439,7 +445,7 @@ impl NewIndexWriter {
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{e:#?}"), ErrorCode::InternalError))?;
|
||||
}
|
||||
println!("tantivy commit elasped: {:#?}", start.elapsed());
|
||||
println!("tantivy commit elapsed: {:#?}", start.elapsed());
|
||||
tracing::info!(count = self.pending, "committed tantivy batch");
|
||||
self.pending = 0;
|
||||
Ok(())
|
||||
@@ -454,16 +460,10 @@ impl NewIndexWriter {
|
||||
("attachment", &mut self.attachment_writer),
|
||||
] {
|
||||
if let Some(writer) = writer_opt.as_mut() {
|
||||
let reader = writer
|
||||
let seg_ids = writer
|
||||
.index()
|
||||
.reader()
|
||||
.searchable_segment_ids()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let seg_ids: Vec<_> = reader
|
||||
.searcher()
|
||||
.segment_readers()
|
||||
.iter()
|
||||
.map(|r| r.segment_id())
|
||||
.collect();
|
||||
println!("merging {} {} segments...", seg_ids.len(), name);
|
||||
if seg_ids.len() > 1 {
|
||||
let _ = writer.merge(&seg_ids);
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use crate::{
|
||||
common::signal::SIGNAL_MANAGER,
|
||||
envelope::extractor::reattach_eml_content,
|
||||
envelope::extractor::reattach_eml_content_self_healing,
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
settings::dir::DATA_DIR_MANAGER,
|
||||
};
|
||||
@@ -68,7 +68,9 @@ impl BlobManager {
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Fjall email_ks error: {:?}", e),
|
||||
_ => {}
|
||||
Ok(true) => {
|
||||
tracing::debug!("Email blob already exists (dedup): {}", &email_hash);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(attachments) = eml.attachments {
|
||||
@@ -80,7 +82,9 @@ impl BlobManager {
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Fjall attach_ks error: {:?}", e),
|
||||
_ => {}
|
||||
Ok(true) => {
|
||||
tracing::debug!("Attachment blob already exists (dedup): {}", &a_hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -145,9 +149,18 @@ impl BlobManager {
|
||||
res = receiver.recv() => {
|
||||
match res {
|
||||
Some(eml) => {
|
||||
Self::process_detached_email(eml, &email_ks, &attach_ks);
|
||||
let mut batch = vec![eml];
|
||||
while let Ok(next_eml) = receiver.try_recv() {
|
||||
Self::process_detached_email(next_eml, &email_ks, &attach_ks);
|
||||
batch.push(next_eml);
|
||||
}
|
||||
let email_ks = email_ks.clone();
|
||||
let attach_ks = attach_ks.clone();
|
||||
if let Err(e) = tokio::task::spawn_blocking(move || {
|
||||
for eml in batch {
|
||||
Self::process_detached_email(eml, &email_ks, &attach_ks);
|
||||
}
|
||||
}).await {
|
||||
tracing::error!("BlobManager: spawn_blocking join error: {:#?}", e);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
@@ -158,16 +171,25 @@ impl BlobManager {
|
||||
}
|
||||
_ = shutdown.recv() => {
|
||||
receiver.close();
|
||||
let remaining = receiver.len();
|
||||
let mut remaining = Vec::new();
|
||||
while let Some(eml) = receiver.recv().await {
|
||||
remaining.push(eml);
|
||||
}
|
||||
tracing::info!(
|
||||
"BlobManager: Shutdown signal received. Processing {} remaining tasks...",
|
||||
remaining
|
||||
remaining.len()
|
||||
);
|
||||
|
||||
while let Some(eml) = receiver.recv().await {
|
||||
Self::process_detached_email(eml, &email_ks, &attach_ks);
|
||||
if !remaining.is_empty() {
|
||||
let email_ks = email_ks.clone();
|
||||
let attach_ks = attach_ks.clone();
|
||||
if let Err(e) = tokio::task::spawn_blocking(move || {
|
||||
for eml in remaining {
|
||||
Self::process_detached_email(eml, &email_ks, &attach_ks);
|
||||
}
|
||||
}).await {
|
||||
tracing::error!("BlobManager: shutdown spawn_blocking join error: {:#?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("BlobManager: All remaining tasks processed. Closing Fjall.");
|
||||
break;
|
||||
}
|
||||
@@ -185,7 +207,9 @@ impl BlobManager {
|
||||
}
|
||||
|
||||
pub async fn queue(&self, email: DetachedEmail) {
|
||||
let _ = self.sender.send(email).await;
|
||||
if let Err(e) = self.sender.send(email).await {
|
||||
tracing::error!("BlobManager channel closed, email lost: {:#?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_email(&self, content_hash: &str) -> BichonResult<Option<Bytes>> {
|
||||
@@ -226,7 +250,13 @@ impl BlobManager {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_reader(account_id: u64, eid: String) -> BichonResult<Cursor<Bytes>> {
|
||||
let (_, data) = reattach_eml_content(account_id, eid)?;
|
||||
/// Returns a reader over the raw EML for an indexed message.
|
||||
///
|
||||
/// If the message's content blob is missing from the blob store, it is fetched
|
||||
/// on demand from the IMAP server, persisted, and returned (self-healing). The
|
||||
/// underlying "content not found" error is only surfaced if that on-demand
|
||||
/// fetch itself fails.
|
||||
pub async fn get_reader(account_id: u64, eid: String) -> BichonResult<Cursor<Bytes>> {
|
||||
let (_, data) = reattach_eml_content_self_healing(account_id, eid).await?;
|
||||
Ok(Cursor::new(data))
|
||||
}
|
||||
|
||||
@@ -38,8 +38,8 @@ use crate::{
|
||||
store::tantivy::{
|
||||
fatal_commit,
|
||||
fields::{
|
||||
F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE, F_SIZE,
|
||||
F_TAGS,
|
||||
F_ATTACHMENT_CATEGORY, F_ATTACHMENT_CONTENT_TYPE, F_ATTACHMENT_EXT, F_DATE,
|
||||
F_INGEST_AT, F_SIZE, F_TAGS,
|
||||
},
|
||||
model::{extract_senders, AttachmentModel},
|
||||
schema::SchemaTools,
|
||||
@@ -78,6 +78,10 @@ pub struct IndexManager {
|
||||
}
|
||||
|
||||
impl IndexManager {
|
||||
pub(crate) fn index_writer(&self) -> &Arc<Mutex<IndexWriter>> {
|
||||
&self.index_writer
|
||||
}
|
||||
|
||||
pub async fn shutdown(&self) {
|
||||
let mut guard = self.handle.lock().await;
|
||||
if let Some(handle) = guard.take() {
|
||||
@@ -150,7 +154,11 @@ impl IndexManager {
|
||||
"Tantivy: Reached threshold ({} docs), committing...",
|
||||
pending_count
|
||||
);
|
||||
fatal_commit(&mut writer);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
tracing::debug!(
|
||||
"Tantivy attach: committed {} docs, pending reset to 0",
|
||||
pending_count
|
||||
);
|
||||
pending_count = 0;
|
||||
commit_interval.reset();
|
||||
}
|
||||
@@ -159,7 +167,7 @@ impl IndexManager {
|
||||
tracing::info!("Tantivy: Receiver closed. Finalizing...");
|
||||
if pending_count > 0 {
|
||||
let mut writer = writer.lock().await;
|
||||
fatal_commit(&mut writer);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
}
|
||||
break;
|
||||
},
|
||||
@@ -168,16 +176,19 @@ impl IndexManager {
|
||||
_ = commit_interval.tick() => {
|
||||
if pending_count > 0 {
|
||||
let mut writer = writer.lock().await;
|
||||
fatal_commit(&mut writer);
|
||||
tracing::debug!(
|
||||
"Tantivy attach: periodic commit ({} docs pending)",
|
||||
pending_count
|
||||
);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
pending_count = 0;
|
||||
tracing::debug!("Tantivy: Periodic commit finished.");
|
||||
}
|
||||
}
|
||||
_ = shutdown.recv() => {
|
||||
tracing::info!("Tantivy: Shutdown signal received. Performing final commit...");
|
||||
if pending_count > 0 {
|
||||
let mut writer = writer.lock().await;
|
||||
fatal_commit(&mut writer);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
}
|
||||
tracing::info!("Tantivy: Shutdown cleanup complete.");
|
||||
break;
|
||||
@@ -256,7 +267,7 @@ impl IndexManager {
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let envelope_id_query = TermQuery::new(
|
||||
Term::from_field_text(SchemaTools::attachment_fields().f_id, aid),
|
||||
Term::from_field_text(SchemaTools::attachment_fields().f_envelope_id, aid),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let boolean_query = BooleanQuery::new(vec![
|
||||
@@ -629,7 +640,7 @@ impl IndexManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_envelopes_multi_account(
|
||||
pub async fn delete_attachments_multi_account(
|
||||
&self,
|
||||
deletes: HashMap<u64, Vec<String>>,
|
||||
) -> BichonResult<()> {
|
||||
@@ -860,6 +871,30 @@ impl IndexManager {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
attachment_docs = size_docs.into_iter().map(|(_, addr)| addr).collect();
|
||||
}
|
||||
// Attachments carry no IMAP INTERNALDATE; fall back to the
|
||||
// attachment's own date field so the sort remains well defined.
|
||||
SortBy::InternalDate => {
|
||||
let date_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
&TopDocs::with_limit(page_size as usize)
|
||||
.and_offset(offset as usize)
|
||||
.order_by_fast_field(F_DATE, order),
|
||||
)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
attachment_docs = date_docs.into_iter().map(|(_, addr)| addr).collect();
|
||||
}
|
||||
SortBy::IngestAt => {
|
||||
let ingest_at_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
&TopDocs::with_limit(page_size as usize)
|
||||
.and_offset(offset as usize)
|
||||
.order_by_fast_field(F_INGEST_AT, order),
|
||||
)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
attachment_docs = ingest_at_docs.into_iter().map(|(_, addr)| addr).collect();
|
||||
}
|
||||
}
|
||||
|
||||
let mut result = Vec::new();
|
||||
|
||||
@@ -0,0 +1,744 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use tantivy::schema::Term;
|
||||
use tantivy::{IndexReader, IndexWriter};
|
||||
|
||||
use crate::common::periodic::{PeriodicTask, TaskHandle};
|
||||
use crate::context::BichonTask;
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::raise_error;
|
||||
use crate::store::tantivy::attachment::ATTACHMENT_MANAGER;
|
||||
use crate::store::tantivy::envelope::ENVELOPE_MANAGER;
|
||||
use crate::store::tantivy::fields::{
|
||||
F_ACCOUNT_ID, F_CONTENT_HASH, F_ID, F_INGEST_AT, F_MAILBOX_ID,
|
||||
};
|
||||
use crate::store::tantivy::schema::SchemaTools;
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// A single document candidate for deduplication.
|
||||
/// Holds just enough information to compare and delete duplicates.
|
||||
struct DedupEntry {
|
||||
/// Unix timestamp (seconds) when this document was ingested.
|
||||
/// Used to determine which copy to keep: we always keep the latest,
|
||||
/// so that a post-uidvalidity-reset uid is preferred over a stale one.
|
||||
ingest_at: i64,
|
||||
/// The email's f_id value, used to delete the duplicate email (via term
|
||||
/// query on f_id) and to cascade-delete attachments whose f_envelope_id
|
||||
/// matches this id.
|
||||
email_id: String,
|
||||
}
|
||||
|
||||
/// Dedup map for one account.
|
||||
/// Key = (mailbox_id, content_hash) — stable identity across uidvalidity resets
|
||||
/// Value = all documents sharing that key, to be reduced to exactly one.
|
||||
type DedupMap = HashMap<(u64, String), Vec<DedupEntry>>;
|
||||
|
||||
// ─── Public entry point ───────────────────────────────────────────────────────
|
||||
|
||||
/// Background deduplication task.
|
||||
///
|
||||
/// Iterates over every account found in the index and removes duplicate emails
|
||||
/// within each (mailbox_id, content_hash) group, keeping the most recently
|
||||
/// ingested copy.
|
||||
///
|
||||
/// For each duplicate email removed, all attachments in the attachment index
|
||||
/// whose f_envelope_id matches the removed email's f_id are also deleted.
|
||||
///
|
||||
/// Why keep the *latest* ingest_at?
|
||||
/// After a uidvalidity reset the server reassigns UIDs. If we kept an old
|
||||
/// copy (lower ingest_at) its uid would be stale, and uid-based incremental
|
||||
/// sync would re-download emails that are already present.
|
||||
///
|
||||
/// Processing is done account-by-account so that peak memory is bounded by
|
||||
/// the largest single account rather than the entire index.
|
||||
pub async fn dedup_task(
|
||||
email_reader: &IndexReader,
|
||||
email_writer: &mut IndexWriter,
|
||||
attachment_writer: &mut IndexWriter,
|
||||
) -> BichonResult<()> {
|
||||
let account_ids = collect_account_ids(email_reader)?;
|
||||
let mut total_deleted = 0u64;
|
||||
|
||||
for account_id in account_ids {
|
||||
total_deleted += dedup_account(email_reader, email_writer, attachment_writer, account_id)?;
|
||||
}
|
||||
|
||||
tracing::info!("dedup: finished, total removed={}", total_deleted);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ─── Periodic task ──────────────────────────────────────────────────────────
|
||||
|
||||
const DEDUP_INTERVAL: std::time::Duration = std::time::Duration::from_secs(12 * 60 * 60);
|
||||
|
||||
/// Periodically scans the email index for duplicate (mailbox_id, content_hash)
|
||||
/// entries and removes redundant copies, keeping the most recently ingested one.
|
||||
/// Attachments belonging to removed emails are cascade-deleted from the
|
||||
/// attachment index.
|
||||
pub struct DedupTask;
|
||||
|
||||
impl BichonTask for DedupTask {
|
||||
fn start() -> TaskHandle {
|
||||
let periodic_task = PeriodicTask::new("index-dedup");
|
||||
|
||||
let task = move |_: Option<u64>| {
|
||||
Box::pin(async move {
|
||||
// Acquire both writers before creating a reader. The fresh reader
|
||||
// sees the last committed state, while the writers ensure we have
|
||||
// exclusive access to perform deletions.
|
||||
let mut email_writer = ENVELOPE_MANAGER.index_writer().lock().await;
|
||||
let mut attach_writer = ATTACHMENT_MANAGER.index_writer().lock().await;
|
||||
let email_reader = ENVELOPE_MANAGER.create_reader()?;
|
||||
|
||||
dedup_task(&email_reader, &mut email_writer, &mut attach_writer).await?;
|
||||
|
||||
// Commit any remaining changes from the dedup pass.
|
||||
// dedup_account commits per-account, but we ensure a final commit
|
||||
// so the attachment index is in sync.
|
||||
crate::store::tantivy::fatal_commit(&mut attach_writer);
|
||||
|
||||
drop(attach_writer);
|
||||
drop(email_writer);
|
||||
Ok(())
|
||||
})
|
||||
};
|
||||
|
||||
periodic_task.start(task, None, DEDUP_INTERVAL, false, false)
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Internals ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Collect the distinct set of account_ids present in the index.
|
||||
///
|
||||
/// Scans only the account_id FAST column — no stored field reads, no I/O
|
||||
/// beyond the column file itself.
|
||||
fn collect_account_ids(reader: &IndexReader) -> BichonResult<Vec<u64>> {
|
||||
let searcher = reader.searcher();
|
||||
let mut ids = std::collections::HashSet::new();
|
||||
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader
|
||||
.fast_fields()
|
||||
.u64(F_ACCOUNT_ID)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let max_doc = segment_reader.max_doc();
|
||||
for doc_id in 0..max_doc {
|
||||
// Skip documents that have already been soft-deleted
|
||||
if segment_reader.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
ids.insert(account_col.values.get_val(doc_id));
|
||||
}
|
||||
}
|
||||
Ok(ids.into_iter().collect())
|
||||
}
|
||||
|
||||
/// Deduplicate all emails for a single account, and cascade-delete their attachments.
|
||||
///
|
||||
/// Strategy:
|
||||
/// 1. Scan FAST columns for (mailbox_id, content_hash, ingest_at, f_id) — no heap reads.
|
||||
/// 2. Group by (mailbox_id, content_hash).
|
||||
/// 3. Within each group, sort descending by ingest_at and soft-delete all
|
||||
/// but the first (most recent) entry.
|
||||
/// 4. For each removed email, delete all attachments in the attachment index
|
||||
/// whose f_envelope_id matches the removed email's f_id.
|
||||
/// 5. Commit both writers once per account so memory is released before the
|
||||
/// next account is processed.
|
||||
///
|
||||
/// Peak memory for this function ≈ account_email_count × ~160 bytes
|
||||
/// (the extra ~80 bytes over previous version comes from storing email_id strings).
|
||||
fn dedup_account(
|
||||
email_reader: &IndexReader,
|
||||
email_writer: &mut IndexWriter,
|
||||
attachment_writer: &mut IndexWriter,
|
||||
account_id: u64,
|
||||
) -> BichonResult<u64> {
|
||||
let searcher = email_reader.searcher();
|
||||
let fields = SchemaTools::email_fields();
|
||||
// eprintln!(
|
||||
// "DEBUG dedup_account: entry account={account_id} f_id_field={:?} f_content_hash_field={:?}",
|
||||
// fields.f_id, fields.f_content_hash
|
||||
// );
|
||||
let mut map: DedupMap = HashMap::new();
|
||||
|
||||
// ── Phase 1: build the dedup map via FAST column scans ──────────────────
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader
|
||||
.fast_fields()
|
||||
.u64(F_ACCOUNT_ID)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let mailbox_col = segment_reader
|
||||
.fast_fields()
|
||||
.u64(F_MAILBOX_ID)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let ingest_col = segment_reader
|
||||
.fast_fields()
|
||||
.i64(F_INGEST_AT)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
// content_hash and f_id are text fields with FAST; stored as dictionary-encoded strings
|
||||
let hash_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_CONTENT_HASH)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
|
||||
.ok_or_else(|| raise_error!(format!("FAST str column '{}' not found in segment; ensure the field is declared with FAST in the schema", F_CONTENT_HASH), ErrorCode::InternalError))?;
|
||||
let id_col = segment_reader
|
||||
.fast_fields()
|
||||
.str(F_ID)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
|
||||
.ok_or_else(|| raise_error!(format!("FAST str column '{}' not found in segment; ensure the field is declared with FAST in the schema", F_ID), ErrorCode::InternalError))?;
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
for doc_id in 0..max_doc {
|
||||
if segment_reader.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
// Filter to the current account without touching stored fields
|
||||
if account_col.values.get_val(doc_id) != account_id {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
let ingest_at = ingest_col.values.get_val(doc_id);
|
||||
|
||||
// Read content_hash from the dictionary-encoded string column
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col
|
||||
.ord_to_str(hash_ord, &mut hash_buf)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let content_hash = hash_buf;
|
||||
|
||||
// Read f_id from the dictionary-encoded string column
|
||||
let id_ord = id_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut id_buf = String::new();
|
||||
id_col
|
||||
.ord_to_str(id_ord, &mut id_buf)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
let email_id = id_buf;
|
||||
|
||||
// eprintln!(
|
||||
// "DEBUG dedup_account: account={account_id} doc_id={doc_id} mailbox={mailbox_id} hash={content_hash:?} id={email_id:?} ingest_at={ingest_at}"
|
||||
// );
|
||||
|
||||
map.entry((mailbox_id, content_hash))
|
||||
.or_default()
|
||||
.push(DedupEntry {
|
||||
ingest_at,
|
||||
email_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── Phase 2: delete duplicate emails and their attachments ───────────────
|
||||
let attachment_fields = SchemaTools::attachment_fields();
|
||||
let mut deleted = 0u64;
|
||||
|
||||
for (_key, mut entries) in map {
|
||||
if entries.len() <= 1 {
|
||||
// No duplicates in this group
|
||||
continue;
|
||||
}
|
||||
|
||||
// Sort descending: the most recently ingested document comes first.
|
||||
// This ensures we keep the copy whose uid reflects the current
|
||||
// uidvalidity, which is required for correct incremental sync.
|
||||
entries.sort_by_key(|e| std::cmp::Reverse(e.ingest_at));
|
||||
|
||||
tracing::debug!(
|
||||
"dedup: account={} mailbox={} hash={}: {} copies, keeping eid={} ingest_at={}, deleting {}",
|
||||
account_id,
|
||||
_key.0,
|
||||
&_key.1,
|
||||
entries.len(),
|
||||
&entries[0].email_id,
|
||||
entries[0].ingest_at,
|
||||
entries.len() - 1
|
||||
);
|
||||
|
||||
// eprintln!(
|
||||
// "DEBUG Phase2: key={_key:?} kept={} deleting={}",
|
||||
// entries[0].email_id,
|
||||
// entries.len() - 1
|
||||
// );
|
||||
// Keep entries[0], soft-delete everything else via term query on f_id
|
||||
for entry in &entries[1..] {
|
||||
eprintln!(
|
||||
"DEBUG Phase2: delete_term f_id={:?} text=\"{}\"",
|
||||
fields.f_id, &entry.email_id
|
||||
);
|
||||
// Remove the duplicate email from the email index
|
||||
let email_term = Term::from_field_text(fields.f_id, &entry.email_id);
|
||||
email_writer.delete_term(email_term);
|
||||
|
||||
// Cascade: remove all attachments belonging to this email.
|
||||
// f_envelope_id in the attachment index mirrors f_id in the email index.
|
||||
let envelope_term =
|
||||
Term::from_field_text(attachment_fields.f_envelope_id, &entry.email_id);
|
||||
attachment_writer.delete_term(envelope_term);
|
||||
|
||||
deleted += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Commit both indexes once per account so the DedupMap memory for this
|
||||
// account can be reclaimed before the next account is processed.
|
||||
if deleted > 0 {
|
||||
email_writer
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
attachment_writer
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
tracing::info!("dedup: account={} removed={}", account_id, deleted);
|
||||
}
|
||||
|
||||
Ok(deleted)
|
||||
}
|
||||
|
||||
// ─── Tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::store::tantivy::fields::AttachmentFields;
|
||||
use crate::store::tantivy::fields::EmailFields;
|
||||
use crate::store::tantivy::schema::SchemaTools;
|
||||
use crate::store::tantivy::tokenizers::EuroTokenizer;
|
||||
use std::collections::HashSet;
|
||||
use std::fmt::Write;
|
||||
use std::fs;
|
||||
use tantivy::Index;
|
||||
use tantivy::TantivyDocument;
|
||||
|
||||
fn temp_dir(prefix: &str) -> std::path::PathBuf {
|
||||
let dir = std::env::temp_dir()
|
||||
.join("bichon-dedup-test")
|
||||
.join(prefix)
|
||||
.join(uuid::Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
fn make_index(dir: &std::path::Path, schema: tantivy::schema::Schema) -> Index {
|
||||
let index = Index::create_in_dir(dir, schema).unwrap();
|
||||
index.tokenizers().register("euro", EuroTokenizer::new());
|
||||
index
|
||||
}
|
||||
|
||||
/// Collect non-deleted f_id values from the email index.
|
||||
fn surviving_email_ids(reader: &IndexReader) -> HashSet<String> {
|
||||
reader.reload().expect("reader reload failed");
|
||||
let searcher = reader.searcher();
|
||||
let mut ids = HashSet::new();
|
||||
let segments = searcher.segment_readers();
|
||||
eprintln!(
|
||||
"DEBUG surviving_email_ids: segment_count={}",
|
||||
segments.len()
|
||||
);
|
||||
for (seg_idx, seg) in segments.iter().enumerate() {
|
||||
let id_col = seg
|
||||
.fast_fields()
|
||||
.str(F_ID)
|
||||
.unwrap()
|
||||
.expect("FAST str column 'id' missing");
|
||||
let max_doc = seg.max_doc();
|
||||
eprintln!("DEBUG surviving_email_ids: seg={seg_idx} max_doc={max_doc}");
|
||||
for doc_id in 0..max_doc {
|
||||
let is_del = seg.is_deleted(doc_id);
|
||||
let ord = id_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut buf = String::new();
|
||||
id_col.ord_to_str(ord, &mut buf).unwrap();
|
||||
eprintln!("DEBUG surviving_email_ids: seg={seg_idx} doc_id={doc_id} is_deleted={is_del} ord={ord} buf={buf:?}");
|
||||
if !is_del {
|
||||
ids.insert(buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
/// Collect non-deleted f_id values from the attachment index.
|
||||
fn surviving_attachment_ids(reader: &IndexReader) -> HashSet<String> {
|
||||
let searcher = reader.searcher();
|
||||
let mut ids = HashSet::new();
|
||||
for seg in searcher.segment_readers() {
|
||||
let id_col = seg
|
||||
.fast_fields()
|
||||
.str(F_ID)
|
||||
.unwrap()
|
||||
.expect("FAST str column 'id' missing in attachment index");
|
||||
let max_doc = seg.max_doc();
|
||||
for doc_id in 0..max_doc {
|
||||
if seg.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
let ord = id_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut buf = String::new();
|
||||
id_col.ord_to_str(ord, &mut buf).unwrap();
|
||||
ids.insert(buf);
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
fn add_email(
|
||||
f: &EmailFields,
|
||||
w: &mut IndexWriter,
|
||||
id: &str,
|
||||
account: u64,
|
||||
mailbox: u64,
|
||||
hash: &str,
|
||||
ingest_at: i64,
|
||||
) {
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_text(f.f_id, id);
|
||||
doc.add_u64(f.f_account_id, account);
|
||||
doc.add_u64(f.f_mailbox_id, mailbox);
|
||||
doc.add_text(f.f_content_hash, hash);
|
||||
doc.add_i64(f.f_ingest_at, ingest_at);
|
||||
w.add_document(doc).unwrap();
|
||||
}
|
||||
|
||||
fn add_attachment(
|
||||
f: &AttachmentFields,
|
||||
w: &mut IndexWriter,
|
||||
id: &str,
|
||||
envelope_id: &str,
|
||||
account: u64,
|
||||
mailbox: u64,
|
||||
) {
|
||||
let mut doc = TantivyDocument::new();
|
||||
doc.add_text(f.f_id, id);
|
||||
doc.add_text(f.f_envelope_id, envelope_id);
|
||||
doc.add_u64(f.f_account_id, account);
|
||||
doc.add_u64(f.f_mailbox_id, mailbox);
|
||||
w.add_document(doc).unwrap();
|
||||
}
|
||||
|
||||
/// Prevent segment merges during dedup so delete operations are isolated
|
||||
/// and test assertions target the exact expected document set.
|
||||
fn apply_no_merge_policy(w: &mut IndexWriter) {
|
||||
let mut mp = tantivy::indexer::LogMergePolicy::default();
|
||||
mp.set_min_num_segments(500);
|
||||
mp.set_max_docs_before_merge(1_000_000);
|
||||
w.set_merge_policy(Box::new(mp));
|
||||
}
|
||||
|
||||
struct Harness;
|
||||
|
||||
impl Harness {
|
||||
async fn run<F>(
|
||||
case: &str,
|
||||
populate: F,
|
||||
expected_emails: &[&str],
|
||||
expected_attachments: &[&str],
|
||||
) where
|
||||
F: FnOnce(&EmailFields, &mut IndexWriter, &AttachmentFields, &mut IndexWriter),
|
||||
{
|
||||
let email_schema = SchemaTools::email_schema();
|
||||
let attach_schema = SchemaTools::attachment_schema();
|
||||
let email_f = SchemaTools::email_fields();
|
||||
let attach_f = SchemaTools::attachment_fields();
|
||||
|
||||
let email_idx = make_index(&temp_dir(case), email_schema);
|
||||
let attach_idx = make_index(&temp_dir(case), attach_schema);
|
||||
|
||||
let mut email_w = email_idx.writer_with_num_threads(1, 50_000_000).unwrap();
|
||||
let mut attach_w = attach_idx.writer_with_num_threads(1, 50_000_000).unwrap();
|
||||
apply_no_merge_policy(&mut email_w);
|
||||
apply_no_merge_policy(&mut attach_w);
|
||||
|
||||
populate(&email_f, &mut email_w, &attach_f, &mut attach_w);
|
||||
|
||||
email_w.commit().unwrap();
|
||||
attach_w.commit().unwrap();
|
||||
drop(email_w);
|
||||
drop(attach_w);
|
||||
|
||||
let mut email_w2 = email_idx.writer_with_num_threads(1, 50_000_000).unwrap();
|
||||
let mut attach_w2 = attach_idx.writer_with_num_threads(1, 50_000_000).unwrap();
|
||||
apply_no_merge_policy(&mut email_w2);
|
||||
apply_no_merge_policy(&mut attach_w2);
|
||||
|
||||
let email_r = email_idx.reader().unwrap();
|
||||
dedup_task(&email_r, &mut email_w2, &mut attach_w2)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let email_r = email_idx.reader().unwrap();
|
||||
let survivors = surviving_email_ids(&email_r);
|
||||
let expected: HashSet<String> = expected_emails.iter().map(|s| s.to_string()).collect();
|
||||
assert_eq!(survivors, expected, "[{case}] email survivors mismatch");
|
||||
|
||||
let attach_r = attach_idx.reader().unwrap();
|
||||
let att_survivors = surviving_attachment_ids(&attach_r);
|
||||
let att_expected: HashSet<String> =
|
||||
expected_attachments.iter().map(|s| s.to_string()).collect();
|
||||
assert_eq!(
|
||||
att_survivors, att_expected,
|
||||
"[{case}] attachment survivors mismatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedup_removes_duplicates_and_cascades_to_attachments() {
|
||||
Harness::run(
|
||||
"basic",
|
||||
|ef, ew, af, aw| {
|
||||
add_email(ef, ew, "dup-old", 1, 200, "hash-dup", 1000);
|
||||
add_email(ef, ew, "dup-new", 1, 200, "hash-dup", 3000);
|
||||
add_email(ef, ew, "unique", 1, 200, "hash-uniq", 1000);
|
||||
add_attachment(af, aw, "att-old", "dup-old", 1, 200);
|
||||
add_attachment(af, aw, "att-new", "dup-new", 1, 200);
|
||||
},
|
||||
&["dup-new", "unique"],
|
||||
&["att-new"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedup_keeps_latest_among_many_duplicates() {
|
||||
Harness::run(
|
||||
"many-dups",
|
||||
|ef, ew, af, aw| {
|
||||
for (i, ts) in [50, 100, 400, 200, 300].iter().enumerate() {
|
||||
let id = format!("dup-{i}");
|
||||
add_email(ef, ew, &id, 1, 1, "H", *ts);
|
||||
add_attachment(af, aw, &format!("att-{i}"), &id, 1, 1);
|
||||
}
|
||||
},
|
||||
&["dup-2"], // ingest_at=400, the latest
|
||||
&["att-2"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedup_no_duplicates_is_noop() {
|
||||
Harness::run(
|
||||
"no-dups",
|
||||
|ef, ew, af, aw| {
|
||||
add_email(ef, ew, "a", 1, 1, "hash-a", 100);
|
||||
add_email(ef, ew, "b", 1, 1, "hash-b", 200);
|
||||
add_email(ef, ew, "c", 1, 1, "hash-c", 300);
|
||||
add_attachment(af, aw, "att-a", "a", 1, 1);
|
||||
add_attachment(af, aw, "att-b", "b", 1, 1);
|
||||
add_attachment(af, aw, "att-c", "c", 1, 1);
|
||||
},
|
||||
&["a", "b", "c"],
|
||||
&["att-a", "att-b", "att-c"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedup_isolates_accounts() {
|
||||
// Same hash, same mailbox, DIFFERENT accounts → no dedup
|
||||
Harness::run(
|
||||
"cross-account",
|
||||
|ef, ew, af, aw| {
|
||||
add_email(ef, ew, "acc1-a", 1, 1, "hash-same", 100);
|
||||
add_email(ef, ew, "acc1-b", 1, 1, "hash-same", 200);
|
||||
add_email(ef, ew, "acc2-a", 2, 1, "hash-same", 100);
|
||||
add_email(ef, ew, "acc2-b", 2, 1, "hash-same", 200);
|
||||
add_attachment(af, aw, "att-1a", "acc1-a", 1, 1);
|
||||
add_attachment(af, aw, "att-1b", "acc1-b", 1, 1);
|
||||
add_attachment(af, aw, "att-2a", "acc2-a", 2, 1);
|
||||
add_attachment(af, aw, "att-2b", "acc2-b", 2, 1);
|
||||
},
|
||||
// Each account keeps its latest: acc1 keeps acc1-b (200>100), acc2 keeps acc2-b
|
||||
&["acc1-b", "acc2-b"],
|
||||
&["att-1b", "att-2b"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedup_isolates_mailboxes() {
|
||||
// Same hash, same account, DIFFERENT mailboxes → no dedup
|
||||
Harness::run(
|
||||
"cross-mailbox",
|
||||
|ef, ew, af, aw| {
|
||||
add_email(ef, ew, "mb1-a", 1, 1, "hash-same", 100);
|
||||
add_email(ef, ew, "mb2-a", 1, 2, "hash-same", 100);
|
||||
add_email(ef, ew, "mb1-b", 1, 1, "hash-same", 200);
|
||||
add_email(ef, ew, "mb2-b", 1, 2, "hash-same", 200);
|
||||
add_attachment(af, aw, "att-1a", "mb1-a", 1, 1);
|
||||
add_attachment(af, aw, "att-1b", "mb1-b", 1, 1);
|
||||
add_attachment(af, aw, "att-2a", "mb2-a", 1, 2);
|
||||
add_attachment(af, aw, "att-2b", "mb2-b", 1, 2);
|
||||
},
|
||||
&["mb1-b", "mb2-b"],
|
||||
&["att-1b", "att-2b"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dedup_multiple_attachments_per_email() {
|
||||
// Deleting an email cascades all its attachments, not just one
|
||||
Harness::run(
|
||||
"multi-att",
|
||||
|ef, ew, af, aw| {
|
||||
add_email(ef, ew, "old", 1, 1, "H", 100);
|
||||
add_email(ef, ew, "new", 1, 1, "H", 200);
|
||||
// The old email has 3 attachments — all should be removed
|
||||
add_attachment(af, aw, "att1", "old", 1, 1);
|
||||
add_attachment(af, aw, "att2", "old", 1, 1);
|
||||
add_attachment(af, aw, "att3", "old", 1, 1);
|
||||
// The kept email has 2 attachments — both should survive
|
||||
add_attachment(af, aw, "att4", "new", 1, 1);
|
||||
add_attachment(af, aw, "att5", "new", 1, 1);
|
||||
},
|
||||
&["new"],
|
||||
&["att4", "att5"],
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Inspects the production email index and reports duplicate counts.
|
||||
///
|
||||
/// A "duplicate" is defined as two or more emails sharing the same
|
||||
/// (account_id, mailbox_id, content_hash) tuple.
|
||||
///
|
||||
/// This test is read-only — it does not modify the index.
|
||||
#[test]
|
||||
fn inspect_production_duplicates() {
|
||||
let index_path = r"E:\bichon-data\bichon-indices\mail_metadata";
|
||||
let report_path = std::path::PathBuf::from(r"E:\bichon\dedup_report.txt");
|
||||
|
||||
let mut report = String::new();
|
||||
let _ = writeln!(report, "opening index at {index_path}...");
|
||||
|
||||
let index = match Index::open_in_dir(index_path) {
|
||||
Ok(idx) => {
|
||||
let _ = writeln!(report, "index opened successfully");
|
||||
idx
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = writeln!(report, "Failed to open index at {index_path}: {e}");
|
||||
let _ = std::fs::write(&report_path, &report);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let reader = match index.reader() {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
let _ = writeln!(report, "Failed to create reader: {e}");
|
||||
let _ = std::fs::write(&report_path, &report);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
reader.reload().expect("reader reload failed");
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let mut total_docs = 0u64;
|
||||
let mut groups: std::collections::HashMap<
|
||||
u64,
|
||||
std::collections::HashMap<(u64, String), u64>,
|
||||
> = std::collections::HashMap::new();
|
||||
|
||||
for segment_reader in searcher.segment_readers() {
|
||||
let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap();
|
||||
let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap();
|
||||
let hash_col = match segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap() {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
let _ = writeln!(
|
||||
report,
|
||||
"Segment has no FAST str column for content_hash, skipping"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let max_doc = segment_reader.max_doc();
|
||||
for doc_id in 0..max_doc {
|
||||
if segment_reader.is_deleted(doc_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let account_id = account_col.values.get_val(doc_id);
|
||||
let mailbox_id = mailbox_col.values.get_val(doc_id);
|
||||
|
||||
let hash_ord = hash_col
|
||||
.ords()
|
||||
.values_for_doc(doc_id as u32)
|
||||
.next()
|
||||
.unwrap_or(0);
|
||||
let mut hash_buf = String::new();
|
||||
hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap();
|
||||
let content_hash = hash_buf;
|
||||
|
||||
total_docs += 1;
|
||||
groups
|
||||
.entry(account_id)
|
||||
.or_default()
|
||||
.entry((mailbox_id, content_hash))
|
||||
.and_modify(|c| *c += 1)
|
||||
.or_insert(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Summarize ──────────────────────────────────────────────────────────
|
||||
let mut total_duplicate_groups = 0u64;
|
||||
let mut total_duplicate_emails = 0u64;
|
||||
|
||||
for (account_id, account_groups) in &groups {
|
||||
let mut account_dup_groups = 0u64;
|
||||
let mut account_dup_emails = 0u64;
|
||||
for ((_mailbox_id, _hash), count) in account_groups {
|
||||
if *count > 1 {
|
||||
account_dup_groups += 1;
|
||||
account_dup_emails += count - 1;
|
||||
}
|
||||
}
|
||||
if account_dup_groups > 0 {
|
||||
let _ = writeln!(
|
||||
report,
|
||||
"account={account_id}: {account_dup_groups} duplicate groups, {account_dup_emails} redundant emails"
|
||||
);
|
||||
}
|
||||
total_duplicate_groups += account_dup_groups;
|
||||
total_duplicate_emails += account_dup_emails;
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
report,
|
||||
"─── Summary ───\n\
|
||||
total_docs = {total_docs}\n\
|
||||
accounts = {}\n\
|
||||
duplicate_groups = {total_duplicate_groups}\n\
|
||||
redundant_emails = {total_duplicate_emails}\n\
|
||||
unique_after_dedup = {}",
|
||||
groups.len(),
|
||||
total_docs - total_duplicate_emails,
|
||||
);
|
||||
|
||||
std::fs::write(&report_path, &report).unwrap();
|
||||
println!("report written to {}", report_path.display());
|
||||
}
|
||||
}
|
||||
@@ -39,10 +39,11 @@ use crate::{
|
||||
blob::BLOB_MANAGER,
|
||||
envelope::Envelope,
|
||||
tantivy::{
|
||||
attachment::ATTACHMENT_MANAGER,
|
||||
fatal_commit,
|
||||
fields::{
|
||||
F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_REGULAR_ATTACHMENT_COUNT, F_SIZE, F_TAGS,
|
||||
F_THREAD_ID, F_UID,
|
||||
F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE,
|
||||
F_REGULAR_ATTACHMENT_COUNT, F_SIZE, F_TAGS, F_THREAD_ID, F_UID,
|
||||
},
|
||||
model::{extract_contacts, EnvelopeWithAttachments},
|
||||
schema::SchemaTools,
|
||||
@@ -86,6 +87,16 @@ pub struct IndexManager {
|
||||
}
|
||||
|
||||
impl IndexManager {
|
||||
pub(crate) fn index_writer(&self) -> &Arc<Mutex<IndexWriter>> {
|
||||
&self.index_writer
|
||||
}
|
||||
|
||||
pub(crate) fn create_reader(&self) -> BichonResult<IndexReader> {
|
||||
self.index
|
||||
.reader()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
|
||||
}
|
||||
|
||||
pub async fn shutdown(&self) {
|
||||
let mut guard = self.handle.lock().await;
|
||||
if let Some(handle) = guard.take() {
|
||||
@@ -158,7 +169,11 @@ impl IndexManager {
|
||||
"Tantivy: Reached threshold ({} docs), committing...",
|
||||
pending_count
|
||||
);
|
||||
fatal_commit(&mut writer);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
tracing::debug!(
|
||||
"Tantivy: committed {} docs, pending reset to 0",
|
||||
pending_count
|
||||
);
|
||||
pending_count = 0;
|
||||
commit_interval.reset();
|
||||
}
|
||||
@@ -167,7 +182,7 @@ impl IndexManager {
|
||||
tracing::info!("Tantivy: Receiver closed. Finalizing...");
|
||||
if pending_count > 0 {
|
||||
let mut writer = writer.lock().await;
|
||||
fatal_commit(&mut writer);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
}
|
||||
break;
|
||||
},
|
||||
@@ -176,16 +191,19 @@ impl IndexManager {
|
||||
_ = commit_interval.tick() => {
|
||||
if pending_count > 0 {
|
||||
let mut writer = writer.lock().await;
|
||||
fatal_commit(&mut writer);
|
||||
tracing::debug!(
|
||||
"Tantivy: periodic commit ({} docs pending)",
|
||||
pending_count
|
||||
);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
pending_count = 0;
|
||||
tracing::debug!("Tantivy: Periodic commit finished.");
|
||||
}
|
||||
}
|
||||
_ = shutdown.recv() => {
|
||||
tracing::info!("Tantivy: Shutdown signal received. Performing final commit...");
|
||||
if pending_count > 0 {
|
||||
let mut writer = writer.lock().await;
|
||||
fatal_commit(&mut writer);
|
||||
tokio::task::block_in_place(|| fatal_commit(&mut writer));
|
||||
}
|
||||
tracing::info!("Tantivy: Shutdown cleanup complete.");
|
||||
break;
|
||||
@@ -203,7 +221,9 @@ impl IndexManager {
|
||||
}
|
||||
|
||||
pub async fn queue(&self, doc: TantivyDocument) {
|
||||
let _ = self.sender.send(doc).await;
|
||||
if let Err(e) = self.sender.send(doc).await {
|
||||
tracing::warn!(error = %e, "Failed to queue document into Tantivy writer channel");
|
||||
}
|
||||
}
|
||||
|
||||
fn open_or_create_index(index_dir: &PathBuf) -> Index {
|
||||
@@ -446,6 +466,40 @@ impl IndexManager {
|
||||
subqueries.push((Occur::Must, Box::new(q)));
|
||||
}
|
||||
|
||||
let start_bound = if let Some(from) = filter.internal_date_since {
|
||||
Bound::Included(Term::from_field_i64(f.f_internal_date, from))
|
||||
} else {
|
||||
Bound::Unbounded
|
||||
};
|
||||
|
||||
let end_bound = if let Some(to) = filter.internal_date_before {
|
||||
Bound::Included(Term::from_field_i64(f.f_internal_date, to))
|
||||
} else {
|
||||
Bound::Unbounded
|
||||
};
|
||||
|
||||
if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded {
|
||||
let q = RangeQuery::new(start_bound, end_bound);
|
||||
subqueries.push((Occur::Must, Box::new(q)));
|
||||
}
|
||||
|
||||
let start_bound = if let Some(from) = filter.ingest_since {
|
||||
Bound::Included(Term::from_field_i64(f.f_ingest_at, from))
|
||||
} else {
|
||||
Bound::Unbounded
|
||||
};
|
||||
|
||||
let end_bound = if let Some(to) = filter.ingest_before {
|
||||
Bound::Included(Term::from_field_i64(f.f_ingest_at, to))
|
||||
} else {
|
||||
Bound::Unbounded
|
||||
};
|
||||
|
||||
if start_bound != Bound::Unbounded || end_bound != Bound::Unbounded {
|
||||
let q = RangeQuery::new(start_bound, end_bound);
|
||||
subqueries.push((Occur::Must, Box::new(q)));
|
||||
}
|
||||
|
||||
if let Some(account_ids) = filter.account_ids {
|
||||
let mut should_queries: Vec<(Occur, Box<dyn Query>)> = Vec::new();
|
||||
for id in account_ids {
|
||||
@@ -643,7 +697,15 @@ impl IndexManager {
|
||||
let agg_res = searcher
|
||||
.search(query.as_ref(), &collector)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
Ok(Self::extract_max_uid(&agg_res))
|
||||
let result = Self::extract_max_uid(&agg_res);
|
||||
tracing::debug!(
|
||||
"[account {}][mailbox {}] get_max_uid = {:?} (num_docs in searcher = {})",
|
||||
account_id,
|
||||
mailbox_id,
|
||||
result,
|
||||
searcher.num_docs()
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn get_account_stats(&self, account_id: u64) -> BichonResult<AccountStats> {
|
||||
@@ -717,8 +779,16 @@ impl IndexManager {
|
||||
.commit()
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
ATTACHMENT_MANAGER
|
||||
.delete_account_attachments(account_id)
|
||||
.await?;
|
||||
|
||||
if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)?;
|
||||
self.cleanup_unused_content(
|
||||
&mut writer,
|
||||
eml_content_hashes,
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -757,7 +827,11 @@ impl IndexManager {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)?;
|
||||
self.cleanup_unused_content(
|
||||
&mut writer,
|
||||
eml_content_hashes,
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -802,9 +876,18 @@ impl IndexManager {
|
||||
|
||||
fn cleanup_unused_content(
|
||||
&self,
|
||||
writer: &mut IndexWriter,
|
||||
eml_content_hashes: HashSet<String>,
|
||||
attachments_content_hashes: HashSet<String>,
|
||||
) -> BichonResult<()> {
|
||||
// Reference-count barrier: commit the writer and reload the reader so the
|
||||
// `Count` below is evaluated against a fully committed, freshly-reloaded
|
||||
// index state. Without this, an envelope that shares a content hash but
|
||||
// is still sitting uncommitted in the writer buffer (e.g. added by the
|
||||
// background ingest task before this delete acquired the writer lock)
|
||||
// would be invisible to the searcher, the count would read 0, and a
|
||||
// still-referenced blob would be deleted.
|
||||
fatal_commit(writer);
|
||||
let searcher = self.create_searcher()?;
|
||||
let fields = SchemaTools::email_fields();
|
||||
let mut eml: HashSet<String> = HashSet::new();
|
||||
@@ -821,11 +904,10 @@ impl IndexManager {
|
||||
eml.insert(content_hash);
|
||||
}
|
||||
}
|
||||
|
||||
let mut attachments: HashSet<String> = HashSet::new();
|
||||
for content_hash in attachments_content_hashes {
|
||||
// Check if any other emails still reference this content hash
|
||||
let hash_term = Term::from_field_text(fields.f_content_hash, &content_hash);
|
||||
let hash_term = Term::from_field_text(fields.f_attachment_content_hash, &content_hash);
|
||||
let hash_query = TermQuery::new(hash_term, IndexRecordOption::Basic);
|
||||
let count = searcher
|
||||
.search(&hash_query, &Count)
|
||||
@@ -885,7 +967,11 @@ impl IndexManager {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
|
||||
if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() {
|
||||
self.cleanup_unused_content(eml_content_hashes, attachments_content_hashes)?;
|
||||
self.cleanup_unused_content(
|
||||
&mut writer,
|
||||
eml_content_hashes,
|
||||
attachments_content_hashes,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1126,6 +1212,31 @@ impl IndexManager {
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
mailbox_docs = size_docs.into_iter().map(|(_, addr)| addr).collect();
|
||||
}
|
||||
SortBy::InternalDate => {
|
||||
let internal_date_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
&TopDocs::with_limit(page_size as usize)
|
||||
.and_offset(offset as usize)
|
||||
.order_by_fast_field(F_INTERNAL_DATE, order),
|
||||
)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
mailbox_docs = internal_date_docs
|
||||
.into_iter()
|
||||
.map(|(_, addr)| addr)
|
||||
.collect();
|
||||
}
|
||||
SortBy::IngestAt => {
|
||||
let ingest_at_docs: Vec<(Option<i64>, DocAddress)> = searcher
|
||||
.search(
|
||||
&query,
|
||||
&TopDocs::with_limit(page_size as usize)
|
||||
.and_offset(offset as usize)
|
||||
.order_by_fast_field(F_INGEST_AT, order),
|
||||
)
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?;
|
||||
mailbox_docs = ingest_at_docs.into_iter().map(|(_, addr)| addr).collect();
|
||||
}
|
||||
}
|
||||
|
||||
let mut result = Vec::new();
|
||||
|
||||
@@ -24,6 +24,7 @@ use crate::{
|
||||
};
|
||||
|
||||
pub mod attachment;
|
||||
pub mod dedup;
|
||||
pub mod envelope;
|
||||
pub mod fields;
|
||||
pub mod filter;
|
||||
|
||||
@@ -19,6 +19,8 @@
|
||||
use crate::common::periodic::TaskHandle;
|
||||
use crate::context::BichonTask;
|
||||
use crate::oauth2::{refresh::OAuth2RefreshTask, task::OAuth2CleanTask};
|
||||
use crate::store::tantivy::dedup::DedupTask;
|
||||
|
||||
pub struct PeriodicTasks {
|
||||
tasks: Vec<TaskHandle>,
|
||||
}
|
||||
@@ -28,6 +30,7 @@ impl PeriodicTasks {
|
||||
let mut tasks = Vec::new();
|
||||
tasks.push(OAuth2CleanTask::start());
|
||||
tasks.push(OAuth2RefreshTask::start());
|
||||
tasks.push(DedupTask::start());
|
||||
Self { tasks }
|
||||
}
|
||||
|
||||
|
||||
@@ -26,10 +26,10 @@ use std::sync::LazyLock;
|
||||
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::settings::cli::SETTINGS;
|
||||
use crate::raise_error;
|
||||
use crate::settings::cli::SETTINGS;
|
||||
|
||||
static ENCRYPT_PASSWORD: LazyLock<String> = LazyLock::new(|| {
|
||||
pub static ENCRYPT_PASSWORD: LazyLock<String> = LazyLock::new(|| {
|
||||
if let Some(file_path) = &SETTINGS.bichon_encrypt_password_file {
|
||||
return fs::read_to_string(file_path)
|
||||
.expect("failed to read the file with the encrypt password")
|
||||
@@ -102,7 +102,10 @@ pub fn internal_encrypt_string(
|
||||
Ok(general_purpose::URL_SAFE.encode(&result))
|
||||
}
|
||||
|
||||
pub fn internal_decrypt_string(password: &str, data: &str) -> Result<String, ring::error::Unspecified> {
|
||||
pub fn internal_decrypt_string(
|
||||
password: &str,
|
||||
data: &str,
|
||||
) -> Result<String, ring::error::Unspecified> {
|
||||
let data = general_purpose::URL_SAFE
|
||||
.decode(data)
|
||||
.map_err(|_| ring::error::Unspecified)?;
|
||||
@@ -146,8 +149,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_wrong_password_fails() {
|
||||
let encrypted =
|
||||
internal_encrypt_string("correct_password", "secret").unwrap();
|
||||
let encrypted = internal_encrypt_string("correct_password", "secret").unwrap();
|
||||
assert!(internal_decrypt_string("wrong_password", &encrypted).is_err());
|
||||
}
|
||||
|
||||
|
||||
@@ -17,9 +17,63 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use regex::Regex;
|
||||
use std::panic;
|
||||
use std::sync::LazyLock;
|
||||
use tracing::error;
|
||||
|
||||
/// Removes remote content references from HTML email body.
|
||||
///
|
||||
/// Strips attributes that load content from http:// or https:// URLs,
|
||||
/// keeping data: URIs and cid: references intact. Does NOT affect
|
||||
/// navigation links (<a href>).
|
||||
pub fn block_remote_content(html: &str) -> String {
|
||||
let mut result = html.to_string();
|
||||
|
||||
// 1. Strip src, poster, data attributes with remote URLs.
|
||||
// These always load content regardless of the tag.
|
||||
static SRC_ATTR_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)\s+(src|poster|data)\s*=\s*["'][^"']*(?:https?://|//)[^"']*["']"#).unwrap()
|
||||
});
|
||||
result = SRC_ATTR_RE.replace_all(&result, "").to_string();
|
||||
|
||||
// 2. Strip srcset attributes with remote URLs.
|
||||
static SRCSET_ATTR_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)\s+srcset\s*=\s*["'][^"']*(?:https?://|//)[^"']*["']"#).unwrap()
|
||||
});
|
||||
result = SRCSET_ATTR_RE.replace_all(&result, "").to_string();
|
||||
|
||||
// 3. Strip href on <link> tags (stylesheets), never <a> links.
|
||||
static LINK_HREF_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)(<link\b[^>]*)\s+href\s*=\s*["'][^"']*(?:https?://|//)[^"']*["']"#).unwrap()
|
||||
});
|
||||
result = LINK_HREF_RE.replace_all(&result, "$1").to_string();
|
||||
|
||||
// 4. Strip CSS url() references with remote URLs in inline styles.
|
||||
static CSS_URL_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)url\(\s*["']?\s*(?:https?://|//)[^)"'\s]*\s*["']?\s*\)"#).unwrap()
|
||||
});
|
||||
result = CSS_URL_RE.replace_all(&result, "").to_string();
|
||||
|
||||
// 5. Strip @import url(...) with remote URLs inside <style> blocks.
|
||||
static IMPORT_URL_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(
|
||||
r#"(?i)@import\s+url\(\s*["']?\s*(?:https?://|//)[^)"'\s]*\s*["']?\s*\)\s*;"#,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
result = IMPORT_URL_RE.replace_all(&result, "").to_string();
|
||||
|
||||
// 6. Strip background attribute on <body> with remote URLs.
|
||||
static BODY_BG_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)(<body\b[^>]*)\s+background\s*=\s*["'][^"']*(?:https?://|//)[^"']*["']"#)
|
||||
.unwrap()
|
||||
});
|
||||
result = BODY_BG_RE.replace_all(&result, "$1").to_string();
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn extract_text(html: String) -> String {
|
||||
let result = panic::catch_unwind(|| {
|
||||
html2text::config::plain()
|
||||
@@ -88,4 +142,132 @@ mod tests {
|
||||
let text = extract_text(html);
|
||||
assert!(text.contains("Click here"));
|
||||
}
|
||||
|
||||
mod block_remote {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strips_img_src_http() {
|
||||
let html = r#"<img src="https://tracker.example.com/pixel.gif" alt="x">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://tracker.example.com"));
|
||||
assert!(result.contains("alt=")); // other attrs preserved
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_img_src_protocol_relative() {
|
||||
let html = r#"<img src="//tracker.example.com/pixel.gif">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("//tracker.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_data_uri() {
|
||||
let html = r#"<img src="data:image/png;base64,ABC123" alt="embedded">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(result.contains("data:image/png;base64,ABC123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_cid_reference() {
|
||||
let html = r#"<img src="cid:abc123@example.com" alt="inline">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(result.contains("cid:abc123@example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_anchor_href() {
|
||||
let html = r#"<a href="https://example.com/page">Click</a>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(result.contains(r#"href="https://example.com/page""#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_link_stylesheet_href() {
|
||||
let html =
|
||||
r#"<link rel="stylesheet" href="https://fonts.example.com/font.css">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://fonts.example.com"));
|
||||
assert!(result.contains("<link")); // tag preserved
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_script_src() {
|
||||
let html = r#"<script src="https://evil.example.com/malware.js"></script>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://evil.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_iframe_src() {
|
||||
let html = r#"<iframe src="https://ads.example.com/banner"></iframe>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://ads.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_css_url_in_style() {
|
||||
let html = r#"<div style="background: url(https://tracker.example.com/bg.jpg)"></div>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://tracker.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_css_url_protocol_relative() {
|
||||
let html = r#"<div style="background: url(//tracker.example.com/bg.jpg)"></div>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("//tracker.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_css_import() {
|
||||
let html =
|
||||
r#"<style>@import url("https://fonts.example.com/font.css");</style>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://fonts.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_video_poster() {
|
||||
let html = r#"<video poster="https://cdn.example.com/thumb.jpg"></video>"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://cdn.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_srcset() {
|
||||
let html =
|
||||
r#"<img srcset="https://cdn.example.com/img1.jpg 1x, https://cdn.example.com/img2.jpg 2x">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://cdn.example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_body_background() {
|
||||
let html = r#"<body background="https://tracker.example.com/bg.jpg">"#;
|
||||
let result = block_remote_content(html);
|
||||
assert!(!result.contains("https://tracker.example.com"));
|
||||
assert!(result.contains("<body"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_mixed_content() {
|
||||
let html = r#"
|
||||
<html>
|
||||
<body>
|
||||
<img src="https://spy.example.com/pixel.gif" width="1" height="1">
|
||||
<img src="data:image/png;base64,OK123" alt="ok">
|
||||
<a href="https://example.com/read-more">Read more</a>
|
||||
<div style="background: url(https://tracker.example.com/bg.jpg) no-repeat"></div>
|
||||
</body>
|
||||
</html>"#;
|
||||
let result = block_remote_content(html);
|
||||
// Remote content gone
|
||||
assert!(!result.contains("spy.example.com"));
|
||||
assert!(!result.contains("tracker.example.com"));
|
||||
// Safe content preserved
|
||||
assert!(result.contains("data:image/png;base64,OK123"));
|
||||
assert!(result.contains(r#"href="https://example.com/read-more""#));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,10 @@ name = "bichon-server"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["embed-web"]
|
||||
embed-web = ["dep:rust-embed"]
|
||||
|
||||
|
||||
[dependencies]
|
||||
bichon-core = { path = "../core", features = ["web-api"] }
|
||||
@@ -17,7 +21,7 @@ poem-openapi = { version = "5.1.16", features = [
|
||||
"swagger-ui",
|
||||
"email",
|
||||
] }
|
||||
rust-embed = "8.11.0"
|
||||
rust-embed = { version = "8.11.0", optional = true }
|
||||
email_address.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -29,7 +33,6 @@ tracing.workspace = true
|
||||
tokio.workspace = true
|
||||
http.workspace = true
|
||||
urlencoding.workspace = true
|
||||
mimalloc.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
poem = { version = "3.1.12", features = ["test"] }
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
pub mod common;
|
||||
pub mod error;
|
||||
pub mod rest;
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use bichon_core::{
|
||||
bichon_version,
|
||||
cache::imap::task::SYNC_TASKS,
|
||||
common::{rustls::BichonTls, signal::SignalManager},
|
||||
context::{executors::BichonContext, Initialize},
|
||||
database::manager::DB_MANAGER,
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
logger,
|
||||
migrate::check_data_status,
|
||||
raise_error,
|
||||
settings::{cli::SETTINGS, dir::DataDirManager},
|
||||
store::{
|
||||
blob::BLOB_MANAGER,
|
||||
tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
|
||||
},
|
||||
tasks::PeriodicTasks,
|
||||
users::manager::UserManager,
|
||||
};
|
||||
use bichon_smtp::server::{start_smtp_server, SmtpServer};
|
||||
use tracing::{error, info};
|
||||
|
||||
pub async fn run() -> BichonResult<()> {
|
||||
logger::initialize_logging();
|
||||
info!(
|
||||
r#"
|
||||
_ _ _
|
||||
| | (_) | |
|
||||
| |__ _ ___ | |__ ___ _ __
|
||||
| '_ \ | | / __|| '_ \ / _ \ | '_ \
|
||||
| |_) || || (__ | | | || (_) || | | |
|
||||
|_.__/ |_| \___||_| |_| \___/ |_| |_|
|
||||
|
||||
"#
|
||||
);
|
||||
info!("Starting bichon-server");
|
||||
info!("Version: {}", bichon_version!());
|
||||
info!("Git: [{}]", env!("GIT_HASH"));
|
||||
info!("GitHub: https://github.com/rustmailer/bichon");
|
||||
|
||||
match check_data_status() {
|
||||
Ok(false) => {
|
||||
error!("Incompatible data format detected.");
|
||||
error!("Your data was created by an older version of Bichon and must be migrated before use.");
|
||||
error!("Please stop the Bichon v0.3.7 service before migration.");
|
||||
error!("Please run: bichon-admin");
|
||||
error!("Documentation: https://github.com/rustmailer/bichon/wiki/Bichon-Data-Migration:-v0.3.7-%E2%86%92-v1.0");
|
||||
return Err(raise_error!(
|
||||
"Legacy data layout detected".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to check data layout: {:#?}", e);
|
||||
return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
|
||||
}
|
||||
Ok(true) => {}
|
||||
}
|
||||
|
||||
if let Err(error) = initialize().await {
|
||||
eprintln!("{:?}", error);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let periodic_tasks = PeriodicTasks::setup();
|
||||
let mut smtp_service: Option<SmtpServer> = None;
|
||||
if SETTINGS.bichon_enable_smtp {
|
||||
info!("SMTP service is enabled, starting...");
|
||||
match start_smtp_server().await {
|
||||
Ok(server) => {
|
||||
info!("SMTP server listening on: {}", server.smtp_addr);
|
||||
smtp_service = Some(server);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to start SMTP server: {}", e);
|
||||
return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info!("SMTP service is disabled by configuration.");
|
||||
}
|
||||
|
||||
rest::start_http_server().await?;
|
||||
periodic_tasks.shutdown().await;
|
||||
|
||||
if let Some(server) = smtp_service {
|
||||
info!("Shutting down SMTP server...");
|
||||
server.stop().await;
|
||||
info!("SMTP server stopped.");
|
||||
}
|
||||
|
||||
SYNC_TASKS.shutdown().await;
|
||||
ENVELOPE_MANAGER.shutdown().await;
|
||||
ATTACHMENT_MANAGER.shutdown().await;
|
||||
BLOB_MANAGER.shutdown().await;
|
||||
DB_MANAGER.flush();
|
||||
info!("Bichon server stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn initialize() -> BichonResult<()> {
|
||||
SignalManager::initialize().await?;
|
||||
DataDirManager::initialize().await?;
|
||||
UserManager::initialize().await?;
|
||||
BichonTls::initialize().await?;
|
||||
BichonContext::initialize().await?;
|
||||
LazyLock::force(&BLOB_MANAGER);
|
||||
LazyLock::force(&ENVELOPE_MANAGER);
|
||||
LazyLock::force(&ATTACHMENT_MANAGER);
|
||||
Ok(())
|
||||
}
|
||||
+2
-194
@@ -16,201 +16,9 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
use bichon_core::error::BichonResult;
|
||||
|
||||
use bichon_core::{
|
||||
bichon_version,
|
||||
cache::imap::task::SYNC_TASKS,
|
||||
common::rustls::BichonTls,
|
||||
context::{executors::BichonContext, Initialize},
|
||||
error::{code::ErrorCode, BichonResult},
|
||||
logger,
|
||||
migrate::check_data_status,
|
||||
raise_error,
|
||||
settings::cli::SETTINGS,
|
||||
store::{
|
||||
blob::BLOB_MANAGER,
|
||||
tantivy::{attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER},
|
||||
},
|
||||
tasks::PeriodicTasks,
|
||||
};
|
||||
use bichon_smtp::server::{start_smtp_server, SmtpServer};
|
||||
use mimalloc::MiMalloc;
|
||||
use tracing::{error, info};
|
||||
|
||||
use bichon_core::{
|
||||
common::signal::SignalManager, settings::dir::DataDirManager, users::manager::UserManager,
|
||||
};
|
||||
|
||||
use crate::rest::start_http_server;
|
||||
|
||||
pub mod common;
|
||||
pub mod error;
|
||||
pub mod rest;
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: MiMalloc = MiMalloc;
|
||||
|
||||
static LOGO: &str = r#"
|
||||
_ _ _
|
||||
| | (_) | |
|
||||
| |__ _ ___ | |__ ___ _ __
|
||||
| '_ \ | | / __|| '_ \ / _ \ | '_ \
|
||||
| |_) || || (__ | | | || (_) || | | |
|
||||
|_.__/ |_| \___||_| |_| \___/ |_| |_|
|
||||
|
||||
"#;
|
||||
#[tokio::main]
|
||||
async fn main() -> BichonResult<()> {
|
||||
logger::initialize_logging();
|
||||
info!("{}", LOGO);
|
||||
info!("Starting bichon-server");
|
||||
info!("Version: {}", bichon_version!());
|
||||
info!("Git: [{}]", env!("GIT_HASH"));
|
||||
info!("GitHub: https://github.com/rustmailer/bichon");
|
||||
|
||||
match check_data_status() {
|
||||
Ok(false) => {
|
||||
error!("Incompatible data format detected.");
|
||||
error!("Your data was created by an older version of Bichon and must be migrated before use.");
|
||||
error!("Please run: bichon-admin");
|
||||
error!("Documentation: https://github.com/rustmailer/bichon/wiki/migration");
|
||||
return Err(raise_error!(
|
||||
"Legacy data layout detected".into(),
|
||||
ErrorCode::InternalError
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to check data layout: {:#?}", e);
|
||||
return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
|
||||
}
|
||||
Ok(true) => {}
|
||||
}
|
||||
|
||||
if let Err(error) = initialize().await {
|
||||
eprintln!("{:?}", error);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let periodic_tasks = PeriodicTasks::setup();
|
||||
let mut smtp_service: Option<SmtpServer> = None;
|
||||
if SETTINGS.bichon_enable_smtp {
|
||||
info!("SMTP service is enabled, starting...");
|
||||
match start_smtp_server().await {
|
||||
Ok(server) => {
|
||||
info!("SMTP server listening on: {}", server.smtp_addr);
|
||||
smtp_service = Some(server);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to start SMTP server: {}", e);
|
||||
return Err(raise_error!(format!("{:#?}", e), ErrorCode::InternalError));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
info!("SMTP service is disabled by configuration.");
|
||||
}
|
||||
|
||||
start_http_server().await?;
|
||||
periodic_tasks.shutdown().await;
|
||||
|
||||
if let Some(server) = smtp_service {
|
||||
info!("Shutting down SMTP server...");
|
||||
server.stop().await;
|
||||
info!("SMTP server stopped.");
|
||||
}
|
||||
|
||||
SYNC_TASKS.shutdown().await;
|
||||
ENVELOPE_MANAGER.shutdown().await;
|
||||
ATTACHMENT_MANAGER.shutdown().await;
|
||||
BLOB_MANAGER.shutdown().await;
|
||||
info!("Bichon server stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Initialize the system by validating settings and starting necessary tasks.
|
||||
async fn initialize() -> BichonResult<()> {
|
||||
SignalManager::initialize().await?;
|
||||
DataDirManager::initialize().await?;
|
||||
UserManager::initialize().await?;
|
||||
BichonTls::initialize().await?;
|
||||
BichonContext::initialize().await?;
|
||||
LazyLock::force(&BLOB_MANAGER);
|
||||
LazyLock::force(&ENVELOPE_MANAGER);
|
||||
LazyLock::force(&ATTACHMENT_MANAGER);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod api_tests {
|
||||
use super::rest::api::create_openapi_service;
|
||||
use poem::test::TestClient;
|
||||
|
||||
#[tokio::test]
|
||||
async fn openapi_spec_json_is_served() {
|
||||
let api_service = create_openapi_service();
|
||||
let spec_endpoint = api_service.spec_endpoint();
|
||||
let cli = TestClient::new(spec_endpoint);
|
||||
|
||||
let resp = cli.get("/").send().await;
|
||||
resp.assert_status_is_ok();
|
||||
|
||||
let body = resp.json().await;
|
||||
let obj = body.value().object();
|
||||
assert!(obj.get_opt("openapi").is_some(), "missing openapi version");
|
||||
assert!(obj.get_opt("info").is_some(), "missing info section");
|
||||
assert!(obj.get_opt("paths").is_some(), "missing paths section");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openapi_spec_yaml_is_served() {
|
||||
let api_service = create_openapi_service();
|
||||
let spec_endpoint = api_service.spec_endpoint_yaml();
|
||||
let cli = TestClient::new(spec_endpoint);
|
||||
|
||||
let resp = cli.get("/").send().await;
|
||||
resp.assert_status_is_ok();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn swagger_ui_is_served() {
|
||||
let api_service = create_openapi_service();
|
||||
let swagger = api_service.swagger_ui();
|
||||
let cli = TestClient::new(swagger);
|
||||
|
||||
let resp = cli.get("/").send().await;
|
||||
resp.assert_status_is_ok();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn openapi_spec_lists_all_tag_groups() {
|
||||
let api_service = create_openapi_service();
|
||||
let spec_endpoint = api_service.spec_endpoint();
|
||||
let cli = TestClient::new(spec_endpoint);
|
||||
|
||||
let resp = cli.get("/").send().await;
|
||||
let body = resp.json().await;
|
||||
let value = body.value();
|
||||
|
||||
let tag_names: Vec<&str> = value
|
||||
.object()
|
||||
.get("tags")
|
||||
.array()
|
||||
.iter()
|
||||
.map(|v| v.object().get("name").string())
|
||||
.collect();
|
||||
|
||||
assert!(tag_names.contains(&"AccessToken"), "missing AccessToken tag");
|
||||
assert!(tag_names.contains(&"Attachment"), "missing Attachment tag");
|
||||
assert!(tag_names.contains(&"AutoConfig"), "missing AutoConfig tag");
|
||||
assert!(tag_names.contains(&"Account"), "missing Account tag");
|
||||
assert!(tag_names.contains(&"System"), "missing System tag");
|
||||
assert!(tag_names.contains(&"Mailbox"), "missing Mailbox tag");
|
||||
assert!(tag_names.contains(&"OAuth2"), "missing OAuth2 tag");
|
||||
assert!(tag_names.contains(&"Message"), "missing Message tag");
|
||||
assert!(tag_names.contains(&"Import"), "missing Import tag");
|
||||
assert!(tag_names.contains(&"Users"), "missing Users tag");
|
||||
}
|
||||
bichon_server::run().await
|
||||
}
|
||||
|
||||
@@ -121,6 +121,8 @@ impl MessageApi {
|
||||
}
|
||||
|
||||
/// Fetches the content of a specific email.
|
||||
/// Set `block_remote_content=true` to strip external images, scripts,
|
||||
/// and other content loaded from http/https URLs.
|
||||
#[oai(
|
||||
path = "/message-content/:account_id/:envelope_id",
|
||||
method = "get",
|
||||
@@ -132,11 +134,18 @@ impl MessageApi {
|
||||
account_id: Path<u64>,
|
||||
/// The ID of the message to fetch.
|
||||
envelope_id: Path<String>,
|
||||
/// Block remote content (http/https URLs) from email body.
|
||||
block_remote_content: Query<Option<bool>>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<FullMessageContent>> {
|
||||
let account_id = account_id.0;
|
||||
let block_remote = block_remote_content.0.unwrap_or(false);
|
||||
context.require_permission(Some(account_id), Permission::DATA_READ)?;
|
||||
Ok(Json(retrieve_email_content(account_id, envelope_id.0)?))
|
||||
Ok(Json(retrieve_email_content(
|
||||
account_id,
|
||||
envelope_id.0,
|
||||
block_remote,
|
||||
)?))
|
||||
}
|
||||
|
||||
/// Retrieves the content of an email embedded as an attachment.
|
||||
@@ -152,15 +161,18 @@ impl MessageApi {
|
||||
/// The ID of the message to fetch.
|
||||
envelope_id: Path<String>,
|
||||
content_hash: Query<String>,
|
||||
block_remote_content: Query<Option<bool>>,
|
||||
context: WrappedContext,
|
||||
) -> ApiResult<Json<FullNestedMessageContent>> {
|
||||
let account_id = account_id.0;
|
||||
let block_remote = block_remote_content.0.unwrap_or(false);
|
||||
context.require_permission(Some(account_id), Permission::DATA_READ)?;
|
||||
let content_hash = content_hash.0.trim();
|
||||
Ok(Json(retrieve_nested_eml_content(
|
||||
account_id,
|
||||
envelope_id.0,
|
||||
content_hash,
|
||||
block_remote,
|
||||
)?))
|
||||
}
|
||||
|
||||
@@ -213,7 +225,7 @@ impl MessageApi {
|
||||
AccountModel::check_account_exists(account_id)?;
|
||||
context.require_permission(Some(account_id), Permission::DATA_RAW_DOWNLOAD)?;
|
||||
let envelope_id = envelope_id.0;
|
||||
let reader = get_reader(account_id, envelope_id.clone())?;
|
||||
let reader = get_reader(account_id, envelope_id.clone()).await?;
|
||||
let body = Body::from_async_read(reader);
|
||||
let attachment = Attachment::new(body)
|
||||
.attachment_type(AttachmentType::Attachment)
|
||||
|
||||
@@ -1,24 +1,11 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#[cfg(feature = "embed-web")]
|
||||
mod inner {
|
||||
use rust_embed::RustEmbed;
|
||||
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "../../web/dist/"]
|
||||
pub struct FrontEndAssets;
|
||||
}
|
||||
|
||||
use rust_embed::RustEmbed;
|
||||
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "../../web/dist/"]
|
||||
pub struct FrontEndAssets;
|
||||
#[cfg(feature = "embed-web")]
|
||||
pub use inner::FrontEndAssets;
|
||||
|
||||
@@ -16,51 +16,49 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::common::auth::ApiGuard;
|
||||
use crate::common::error::ErrorCapture;
|
||||
use crate::common::log::Tracing;
|
||||
use crate::common::tls::rustls_config;
|
||||
use crate::common::timeout::{Timeout, TIMEOUT_HEADER};
|
||||
use crate::error::handler::error_handler;
|
||||
use crate::rest::public::features::get_features;
|
||||
use crate::rest::public::login::login;
|
||||
use crate::rest::public::status::get_status;
|
||||
use bichon_core::common::signal::SIGNAL_MANAGER;
|
||||
use bichon_core::error::code::ErrorCode;
|
||||
use bichon_core::error::BichonResult;
|
||||
use bichon_core::raise_error;
|
||||
use bichon_core::settings::cli::SETTINGS;
|
||||
|
||||
use super::error::ApiErrorResponse;
|
||||
use crate::common::auth::ApiGuard;
|
||||
use crate::common::timeout::{Timeout, TIMEOUT_HEADER};
|
||||
use api::create_openapi_service;
|
||||
use assets::FrontEndAssets;
|
||||
use bichon_core::raise_error;
|
||||
use http::{HeaderValue, Method};
|
||||
use poem::endpoint::EmbeddedFilesEndpoint;
|
||||
use http::Method;
|
||||
use poem::listener::{Listener, TcpListener};
|
||||
use poem::middleware::{CatchPanic, Compression, SetHeader};
|
||||
use poem::{get, handler, post, IntoResponse};
|
||||
use poem::{middleware::Cors, EndpointExt, Route, Server};
|
||||
use poem::middleware::{CatchPanic, Compression, Cors};
|
||||
use poem::{get, post, Endpoint, EndpointExt, Route, Server};
|
||||
use public::oauth2::oauth2_callback;
|
||||
use std::collections::HashSet;
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(feature = "embed-web")]
|
||||
use {
|
||||
assets::FrontEndAssets,
|
||||
http::HeaderValue,
|
||||
poem::{handler, endpoint::EmbeddedFilesEndpoint, IntoResponse},
|
||||
poem::middleware::SetHeader,
|
||||
};
|
||||
|
||||
pub mod api;
|
||||
pub mod assets;
|
||||
pub mod public;
|
||||
|
||||
pub type ApiResult<T, E = ApiErrorResponse> = std::result::Result<T, E>;
|
||||
|
||||
pub async fn start_http_server() -> BichonResult<()> {
|
||||
let listener = TcpListener::bind((
|
||||
SETTINGS.bichon_bind_ip.clone().unwrap_or("0.0.0.0".into()),
|
||||
SETTINGS.bichon_http_port as u16,
|
||||
));
|
||||
|
||||
let listener = if SETTINGS.bichon_enable_rest_https {
|
||||
listener.rustls(rustls_config()?).boxed()
|
||||
} else {
|
||||
listener.boxed()
|
||||
};
|
||||
use super::error::ApiErrorResponse;
|
||||
|
||||
/// Build the community route tree. Pro/Enterprise servers can call this
|
||||
/// and then add their own routes before passing the tree to the server.
|
||||
pub fn build_routes() -> impl Endpoint {
|
||||
let api_service = create_openapi_service()
|
||||
.summary("A lightweight, high-performance Rust email archiver with WebUI");
|
||||
|
||||
@@ -79,7 +77,6 @@ pub async fn start_http_server() -> BichonResult<()> {
|
||||
.with(Tracing);
|
||||
|
||||
let cors_origins: Option<HashSet<String>> = SETTINGS.bichon_cors_origins.clone();
|
||||
|
||||
let cors_origins: Vec<String> = cors_origins.unwrap_or_default().into_iter().collect();
|
||||
|
||||
let cors = Cors::new()
|
||||
@@ -92,7 +89,6 @@ pub async fn start_http_server() -> BichonResult<()> {
|
||||
}
|
||||
cors_origins.iter().any(|o| o == origin)
|
||||
})
|
||||
//.allow_origins(cors_origins)
|
||||
.allow_credentials(true)
|
||||
.allow_methods(&[
|
||||
Method::GET,
|
||||
@@ -107,13 +103,6 @@ pub async fn start_http_server() -> BichonResult<()> {
|
||||
.expose_headers(vec!["Accept"])
|
||||
.max_age(SETTINGS.bichon_cors_max_age);
|
||||
|
||||
let cache_static = || {
|
||||
SetHeader::new().overriding(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("max-age=86400"),
|
||||
)
|
||||
};
|
||||
|
||||
let app_logic = Route::new()
|
||||
.nest("/api-docs/swagger", swagger)
|
||||
.nest("/api-docs/redoc", redoc)
|
||||
@@ -122,42 +111,43 @@ pub async fn start_http_server() -> BichonResult<()> {
|
||||
.nest("/api-docs/spec.json", spec_json)
|
||||
.nest("/api-docs/spec.yaml", spec_yaml)
|
||||
.nest("/oauth2/callback", get(oauth2_callback))
|
||||
.nest("/api/v1/features", get(get_features))
|
||||
.nest("/api/status", get(get_status))
|
||||
.nest("/api/login", post(login))
|
||||
.nest_no_strip("/api/v1", open_api_route)
|
||||
.nest_no_strip("/api/v1", open_api_route);
|
||||
|
||||
let app_logic = add_web_assets(app_logic);
|
||||
|
||||
Route::new()
|
||||
.nest(&SETTINGS.bichon_base_url, app_logic)
|
||||
.with(cors)
|
||||
.with_if(SETTINGS.bichon_http_compression_enabled, Compression::new())
|
||||
.with(CatchPanic::new())
|
||||
}
|
||||
|
||||
#[cfg(feature = "embed-web")]
|
||||
fn add_web_assets(route: Route) -> impl Endpoint {
|
||||
let cache_static = || {
|
||||
SetHeader::new().overriding(
|
||||
http::header::CACHE_CONTROL,
|
||||
HeaderValue::from_static("max-age=86400"),
|
||||
)
|
||||
};
|
||||
|
||||
route
|
||||
.nest_no_strip(
|
||||
"/assets",
|
||||
EmbeddedFilesEndpoint::<FrontEndAssets>::new().with(cache_static()),
|
||||
)
|
||||
.at("/*", serve_index_with_base);
|
||||
|
||||
let route = Route::new()
|
||||
.nest(&SETTINGS.bichon_base_url, app_logic)
|
||||
.with(cors)
|
||||
.with_if(SETTINGS.bichon_http_compression_enabled, Compression::new())
|
||||
.with(CatchPanic::new());
|
||||
|
||||
let mut rx = SIGNAL_MANAGER.subscribe();
|
||||
let shutdown_fut = async move {
|
||||
let _ = rx.recv().await;
|
||||
};
|
||||
let server = Server::new(listener)
|
||||
.name("Bichon Service")
|
||||
.idle_timeout(Duration::from_secs(60))
|
||||
.run_with_graceful_shutdown(
|
||||
route.catch_all_error(error_handler),
|
||||
shutdown_fut,
|
||||
Some(Duration::from_secs(5)),
|
||||
);
|
||||
println!(
|
||||
"Bichon Service is now running on port {}.",
|
||||
SETTINGS.bichon_http_port
|
||||
);
|
||||
server
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
|
||||
.at("/*", serve_index_with_base)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "embed-web"))]
|
||||
fn add_web_assets(route: Route) -> Route {
|
||||
route
|
||||
}
|
||||
|
||||
#[cfg(feature = "embed-web")]
|
||||
#[handler]
|
||||
async fn serve_index_with_base() -> impl IntoResponse {
|
||||
let mut html =
|
||||
@@ -180,3 +170,38 @@ async fn serve_index_with_base() -> impl IntoResponse {
|
||||
.content_type("text/html; charset=utf-8")
|
||||
.body(html)
|
||||
}
|
||||
|
||||
pub async fn start_http_server() -> BichonResult<()> {
|
||||
let listener = TcpListener::bind((
|
||||
SETTINGS.bichon_bind_ip.clone().unwrap_or("0.0.0.0".into()),
|
||||
SETTINGS.bichon_http_port as u16,
|
||||
));
|
||||
|
||||
let listener = if SETTINGS.bichon_enable_rest_https {
|
||||
listener.rustls(rustls_config()?).boxed()
|
||||
} else {
|
||||
listener.boxed()
|
||||
};
|
||||
|
||||
let route = build_routes();
|
||||
|
||||
let mut rx = SIGNAL_MANAGER.subscribe();
|
||||
let shutdown_fut = async move {
|
||||
let _ = rx.recv().await;
|
||||
};
|
||||
let server = Server::new(listener)
|
||||
.name("Bichon Service")
|
||||
.idle_timeout(Duration::from_secs(60))
|
||||
.run_with_graceful_shutdown(
|
||||
route.catch_all_error(error_handler),
|
||||
shutdown_fut,
|
||||
Some(Duration::from_secs(5)),
|
||||
);
|
||||
println!(
|
||||
"Bichon Service is now running on port {}.",
|
||||
SETTINGS.bichon_http_port
|
||||
);
|
||||
server
|
||||
.await
|
||||
.map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use poem::{handler, web::Json, IntoResponse};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct FeaturesResponse {
|
||||
features: Vec<String>,
|
||||
edition: &'static str,
|
||||
version: String,
|
||||
}
|
||||
|
||||
#[handler]
|
||||
pub async fn get_features() -> impl IntoResponse {
|
||||
Json(FeaturesResponse {
|
||||
features: vec![],
|
||||
edition: "community",
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
})
|
||||
}
|
||||
@@ -17,6 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
pub mod features;
|
||||
pub mod login;
|
||||
pub mod oauth2;
|
||||
pub mod status;
|
||||
|
||||
@@ -628,6 +628,7 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> {
|
||||
unseen: None,
|
||||
uid_next: None,
|
||||
uid_validity: None,
|
||||
highest_uid: None,
|
||||
};
|
||||
let mailbox_id = mailbox.id;
|
||||
|
||||
|
||||
+12
-2
@@ -10,7 +10,10 @@
|
||||
"preview": "vite preview",
|
||||
"format:check": "prettier --check .",
|
||||
"format": "prettier --write .",
|
||||
"knip": "knip"
|
||||
"knip": "knip",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage"
|
||||
},
|
||||
"dependencies": {
|
||||
"@emotion/react": "^11.14.0",
|
||||
@@ -81,6 +84,9 @@
|
||||
"@tanstack/react-query-devtools": "^5.62.3",
|
||||
"@tanstack/router-devtools": "^1.86.1",
|
||||
"@tanstack/router-plugin": "^1.86.0",
|
||||
"@testing-library/jest-dom": "^6.9.1",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@testing-library/user-event": "^14.6.1",
|
||||
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
|
||||
"@types/file-saver": "^2.0.7",
|
||||
"@types/js-cookie": "^3.0.6",
|
||||
@@ -88,18 +94,22 @@
|
||||
"@types/react": "^18.3.18",
|
||||
"@types/react-dom": "^18.3.5",
|
||||
"@vitejs/plugin-react-swc": "^3.7.2",
|
||||
"@vitest/coverage-v8": "^4.1.7",
|
||||
"autoprefixer": "^10.4.20",
|
||||
"eslint": "^9.16.0",
|
||||
"eslint-plugin-react-hooks": "^5.1.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.16",
|
||||
"globals": "^15.13.0",
|
||||
"jsdom": "^29.1.1",
|
||||
"knip": "^5.41.1",
|
||||
"msw": "^2.14.6",
|
||||
"postcss": "^8.4.49",
|
||||
"prettier": "^3.4.2",
|
||||
"prettier-plugin-tailwindcss": "^0.6.9",
|
||||
"tailwindcss": "^3.4.16",
|
||||
"typescript": "~5.7.2",
|
||||
"typescript-eslint": "^8.17.0",
|
||||
"vite": "^6.0.11"
|
||||
"vite": "^6.0.11",
|
||||
"vitest": "^4.1.7"
|
||||
}
|
||||
}
|
||||
Generated
+1182
-10
File diff suppressed because it is too large
Load Diff
@@ -128,7 +128,6 @@ export interface AccountModel {
|
||||
capabilities?: string[];
|
||||
date_since?: DateSelection;
|
||||
date_before?: RelativeDate;
|
||||
folder_limit?: number,
|
||||
download_folders: string[];
|
||||
download_interval_min?: number;
|
||||
download_batch_size?: number;
|
||||
@@ -143,6 +142,7 @@ export interface AccountModel {
|
||||
imap_quota_window?: QuotaWindow;
|
||||
imap_quota_bytes?: number;
|
||||
auto_download_new_mailboxes?: boolean;
|
||||
download_schedule?: string;
|
||||
}
|
||||
|
||||
export const download_state = async (account_id: number) => {
|
||||
|
||||
@@ -64,7 +64,8 @@ export interface AttachmentInfo {
|
||||
export interface MessageContentResponse {
|
||||
text?: string;
|
||||
html?: string;
|
||||
attachments?: AttachmentInfo[]
|
||||
attachments?: AttachmentInfo[];
|
||||
has_remote_content?: boolean;
|
||||
}
|
||||
|
||||
export interface NestedMessageContentResponse {
|
||||
@@ -72,6 +73,7 @@ export interface NestedMessageContentResponse {
|
||||
html?: string;
|
||||
attachments?: AttachmentInfo[];
|
||||
envelope: EmailEnvelope;
|
||||
has_remote_content?: boolean;
|
||||
}
|
||||
|
||||
export const getContent = (messageContent: MessageContentResponse): string | null => {
|
||||
@@ -83,13 +85,25 @@ export const getContent = (messageContent: MessageContentResponse): string | nul
|
||||
return null;
|
||||
};
|
||||
|
||||
export const load_message = async (accountId: number, id: string) => {
|
||||
const response = await axiosInstance.get<MessageContentResponse>(`api/v1/message-content/${accountId}/${id}`);
|
||||
export const load_message = async (accountId: number, id: string, blockRemoteContent = false) => {
|
||||
const params = new URLSearchParams();
|
||||
if (blockRemoteContent) {
|
||||
params.set('block_remote_content', 'true');
|
||||
}
|
||||
const qs = params.toString();
|
||||
const url = `api/v1/message-content/${accountId}/${id}${qs ? '?' + qs : ''}`;
|
||||
const response = await axiosInstance.get<MessageContentResponse>(url);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
export const load_nested_message = async (accountId: number, id: string, content_hash: string) => {
|
||||
const response = await axiosInstance.get<NestedMessageContentResponse>(`api/v1/nested-message-content/${accountId}/${id}?content_hash=${content_hash}`);
|
||||
export const load_nested_message = async (accountId: number, id: string, content_hash: string, blockRemoteContent = false) => {
|
||||
const params = new URLSearchParams({ content_hash });
|
||||
if (blockRemoteContent) {
|
||||
params.set('block_remote_content', 'true');
|
||||
}
|
||||
const response = await axiosInstance.get<NestedMessageContentResponse>(
|
||||
`api/v1/nested-message-content/${accountId}/${id}?${params.toString()}`
|
||||
);
|
||||
return response.data;
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// Temporary mock data for download-folders responsive testing
|
||||
import { MailboxData, MailboxListResponse } from './api';
|
||||
|
||||
let _id = 1;
|
||||
const m = (overrides: Partial<MailboxData>): MailboxData => ({
|
||||
account_id: 1,
|
||||
attributes: [],
|
||||
delimiter: '/',
|
||||
exists: Math.floor(Math.random() * 5000) + 100,
|
||||
id: _id++,
|
||||
name: '',
|
||||
uid_next: null,
|
||||
uid_validity: null,
|
||||
unseen: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const SHORT = [
|
||||
m({ name: 'INBOX' }),
|
||||
m({ name: 'INBOX/Drafts' }),
|
||||
m({ name: 'INBOX/Sent' }),
|
||||
m({ name: 'INBOX/Trash' }),
|
||||
m({ name: 'INBOX/Archive' }),
|
||||
m({ name: 'INBOX/Spam' }),
|
||||
m({ name: 'INBOX/Templates' }),
|
||||
];
|
||||
|
||||
const PROJECTS = [
|
||||
m({ name: 'INBOX/Projects' }),
|
||||
m({ name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review' }),
|
||||
m({ name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review/Drafts' }),
|
||||
m({
|
||||
name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review/Drafts/Revision 3 - Updated Forecast Models and Department Sign-off Required',
|
||||
attributes: [{ attr: 'HasChildren', extension: null }],
|
||||
}),
|
||||
m({
|
||||
name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review/Drafts/Revision 3 - Updated Forecast Models and Department Sign-off Required/Comments from CFO',
|
||||
}),
|
||||
m({
|
||||
name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review/Drafts/Revision 3 - Updated Forecast Models and Department Sign-off Required/Attachments',
|
||||
}),
|
||||
m({ name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review/Final' }),
|
||||
m({ name: 'INBOX/Projects/Q4 2025 Financial Reports and Annual Budget Planning Review/Final/Approved with Amendments' }),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection' }),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Kickoff' }),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Kickoff/Meeting Minutes and Action Items' }),
|
||||
m({
|
||||
name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Vendor Proposals',
|
||||
attributes: [{ attr: 'HasNoChildren', extension: null }],
|
||||
}),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Vendor Proposals/AWS Proposal Package' }),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Vendor Proposals/Azure Proposal Package' }),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Vendor Proposals/GCP Proposal Package' }),
|
||||
m({ name: 'INBOX/Projects/Q1 2026 Strategic Initiative - Cloud Infrastructure Migration Assessment and Vendor Selection/Internal Review and Scoring Committee' }),
|
||||
m({ name: 'INBOX/Projects/HR Portal Redesign - Employee Self-Service Platform Modernization' }),
|
||||
m({ name: 'INBOX/Projects/HR Portal Redesign - Employee Self-Service Platform Modernization/Wireframes and Mockups' }),
|
||||
m({ name: 'INBOX/Projects/HR Portal Redesign - Employee Self-Service Platform Modernization/Wireframes and Mockups/Iteration 1' }),
|
||||
m({ name: 'INBOX/Projects/HR Portal Redesign - Employee Self-Service Platform Modernization/Wireframes and Mockups/Iteration 2' }),
|
||||
m({ name: 'INBOX/Projects/HR Portal Redesign - Employee Self-Service Platform Modernization/Usability Testing Results and Feedback Compilation' }),
|
||||
];
|
||||
|
||||
const CLIENTS = [
|
||||
m({ name: 'INBOX/Clients' }),
|
||||
m({
|
||||
name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026',
|
||||
attributes: [{ attr: 'HasChildren', extension: null }],
|
||||
}),
|
||||
m({ name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026/Contract Documents' }),
|
||||
m({ name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026/Contract Documents/Redlined Versions' }),
|
||||
m({
|
||||
name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026/Contract Documents/Redlined Versions/Legal Review Round 1',
|
||||
}),
|
||||
m({
|
||||
name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026/Contract Documents/Redlined Versions/Legal Review Round 2 - Final',
|
||||
}),
|
||||
m({
|
||||
name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026/Invoices and Payment Records',
|
||||
}),
|
||||
m({ name: 'INBOX/Clients/Acme Corporation - Enterprise Software Licensing and Support Agreement Renewal 2026/Support Tickets and Correspondence' }),
|
||||
m({
|
||||
name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement',
|
||||
attributes: [{ attr: 'HasChildren', extension: null }],
|
||||
}),
|
||||
m({ name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement/Phase 1 Discovery and Assessment' }),
|
||||
m({ name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement/Phase 1 Discovery and Assessment/Stakeholder Interviews' }),
|
||||
m({ name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement/Phase 1 Discovery and Assessment/Current State Architecture Documentation' }),
|
||||
m({ name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement/Phase 2 Implementation Roadmap' }),
|
||||
m({ name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement/Phase 2 Implementation Roadmap/Sprint Planning and Resource Allocation' }),
|
||||
m({ name: 'INBOX/Clients/Globex Industries - Multi-Year Digital Transformation Consulting Engagement/Phase 2 Implementation Roadmap/Risk Assessment and Mitigation Strategies' }),
|
||||
m({ name: 'INBOX/Clients/Initech Solutions - Cybersecurity Audit and Compliance Remediation Program 2026' }),
|
||||
m({ name: 'INBOX/Clients/Initech Solutions - Cybersecurity Audit and Compliance Remediation Program 2026/Penetration Testing Reports' }),
|
||||
m({ name: 'INBOX/Clients/Initech Solutions - Cybersecurity Audit and Compliance Remediation Program 2026/Penetration Testing Reports/External Network Assessment' }),
|
||||
m({ name: 'INBOX/Clients/Initech Solutions - Cybersecurity Audit and Compliance Remediation Program 2026/Penetration Testing Reports/Internal Network Assessment' }),
|
||||
m({ name: 'INBOX/Clients/Initech Solutions - Cybersecurity Audit and Compliance Remediation Program 2026/Penetration Testing Reports/Web Application Security Scan Results' }),
|
||||
m({ name: 'INBOX/Clients/Initech Solutions - Cybersecurity Audit and Compliance Remediation Program 2026/Compliance Gap Analysis and Remediation Tracking' }),
|
||||
m({ name: 'INBOX/Clients/Massive Dynamic - Research Collaboration on Advanced Machine Learning Applications in Healthcare Informatics' }),
|
||||
m({ name: 'INBOX/Clients/Massive Dynamic - Research Collaboration on Advanced Machine Learning Applications in Healthcare Informatics/Data Sharing Agreements and Ethics Board Approvals' }),
|
||||
m({ name: 'INBOX/Clients/Massive Dynamic - Research Collaboration on Advanced Machine Learning Applications in Healthcare Informatics/Literature Review and Prior Art Analysis' }),
|
||||
m({ name: 'INBOX/Clients/Massive Dynamic - Research Collaboration on Advanced Machine Learning Applications in Healthcare Informatics/Model Training Datasets and Validation Results' }),
|
||||
];
|
||||
|
||||
const NOTIFICATIONS = [
|
||||
m({ name: 'INBOX/Notifications' }),
|
||||
m({ name: 'INBOX/Notifications/GitHub Enterprise - Pull Request Reviews and CI/CD Pipeline Status Updates' }),
|
||||
m({ name: 'INBOX/Notifications/Jira Service Management - Incident Response Alerts and Escalation Notifications' }),
|
||||
m({ name: 'INBOX/Notifications/Confluence - Documentation Updates and Page Modification Summaries' }),
|
||||
m({ name: 'INBOX/Notifications/Slack Workspace - Channel Highlights and Direct Message Digest Compilation' }),
|
||||
m({ name: 'INBOX/Notifications/Datadog - Application Performance Monitoring Alerts and Anomaly Detection Reports' }),
|
||||
m({ name: 'INBOX/Notifications/PagerDuty - On-Call Rotation Schedule and Incident Acknowledgment Confirmations' }),
|
||||
m({ name: 'INBOX/Notifications/Microsoft 365 - Calendar Invitations and Meeting Room Booking Confirmations' }),
|
||||
];
|
||||
|
||||
const NEWSLETTERS = [
|
||||
m({ name: 'INBOX/Newsletters' }),
|
||||
m({ name: 'INBOX/Newsletters/Rust Weekly - Community Updates, Crate Highlights, and RFC Progress Tracking Digest' }),
|
||||
m({
|
||||
name: 'INBOX/Newsletters/Systems Programming Insider - Deep Dive Articles on Memory Management and Concurrency Patterns',
|
||||
attributes: [{ attr: 'HasNoChildren', extension: null }],
|
||||
}),
|
||||
m({ name: 'INBOX/Newsletters/Cloud Native Computing Foundation - Kubernetes Ecosystem Updates and Project Maturity Reports' }),
|
||||
m({ name: 'INBOX/Newsletters/Software Architecture Monthly - Case Studies in Distributed Systems Design and Microservices Patterns' }),
|
||||
m({ name: 'INBOX/Newsletters/DevOps Weekly Digest - Tool Reviews, Pipeline Optimization Techniques, and Platform Engineering Insights' }),
|
||||
m({ name: 'INBOX/Newsletters/Information Security Briefing - CVE Disclosures, Threat Intelligence Reports, and Zero-Day Advisories' }),
|
||||
m({ name: 'INBOX/Newsletters/Tech Leadership Forum - Engineering Management Best Practices and Organizational Scaling Strategies' }),
|
||||
];
|
||||
|
||||
export const MOCK_MAILBOX_LIST: MailboxListResponse = {
|
||||
status: 'ready',
|
||||
mailboxes: [
|
||||
...SHORT,
|
||||
...PROJECTS,
|
||||
...CLIENTS,
|
||||
...NOTIFICATIONS,
|
||||
...NEWSLETTERS,
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,104 @@
|
||||
// Temporary mock data for dashboard responsive testing
|
||||
import { DashboardStats } from './api';
|
||||
|
||||
const NOW = Date.now();
|
||||
const DAY = 86400000;
|
||||
|
||||
export const MOCK_DASHBOARD_STATS: DashboardStats = {
|
||||
account_count: 12,
|
||||
email_count: 145892,
|
||||
attachment_count: 34201,
|
||||
total_size_bytes: 128_849_018_880, // ~120 GB logical
|
||||
storage_usage_bytes: 85_899_345_920, // ~80 GB blob
|
||||
index_usage_bytes: 12_884_901_888, // ~12 GB index
|
||||
recent_activity: Array.from({ length: 30 }, (_, i) => ({
|
||||
timestamp_ms: NOW - (29 - i) * DAY,
|
||||
count: Math.floor(Math.random() * 2000) + 200,
|
||||
})),
|
||||
top_senders: [
|
||||
{ key: 'alexander.hamilton@verylongemaildomain-truncation-test.com', count: 4523 },
|
||||
{ key: 'noreply@github-enterprise-notifications.system.example.org', count: 3891 },
|
||||
{ key: 'jane.doe+project-alpha-beta-gamma@company-with-long-name.io', count: 3102 },
|
||||
{ key: 'newsletter-subscriptions@really-long-marketing-domain.co.uk', count: 2845 },
|
||||
{ key: 'support-tickets+priority-high@helpdesk.corporate.example.com', count: 2100 },
|
||||
{ key: 'bot-pipeline-ci-cd-failures@devops.internal.long-subdomain.net', count: 1789 },
|
||||
{ key: 'short@s.dev', count: 1500 },
|
||||
{ key: 'alerts-monitoring-productions-east-us@observability-platform.com', count: 1256 },
|
||||
{ key: 'weekly-digest-no-reply@newsletter.huge-media-conglomerate.org', count: 980 },
|
||||
{ key: 'invitations-events-calendar-reminders@social-network-app.io', count: 760 },
|
||||
],
|
||||
top_accounts: [
|
||||
{ key: 'primary.work.mailbox@enterprise-long-domain-name.com', count: 78500 },
|
||||
{ key: 'personal.archive+all@very-lengthy-personal-domain.me', count: 42300 },
|
||||
{ key: 'secondary.backup@another-extremely-long-domain.co', count: 15100 },
|
||||
{ key: 'team-leads@department-of-engineering.corp.example.org', count: 8992 },
|
||||
{ key: 'short@x.co', count: 1000 },
|
||||
],
|
||||
with_attachment_count: 18500,
|
||||
without_attachment_count: 15701,
|
||||
top_largest_emails: [
|
||||
{
|
||||
id: 'msg-001',
|
||||
subject: 'RE: [EXTERNAL] Q4 Financial Reports & Budget Planning Documents for Review - Please Provide Feedback by EOD Friday with Department Head Sign-off Required',
|
||||
size_bytes: 52_428_800,
|
||||
},
|
||||
{
|
||||
id: 'msg-002',
|
||||
subject: 'Fwd: Urgent: Client Presentation Draft - Version 7 Final (With Legal Team Amendments and Compliance Review Attached)',
|
||||
size_bytes: 48_234_496,
|
||||
},
|
||||
{
|
||||
id: 'msg-003',
|
||||
subject: 'Meeting Minutes: Cross-Functional Architecture Review Session - Microservices Migration Strategy and Timeline Discussion (Part 3 of 5)',
|
||||
size_bytes: 41_943_040,
|
||||
},
|
||||
{
|
||||
id: 'msg-004',
|
||||
subject: 'Invoice #INV-2026-04582 - Professional Services Engagement: Cloud Infrastructure Assessment and Remediation Planning Phase II Deliverables',
|
||||
size_bytes: 38_797_312,
|
||||
},
|
||||
{
|
||||
id: 'msg-005',
|
||||
subject: '[ACTION REQUIRED] Security Incident Response: Post-Mortem Analysis and Remediation Steps for CVE-2026-12345 - Department-Wide Mandatory Review',
|
||||
size_bytes: 35_651_584,
|
||||
},
|
||||
{
|
||||
id: 'msg-006',
|
||||
subject: 'Monthly Newsletter: Engineering Blog Digest - Articles on Distributed Systems, Rust Async Runtime Internals, and Performance Optimization Techniques',
|
||||
size_bytes: 31_457_280,
|
||||
},
|
||||
{
|
||||
id: 'msg-007',
|
||||
subject: 'Contract Review: Master Service Agreement Amendment #7 with Third-Party Vendor Integration Services for Payment Processing Platform',
|
||||
size_bytes: 28_311_552,
|
||||
},
|
||||
{
|
||||
id: 'msg-008',
|
||||
subject: 'Travel Itinerary & Expense Report: International Conference on Systems Programming - Accommodation, Flight, and Per Diem Documentation Package',
|
||||
size_bytes: 25_165_824,
|
||||
},
|
||||
{
|
||||
id: 'msg-009',
|
||||
subject: 'Re: [INTERNAL] Employee Onboarding Documentation Package - Benefits Enrollment, Tax Forms, Direct Deposit Setup, and IT Access Request Forms Bundle',
|
||||
size_bytes: 22_020_096,
|
||||
},
|
||||
{
|
||||
id: 'msg-010',
|
||||
subject: 'Data Export Request: Complete Transaction History 2024-2026 with Audit Trail and Compliance Certification for External Regulatory Review Board',
|
||||
size_bytes: 18_874_368,
|
||||
},
|
||||
],
|
||||
top_largest_attachments: [
|
||||
{ id: 'att-001', name: 'Q4_2025_Financial_Statements_Audited_with_Supporting_Schedules_and_Notes_v3_FINAL.xlsx', size_bytes: 45_254_100 },
|
||||
{ id: 'att-002', name: 'project_deliverables_package_phase_2_with_test_results_coverage_report_and_deployment_guide.zip', size_bytes: 38_900_500 },
|
||||
{ id: 'att-003', name: '2026-01-15_production_database_backup_full_with_transaction_logs_and_stored_procedures.sql.gz', size_bytes: 35_200_000 },
|
||||
{ id: 'att-004', name: 'client_presentation_deck_v7_final_approved_with_speaker_notes_and_embedded_video_demo.pptx', size_bytes: 31_000_000 },
|
||||
{ id: 'att-005', name: 'system_architecture_diagrams_microservices_v2_with_sequence_flows_and_deployment_topology.pdf', size_bytes: 28_500_000 },
|
||||
{ id: 'att-006', name: 'annual_company_event_photograph_high_resolution_group_photo_panorama_2026.jpg', size_bytes: 25_000_000 },
|
||||
{ id: 'att-007', name: 'complete_source_code_archive_feature_branch_refactor_auth_module_2026_01_15.tar.gz', size_bytes: 22_800_000 },
|
||||
{ id: 'att-008', name: 'product_demo_screencast_walkthrough_new_features_2026_release_candidate.mp4', size_bytes: 19_500_000 },
|
||||
{ id: 'att-009', name: 'legal_contract_review_package_with_redlined_amendments_and_counsel_opinion_letters.pdf', size_bytes: 16_200_000 },
|
||||
{ id: 'att-010', name: 'employee_training_module_compliance_and_security_awareness_2026_v2_interactive.iso', size_bytes: 12_800_000 },
|
||||
],
|
||||
system_version: '1.0.1',
|
||||
};
|
||||
@@ -31,7 +31,7 @@ const EmailIframe: React.FC<EmailIframeProps> = ({ emailHtml, height }) => {
|
||||
return (
|
||||
<iframe
|
||||
src={iframeSrc}
|
||||
sandbox="allow-scripts"
|
||||
sandbox=""
|
||||
className="w-full border-none"
|
||||
title="Email Content"
|
||||
style={{ height: height ?? '4000px' }}
|
||||
|
||||
@@ -202,7 +202,7 @@ const Sidebar = React.forwardRef<
|
||||
<SheetContent
|
||||
data-sidebar='sidebar'
|
||||
data-mobile='true'
|
||||
className='w-[--sidebar-width] bg-sidebar p-0 text-sidebar-foreground [&>button]:hidden'
|
||||
className='w-[--sidebar-width] !bg-sidebar p-0 text-sidebar-foreground [&>button]:hidden'
|
||||
style={
|
||||
{
|
||||
'--sidebar-width': SIDEBAR_WIDTH_MOBILE,
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { getAccountSchema } from '../schema'
|
||||
|
||||
const t = (key: string) => key
|
||||
|
||||
const baseData = {
|
||||
email: 'test@example.com',
|
||||
imap: {
|
||||
host: 'imap.example.com',
|
||||
port: 993,
|
||||
encryption: 'Ssl' as const,
|
||||
auth: {
|
||||
auth_type: 'Password' as const,
|
||||
password: 'mypassword',
|
||||
},
|
||||
},
|
||||
enabled: true,
|
||||
use_dangerous: false,
|
||||
download_interval_min: 60,
|
||||
download_batch_size: 30,
|
||||
auto_download_new_mailboxes: true,
|
||||
}
|
||||
|
||||
describe('Account Schema - date_since validation', () => {
|
||||
const schema = getAccountSchema(false, t)
|
||||
|
||||
it('accepts fixed date_since', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_since: { fixed: '2024-01-01' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts relative date_since', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_since: { relative: { unit: 'Months', value: 6 } },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts undefined date_since', () => {
|
||||
const result = schema.safeParse(baseData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects relative date_since with value 0', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_since: { relative: { unit: 'Months', value: 0 } },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects relative date_since with negative value', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_since: { relative: { unit: 'Months', value: -1 } },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects relative date_since with non-integer value', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_since: { relative: { unit: 'Months', value: 1.5 } },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects fixed date_since with empty string', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_since: { fixed: '' },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Account Schema - date_before validation', () => {
|
||||
const schema = getAccountSchema(false, t)
|
||||
|
||||
it('accepts valid date_before', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_before: { unit: 'Days', value: 30 },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts undefined date_before', () => {
|
||||
const result = schema.safeParse(baseData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects date_before with value 0', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
date_before: { unit: 'Days', value: 0 },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Account Schema - use_dangerous and enabled flags', () => {
|
||||
const schema = getAccountSchema(false, t)
|
||||
|
||||
it('accepts use_dangerous: true', () => {
|
||||
const result = schema.safeParse({ ...baseData, use_dangerous: true })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts enabled: false', () => {
|
||||
const result = schema.safeParse({ ...baseData, enabled: false })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts auto_download_new_mailboxes: false', () => {
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
auto_download_new_mailboxes: false,
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Account Schema - missing required nested fields', () => {
|
||||
const schema = getAccountSchema(false, t)
|
||||
|
||||
it('rejects missing imap entirely', () => {
|
||||
const { imap, ...noImap } = baseData
|
||||
const result = schema.safeParse(noImap)
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects missing imap.auth', () => {
|
||||
const { auth, ...noAuth } = baseData.imap
|
||||
const result = schema.safeParse({
|
||||
...baseData,
|
||||
imap: noAuth,
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,332 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { getAccountSchema, getAuthConfigSchema } from '../schema'
|
||||
|
||||
const t = (key: string) => key
|
||||
|
||||
const validAccountData = {
|
||||
email: 'user@example.com',
|
||||
imap: {
|
||||
host: 'imap.example.com',
|
||||
port: 993,
|
||||
encryption: 'Ssl' as const,
|
||||
auth: {
|
||||
auth_type: 'Password' as const,
|
||||
password: 'mypassword',
|
||||
},
|
||||
},
|
||||
enabled: true,
|
||||
use_dangerous: false,
|
||||
download_interval_min: 60,
|
||||
download_batch_size: 30,
|
||||
auto_download_new_mailboxes: true,
|
||||
}
|
||||
|
||||
describe('Account Form Schema', () => {
|
||||
describe('email field', () => {
|
||||
const schema = getAccountSchema(false, t)
|
||||
|
||||
it('rejects empty email', () => {
|
||||
const result = schema.safeParse({ ...validAccountData, email: '' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects invalid email format', () => {
|
||||
const result = schema.safeParse({
|
||||
...validAccountData,
|
||||
email: 'not-an-email',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects email without @', () => {
|
||||
const result = schema.safeParse({
|
||||
...validAccountData,
|
||||
email: 'username',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid email', () => {
|
||||
const result = schema.safeParse(validAccountData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('imap.host field', () => {
|
||||
it('rejects empty IMAP host', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, host: '' },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid hostname', () => {
|
||||
const result = getAccountSchema(false, t).safeParse(validAccountData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts IP address as host', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, host: '192.168.1.1' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('imap.port field', () => {
|
||||
it('accepts port 993 (standard IMAP SSL)', () => {
|
||||
const result = getAccountSchema(false, t).safeParse(validAccountData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts port 143 (standard IMAP)', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, port: 143 },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts port 0 (auto-detect)', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, port: 0 },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects negative port', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, port: -1 },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects port > 65535', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, port: 99999 },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects non-integer port', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, port: 993.5 },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('imap.encryption field', () => {
|
||||
it('accepts Ssl', () => {
|
||||
const result = getAccountSchema(false, t).safeParse(validAccountData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts StartTls', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, encryption: 'StartTls' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts None', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, encryption: 'None' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects invalid encryption value', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
imap: { ...validAccountData.imap, encryption: 'TLS' },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('download_interval_min field', () => {
|
||||
it('rejects value less than 10', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_interval_min: 5,
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts value of exactly 10', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_interval_min: 10,
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects non-integer value', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_interval_min: 30.5,
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('download_batch_size field', () => {
|
||||
it('rejects value less than 10', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_batch_size: 5,
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects value greater than 200', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_batch_size: 500,
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts value of exactly 10', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_batch_size: 10,
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts value of exactly 200', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_batch_size: 200,
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('account_name and login_name fields', () => {
|
||||
it('accepts undefined account_name and login_name', () => {
|
||||
const result = getAccountSchema(false, t).safeParse(validAccountData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts provided account_name', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
account_name: 'My Work Email',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts provided login_name', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
login_name: 'username',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('download_schedule field', () => {
|
||||
it('accepts undefined download_schedule', () => {
|
||||
const result = getAccountSchema(false, t).safeParse(validAccountData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts valid 6-field cron expression', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_schedule: '0 0 0 * * *',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts cron with */step syntax', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_schedule: '0 */30 8-17 * * 1-5',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects cron with too few fields', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_schedule: '0 0 *',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts empty string cron (treated as not set)', () => {
|
||||
const result = getAccountSchema(false, t).safeParse({
|
||||
...validAccountData,
|
||||
download_schedule: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('Auth Config Schema (password validation)', () => {
|
||||
describe('when creating (isEdit = false)', () => {
|
||||
const schema = getAuthConfigSchema(false, t)
|
||||
|
||||
it('requires password when auth_type is Password', () => {
|
||||
const result = schema.safeParse({
|
||||
auth_type: 'Password',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('requires password when auth_type is Password and password undefined', () => {
|
||||
const result = schema.safeParse({
|
||||
auth_type: 'Password',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid password with Password auth', () => {
|
||||
const result = schema.safeParse({
|
||||
auth_type: 'Password',
|
||||
password: 'mypassword',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('does not require password when auth_type is OAuth2', () => {
|
||||
const result = schema.safeParse({
|
||||
auth_type: 'OAuth2',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('when editing (isEdit = true)', () => {
|
||||
const schema = getAuthConfigSchema(true, t)
|
||||
|
||||
it('does not require password even with Password auth', () => {
|
||||
const result = schema.safeParse({
|
||||
auth_type: 'Password',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts with undefined password', () => {
|
||||
const result = schema.safeParse({
|
||||
auth_type: 'Password',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -133,8 +133,8 @@ export function AccountDetailDrawer({ open, onOpenChange, currentRow }: Props) {
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="text-muted-foreground">{t('accounts.folderLimit')}:</span>
|
||||
<span>{currentRow.folder_limit ? currentRow.folder_limit : t('accounts.notAvailable')}</span>
|
||||
<span className="text-muted-foreground">{t('accounts.downloadSchedule')}:</span>
|
||||
<span>{currentRow.download_schedule || t('accounts.notAvailable')}</span>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
import { zodResolver } from '@hookform/resolvers/zod';
|
||||
import * as React from 'react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { z } from 'zod';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Form } from '@/components/ui/form';
|
||||
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog';
|
||||
@@ -35,112 +34,9 @@ import { ToastAction } from '@/components/ui/toast';
|
||||
import { AxiosError } from 'axios';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { cn } from "@/lib/utils";
|
||||
import { getAccountSchema, type AccountFormValues } from './schema';
|
||||
|
||||
const encryptionSchema = z.union([
|
||||
z.literal('Ssl'),
|
||||
z.literal('StartTls'),
|
||||
z.literal('None'),
|
||||
]);
|
||||
|
||||
const authTypeSchema = z.union([
|
||||
z.literal('Password'),
|
||||
z.literal('OAuth2'),
|
||||
]);
|
||||
|
||||
const getAuthConfigSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z.object({
|
||||
auth_type: authTypeSchema,
|
||||
password: z.string().optional(),
|
||||
}).refine(
|
||||
(data) => {
|
||||
if (data.auth_type === 'Password' && !isEdit) {
|
||||
return !!data.password?.trim();
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{
|
||||
message: t('validation.passwordRequired'),
|
||||
path: ['password'],
|
||||
}
|
||||
);
|
||||
|
||||
const getImapConfigSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z.object({
|
||||
host: z.string({ required_error: t('validation.imapHostRequired') }).min(1, { message: t('validation.imapHostCannotBeEmpty') }),
|
||||
port: z.number().int().min(0, { message: t('validation.imapPortMustBePositive') }).max(65535, { message: t('validation.imapPortMustBeLessThan65536') }),
|
||||
encryption: encryptionSchema,
|
||||
auth: getAuthConfigSchema(isEdit, t),
|
||||
use_proxy: z.number().optional(),
|
||||
});
|
||||
|
||||
const getRelativeDateSchema = (t: (key: string) => string) => z.object({
|
||||
unit: z.enum(["Days", "Months", "Years"], { message: t('accounts.selectUnit') }),
|
||||
value: z.number({ message: t('accounts.enterValue') }).int().min(1, t('accounts.mustBeAtLeast1')),
|
||||
});
|
||||
|
||||
const getDateSelectionSchema = (t: (key: string) => string) => z.union([
|
||||
z.object({ fixed: z.string({ message: t('accounts.selectDate') }) }),
|
||||
z.object({ relative: getRelativeDateSchema(t) }),
|
||||
z.undefined(),
|
||||
]);
|
||||
|
||||
export type Account = {
|
||||
login_name?: string;
|
||||
account_name?: string;
|
||||
email: string;
|
||||
imap: {
|
||||
host: string;
|
||||
port: number;
|
||||
encryption: 'Ssl' | 'StartTls' | 'None';
|
||||
auth: {
|
||||
auth_type: 'Password' | 'OAuth2';
|
||||
password?: string;
|
||||
};
|
||||
use_proxy?: number;
|
||||
};
|
||||
enabled: boolean;
|
||||
use_dangerous: boolean;
|
||||
date_since?: {
|
||||
fixed?: string;
|
||||
relative?: {
|
||||
unit?: 'Days' | 'Months' | 'Years';
|
||||
value?: number;
|
||||
};
|
||||
};
|
||||
date_before?: {
|
||||
unit?: 'Days' | 'Months' | 'Years';
|
||||
value?: number;
|
||||
};
|
||||
folder_limit?: number;
|
||||
download_interval_min: number;
|
||||
download_batch_size: number;
|
||||
auto_download_new_mailboxes: boolean;
|
||||
};
|
||||
|
||||
const getAccountSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z.object({
|
||||
account_name: z.string().optional(),
|
||||
login_name: z.string().optional(),
|
||||
email: z.string({ required_error: t('validation.emailRequired') }).email({ message: t('validation.invalidEmail') }),
|
||||
imap: getImapConfigSchema(isEdit, t),
|
||||
enabled: z.boolean(),
|
||||
use_dangerous: z.boolean(),
|
||||
date_since: getDateSelectionSchema(t).optional(),
|
||||
date_before: getRelativeDateSchema(t).optional(),
|
||||
folder_limit: z
|
||||
.number({ invalid_type_error: t('validation.folderLimitMustBeNumber') })
|
||||
.int()
|
||||
.min(100, { message: t('validation.folderLimitMustBeAtLeast100') })
|
||||
.nullable()
|
||||
.optional(),
|
||||
download_interval_min: z.number({ invalid_type_error: t('validation.incrementalSyncMustBeNumber') }).int().min(10, { message: t('validation.incrementalSyncMustBeAtLeast10') }),
|
||||
download_batch_size: z
|
||||
.number({ invalid_type_error: t('validation.singleRequestBatchSizeMustBeNumber') })
|
||||
.int()
|
||||
.min(10, { message: t('validation.singleRequestBatchSizeTooSmall') })
|
||||
.max(200, { message: t('validation.singleRequestBatchSizeTooLarge') }),
|
||||
auto_download_new_mailboxes: z.boolean(),
|
||||
});
|
||||
export type Account = AccountFormValues;
|
||||
|
||||
type Step = {
|
||||
id: `step-${number}`;
|
||||
@@ -153,7 +49,7 @@ export type Steps = [...Step[]];
|
||||
const getSteps = (t: (key: string) => string): Steps => [
|
||||
{ id: "step-1", name: t('accounts.steps.emailAddress'), fields: ["email", "account_name"] },
|
||||
{ id: "step-2", name: t('accounts.steps.imap'), fields: ["imap", "use_dangerous", "login_name"] },
|
||||
{ id: "step-3", name: t('accounts.steps.syncPreferences'), fields: ["enabled", "date_since", "date_before", "folder_limit", "download_interval_min", "download_batch_size", "auto_download_new_mailboxes"] },
|
||||
{ id: "step-3", name: t('accounts.steps.syncPreferences'), fields: ["enabled", "date_since", "date_before", "download_interval_min", "download_batch_size", "auto_download_new_mailboxes", "download_schedule"] },
|
||||
{ id: "step-4", name: t('accounts.steps.summary'), fields: [] },
|
||||
];
|
||||
|
||||
@@ -183,10 +79,10 @@ const defaultValues: Account = {
|
||||
use_dangerous: false,
|
||||
date_since: undefined,
|
||||
date_before: undefined,
|
||||
folder_limit: undefined,
|
||||
download_interval_min: 60,
|
||||
download_batch_size: 30,
|
||||
auto_download_new_mailboxes: true,
|
||||
download_schedule: undefined,
|
||||
};
|
||||
|
||||
const emptyImap: ImapConfig = {
|
||||
@@ -205,6 +101,7 @@ const mapCurrentRowToFormValues = (currentRow: AccountModel): Account => {
|
||||
}
|
||||
|
||||
return {
|
||||
account_name: currentRow.account_name ?? undefined,
|
||||
login_name: currentRow.login_name ?? undefined,
|
||||
email: currentRow.email,
|
||||
imap,
|
||||
@@ -212,10 +109,10 @@ const mapCurrentRowToFormValues = (currentRow: AccountModel): Account => {
|
||||
use_dangerous: currentRow.use_dangerous,
|
||||
date_since: currentRow.date_since ?? undefined,
|
||||
date_before: currentRow.date_before ?? undefined,
|
||||
folder_limit: currentRow.folder_limit ?? undefined,
|
||||
download_interval_min: currentRow.download_interval_min ?? 60,
|
||||
download_batch_size: currentRow.download_batch_size ?? 30,
|
||||
auto_download_new_mailboxes: currentRow.auto_download_new_mailboxes ?? true,
|
||||
download_schedule: currentRow.download_schedule ?? undefined,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -294,18 +191,18 @@ export function AccountActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
use_dangerous: data.use_dangerous,
|
||||
date_since: data.date_since,
|
||||
date_before: data.date_before,
|
||||
folder_limit: data.folder_limit,
|
||||
download_interval_min: data.download_interval_min,
|
||||
download_batch_size: data.download_batch_size,
|
||||
auto_download_new_mailboxes: data.auto_download_new_mailboxes,
|
||||
download_schedule: data.download_schedule || null,
|
||||
};
|
||||
if (isEdit) {
|
||||
const isAllMode = !data.date_since && !data.date_before;
|
||||
const clear_folder_limit = !data.folder_limit;
|
||||
const clear_download_schedule = !data.download_schedule && currentRow?.download_schedule;
|
||||
updateMutation.mutate({
|
||||
...commonData,
|
||||
...(isAllMode ? { clear_date_range: true } : {}),
|
||||
...(clear_folder_limit ? { clear_folder_limit: true } : {})
|
||||
...(clear_download_schedule ? { clear_download_schedule: true } : {})
|
||||
});
|
||||
} else {
|
||||
createMutation.mutate({ ...commonData, account_type: "IMAP" });
|
||||
|
||||
@@ -106,9 +106,12 @@ export function useColumns(): ColumnDef<AccountModel>[] {
|
||||
if (account_type === "NoSync") {
|
||||
return <LongText className="text-center">n/a</LongText>
|
||||
}
|
||||
if (row.original.download_schedule) {
|
||||
return <LongText className="text-center">{row.original.download_schedule}</LongText>
|
||||
}
|
||||
return <LongText className="text-center">{row.original.download_interval_min} min</LongText>
|
||||
},
|
||||
meta: { className: 'text-center max-w-[120px]' },
|
||||
meta: { className: 'text-center max-w-[160px]' },
|
||||
enableHiding: false,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -98,26 +98,6 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align='end' className='w-[220px]'>
|
||||
|
||||
{showDownload && (
|
||||
<DropdownMenuItem onClick={handleStartDownload}>
|
||||
{t('accounts.startDownload')}
|
||||
<DropdownMenuShortcut>
|
||||
<IconPlayerPlay size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
|
||||
|
||||
{showDownload && (
|
||||
<DropdownMenuItem onClick={handleCancelDownload}>
|
||||
{t('accounts.cancelDownload')}
|
||||
<DropdownMenuShortcut>
|
||||
<IconPlayerStop size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{showDownload && <DropdownMenuSeparator />}
|
||||
{hasPermission && <DropdownMenuItem
|
||||
onClick={() => {
|
||||
setCurrentRow(row.original)
|
||||
@@ -169,6 +149,27 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) {
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>}
|
||||
{hasPermission && <DropdownMenuSeparator />}
|
||||
|
||||
{showDownload && (
|
||||
<DropdownMenuItem onClick={handleStartDownload}>
|
||||
{t('accounts.startDownload')}
|
||||
<DropdownMenuShortcut>
|
||||
<IconPlayerPlay size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
|
||||
|
||||
{showDownload && (
|
||||
<DropdownMenuItem onClick={handleCancelDownload}>
|
||||
{t('accounts.cancelDownload')}
|
||||
<DropdownMenuShortcut>
|
||||
<IconPlayerStop size={16} />
|
||||
</DropdownMenuShortcut>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{showDownload && <DropdownMenuSeparator />}
|
||||
|
||||
{hasPermission && <DropdownMenuItem
|
||||
onClick={() => {
|
||||
setCurrentRow(row.original)
|
||||
|
||||
@@ -31,13 +31,14 @@ import { Loader2, CheckSquare, Square } from 'lucide-react'
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import { toast } from '@/hooks/use-toast'
|
||||
import { list_mailboxes, MailboxData } from '@/api/mailbox/api'
|
||||
//import { MOCK_MAILBOX_LIST } from '@/api/mailbox/mock-mailboxes'
|
||||
import { buildTree, ExtendedTreeItemProps } from '@/lib/build-tree'
|
||||
import { Skeleton } from '@/components/ui/skeleton'
|
||||
import { AccountModel, update_account } from '@/api/account/api'
|
||||
import { ToastAction } from '@/components/ui/toast'
|
||||
import axios, { AxiosError } from 'axios'
|
||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { ScrollArea } from '@/components/ui/scroll-area'
|
||||
import { RichTreeView } from '@mui/x-tree-view/RichTreeView';
|
||||
import { useTheme } from '@/context/theme-context'
|
||||
import React from 'react'
|
||||
@@ -108,7 +109,7 @@ function CustomLabel({
|
||||
</div>
|
||||
{exists !== undefined && (
|
||||
<span
|
||||
className="text-sm opacity-60 min-w-[40px] text-right text-inherit"
|
||||
className="text-sm opacity-60 min-w-[40px] text-right text-inherit mr-4"
|
||||
>
|
||||
{exists}
|
||||
</span>
|
||||
@@ -181,7 +182,7 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props)
|
||||
setItemsWithChildren(itemsWithChildren);
|
||||
setExpandedItems(itemsWithChildren);
|
||||
const download_folders = data
|
||||
.filter(mailbox => currentRow.download_folders.includes(mailbox.name))
|
||||
.filter(mailbox => currentRow.download_folders?.includes(mailbox.name))
|
||||
.map(mailbox => mailbox.id.toString());
|
||||
setSelectedItems(download_folders);
|
||||
};
|
||||
@@ -404,7 +405,7 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props)
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className="sm:max-w-3xl max-h-[90vh] flex flex-col">
|
||||
<DialogContent className="max-w-[95vw] sm:max-w-3xl max-h-[90vh] flex flex-col overflow-hidden">
|
||||
<DialogHeader className="flex-shrink-0">
|
||||
<DialogTitle>{t('accounts.selectMailboxes')}</DialogTitle>
|
||||
<DialogDescription>
|
||||
@@ -412,8 +413,8 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props)
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div className="flex flex-col pt-2 gap-2">
|
||||
<div className="flex-1 min-h-0 grid gap-4" style={{ gridTemplateRows: 'auto 1fr' }}>
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex gap-2 flex-wrap">
|
||||
<Button
|
||||
variant="outline"
|
||||
@@ -479,7 +480,7 @@ export function DownloadFoldersDialog({ currentRow, open, onOpenChange }: Props)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ScrollArea className="h-[32rem] flex-1 min-h-0 w-full pr-4 -mr-4 py-1">
|
||||
<ScrollArea className="min-h-0 w-full py-1">
|
||||
{isLoading && (
|
||||
<div className="p-8 space-y-8">
|
||||
<div className="flex flex-col items-center gap-3 text-muted-foreground">
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
const encryptionSchema = z.union([
|
||||
z.literal('Ssl'),
|
||||
z.literal('StartTls'),
|
||||
z.literal('None'),
|
||||
])
|
||||
|
||||
const authTypeSchema = z.union([
|
||||
z.literal('Password'),
|
||||
z.literal('OAuth2'),
|
||||
])
|
||||
|
||||
export const getAuthConfigSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z
|
||||
.object({
|
||||
auth_type: authTypeSchema,
|
||||
password: z.string().optional(),
|
||||
})
|
||||
.refine(
|
||||
(data) => {
|
||||
if (data.auth_type === 'Password' && !isEdit) {
|
||||
return !!data.password?.trim()
|
||||
}
|
||||
return true
|
||||
},
|
||||
{
|
||||
message: t('validation.passwordRequired'),
|
||||
path: ['password'],
|
||||
}
|
||||
)
|
||||
|
||||
export const getImapConfigSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z.object({
|
||||
host: z
|
||||
.string({ required_error: t('validation.imapHostRequired') })
|
||||
.min(1, { message: t('validation.imapHostCannotBeEmpty') }),
|
||||
port: z
|
||||
.number()
|
||||
.int()
|
||||
.min(0, { message: t('validation.imapPortMustBePositive') })
|
||||
.max(65535, { message: t('validation.imapPortMustBeLessThan65536') }),
|
||||
encryption: encryptionSchema,
|
||||
auth: getAuthConfigSchema(isEdit, t),
|
||||
use_proxy: z.number().optional(),
|
||||
})
|
||||
|
||||
const relativeDateSchema = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
unit: z.enum(['Days', 'Months', 'Years'], {
|
||||
message: t('accounts.selectUnit'),
|
||||
}),
|
||||
value: z
|
||||
.number({ message: t('accounts.enterValue') })
|
||||
.int()
|
||||
.min(1, t('accounts.mustBeAtLeast1')),
|
||||
})
|
||||
|
||||
const dateSelectionSchema = (t: (key: string) => string) =>
|
||||
z
|
||||
.object({
|
||||
fixed: z
|
||||
.string({ message: t('accounts.selectDate') })
|
||||
.min(1, { message: t('accounts.selectDate') })
|
||||
.optional(),
|
||||
relative: relativeDateSchema(t).optional(),
|
||||
})
|
||||
.optional()
|
||||
|
||||
export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) =>
|
||||
z.object({
|
||||
account_name: z.string().optional(),
|
||||
login_name: z.string().optional(),
|
||||
email: z
|
||||
.string({ required_error: t('validation.emailRequired') })
|
||||
.email({ message: t('validation.invalidEmail') }),
|
||||
imap: getImapConfigSchema(isEdit, t),
|
||||
enabled: z.boolean(),
|
||||
use_dangerous: z.boolean(),
|
||||
date_since: dateSelectionSchema(t).optional(),
|
||||
date_before: relativeDateSchema(t).optional(),
|
||||
download_interval_min: z
|
||||
.number({
|
||||
invalid_type_error: t('validation.incrementalSyncMustBeNumber'),
|
||||
})
|
||||
.int()
|
||||
.min(10, {
|
||||
message: t('validation.incrementalSyncMustBeAtLeast10'),
|
||||
}),
|
||||
download_batch_size: z
|
||||
.number({
|
||||
invalid_type_error: t(
|
||||
'validation.singleRequestBatchSizeMustBeNumber'
|
||||
),
|
||||
})
|
||||
.int()
|
||||
.min(10, {
|
||||
message: t('validation.singleRequestBatchSizeTooSmall'),
|
||||
})
|
||||
.max(200, {
|
||||
message: t('validation.singleRequestBatchSizeTooLarge'),
|
||||
}),
|
||||
auto_download_new_mailboxes: z.boolean(),
|
||||
download_schedule: z
|
||||
.string()
|
||||
.optional()
|
||||
.refine(
|
||||
(val) => {
|
||||
if (!val || val.trim() === '') return true;
|
||||
const fields = val.trim().split(/\s+/);
|
||||
if (fields.length < 6) return false;
|
||||
return true;
|
||||
},
|
||||
{ message: t('validation.invalidCronExpression') }
|
||||
),
|
||||
})
|
||||
|
||||
export type AccountFormValues = z.infer<
|
||||
ReturnType<typeof getAccountSchema>
|
||||
>
|
||||
@@ -48,6 +48,72 @@ import i18n from "@/i18n";
|
||||
|
||||
|
||||
type SyncMode = 'all' | 'since_fixed' | 'since_relative' | 'before_relative';
|
||||
type ScheduleMode = 'interval' | 'cron';
|
||||
type CronMode = 'simple' | 'advanced';
|
||||
type CronFrequency = 'daily' | 'weekly' | 'monthly';
|
||||
|
||||
interface CronSimpleState {
|
||||
frequency: CronFrequency;
|
||||
hour: number;
|
||||
minute: number;
|
||||
dayOfWeek: number;
|
||||
dayOfMonth: number;
|
||||
}
|
||||
|
||||
const DEFAULT_CRON_SIMPLE: CronSimpleState = {
|
||||
frequency: 'daily',
|
||||
hour: 0,
|
||||
minute: 0,
|
||||
dayOfWeek: 1,
|
||||
dayOfMonth: 1,
|
||||
};
|
||||
|
||||
function buildCronFromSimple(s: CronSimpleState): string {
|
||||
switch (s.frequency) {
|
||||
case 'daily':
|
||||
return `0 ${s.minute} ${s.hour} * * *`;
|
||||
case 'weekly':
|
||||
return `0 ${s.minute} ${s.hour} * * ${s.dayOfWeek}`;
|
||||
case 'monthly':
|
||||
return `0 ${s.minute} ${s.hour} ${s.dayOfMonth} * *`;
|
||||
}
|
||||
}
|
||||
|
||||
function tryParseCronToSimple(cron: string): CronSimpleState | null {
|
||||
const fields = cron.trim().split(/\s+/);
|
||||
if (fields.length < 6) return null;
|
||||
|
||||
const sec = fields[0];
|
||||
const min = fields[1];
|
||||
const hour = fields[2];
|
||||
const dom = fields[3];
|
||||
const month = fields[4];
|
||||
const dow = fields[5];
|
||||
|
||||
if (sec !== '0') return null;
|
||||
if (month !== '*') return null;
|
||||
|
||||
const minuteVal = parseInt(min, 10);
|
||||
const hourVal = parseInt(hour, 10);
|
||||
if (isNaN(minuteVal) || isNaN(hourVal)) return null;
|
||||
|
||||
if (dom === '*' && dow === '*') {
|
||||
return { frequency: 'daily', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: 1 };
|
||||
}
|
||||
if (dom === '*') {
|
||||
const dowVal = parseInt(dow, 10);
|
||||
if (!isNaN(dowVal)) {
|
||||
return { frequency: 'weekly', hour: hourVal, minute: minuteVal, dayOfWeek: dowVal, dayOfMonth: 1 };
|
||||
}
|
||||
}
|
||||
if (dow === '*') {
|
||||
const domVal = parseInt(dom, 10);
|
||||
if (!isNaN(domVal)) {
|
||||
return { frequency: 'monthly', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: domVal };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function Step3() {
|
||||
const { t } = useTranslation();
|
||||
@@ -61,6 +127,31 @@ export default function Step3() {
|
||||
return 'all';
|
||||
});
|
||||
|
||||
const [scheduleMode, setScheduleMode] = useState<ScheduleMode>(() => {
|
||||
if (current.download_schedule) return 'cron';
|
||||
return 'interval';
|
||||
});
|
||||
|
||||
const [cronMode, setCronMode] = useState<CronMode>(() => {
|
||||
if (current.download_schedule && tryParseCronToSimple(current.download_schedule)) {
|
||||
return 'simple';
|
||||
}
|
||||
if (current.download_schedule) return 'advanced';
|
||||
return 'simple';
|
||||
});
|
||||
|
||||
const [cronSimple, setCronSimple] = useState<CronSimpleState>(() => {
|
||||
if (current.download_schedule) {
|
||||
return tryParseCronToSimple(current.download_schedule) ?? DEFAULT_CRON_SIMPLE;
|
||||
}
|
||||
return DEFAULT_CRON_SIMPLE;
|
||||
});
|
||||
|
||||
const updateCronFromSimple = (partial: Partial<CronSimpleState>) => {
|
||||
const next = { ...cronSimple, ...partial };
|
||||
setCronSimple(next);
|
||||
setValue('download_schedule', buildCronFromSimple(next));
|
||||
};
|
||||
|
||||
const handleModeChange = (mode: SyncMode) => {
|
||||
setSyncMode(mode);
|
||||
@@ -77,41 +168,231 @@ export default function Step3() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleScheduleModeChange = (mode: ScheduleMode) => {
|
||||
setScheduleMode(mode);
|
||||
if (mode === 'interval') {
|
||||
setValue("download_schedule", undefined);
|
||||
} else {
|
||||
setValue("download_interval_min", 60);
|
||||
if (cronMode === 'simple') {
|
||||
updateCronFromSimple(cronSimple);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-8">
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_interval_min"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadInterval')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.downloadIntervalPlaceholder')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
<div className="space-y-4">
|
||||
<FormItem>
|
||||
<FormLabel className="text-base font-semibold">{t('accounts.scheduleMode')}</FormLabel>
|
||||
<FormDescription>
|
||||
{t('accounts.scheduleModeDescription')}
|
||||
</FormDescription>
|
||||
<Select value={scheduleMode} onValueChange={(v) => handleScheduleModeChange(v as ScheduleMode)}>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="interval">{t('accounts.scheduleModeInterval')}</SelectItem>
|
||||
<SelectItem value="cron">{t('accounts.scheduleModeCron')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
{scheduleMode === 'interval' ? (
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_interval_min"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadInterval')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.downloadIntervalPlaceholder')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
) : (
|
||||
<div className="space-y-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{/* <FormLabel className="text-sm font-medium">{t('accounts.downloadSchedule')}</FormLabel> */}
|
||||
<div className="flex items-center rounded-md border text-xs">
|
||||
<button
|
||||
type="button"
|
||||
className={`px-2 py-1 rounded-l-md ${cronMode === 'simple' ? 'bg-primary text-primary-foreground' : 'text-muted-foreground hover:text-foreground'}`}
|
||||
onClick={() => setCronMode('simple')}
|
||||
>
|
||||
{t('accounts.cronSimple')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`px-2 py-1 rounded-r-md ${cronMode === 'advanced' ? 'bg-primary text-primary-foreground' : 'text-muted-foreground hover:text-foreground'}`}
|
||||
onClick={() => setCronMode('advanced')}
|
||||
>
|
||||
{t('accounts.cronAdvanced')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{cronMode === 'simple' ? (
|
||||
<div className="flex flex-wrap items-end gap-3">
|
||||
<FormItem className="w-[140px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronFrequency')}</FormLabel>
|
||||
<Select
|
||||
value={cronSimple.frequency}
|
||||
onValueChange={(v) => updateCronFromSimple({ frequency: v as CronFrequency })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="daily">{t('accounts.cronDaily')}</SelectItem>
|
||||
<SelectItem value="weekly">{t('accounts.cronWeekly')}</SelectItem>
|
||||
<SelectItem value="monthly">{t('accounts.cronMonthly')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<FormItem className="w-[80px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronHour')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.hour)}
|
||||
onValueChange={(v) => updateCronFromSimple({ hour: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{Array.from({ length: 24 }, (_, i) => (
|
||||
<SelectItem key={i} value={String(i)}>
|
||||
{String(i).padStart(2, '0')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
<span className="text-muted-foreground pb-2">:</span>
|
||||
|
||||
<FormItem className="w-[80px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronMinute')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.minute)}
|
||||
onValueChange={(v) => updateCronFromSimple({ minute: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{[0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55].map((m) => (
|
||||
<SelectItem key={m} value={String(m)}>
|
||||
{String(m).padStart(2, '0')}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
|
||||
{cronSimple.frequency === 'weekly' && (
|
||||
<FormItem className="w-[140px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronDayOfWeek')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.dayOfWeek)}
|
||||
onValueChange={(v) => updateCronFromSimple({ dayOfWeek: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="1">{t('accounts.cronMonday')}</SelectItem>
|
||||
<SelectItem value="2">{t('accounts.cronTuesday')}</SelectItem>
|
||||
<SelectItem value="3">{t('accounts.cronWednesday')}</SelectItem>
|
||||
<SelectItem value="4">{t('accounts.cronThursday')}</SelectItem>
|
||||
<SelectItem value="5">{t('accounts.cronFriday')}</SelectItem>
|
||||
<SelectItem value="6">{t('accounts.cronSaturday')}</SelectItem>
|
||||
<SelectItem value="0">{t('accounts.cronSunday')}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
|
||||
{cronSimple.frequency === 'monthly' && (
|
||||
<FormItem className="w-[90px]">
|
||||
<FormLabel className="text-xs">{t('accounts.cronDayOfMonth')}</FormLabel>
|
||||
<Select
|
||||
value={String(cronSimple.dayOfMonth)}
|
||||
onValueChange={(v) => updateCronFromSimple({ dayOfMonth: parseInt(v, 10) })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="max-h-[200px]">
|
||||
{Array.from({ length: 28 }, (_, i) => i + 1).map((d) => (
|
||||
<SelectItem key={d} value={String(d)}>
|
||||
{d}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormItem>
|
||||
)}
|
||||
|
||||
<div className="text-xs text-muted-foreground pb-2 font-mono">
|
||||
= {buildCronFromSimple(cronSimple)}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_schedule"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormControl>
|
||||
<Input
|
||||
{...field}
|
||||
value={field.value ?? ''}
|
||||
placeholder={t('accounts.downloadSchedulePlaceholder')}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormDescription>
|
||||
{t('accounts.downloadScheduleDescription')}
|
||||
</FormDescription>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{cronMode === 'simple' && (
|
||||
<FormDescription>{t('accounts.downloadScheduleDescription')}</FormDescription>
|
||||
)}
|
||||
|
||||
<div className="text-xs text-muted-foreground mt-1 flex items-center gap-1">
|
||||
<span>{t('accounts.cronTimezoneNote')}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_batch_size"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadBatchSize')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.downloadBatchSizeDescription')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={control}
|
||||
name="download_batch_size"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.downloadBatchSize')}</FormLabel>
|
||||
<FormControl>
|
||||
<Input type="number" {...field} onChange={(e) => field.onChange(parseInt(e.target.value, 10))} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
<FormDescription>
|
||||
{t('accounts.downloadBatchSizeDescription')}
|
||||
</FormDescription>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<FormField
|
||||
@@ -249,29 +530,6 @@ export default function Step3() {
|
||||
/>
|
||||
|
||||
<hr className="my-4" />
|
||||
|
||||
<FormField
|
||||
control={control}
|
||||
name="folder_limit"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{t('accounts.folderLimit')}</FormLabel>
|
||||
<FormDescription>{t('accounts.folderLimitDescription')}</FormDescription>
|
||||
<FormControl>
|
||||
<Input
|
||||
type="number"
|
||||
placeholder={t('accounts.folderLimitPlaceholder')}
|
||||
value={field.value ?? ''}
|
||||
onChange={(e) => {
|
||||
const value = e.target.value;
|
||||
field.onChange(value === '' ? null : Number(value));
|
||||
}}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -50,7 +50,7 @@ export default function Step4() {
|
||||
|
||||
return (
|
||||
<div className="rounded-xl">
|
||||
<Accordion type="multiple" defaultValue={['email', 'account_name', 'login_name', 'imap', 'date_since', 'folder_limit', 'sync_interval', 'sync_scope', 'sync_batch_size']}>
|
||||
<Accordion type="multiple" defaultValue={['email', 'account_name', 'login_name', 'imap', 'date_since', 'sync_interval', 'sync_scope', 'sync_batch_size', 'download_schedule']}>
|
||||
<AccordionItem key="email" value="email">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.email')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.email}</AccordionContent>
|
||||
@@ -154,11 +154,6 @@ export default function Step4() {
|
||||
</AccordionItem>
|
||||
|
||||
|
||||
<AccordionItem key="folder_limit" value="folder_limit">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.folderLimit')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.folder_limit ?? t('accounts.notAvailable')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="sync_interval" value="sync_interval">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.downloadInterval')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.download_interval_min} {t('accounts.minutes')}</AccordionContent>
|
||||
@@ -169,6 +164,11 @@ export default function Step4() {
|
||||
<AccordionContent>{summaryData.download_batch_size}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="download_schedule" value="download_schedule">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.downloadSchedule')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.download_schedule || t('accounts.notAvailable')}</AccordionContent>
|
||||
</AccordionItem>
|
||||
|
||||
<AccordionItem key="auto_download_new_mailboxes" value="auto_download_new_mailboxes">
|
||||
<AccordionTrigger className="font-medium capitalize text-gray-600">{t('accounts.autoDownloadNewMailboxes')}:</AccordionTrigger>
|
||||
<AccordionContent>{summaryData.auto_download_new_mailboxes ? t('common.yes') : t('common.no')}</AccordionContent>
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { Loader, Download, Trash2, MessageSquareMore, FileText, FileImage, FileVideo, FileArchive, FileSpreadsheet, FileCode, FileIcon, FileAudio, Upload } from 'lucide-react';
|
||||
import { Loader, Download, Trash2, MessageSquareMore, FileText, FileImage, FileVideo, FileArchive, FileSpreadsheet, FileCode, FileIcon, FileAudio, Upload, ShieldCheck } from 'lucide-react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Separator } from '@/components/ui/separator';
|
||||
@@ -121,6 +121,12 @@ export function MailMessageView({
|
||||
const [nestedEmlFile, setNestedEmlFile] = useState<AttachmentInfo | null>(null);
|
||||
const { getEmailById } = useMinimalAccountList();
|
||||
const [threadOpen, setThreadOpen] = useState(false);
|
||||
const [blockRemote, setBlockRemote] = useState(true);
|
||||
const [hasRemoteContent, setHasRemoteContent] = useState(false);
|
||||
|
||||
const toggleBlockRemote = () => {
|
||||
setBlockRemote((prev) => !prev);
|
||||
};
|
||||
|
||||
const downloadAttachmentMutation = useMutation({
|
||||
mutationFn: ({ content_hash }: { content_hash: string }) =>
|
||||
@@ -137,12 +143,13 @@ export function MailMessageView({
|
||||
});
|
||||
|
||||
const loadMessageMutation = useMutation({
|
||||
mutationFn: () => load_message(envelope.account_id, envelope.id),
|
||||
mutationFn: () => load_message(envelope.account_id, envelope.id, blockRemote),
|
||||
onSuccess: (data) => {
|
||||
setLoading(false);
|
||||
setContent(getContent(data));
|
||||
if (data.attachments) setAttachments(data.attachments);
|
||||
setContentType(data.html ? 'Html' : 'Plain');
|
||||
setHasRemoteContent(!!data.has_remote_content);
|
||||
},
|
||||
onError: (error: any) => {
|
||||
setLoading(false);
|
||||
@@ -154,10 +161,14 @@ export function MailMessageView({
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
setBlockRemote(true);
|
||||
}, [envelope.id]);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
loadMessageMutation.mutate();
|
||||
}, [envelope.id]);
|
||||
}, [envelope.id, blockRemote]);
|
||||
|
||||
|
||||
const handleViewNestedEml = (attachment: AttachmentInfo) => {
|
||||
@@ -376,6 +387,30 @@ export function MailMessageView({
|
||||
</div>
|
||||
)}
|
||||
{showAttachments && <Separator className="mb-2" />}
|
||||
{hasRemoteContent && (
|
||||
<div className="flex items-center justify-between bg-muted border px-3 py-1.5 mb-3 text-xs">
|
||||
<div className="flex items-center gap-1.5 min-w-0">
|
||||
<ShieldCheck className="h-3.5 w-3.5 text-muted-foreground shrink-0" />
|
||||
{blockRemote ? (
|
||||
<span className="text-muted-foreground truncate">
|
||||
{t('mail.remoteBlocked', 'To protect your privacy, Bichon has blocked remote content in this message.')}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-muted-foreground truncate">
|
||||
{t('mail.remoteShown', 'Remote content is now shown.')}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<span
|
||||
className="underline cursor-pointer hover:no-underline text-muted-foreground text-[11px] font-medium shrink-0 ml-2 select-none"
|
||||
onClick={toggleBlockRemote}
|
||||
>
|
||||
{blockRemote
|
||||
? t('mail.showRemoteContent', 'Show remote content')
|
||||
: t('mail.blockRemoteAgain', 'Block again')}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex-1 overflow-auto">
|
||||
{loading ? (
|
||||
<div className="flex justify-center items-center py-8">
|
||||
|
||||
@@ -150,7 +150,7 @@ export function NestedEmailDialog({ open, onOpenChange }: any) {
|
||||
|
||||
const { data, isLoading } = useQuery({
|
||||
queryKey: ['nested-message', currentAttachment?.account_id!, currentAttachment?.envelope_id!, currentAttachment?.content_hash!],
|
||||
queryFn: () => load_nested_message(currentAttachment?.account_id!, currentAttachment?.envelope_id!, currentAttachment?.content_hash!),
|
||||
queryFn: () => load_nested_message(currentAttachment?.account_id!, currentAttachment?.envelope_id!, currentAttachment?.content_hash!, true),
|
||||
enabled: open && !!currentAttachment,
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { getFormSchema } from '../schema'
|
||||
|
||||
// Simple mock t function that returns the key
|
||||
const t = (key: string, _options?: Record<string, any>) => key
|
||||
|
||||
describe('Login Form Schema', () => {
|
||||
const schema = getFormSchema(t)
|
||||
|
||||
describe('username field', () => {
|
||||
it('rejects empty username', () => {
|
||||
const result = schema.safeParse({ username: '', password: 'abcd' })
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
const usernameErrors = result.error.issues.filter(
|
||||
(i) => i.path[0] === 'username'
|
||||
)
|
||||
expect(usernameErrors.length).toBeGreaterThan(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('accepts valid username with password', () => {
|
||||
const result = schema.safeParse({ username: 'admin', password: 'pass1234' })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts email as username', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'user@example.com',
|
||||
password: 'mypassword',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('password field', () => {
|
||||
it('rejects empty password', () => {
|
||||
const result = schema.safeParse({ username: 'admin', password: '' })
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
const passwordErrors = result.error.issues.filter(
|
||||
(i) => i.path[0] === 'password'
|
||||
)
|
||||
expect(passwordErrors.length).toBeGreaterThan(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects password shorter than 4 characters', () => {
|
||||
const result = schema.safeParse({ username: 'admin', password: 'ab' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts password of exactly 4 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'admin',
|
||||
password: 'abcd',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts long password', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'admin',
|
||||
password: 'a'.repeat(256),
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('missing fields', () => {
|
||||
it('rejects empty object', () => {
|
||||
const result = schema.safeParse({})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects object with only username', () => {
|
||||
const result = schema.safeParse({ username: 'admin' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,16 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const getFormSchema = (
|
||||
t: (key: string, options?: Record<string, any>) => string
|
||||
) =>
|
||||
z.object({
|
||||
username: z
|
||||
.string()
|
||||
.min(1, { message: t('validation.pleaseEnterUsernameOrEmail') }),
|
||||
password: z
|
||||
.string()
|
||||
.min(1, { message: t('validation.pleaseEnterPassword') })
|
||||
.min(4, { message: t('validation.passwordMinLength', { min: 4 }) }),
|
||||
})
|
||||
|
||||
export type LoginFormValues = z.infer<ReturnType<typeof getFormSchema>>
|
||||
@@ -18,10 +18,10 @@
|
||||
|
||||
|
||||
import { HTMLAttributes, useState } from 'react'
|
||||
import { z } from 'zod'
|
||||
import { useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { cn, toSearchParams } from '@/lib/utils'
|
||||
import { getFormSchema, type LoginFormValues } from './schema'
|
||||
import {
|
||||
Form,
|
||||
FormControl,
|
||||
@@ -47,17 +47,6 @@ import { useTheme } from '@/context/theme-context'
|
||||
|
||||
type UserAuthFormProps = HTMLAttributes<HTMLDivElement>
|
||||
|
||||
const getFormSchema = (t: (key: string, options?: Record<string, any>) => string) =>
|
||||
z.object({
|
||||
username: z
|
||||
.string()
|
||||
.min(1, { message: t('validation.pleaseEnterUsernameOrEmail') }),
|
||||
password: z
|
||||
.string()
|
||||
.min(1, { message: t('validation.pleaseEnterPassword') })
|
||||
.min(4, { message: t('validation.passwordMinLength', { min: 4 }) }),
|
||||
});
|
||||
|
||||
export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
const { setTheme } = useTheme();
|
||||
@@ -68,7 +57,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
|
||||
const redirect = toSearchParams(search).get('redirect') || '/';
|
||||
|
||||
const formSchema = getFormSchema(t)
|
||||
const form = useForm<z.infer<typeof formSchema>>({
|
||||
const form = useForm<LoginFormValues>({
|
||||
resolver: zodResolver(formSchema),
|
||||
defaultValues: {
|
||||
username: '',
|
||||
@@ -81,7 +70,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
|
||||
retry: 0,
|
||||
});
|
||||
|
||||
async function onSubmit(data: z.infer<typeof formSchema>) {
|
||||
async function onSubmit(data: LoginFormValues) {
|
||||
setIsLoading(true)
|
||||
|
||||
mutation.mutate(data, {
|
||||
|
||||
@@ -17,9 +17,11 @@ import { Mail, Users, Inbox, Zap, Paperclip } from 'lucide-react';
|
||||
import { formatBytes, formatNumber } from '@/lib/utils';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { get_dashboard_stats, INITIAL_DASHBOARD_STATS, TimeBucket } from '@/api/system/api';
|
||||
//import { MOCK_DASHBOARD_STATS } from '@/api/system/mock-dashboard';
|
||||
import { Main } from '@/components/layout/main';
|
||||
import { FixedHeader } from '@/components/layout/fixed-header';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import LongText from '@/components/long-text';
|
||||
import { getToken } from '@/stores/authStore';
|
||||
import { useNavigate } from '@tanstack/react-router';
|
||||
import useMinimalAccountList from '@/hooks/use-minimal-account-list';
|
||||
@@ -346,18 +348,18 @@ export default function MailArchiveDashboard() {
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
<div className="grid gap-6 grid-cols-1 md:grid-cols-2 lg:grid-cols-4">
|
||||
<Card>
|
||||
<div className="grid gap-6 grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4">
|
||||
<Card className="overflow-hidden">
|
||||
<CardHeader className="!px-4 !pt-4 !pb-1">
|
||||
<CardTitle className="text-xs font-bold uppercase tracking-wider">{t('dashboard.top10Senders')}</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="p-0">
|
||||
{hasTopSenders ? (
|
||||
<Table>
|
||||
<Table className="table-fixed">
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead className="text-xs">{t('dashboard.sender')}</TableHead>
|
||||
<TableHead className="text-right text-xs">{t('dashboard.count')}</TableHead>
|
||||
<TableHead className="w-[96px] text-right text-xs">{t('dashboard.count')}</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
@@ -368,16 +370,18 @@ export default function MailArchiveDashboard() {
|
||||
<div className="absolute left-0 top-0 bottom-0 w-[2px] bg-primary opacity-0 group-hover:opacity-100 transition-opacity" />
|
||||
<div className="text-xs flex flex-wrap gap-x-1 min-w-0 flex-1">
|
||||
<span className="flex items-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ from: s.key })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors truncate max-w-[258px]"
|
||||
>
|
||||
{s.key}
|
||||
</button>
|
||||
<LongText className="max-w-[180px] md:max-w-[160px] lg:max-w-[200px] xl:max-w-[220px]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ from: s.key })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors"
|
||||
>
|
||||
{s.key}
|
||||
</button>
|
||||
</LongText>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -392,17 +396,17 @@ export default function MailArchiveDashboard() {
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<Card className="overflow-hidden">
|
||||
<CardHeader className="!px-4 !pt-4 !pb-1">
|
||||
<CardTitle className="text-xs font-bold uppercase tracking-wider">{t('dashboard.top10LargestEmails')}</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="p-0">
|
||||
{hasTopEmails ? (
|
||||
<Table>
|
||||
<Table className="table-fixed">
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead className="text-xs">{t('dashboard.subject')}</TableHead>
|
||||
<TableHead className="text-right text-xs">{t('dashboard.size')}</TableHead>
|
||||
<TableHead className="w-[96px] text-right text-xs">{t('dashboard.size')}</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
@@ -413,16 +417,18 @@ export default function MailArchiveDashboard() {
|
||||
<div className="absolute left-0 top-0 bottom-0 w-[2px] bg-primary opacity-0 group-hover:opacity-100 transition-opacity" />
|
||||
<div className="text-xs flex flex-wrap gap-x-1 min-w-0 flex-1">
|
||||
<span className="flex items-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ id: m.id })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors truncate max-w-[258px]"
|
||||
>
|
||||
{m.subject || t('dashboard.noSubject')}
|
||||
</button>
|
||||
<LongText className="max-w-[180px] md:max-w-[160px] lg:max-w-[200px] xl:max-w-[220px]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ id: m.id })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors"
|
||||
>
|
||||
{m.subject || t('dashboard.noSubject')}
|
||||
</button>
|
||||
</LongText>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -437,7 +443,7 @@ export default function MailArchiveDashboard() {
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<Card className="overflow-hidden">
|
||||
<CardHeader className="!px-4 !pt-4 !pb-1">
|
||||
<CardTitle className="text-xs font-bold uppercase tracking-wider">
|
||||
{t('dashboard.top10LargestAttachments')}
|
||||
@@ -445,11 +451,11 @@ export default function MailArchiveDashboard() {
|
||||
</CardHeader>
|
||||
<CardContent className="p-0">
|
||||
{stats?.top_largest_attachments?.length ? (
|
||||
<Table>
|
||||
<Table className="table-fixed">
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead className="text-xs">{t('attachment.name')}</TableHead>
|
||||
<TableHead className="text-right text-xs">{t('dashboard.size')}</TableHead>
|
||||
<TableHead className="w-[96px] text-right text-xs">{t('dashboard.size')}</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
@@ -463,16 +469,18 @@ export default function MailArchiveDashboard() {
|
||||
<div className="absolute left-0 top-0 bottom-0 w-[2px] bg-primary opacity-0 group-hover:opacity-100 transition-opacity" />
|
||||
<div className="text-xs flex flex-wrap gap-x-1 min-w-0 flex-1">
|
||||
<span className="flex items-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickAttachmentSearch({ id: a.id })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors truncate max-w-[238px]"
|
||||
>
|
||||
{a.name || 'Unnamed'}
|
||||
</button>
|
||||
<LongText className="max-w-[160px] md:max-w-[140px] lg:max-w-[180px] xl:max-w-[200px]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickAttachmentSearch({ id: a.id })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors"
|
||||
>
|
||||
{a.name || 'Unnamed'}
|
||||
</button>
|
||||
</LongText>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -489,17 +497,17 @@ export default function MailArchiveDashboard() {
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card>
|
||||
<Card className="overflow-hidden">
|
||||
<CardHeader className="!px-4 !pt-4 !pb-1">
|
||||
<CardTitle className="text-xs font-bold uppercase tracking-wider">{t('dashboard.top10Accounts')}</CardTitle>
|
||||
</CardHeader>
|
||||
<CardContent className="p-0">
|
||||
{hasTopAccounts ? (
|
||||
<Table>
|
||||
<Table className="table-fixed">
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead className="text-xs">{t('dashboard.account')}</TableHead>
|
||||
<TableHead className="text-right text-xs">{t('dashboard.emails')}</TableHead>
|
||||
<TableHead className="w-[96px] text-right text-xs">{t('dashboard.emails')}</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
@@ -510,16 +518,18 @@ export default function MailArchiveDashboard() {
|
||||
<div className="absolute left-0 top-0 bottom-0 w-[2px] bg-primary opacity-0 group-hover:opacity-100 transition-opacity" />
|
||||
<div className="text-xs flex flex-wrap gap-x-1 min-w-0 flex-1">
|
||||
<span className="flex items-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ account_ids: [getAccountIdByEmail(acc.key) || 0] })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors truncate max-w-[258px]"
|
||||
>
|
||||
{acc.key}
|
||||
</button>
|
||||
<LongText className="max-w-[180px] md:max-w-[160px] lg:max-w-[200px] xl:max-w-[220px]">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
handleQuickSearch({ account_ids: [getAccountIdByEmail(acc.key) || 0] })
|
||||
}}
|
||||
className="hover:text-primary hover:underline transition-colors"
|
||||
>
|
||||
{acc.key}
|
||||
</button>
|
||||
</LongText>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { getOAuth2Schema } from '../schema'
|
||||
|
||||
const t = (key: string) => key
|
||||
|
||||
describe('OAuth2 Form Schema', () => {
|
||||
const schema = getOAuth2Schema(t)
|
||||
|
||||
const validData = {
|
||||
client_id: 'my-client-id',
|
||||
auth_url: 'https://accounts.example.com/o/oauth2/auth',
|
||||
token_url: 'https://oauth2.example.com/token',
|
||||
redirect_uri: 'https://myapp.example.com/oauth2/callback',
|
||||
enabled: true,
|
||||
}
|
||||
|
||||
describe('client_id field', () => {
|
||||
it('rejects empty client_id', () => {
|
||||
const result = schema.safeParse({ ...validData, client_id: '' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid client_id', () => {
|
||||
const result = schema.safeParse(validData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('client_secret field', () => {
|
||||
it('accepts undefined client_secret', () => {
|
||||
const result = schema.safeParse(validData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts provided client_secret', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
client_secret: 'my-secret',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('auth_url field', () => {
|
||||
it('rejects empty auth_url', () => {
|
||||
const result = schema.safeParse({ ...validData, auth_url: '' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects invalid URL format for auth_url', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
auth_url: 'not-a-url',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid auth_url', () => {
|
||||
const result = schema.safeParse(validData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('token_url field', () => {
|
||||
it('rejects empty token_url', () => {
|
||||
const result = schema.safeParse({ ...validData, token_url: '' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects invalid URL format for token_url', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
token_url: 'not-a-url',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('redirect_uri field', () => {
|
||||
it('rejects empty redirect_uri', () => {
|
||||
const result = schema.safeParse({ ...validData, redirect_uri: '' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects invalid URL format for redirect_uri', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
redirect_uri: 'not-a-url',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('scopes field', () => {
|
||||
it('accepts empty scopes array', () => {
|
||||
const result = schema.safeParse({ ...validData, scopes: [] })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts valid scopes', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
scopes: [{ value: 'https://mail.google.com/' }],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects scope with empty value', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
scopes: [{ value: '' }],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('extra_params field', () => {
|
||||
it('accepts empty extra_params array', () => {
|
||||
const result = schema.safeParse({ ...validData, extra_params: [] })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts valid extra_params', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
extra_params: [{ key: 'access_type', value: 'offline' }],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects param with empty key', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
extra_params: [{ key: '', value: 'offline' }],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects param with empty value', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
extra_params: [{ key: 'access_type', value: '' }],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('enabled field', () => {
|
||||
it('accepts enabled: true', () => {
|
||||
const result = schema.safeParse(validData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts enabled: false', () => {
|
||||
const result = schema.safeParse({ ...validData, enabled: false })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('description field', () => {
|
||||
it('rejects description longer than 255 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
description: 'a'.repeat(256),
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts description of exactly 255 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validData,
|
||||
description: 'a'.repeat(255),
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('use_proxy field', () => {
|
||||
it('accepts undefined use_proxy', () => {
|
||||
const result = schema.safeParse(validData)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts numeric use_proxy', () => {
|
||||
const result = schema.safeParse({ ...validData, use_proxy: 1 })
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -17,7 +17,6 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { z } from 'zod'
|
||||
import { useFieldArray, useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { toast } from '@/hooks/use-toast'
|
||||
@@ -53,115 +52,22 @@ import { AxiosError } from 'axios'
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'
|
||||
import useProxyList from '@/hooks/use-proxy'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
const getParamSchema = (t: (key: string) => string) => z.object({
|
||||
key: z.string({ required_error: t('oauth2.keyIsRequired') }).min(1, t('oauth2.keyCannotBeEmpty')),
|
||||
value: z.string({ required_error: t('oauth2.valueIsRequired') }).min(1, t('oauth2.valueCannotBeEmpty')),
|
||||
});
|
||||
|
||||
const paramSchema = z.object({
|
||||
key: z.string({ required_error: 'Key is required' }).min(1, "Key cannot be empty"),
|
||||
value: z.string({ required_error: 'Value is required' }).min(1, "Value cannot be empty"),
|
||||
});
|
||||
|
||||
const getScopeSchema = (t: (key: string) => string) => z.object({
|
||||
value: z.string({ required_error: t('oauth2.valueIsRequired') }).min(1, t('oauth2.valueCannotBeEmpty')),
|
||||
});
|
||||
|
||||
const scopeSchema = z.object({
|
||||
value: z.string({ required_error: 'Value is required' }).min(1, "Value cannot be empty"),
|
||||
});
|
||||
|
||||
const extraparamSchema = z.record(z.string()).optional();
|
||||
const authorizescopeSchema = z.array(z.string()).optional();
|
||||
|
||||
import { getOAuth2Schema, type OAuth2FormValues } from './schema'
|
||||
|
||||
function convertToExtraParamsSchema(
|
||||
record: z.infer<typeof extraparamSchema>
|
||||
): z.infer<typeof paramSchema>[] {
|
||||
if (!record) {
|
||||
return [];
|
||||
}
|
||||
return Object.entries(record).map(([key, value]) => ({
|
||||
key,
|
||||
value,
|
||||
}));
|
||||
record: Record<string, string> | undefined
|
||||
): { key: string; value: string }[] {
|
||||
if (!record) return []
|
||||
return Object.entries(record).map(([key, value]) => ({ key, value }))
|
||||
}
|
||||
|
||||
|
||||
function convertToScopeSchema(authorizeScopes: z.infer<typeof authorizescopeSchema>): z.infer<typeof scopeSchema>[] {
|
||||
if (!authorizeScopes || authorizeScopes.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return authorizeScopes.map((scope) => ({
|
||||
value: scope,
|
||||
}));
|
||||
function convertToScopeSchema(
|
||||
scopes: string[] | undefined
|
||||
): { value: string }[] {
|
||||
if (!scopes || scopes.length === 0) return []
|
||||
return scopes.map((scope) => ({ value: scope }))
|
||||
}
|
||||
|
||||
const getOAuth2Schema = (t: (key: string) => string) => z.object({
|
||||
description: z.string().max(255, { message: t('oauth2.descriptionMustNotExceed255Characters') }).optional(),
|
||||
client_id: z.string({
|
||||
required_error: t('oauth2.clientIdIsRequired'),
|
||||
}).min(1, { message: t('oauth2.clientIdCannotBeEmpty') }),
|
||||
client_secret: z.string().optional(),
|
||||
auth_url: z.string({
|
||||
required_error: t('oauth2.authorizationUrlIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.authorizationUrlCannotBeEmpty') })
|
||||
.url({ message: t('oauth2.invalidAuthorizationUrlFormat') }),
|
||||
|
||||
token_url: z.string({
|
||||
required_error: t('oauth2.tokenUrlIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.tokenUrlCannotBeEmpty') })
|
||||
.url({ message: t('oauth2.invalidTokenUrlFormat') }),
|
||||
|
||||
redirect_uri: z.string({
|
||||
required_error: t('oauth2.redirectUriIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.redirectUriCannotBeEmpty') })
|
||||
.url({ message: t('oauth2.invalidRedirectUriFormat') }),
|
||||
|
||||
scopes: z.array(getScopeSchema(t)).optional(),
|
||||
extra_params: z.array(getParamSchema(t)).optional(),
|
||||
enabled: z.boolean(),
|
||||
use_proxy: z.number().optional(),
|
||||
});
|
||||
|
||||
const oauth2Schema = z.object({
|
||||
description: z.string().max(255, { message: "Description must not exceed 255 characters." }).optional(),
|
||||
client_id: z.string({
|
||||
required_error: "Client ID is required",
|
||||
}).min(1, { message: "Client ID cannot be empty" }),
|
||||
client_secret: z.string().optional(),
|
||||
auth_url: z.string({
|
||||
required_error: "Authorization URL is required",
|
||||
})
|
||||
.min(1, { message: "Authorization URL cannot be empty" })
|
||||
.url({ message: "Invalid Authorization URL format" }),
|
||||
|
||||
token_url: z.string({
|
||||
required_error: "Token URL is required",
|
||||
})
|
||||
.min(1, { message: "Token URL cannot be empty" })
|
||||
.url({ message: "Invalid Token URL format" }),
|
||||
|
||||
redirect_uri: z.string({
|
||||
required_error: "Redirect URI is required",
|
||||
})
|
||||
.min(1, { message: "Redirect URI cannot be empty" })
|
||||
.url({ message: "Invalid Redirect URI format" }),
|
||||
|
||||
scopes: z.array(scopeSchema).optional(),
|
||||
extra_params: z.array(paramSchema).optional(),
|
||||
enabled: z.boolean(),
|
||||
use_proxy: z.number().optional(),
|
||||
});
|
||||
|
||||
export type OAuth2Form = z.infer<typeof oauth2Schema>;
|
||||
|
||||
|
||||
interface Props {
|
||||
currentRow?: OAuth2Entity
|
||||
open: boolean
|
||||
@@ -185,7 +91,7 @@ const defaultValues = {
|
||||
export function ActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
const { t } = useTranslation()
|
||||
const isEdit = !!currentRow
|
||||
const form = useForm<OAuth2Form>({
|
||||
const form = useForm<OAuth2FormValues>({
|
||||
resolver: zodResolver(getOAuth2Schema(t)),
|
||||
defaultValues: isEdit
|
||||
? {
|
||||
@@ -255,7 +161,7 @@ export function ActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
const onSubmit = (values: OAuth2Form) => {
|
||||
const onSubmit = (values: OAuth2FormValues) => {
|
||||
if (!isEdit) {
|
||||
if (!values.client_secret) {
|
||||
form.setError('client_secret', {
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
const paramEntry = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
key: z
|
||||
.string({ required_error: t('oauth2.keyIsRequired') })
|
||||
.min(1, t('oauth2.keyCannotBeEmpty')),
|
||||
value: z
|
||||
.string({ required_error: t('oauth2.valueIsRequired') })
|
||||
.min(1, t('oauth2.valueCannotBeEmpty')),
|
||||
})
|
||||
|
||||
const scopeEntry = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
value: z
|
||||
.string({ required_error: t('oauth2.valueIsRequired') })
|
||||
.min(1, t('oauth2.valueCannotBeEmpty')),
|
||||
})
|
||||
|
||||
export const getOAuth2Schema = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
description: z
|
||||
.string()
|
||||
.max(255, { message: t('oauth2.descriptionMustNotExceed255Characters') })
|
||||
.optional(),
|
||||
client_id: z
|
||||
.string({
|
||||
required_error: t('oauth2.clientIdIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.clientIdCannotBeEmpty') }),
|
||||
client_secret: z.string().optional(),
|
||||
auth_url: z
|
||||
.string({
|
||||
required_error: t('oauth2.authorizationUrlIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.authorizationUrlCannotBeEmpty') })
|
||||
.url({ message: t('oauth2.invalidAuthorizationUrlFormat') }),
|
||||
token_url: z
|
||||
.string({
|
||||
required_error: t('oauth2.tokenUrlIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.tokenUrlCannotBeEmpty') })
|
||||
.url({ message: t('oauth2.invalidTokenUrlFormat') }),
|
||||
redirect_uri: z
|
||||
.string({
|
||||
required_error: t('oauth2.redirectUriIsRequired'),
|
||||
})
|
||||
.min(1, { message: t('oauth2.redirectUriCannotBeEmpty') })
|
||||
.url({ message: t('oauth2.invalidRedirectUriFormat') }),
|
||||
scopes: z.array(scopeEntry(t)).optional(),
|
||||
extra_params: z.array(paramEntry(t)).optional(),
|
||||
enabled: z.boolean(),
|
||||
use_proxy: z.number().optional(),
|
||||
})
|
||||
|
||||
export type OAuth2FormValues = z.infer<ReturnType<typeof getOAuth2Schema>>
|
||||
@@ -19,7 +19,7 @@
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { Loader, Download, Trash2, MessageSquareMore, FileText, FileImage, FileVideo, FileArchive, FileSpreadsheet, FileCode, FileIcon, FileAudio, Upload } from 'lucide-react';
|
||||
import { Loader, Download, Trash2, MessageSquareMore, FileText, FileImage, FileVideo, FileArchive, FileSpreadsheet, FileCode, FileIcon, FileAudio, Upload, ShieldCheck } from 'lucide-react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Separator } from '@/components/ui/separator';
|
||||
@@ -129,6 +129,12 @@ export function MailMessageView({
|
||||
const [nestedEmlFile, setNestedEmlFile] = useState<AttachmentInfo | null>(null);
|
||||
const { getEmailById } = useMinimalAccountList();
|
||||
const [threadOpen, setThreadOpen] = useState(false);
|
||||
const [blockRemote, setBlockRemote] = useState(true);
|
||||
const [hasRemoteContent, setHasRemoteContent] = useState(false);
|
||||
|
||||
const toggleBlockRemote = () => {
|
||||
setBlockRemote((prev) => !prev);
|
||||
};
|
||||
|
||||
const downloadAttachmentMutation = useMutation({
|
||||
mutationFn: ({ content_hash }: { content_hash: string }) =>
|
||||
@@ -145,12 +151,13 @@ export function MailMessageView({
|
||||
});
|
||||
|
||||
const loadMessageMutation = useMutation({
|
||||
mutationFn: () => load_message(envelope.account_id, envelope.id),
|
||||
mutationFn: () => load_message(envelope.account_id, envelope.id, blockRemote),
|
||||
onSuccess: (data) => {
|
||||
setLoading(false);
|
||||
setContent(getContent(data));
|
||||
if (data.attachments) setAttachments(data.attachments);
|
||||
setContentType(data.html ? 'Html' : 'Plain');
|
||||
setHasRemoteContent(!!data.has_remote_content);
|
||||
},
|
||||
onError: (error: any) => {
|
||||
setLoading(false);
|
||||
@@ -162,10 +169,14 @@ export function MailMessageView({
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
setBlockRemote(true);
|
||||
}, [envelope.id]);
|
||||
|
||||
useEffect(() => {
|
||||
setLoading(true);
|
||||
loadMessageMutation.mutate();
|
||||
}, [envelope.id]);
|
||||
}, [envelope.id, blockRemote]);
|
||||
|
||||
|
||||
const handleViewNestedEml = (attachment: AttachmentInfo) => {
|
||||
@@ -384,6 +395,30 @@ export function MailMessageView({
|
||||
</div>
|
||||
)}
|
||||
{showAttachments && <Separator className="mb-2" />}
|
||||
{hasRemoteContent && (
|
||||
<div className="flex items-center justify-between bg-muted border px-3 py-1.5 mb-3 text-xs">
|
||||
<div className="flex items-center gap-1.5 min-w-0">
|
||||
<ShieldCheck className="h-3.5 w-3.5 text-muted-foreground shrink-0" />
|
||||
{blockRemote ? (
|
||||
<span className="text-muted-foreground truncate">
|
||||
{t('mail.remoteBlocked', 'To protect your privacy, Bichon has blocked remote content in this message.')}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-muted-foreground truncate">
|
||||
{t('mail.remoteShown', 'Remote content is now shown.')}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<span
|
||||
className="underline cursor-pointer hover:no-underline text-muted-foreground text-[11px] font-medium shrink-0 ml-2 select-none"
|
||||
onClick={toggleBlockRemote}
|
||||
>
|
||||
{blockRemote
|
||||
? t('mail.showRemoteContent', 'Show remote content')
|
||||
: t('mail.blockRemoteAgain', 'Block again')}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex-1 overflow-auto">
|
||||
{loading ? (
|
||||
<div className="flex justify-center items-center py-8">
|
||||
|
||||
@@ -148,7 +148,7 @@ export function NestedEmailDialog({ open, onOpenChange, accountId, envelopeId, f
|
||||
|
||||
const { data, isLoading } = useQuery({
|
||||
queryKey: ['nested-message', accountId, envelopeId, content_hash],
|
||||
queryFn: () => load_nested_message(accountId, envelopeId, content_hash),
|
||||
queryFn: () => load_nested_message(accountId, envelopeId, content_hash, true),
|
||||
enabled: open && !!content_hash,
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { profileSchema } from '../schema'
|
||||
|
||||
const t = (key: string) => key
|
||||
|
||||
describe('Profile Form Schema', () => {
|
||||
const schema = profileSchema(t)
|
||||
|
||||
describe('username field', () => {
|
||||
it('rejects empty username', () => {
|
||||
const result = schema.safeParse({
|
||||
username: '',
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
const errors = result.error.issues.filter(
|
||||
(i) => i.path[0] === 'username'
|
||||
)
|
||||
expect(errors.length).toBeGreaterThan(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects username shorter than 3 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'ab',
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts username of exactly 3 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'abc',
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects username longer than 32 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'a'.repeat(33),
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts username of exactly 32 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'a'.repeat(32),
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('email field', () => {
|
||||
it('rejects empty email', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: '',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects invalid email format', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'not-an-email',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects email without domain', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid email', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('password field', () => {
|
||||
it('accepts empty password (keep current)', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@example.com',
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
// Empty password should be transformed to undefined
|
||||
expect(result.data.password).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects password shorter than 8 characters when provided', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@example.com',
|
||||
password: 'short',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts password of exactly 8 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@example.com',
|
||||
password: '12345678',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects password longer than 256 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@example.com',
|
||||
password: 'a'.repeat(257),
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('transforms non-empty password to the string value', () => {
|
||||
const result = schema.safeParse({
|
||||
username: 'validuser',
|
||||
email: 'user@example.com',
|
||||
password: 'myNewPassword123',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.password).toBe('myNewPassword123')
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -16,7 +16,6 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { z } from 'zod'
|
||||
import { useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
@@ -42,41 +41,7 @@ import { Badge } from '@/components/ui/badge'
|
||||
import { FileWithPreview } from '@/hooks/use-file-upload'
|
||||
import AvatarUpload from './avatar-upload'
|
||||
import { PermissionsDialog } from '../access/permissions-dialog'
|
||||
|
||||
const profileSchema = (t: (key: string) => string) => z.object({
|
||||
username: z
|
||||
.string({
|
||||
required_error: t('settings.profile.validation.username.required'),
|
||||
})
|
||||
.min(3, {
|
||||
message: t('settings.profile.validation.username.min'),
|
||||
})
|
||||
.max(32, {
|
||||
message: t('settings.profile.validation.username.max'),
|
||||
}),
|
||||
|
||||
email: z
|
||||
.string({
|
||||
required_error: t('settings.profile.validation.email.required'),
|
||||
})
|
||||
.email({
|
||||
message: t('settings.profile.validation.email.invalid'),
|
||||
}),
|
||||
|
||||
password: z
|
||||
.string()
|
||||
.min(8, {
|
||||
message: t('settings.profile.validation.password.min'),
|
||||
})
|
||||
.max(256, {
|
||||
message: t('settings.profile.validation.password.max'),
|
||||
})
|
||||
.or(z.literal(''))
|
||||
.optional()
|
||||
.transform((v) => (v ? v : undefined)),
|
||||
})
|
||||
|
||||
export type ProfileFormValues = z.infer<ReturnType<typeof profileSchema>>
|
||||
import { profileSchema, type ProfileFormValues } from './schema'
|
||||
|
||||
function fileToBase64(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const profileSchema = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
username: z
|
||||
.string({
|
||||
required_error: t('settings.profile.validation.username.required'),
|
||||
})
|
||||
.min(3, {
|
||||
message: t('settings.profile.validation.username.min'),
|
||||
})
|
||||
.max(32, {
|
||||
message: t('settings.profile.validation.username.max'),
|
||||
}),
|
||||
|
||||
email: z
|
||||
.string({
|
||||
required_error: t('settings.profile.validation.email.required'),
|
||||
})
|
||||
.email({
|
||||
message: t('settings.profile.validation.email.invalid'),
|
||||
}),
|
||||
|
||||
password: z
|
||||
.string()
|
||||
.min(8, {
|
||||
message: t('settings.profile.validation.password.min'),
|
||||
})
|
||||
.max(256, {
|
||||
message: t('settings.profile.validation.password.max'),
|
||||
})
|
||||
.or(z.literal(''))
|
||||
.optional()
|
||||
.transform((v) => (v ? v : undefined)),
|
||||
})
|
||||
|
||||
export type ProfileFormValues = z.infer<ReturnType<typeof profileSchema>>
|
||||
@@ -0,0 +1,173 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { proxyFormSchema } from '../schema'
|
||||
|
||||
describe('Proxy Form Schema', () => {
|
||||
describe('url field - basic validation', () => {
|
||||
it('rejects empty URL', () => {
|
||||
const result = proxyFormSchema.safeParse({ url: '' })
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid socks5 URL', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts valid http URL', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'http://proxy.example.com:8080',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('url field - protocol validation', () => {
|
||||
it('rejects https protocol', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'https://proxy.example.com:443',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('http:// or socks5://')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects ftp protocol', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'ftp://files.example.com',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects URL without protocol', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: '127.0.0.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('Invalid URL format')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('url field - port validation', () => {
|
||||
it('rejects port 0', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1:0',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects port > 65535', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1:99999',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts port 65535', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1:65535',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts port 1', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1:1',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('defaults to port 1080 when no port specified', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('url field - hostname validation', () => {
|
||||
it('accepts IP address hostname', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://192.168.1.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts domain hostname', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://proxy.internal:1080',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects hostname with invalid characters', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://proxy_host:1080',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('Hostname contains invalid characters')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('url field - auth validation', () => {
|
||||
it('rejects username without password', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://user@127.0.0.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('Password cannot be empty')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects short password when username provided', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://user:short@127.0.0.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) {
|
||||
expect(
|
||||
result.error.issues.some((i) =>
|
||||
i.message?.includes('Password must be at least 8')
|
||||
)
|
||||
).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('accepts valid auth credentials', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://user:password123@127.0.0.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts URL without auth (no credentials)', () => {
|
||||
const result = proxyFormSchema.safeParse({
|
||||
url: 'socks5://127.0.0.1:1080',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -17,7 +17,6 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import { z } from 'zod'
|
||||
import { useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { toast } from '@/hooks/use-toast'
|
||||
@@ -47,75 +46,7 @@ import { Loader2 } from 'lucide-react'
|
||||
import { add_proxy, update_proxy } from '@/api/system/api'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Proxy } from '@/api/system/api'
|
||||
|
||||
const proxyFormSchema = z.object({
|
||||
url: z.string()
|
||||
.min(1, "Proxy address cannot be empty")
|
||||
.superRefine((value, ctx) => {
|
||||
|
||||
if (value.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch (e) {
|
||||
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Invalid URL format",
|
||||
path: [],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if (url.protocol !== 'socks5:' && url.protocol !== 'http:') {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "URL must start with http:// or socks5://",
|
||||
path: [],
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
if (!/^[a-zA-Z0-9\-\.]+$/.test(url.hostname)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Hostname contains invalid characters",
|
||||
path: [],
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
const port = parseInt(url.port || '1080');
|
||||
if (port <= 0 || port > 65535) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Port must be between 1-65535",
|
||||
path: [],
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
if (url.username && !url.password) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Password cannot be empty when username is provided",
|
||||
path: [],
|
||||
});
|
||||
} else if (url.password && url.password.length < 8) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Password must be at least 8 characters",
|
||||
path: [],
|
||||
});
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
export type ProxyForm = z.infer<typeof proxyFormSchema>;
|
||||
import { proxyFormSchema, type ProxyFormValues } from './schema'
|
||||
|
||||
|
||||
interface Props {
|
||||
@@ -140,7 +71,7 @@ export function ProxyActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
const { t } = useTranslation()
|
||||
const isEdit = !!currentRow
|
||||
const queryClient = useQueryClient();
|
||||
const form = useForm<ProxyForm>({
|
||||
const form = useForm<ProxyFormValues>({
|
||||
resolver: zodResolver(proxyFormSchema),
|
||||
defaultValues: isEdit
|
||||
? mapCurrentRowToFormValues(currentRow)
|
||||
@@ -186,7 +117,7 @@ export function ProxyActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
}
|
||||
|
||||
|
||||
const onSubmit = (values: ProxyForm) => {
|
||||
const onSubmit = (values: ProxyFormValues) => {
|
||||
const url = values.url;
|
||||
if (isEdit) {
|
||||
updateMutation.mutate(url);
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const proxyFormSchema = z.object({
|
||||
url: z
|
||||
.string()
|
||||
.min(1, 'Proxy address cannot be empty')
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(value)
|
||||
} catch (_e) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Invalid URL format',
|
||||
path: [],
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (url.protocol !== 'socks5:' && url.protocol !== 'http:') {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'URL must start with http:// or socks5://',
|
||||
path: [],
|
||||
})
|
||||
}
|
||||
|
||||
if (!/^[a-zA-Z0-9\-\.]+$/.test(url.hostname)) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Hostname contains invalid characters',
|
||||
path: [],
|
||||
})
|
||||
}
|
||||
|
||||
const port = parseInt(url.port || '1080')
|
||||
if (port <= 0 || port > 65535) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Port must be between 1-65535',
|
||||
path: [],
|
||||
})
|
||||
}
|
||||
|
||||
if (url.username && !url.password) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Password cannot be empty when username is provided',
|
||||
path: [],
|
||||
})
|
||||
} else if (url.password && url.password.length < 8) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: 'Password must be at least 8 characters',
|
||||
path: [],
|
||||
})
|
||||
}
|
||||
}),
|
||||
})
|
||||
|
||||
export type ProxyFormValues = z.infer<typeof proxyFormSchema>
|
||||
@@ -0,0 +1,129 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { getRoleFormSchema } from '../schema'
|
||||
|
||||
const t = (key: string) => key
|
||||
|
||||
describe('Role Form Schema', () => {
|
||||
const schema = getRoleFormSchema(t)
|
||||
|
||||
describe('name field', () => {
|
||||
it('rejects empty name', () => {
|
||||
const result = schema.safeParse({
|
||||
name: '',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read'],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid name', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Viewer',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read'],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('role_type field', () => {
|
||||
it('accepts Global role type', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Admin',
|
||||
role_type: 'Global',
|
||||
permissions: ['system:access'],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts Account role type', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Viewer',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read'],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects invalid role type', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Test',
|
||||
role_type: 'Invalid',
|
||||
permissions: ['data:read'],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('permissions field', () => {
|
||||
it('rejects empty permissions array', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Viewer',
|
||||
role_type: 'Account',
|
||||
permissions: [],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts single permission', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Viewer',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read'],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts multiple permissions', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Manager',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read', 'data:manage', 'account:manage'],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts all available permissions', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Super Admin',
|
||||
role_type: 'Global',
|
||||
permissions: [
|
||||
'system:access',
|
||||
'system:root',
|
||||
'user:manage',
|
||||
'user:view',
|
||||
'token:manage',
|
||||
'account:create',
|
||||
'account:manage:all',
|
||||
'data:read:all',
|
||||
'data:manage:all',
|
||||
'data:raw:download:all',
|
||||
'data:delete:all',
|
||||
'data:export:batch:all',
|
||||
],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('description field', () => {
|
||||
it('accepts undefined description', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Viewer',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read'],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts description string', () => {
|
||||
const result = schema.safeParse({
|
||||
name: 'Viewer',
|
||||
role_type: 'Account',
|
||||
permissions: ['data:read'],
|
||||
description: 'Read-only access to data',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -16,7 +16,6 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { z } from 'zod'
|
||||
import { useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
import { toast } from '@/hooks/use-toast'
|
||||
@@ -49,6 +48,7 @@ import {
|
||||
} from '@/components/ui/radio-group'
|
||||
import { cn } from '@/lib/utils'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { getRoleFormSchema, type RoleFormValues } from './schema'
|
||||
|
||||
interface Props {
|
||||
currentRow?: UserRole
|
||||
@@ -97,16 +97,9 @@ export function RoleActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
const queryClient = useQueryClient()
|
||||
const { t } = useTranslation()
|
||||
|
||||
const roleFormSchema = z.object({
|
||||
name: z.string().min(1, t('roles.validation.name_required')),
|
||||
role_type: z.enum(['Global', 'Account']),
|
||||
permissions: z.array(z.string()).min(1, t('roles.validation.perm_required')),
|
||||
description: z.string().optional(),
|
||||
})
|
||||
const roleFormSchema = getRoleFormSchema(t)
|
||||
|
||||
type RoleForm = z.infer<typeof roleFormSchema>
|
||||
|
||||
const form = useForm<RoleForm>({
|
||||
const form = useForm<RoleFormValues>({
|
||||
resolver: zodResolver(roleFormSchema),
|
||||
defaultValues: {
|
||||
name: isEdit ? currentRow.name : '',
|
||||
@@ -117,7 +110,7 @@ export function RoleActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
})
|
||||
|
||||
const mutation = useMutation({
|
||||
mutationFn: (values: RoleForm) =>
|
||||
mutationFn: (values: RoleFormValues) =>
|
||||
isEdit ? update_role(currentRow!.id, values) : create_role(values),
|
||||
onSuccess: () => {
|
||||
toast({ title: t(isEdit ? 'roles.actions.success_update' : 'roles.actions.success_create') })
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const getRoleFormSchema = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
name: z.string().min(1, t('roles.validation.name_required')),
|
||||
role_type: z.enum(['Global', 'Account']),
|
||||
permissions: z.array(z.string()).min(1, t('roles.validation.perm_required')),
|
||||
description: z.string().optional(),
|
||||
})
|
||||
|
||||
export type RoleFormValues = z.infer<ReturnType<typeof getRoleFormSchema>>
|
||||
@@ -0,0 +1,308 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { getCreateUserSchema, getUpdateUserSchema } from '../schema'
|
||||
|
||||
const t = (key: string) => key
|
||||
|
||||
const validBaseUser = {
|
||||
username: 'johndoe',
|
||||
email: 'john@example.com',
|
||||
global_roles: [1],
|
||||
}
|
||||
|
||||
const validCreateUser = {
|
||||
...validBaseUser,
|
||||
password: 'securePassword123',
|
||||
}
|
||||
|
||||
const invalidCases = [
|
||||
{ desc: 'empty username', data: { ...validCreateUser, username: '' } },
|
||||
{
|
||||
desc: 'username shorter than 3',
|
||||
data: { ...validCreateUser, username: 'ab' },
|
||||
},
|
||||
{
|
||||
desc: 'username longer than 32',
|
||||
data: { ...validCreateUser, username: 'a'.repeat(33) },
|
||||
},
|
||||
{ desc: 'empty email', data: { ...validCreateUser, email: '' } },
|
||||
{
|
||||
desc: 'invalid email format',
|
||||
data: { ...validCreateUser, email: 'not-an-email' },
|
||||
},
|
||||
{
|
||||
desc: 'empty global_roles',
|
||||
data: { ...validCreateUser, global_roles: [] },
|
||||
},
|
||||
{
|
||||
desc: 'empty password on create',
|
||||
data: { ...validBaseUser, password: '' },
|
||||
},
|
||||
{
|
||||
desc: 'short password on create',
|
||||
data: { ...validBaseUser, password: 'short' },
|
||||
},
|
||||
{
|
||||
desc: 'password longer than 256 on create',
|
||||
data: { ...validBaseUser, password: 'a'.repeat(257) },
|
||||
},
|
||||
]
|
||||
|
||||
describe('Create User Schema', () => {
|
||||
const schema = getCreateUserSchema(t)
|
||||
|
||||
it('accepts valid user data', () => {
|
||||
const result = schema.safeParse(validCreateUser)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it.each(invalidCases)('rejects $desc', ({ data }) => {
|
||||
const result = schema.safeParse(data)
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts username of exactly 3 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
username: 'abc',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts username of exactly 32 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
username: 'a'.repeat(32),
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts password of exactly 8 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validBaseUser,
|
||||
password: '12345678',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts password of exactly 256 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validBaseUser,
|
||||
password: 'a'.repeat(256),
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Update User Schema', () => {
|
||||
const schema = getUpdateUserSchema(t)
|
||||
|
||||
it('accepts empty password (keep current)', () => {
|
||||
const result = schema.safeParse({
|
||||
...validBaseUser,
|
||||
password: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.password).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('accepts undefined password', () => {
|
||||
const result = schema.safeParse(validBaseUser)
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.password).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects short password when provided', () => {
|
||||
const result = schema.safeParse({
|
||||
...validBaseUser,
|
||||
password: 'short',
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts valid password when provided', () => {
|
||||
const result = schema.safeParse({
|
||||
...validBaseUser,
|
||||
password: 'newPassword123',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.password).toBe('newPassword123')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('User Schema - ACL', () => {
|
||||
const schema = getCreateUserSchema(t)
|
||||
|
||||
describe('ip_whitelist validation', () => {
|
||||
it('accepts valid IPv4 addresses', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: '192.168.1.1\n10.0.0.1' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts valid IPv6 address', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: '2001:0db8:85a3:0000:0000:8a2e:0370:7334' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects invalid IP format', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: 'not-an-ip' },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects invalid IP with too many octets', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: '192.168.1.1.1' },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects IP with octet > 255', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: '300.1.1.1' },
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts empty ACL (no security policies)', () => {
|
||||
const result = schema.safeParse(validCreateUser)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('rate_limit validation', () => {
|
||||
it('accepts valid rate_limit', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: {
|
||||
rate_limit: { quota: 100, interval: 60 },
|
||||
},
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('transforms ACL with only rate_limit', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: {
|
||||
rate_limit: { quota: 100, interval: 60 },
|
||||
},
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success && result.data.acl) {
|
||||
expect(result.data.acl.rate_limit).toBeDefined()
|
||||
expect(result.data.acl.rate_limit!.quota).toBe(100)
|
||||
expect(result.data.acl.ip_whitelist).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('returns undefined for ACL with only empty ip_whitelist', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: '' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.acl).toBeUndefined()
|
||||
}
|
||||
})
|
||||
|
||||
it('returns undefined for ACL with no data', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
acl: { ip_whitelist: '\n\n' },
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.acl).toBeUndefined()
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('User Schema - account_access_entries', () => {
|
||||
const schema = getCreateUserSchema(t)
|
||||
|
||||
it('accepts empty account_access_entries (defaults to [])', () => {
|
||||
const result = schema.safeParse(validCreateUser)
|
||||
expect(result.success).toBe(true)
|
||||
if (result.success) {
|
||||
expect(result.data.account_access_entries).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
it('accepts valid account access entries', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
account_access_entries: [
|
||||
{ accountId: 1, roleId: 2 },
|
||||
{ accountId: 3, roleId: 4 },
|
||||
],
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects entry with accountId 0', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
account_access_entries: [{ accountId: 0, roleId: 1 }],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects entry with roleId 0', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
account_access_entries: [{ accountId: 1, roleId: 0 }],
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('User Schema - description', () => {
|
||||
const schema = getCreateUserSchema(t)
|
||||
|
||||
it('accepts undefined description', () => {
|
||||
const result = schema.safeParse(validCreateUser)
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts empty string description', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
description: '',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('accepts valid description', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
description: 'A test user account',
|
||||
})
|
||||
expect(result.success).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects description longer than 256 characters', () => {
|
||||
const result = schema.safeParse({
|
||||
...validCreateUser,
|
||||
description: 'a'.repeat(257),
|
||||
})
|
||||
expect(result.success).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -16,7 +16,6 @@
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import { z } from 'zod'
|
||||
import { useState, useMemo } from 'react'
|
||||
import { useFieldArray, useForm } from 'react-hook-form'
|
||||
import { zodResolver } from '@hookform/resolvers/zod'
|
||||
@@ -56,75 +55,9 @@ import { useRoles } from '@/hooks/use-roles'
|
||||
import { PasswordInput } from '@/components/password-input'
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { getCreateUserSchema, getUpdateUserSchema, type UserFormValues } from './schema'
|
||||
|
||||
const isValidIP = (ip: string) => {
|
||||
const ipv4 = /^(?:(?:\d{1,3}\.){3}\d{1,3})$/
|
||||
const ipv6 = /^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$/
|
||||
return ipv4.test(ip) || ipv6.test(ip)
|
||||
}
|
||||
|
||||
const accountAccessEntry = (t: any) => z.object({
|
||||
accountId: z.number().min(1, t('users.actions.schema.account_required')),
|
||||
roleId: z.number().min(1, t('users.actions.schema.role_required'))
|
||||
});
|
||||
|
||||
const baseUserSchema = (t: any) => ({
|
||||
username: z.string()
|
||||
.min(1, t('users.actions.schema.username_required'))
|
||||
.min(3, t('users.actions.schema.username_min'))
|
||||
.max(32, t('users.actions.schema.username_max')),
|
||||
email: z.string()
|
||||
.min(1, t('users.actions.schema.email_required'))
|
||||
.email(t('users.actions.schema.email_invalid')),
|
||||
global_roles: z.array(z.number()).min(1, t('users.actions.schema.global_role_required')),
|
||||
account_access_entries: z.array(accountAccessEntry(t)).optional().default([]),
|
||||
description: z.string().max(256, t('users.actions.schema.description_max')).optional().or(z.literal('')),
|
||||
acl: z.object({
|
||||
ip_whitelist: z.string().optional(),
|
||||
rate_limit: z.object({
|
||||
quota: z.number().positive().optional(),
|
||||
interval: z.number().positive().optional(),
|
||||
}).optional()
|
||||
}).optional().transform((data) => {
|
||||
if (!data) return undefined;
|
||||
const ips = data.ip_whitelist?.split('\n').map(v => v.trim()).filter(Boolean) || [];
|
||||
const finalRateLimit = (data.rate_limit?.quota && data.rate_limit?.interval)
|
||||
? data.rate_limit
|
||||
: undefined;
|
||||
if (ips.length === 0 && !finalRateLimit) return undefined;
|
||||
return {
|
||||
ip_whitelist: ips.length > 0 ? ips.join('\n') : undefined,
|
||||
rate_limit: finalRateLimit
|
||||
};
|
||||
})
|
||||
.refine((data) => {
|
||||
if (!data?.ip_whitelist) return true;
|
||||
return data.ip_whitelist.split('\n').every(isValidIP);
|
||||
}, {
|
||||
message: t('users.actions.schema.ip_invalid'),
|
||||
path: ["ip_whitelist"]
|
||||
})
|
||||
});
|
||||
|
||||
const createUserSchema = (t: any) => z.object({
|
||||
...baseUserSchema(t),
|
||||
password: z.string()
|
||||
.min(1, t('users.actions.schema.password_required'))
|
||||
.min(8, t('users.actions.schema.password_min'))
|
||||
.max(256, t('users.actions.schema.password_max')),
|
||||
});
|
||||
|
||||
const updateUserSchema = (t: any) => z.object({
|
||||
...baseUserSchema(t),
|
||||
password: z.string()
|
||||
.min(8, t('users.actions.schema.password_min'))
|
||||
.max(256, t('users.actions.schema.password_max'))
|
||||
.or(z.literal(''))
|
||||
.optional()
|
||||
.transform(v => v || undefined),
|
||||
});
|
||||
|
||||
export type UserForm = z.infer<ReturnType<typeof createUserSchema>> | z.infer<ReturnType<typeof updateUserSchema>>
|
||||
export type UserForm = UserFormValues
|
||||
|
||||
interface Props {
|
||||
currentRow?: User
|
||||
@@ -142,7 +75,7 @@ export function UserActionDialog({ currentRow, open, onOpenChange }: Props) {
|
||||
const { minimalList: allAccounts } = useMinimalAccountList()
|
||||
|
||||
const form = useForm<UserForm>({
|
||||
resolver: zodResolver(isEdit ? updateUserSchema(t) : createUserSchema(t)),
|
||||
resolver: zodResolver(isEdit ? getUpdateUserSchema(t) : getCreateUserSchema(t)),
|
||||
defaultValues: useMemo(() => {
|
||||
if (isEdit && currentRow) {
|
||||
const accessEntries = currentRow.account_access_map
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
export const accountAccessEntry = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
accountId: z.number().min(1, t('users.actions.schema.account_required')),
|
||||
roleId: z.number().min(1, t('users.actions.schema.role_required')),
|
||||
})
|
||||
|
||||
const isValidIPv4 = (ip: string): boolean => {
|
||||
const parts = ip.split('.')
|
||||
if (parts.length !== 4) return false
|
||||
return parts.every((part) => {
|
||||
const num = Number(part)
|
||||
return part === String(num) && num >= 0 && num <= 255
|
||||
})
|
||||
}
|
||||
|
||||
const isValidIP = (ip: string) => {
|
||||
const ipv6 = /^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$/
|
||||
return isValidIPv4(ip) || ipv6.test(ip)
|
||||
}
|
||||
|
||||
export const getBaseUserSchema = (t: (key: string) => string) =>
|
||||
z.object({
|
||||
username: z
|
||||
.string()
|
||||
.min(1, t('users.actions.schema.username_required'))
|
||||
.min(3, t('users.actions.schema.username_min'))
|
||||
.max(32, t('users.actions.schema.username_max')),
|
||||
email: z
|
||||
.string()
|
||||
.min(1, t('users.actions.schema.email_required'))
|
||||
.email(t('users.actions.schema.email_invalid')),
|
||||
global_roles: z
|
||||
.array(z.number())
|
||||
.min(1, t('users.actions.schema.global_role_required')),
|
||||
account_access_entries: z
|
||||
.array(accountAccessEntry(t))
|
||||
.optional()
|
||||
.default([]),
|
||||
description: z
|
||||
.string()
|
||||
.max(256, t('users.actions.schema.description_max'))
|
||||
.optional()
|
||||
.or(z.literal('')),
|
||||
acl: z
|
||||
.object({
|
||||
ip_whitelist: z.string().optional(),
|
||||
rate_limit: z
|
||||
.object({
|
||||
quota: z.number().positive().optional(),
|
||||
interval: z.number().positive().optional(),
|
||||
})
|
||||
.optional(),
|
||||
})
|
||||
.optional()
|
||||
.transform((data) => {
|
||||
if (!data) return undefined
|
||||
const ips =
|
||||
data.ip_whitelist
|
||||
?.split('\n')
|
||||
.map((v) => v.trim())
|
||||
.filter(Boolean) || []
|
||||
const finalRateLimit =
|
||||
data.rate_limit?.quota && data.rate_limit?.interval
|
||||
? data.rate_limit
|
||||
: undefined
|
||||
if (ips.length === 0 && !finalRateLimit) return undefined
|
||||
return {
|
||||
ip_whitelist: ips.length > 0 ? ips.join('\n') : undefined,
|
||||
rate_limit: finalRateLimit,
|
||||
}
|
||||
})
|
||||
.refine(
|
||||
(data) => {
|
||||
if (!data?.ip_whitelist) return true
|
||||
return data.ip_whitelist.split('\n').every(isValidIP)
|
||||
},
|
||||
{
|
||||
message: t('users.actions.schema.ip_invalid'),
|
||||
path: ['ip_whitelist'],
|
||||
}
|
||||
),
|
||||
})
|
||||
|
||||
export const getCreateUserSchema = (t: (key: string) => string) =>
|
||||
getBaseUserSchema(t).extend({
|
||||
password: z
|
||||
.string()
|
||||
.min(1, t('users.actions.schema.password_required'))
|
||||
.min(8, t('users.actions.schema.password_min'))
|
||||
.max(256, t('users.actions.schema.password_max')),
|
||||
})
|
||||
|
||||
export const getUpdateUserSchema = (t: (key: string) => string) =>
|
||||
getBaseUserSchema(t).extend({
|
||||
password: z
|
||||
.string()
|
||||
.min(8, t('users.actions.schema.password_min'))
|
||||
.max(256, t('users.actions.schema.password_max'))
|
||||
.or(z.literal(''))
|
||||
.optional()
|
||||
.transform((v) => v || undefined),
|
||||
})
|
||||
|
||||
export type UserFormValues = z.infer<ReturnType<typeof getCreateUserSchema>>
|
||||
+32
-6
@@ -100,6 +100,7 @@
|
||||
"auth": "المصادقة",
|
||||
"authType": "نوع_المصادقة",
|
||||
"autoConfiguring": "جارٍ التكوين التلقائي...",
|
||||
"autoDownloadNewMailboxes": "إضافة المجلدات الجديدة تلقائيًا",
|
||||
"autoDownloadNewMailboxesDescription": "إضافة المجلدات الجديدة المكتشفة تلقائيًا إلى قائمة التنزيل.",
|
||||
"beforeRelative": "تنزيل رسائل البريد الإلكتروني القديمة فقط",
|
||||
"beforeRelativeValue": "تنزيل رسائل البريد الإلكتروني قبل {{value}} {{unit}} مضت",
|
||||
@@ -114,6 +115,24 @@
|
||||
"continue": "متابعة",
|
||||
"createdAt": "تاريخ الإنشاء",
|
||||
"creationFailed": "فشل الإنشاء، يرجى المحاولة مرة أخرى لاحقًا",
|
||||
"cronAdvanced": "تعبير متقدم",
|
||||
"cronDaily": "يومياً",
|
||||
"cronDayOfMonth": "اليوم من الأسبوع",
|
||||
"cronDayOfWeek": "اليوم من الأسبوع",
|
||||
"cronFrequency": "التكرار",
|
||||
"cronFriday": "الجمعة",
|
||||
"cronHour": "الساعة",
|
||||
"cronMinute": "الدقيقة",
|
||||
"cronMonday": "الإثنين",
|
||||
"cronMonthly": "شهرياً",
|
||||
"cronSaturday": "السبت",
|
||||
"cronSimple": "تعبير بسيط",
|
||||
"cronSunday": "الأحد",
|
||||
"cronThursday": "الخميس",
|
||||
"cronTimezoneNote": "جميع الأوقات بالتوقيت المحلي للخادم.",
|
||||
"cronTuesday": "الثلاثاء",
|
||||
"cronWednesday": "الأربعاء",
|
||||
"cronWeekly": "أسبوعياً",
|
||||
"dateSelection": "تحديد التاريخ",
|
||||
"dateSince": "التاريخ منذ",
|
||||
"days": "أيام",
|
||||
@@ -132,6 +151,9 @@
|
||||
"downloadFailed": "فشل بدء مهمة التنزيل",
|
||||
"downloadInterval": "دورة التنزيل (بالدقائق)",
|
||||
"downloadIntervalPlaceholder": "أدخل الدقائق",
|
||||
"downloadSchedule": "Cron Schedule",
|
||||
"downloadScheduleDescription": "كرون من 6 حقول (ثانية-دقائق-ساعات-يوم-شهر-أسبوع) بتوقيت الخادم. يتجاوز الفاصل الزمني.",
|
||||
"downloadSchedulePlaceholder": "مثال: 0 0 * * *",
|
||||
"downloadScope": "استراتيجية التنزيل",
|
||||
"downloadScopeDescription": "اختر رسائل البريد الإلكتروني التي يجب فهرستها وتنزيلها.",
|
||||
"downloadStarted": "بدأت مهمة التنزيل",
|
||||
@@ -155,9 +177,6 @@
|
||||
"everyMinutes": "كل {{minutes}} دقيقة",
|
||||
"field": "الحقل",
|
||||
"fixed": "ثابت",
|
||||
"folderLimit": "حد المجلد",
|
||||
"folderLimitDescription": "حدد عدد رسائل البريد الإلكتروني للمزامنة لكل مجلد (الحد الأدنى 100). اتركه فارغًا بلا حدود.",
|
||||
"folderLimitPlaceholder": "مثال: 1000",
|
||||
"folderSync": {
|
||||
"autoSelectDescendants": "تحديد العناصر الفرعية تلقائيًا",
|
||||
"autoSelectParents": "تحديد العناصر الأصلية تلقائيًا",
|
||||
@@ -246,6 +265,10 @@
|
||||
}
|
||||
},
|
||||
"saveChanges": "حفظ التغييرات",
|
||||
"scheduleMode": "جدولة التنزيل",
|
||||
"scheduleModeCron": "تعبير Cron",
|
||||
"scheduleModeDescription": "تعيين التنزيل بفواصل زمنية ثابتة أو عبر Cron.",
|
||||
"scheduleModeInterval": "فواصل زمنية ثابتة",
|
||||
"selectAccountType": "اختر نوع الحساب",
|
||||
"selectAtLeastOneFolder": "يرجى تحديد مجلد واحد على الأقل",
|
||||
"selectAuthMethod": "اختر طريقة مصادقة",
|
||||
@@ -556,6 +579,7 @@
|
||||
"account": "الحساب",
|
||||
"attachments": "المرفقات",
|
||||
"bcc": "نسخة مخفية",
|
||||
"blockRemoteAgain": "حظر مجدداً",
|
||||
"cc": "نسخة",
|
||||
"clickToDownload": "انقر للتنزيل",
|
||||
"date": "التاريخ",
|
||||
@@ -575,8 +599,11 @@
|
||||
"noMessageSelected": "لم يتم تحديد رسالة",
|
||||
"noTagsYet": "لا توجد علامات بعد",
|
||||
"onlyNonInlineAttachments": "يتم عرض المرفقات غير المضمنة فقط هنا.",
|
||||
"remoteBlocked": "لحماية خصوصيتك، قام Bichon بحظر المحتوى الخارجي في هذه الرسالة.",
|
||||
"remoteShown": "تم عرض المحتوى الخارجي.",
|
||||
"showLess": "إظهار أقل",
|
||||
"showMore": "إظهار المزيد...",
|
||||
"showRemoteContent": "عرض المحتوى الخارجي",
|
||||
"subject": "الموضوع",
|
||||
"tags": "العلامات",
|
||||
"to": "إلى",
|
||||
@@ -1645,14 +1672,13 @@
|
||||
},
|
||||
"validation": {
|
||||
"emailRequired": "البريد الإلكتروني مطلوب",
|
||||
"folderLimitMustBeAtLeast100": "يجب أن يكون حد المجلد 100 على الأقل",
|
||||
"folderLimitMustBeNumber": "يجب أن يكون حد المجلد رقمًا",
|
||||
"imapHostCannotBeEmpty": "لا يمكن أن يكون مُضيف IMAP فارغًا",
|
||||
"imapHostRequired": "مُضيف IMAP مطلوب",
|
||||
"imapPortMustBeLessThan65536": "يجب أن يكون منفذ IMAP أقل من 65536",
|
||||
"imapPortMustBePositive": "يجب أن يكون منفذ IMAP عددًا صحيحًا موجبًا",
|
||||
"incrementalSyncMustBeAtLeast10": "يجب أن يكون فاصل المزامنة التزايدية 10 دقائق على الأقل",
|
||||
"incrementalSyncMustBeNumber": "يجب أن يكون فاصل المزامنة التزايدية رقمًا",
|
||||
"invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')",
|
||||
"invalidEmail": "عنوان بريد إلكتروني غير صالح",
|
||||
"invalidUrl": "عنوان URL غير صالح",
|
||||
"passwordMinLength": "يجب أن تتكون كلمة المرور من {{min}} أحرف على الأقل",
|
||||
@@ -1664,4 +1690,4 @@
|
||||
"singleRequestBatchSizeTooLarge": "يجب أن يكون حجم الدُفعة على الأكثر 200",
|
||||
"singleRequestBatchSizeTooSmall": "يجب أن يكون حجم الدُفعة على الأقل 10"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+32
-6
@@ -100,6 +100,7 @@
|
||||
"auth": "Godkendelse",
|
||||
"authType": "godkendelsestype",
|
||||
"autoConfiguring": "Konfigurerer automatisk...",
|
||||
"autoDownloadNewMailboxes": "Tilføj automatisk nye mapper",
|
||||
"autoDownloadNewMailboxesDescription": "Føj automatisk nye mapper til downloadlisten.",
|
||||
"beforeRelative": "Download kun gamle e-mails",
|
||||
"beforeRelativeValue": "Download e-mails før {{value}} {{unit}} siden",
|
||||
@@ -114,6 +115,24 @@
|
||||
"continue": "Fortsæt",
|
||||
"createdAt": "Oprettet",
|
||||
"creationFailed": "Oprettelse mislykkedes, prøv venligst igen senere",
|
||||
"cronAdvanced": "Avanceret udtryk",
|
||||
"cronDaily": "Dagligt",
|
||||
"cronDayOfMonth": "Ugedag",
|
||||
"cronDayOfWeek": "Ugedag",
|
||||
"cronFrequency": "Frekvens",
|
||||
"cronFriday": "Fredag",
|
||||
"cronHour": "Time",
|
||||
"cronMinute": "Minut",
|
||||
"cronMonday": "Mandag",
|
||||
"cronMonthly": "Månedligt",
|
||||
"cronSaturday": "Lørdag",
|
||||
"cronSimple": "Simpelt udtryk",
|
||||
"cronSunday": "Søndag",
|
||||
"cronThursday": "Torsdag",
|
||||
"cronTimezoneNote": "Alle tider er serverens lokale tid.",
|
||||
"cronTuesday": "Tirsdag",
|
||||
"cronWednesday": "Onsdag",
|
||||
"cronWeekly": "Ugentligt",
|
||||
"dateSelection": "Datovalg",
|
||||
"dateSince": "Dato fra",
|
||||
"days": "Dage",
|
||||
@@ -132,6 +151,9 @@
|
||||
"downloadFailed": "Kunne ikke starte download-opgave",
|
||||
"downloadInterval": "Downloadinterval (minutter)",
|
||||
"downloadIntervalPlaceholder": "Indtast minutter",
|
||||
"downloadSchedule": "Cron Schedule",
|
||||
"downloadScheduleDescription": "6-felters Cron (sek min time dag mdr ugedag) i servertid. Tilsidesætter interval.",
|
||||
"downloadSchedulePlaceholder": "F.eks.: 0 0 * * *",
|
||||
"downloadScope": "Downloadstrategi",
|
||||
"downloadScopeDescription": "Vælg hvilke e-mails der skal indekseres og downloades.",
|
||||
"downloadStarted": "Download-opgave startet",
|
||||
@@ -155,9 +177,6 @@
|
||||
"everyMinutes": "hvert {{minutes}} minut",
|
||||
"field": "Felt",
|
||||
"fixed": "Fast",
|
||||
"folderLimit": "Mappegrænse",
|
||||
"folderLimitDescription": "Begræns antallet af e-mails, der skal synkroniseres pr. mappe (minimum 100). Lad stå tomt for ingen grænse.",
|
||||
"folderLimitPlaceholder": "f.eks. 1000",
|
||||
"folderSync": {
|
||||
"autoSelectDescendants": "Vælg efterkommere automatisk",
|
||||
"autoSelectParents": "Vælg forældre automatisk",
|
||||
@@ -246,6 +265,10 @@
|
||||
}
|
||||
},
|
||||
"saveChanges": "Gem ændringer",
|
||||
"scheduleMode": "Planlægning af download",
|
||||
"scheduleModeCron": "Cron-udtryk",
|
||||
"scheduleModeDescription": "Download via faste intervaller eller Cron.",
|
||||
"scheduleModeInterval": "Fast interval",
|
||||
"selectAccountType": "Vælg kontotype",
|
||||
"selectAtLeastOneFolder": "Vælg venligst mindst én mappe",
|
||||
"selectAuthMethod": "Vælg en godkendelsesmetode",
|
||||
@@ -556,6 +579,7 @@
|
||||
"account": "Konto",
|
||||
"attachments": "Vedhæftninger",
|
||||
"bcc": "Blindkopi",
|
||||
"blockRemoteAgain": "Bloker igen",
|
||||
"cc": "Kopi",
|
||||
"clickToDownload": "Klik for at downloade",
|
||||
"date": "Dato",
|
||||
@@ -575,8 +599,11 @@
|
||||
"noMessageSelected": "Ingen meddelelse valgt",
|
||||
"noTagsYet": "Ingen etiketter endnu",
|
||||
"onlyNonInlineAttachments": "Kun ikke-indlejrede vedhæftninger vises her.",
|
||||
"remoteBlocked": "For at beskytte dit privatliv har Bichon blokeret eksternt indhold i denne meddelelse.",
|
||||
"remoteShown": "Eksternt indhold vises nu.",
|
||||
"showLess": "vis mindre",
|
||||
"showMore": "vis mere...",
|
||||
"showRemoteContent": "Vis eksternt indhold",
|
||||
"subject": "Emne",
|
||||
"tags": "Etiketter",
|
||||
"to": "Til",
|
||||
@@ -1645,14 +1672,13 @@
|
||||
},
|
||||
"validation": {
|
||||
"emailRequired": "E-mail er påkrævet",
|
||||
"folderLimitMustBeAtLeast100": "Mappegrænse skal være mindst 100",
|
||||
"folderLimitMustBeNumber": "Mappegrænse skal være et tal",
|
||||
"imapHostCannotBeEmpty": "IMAP-vært kan ikke være tom",
|
||||
"imapHostRequired": "IMAP-vært er påkrævet",
|
||||
"imapPortMustBeLessThan65536": "IMAP-port skal være mindre end 65536",
|
||||
"imapPortMustBePositive": "IMAP-port skal være et positivt heltal",
|
||||
"incrementalSyncMustBeAtLeast10": "Inkrementelt synkroniseringsinterval skal være mindst 10 minutter",
|
||||
"incrementalSyncMustBeNumber": "Inkrementelt synkroniseringsinterval skal være et tal",
|
||||
"invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')",
|
||||
"invalidEmail": "Ugyldig e-mailadresse",
|
||||
"invalidUrl": "Ugyldig URL",
|
||||
"passwordMinLength": "Adgangskoden skal være mindst {{min}} tegn lang",
|
||||
@@ -1664,4 +1690,4 @@
|
||||
"singleRequestBatchSizeTooLarge": "Batch‑størrelse skal være højst 200",
|
||||
"singleRequestBatchSizeTooSmall": "Batch‑størrelse skal være mindst 10"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user