From fa70437a62dfb7d3c01e47b51191207eb453f5de Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sat, 25 Apr 2026 05:39:58 +0800 Subject: [PATCH] feat: support export account emails to a single mbox file --- Cargo.lock | 26 ++- Cargo.toml | 2 +- config.toml | 2 +- crates/cli/Cargo.toml | 5 +- crates/cli/src/api/download.rs | 90 ++++++++ crates/cli/src/api/mod.rs | 4 + crates/cli/src/api/search.rs | 54 +++++ crates/cli/src/{ => api}/sender.rs | 0 crates/cli/src/api/stats.rs | 48 ++++ crates/cli/src/auth.rs | 10 +- crates/cli/src/eml/mod.rs | 2 +- crates/cli/src/export/mod.rs | 206 ++++++++++++++++++ crates/cli/src/main.rs | 56 +++-- crates/cli/src/mbox/mod.rs | 2 +- crates/cli/src/pst/mod.rs | 4 +- crates/core/src/account/mod.rs | 7 +- crates/core/src/account/stats.rs | 8 + crates/core/src/message/search.rs | 25 ++- crates/core/src/store/tantivy/envelope.rs | 91 ++++++-- crates/server/src/rest/api/account.rs | 29 ++- web/src/api/account/api.ts | 4 +- .../components/running-state-dialog.tsx | 4 +- 22 files changed, 610 insertions(+), 69 deletions(-) create mode 100644 crates/cli/src/api/download.rs create mode 100644 crates/cli/src/api/mod.rs create mode 100644 crates/cli/src/api/search.rs rename crates/cli/src/{ => api}/sender.rs (100%) create mode 100644 crates/cli/src/api/stats.rs create mode 100644 crates/cli/src/export/mod.rs create mode 100644 crates/core/src/account/stats.rs diff --git a/Cargo.lock b/Cargo.lock index ab5f548..3cb59f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -481,12 +481,15 @@ dependencies = [ "console", "dialoguer", "hex", + "indicatif", "mail-parser", "mail-send", "memmap2", "outlook-pst", "reqwest", "serde", + "serde_json", + "sysinfo", "tokio", "toml", ] @@ -2460,6 +2463,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "infer" version = "0.2.3" @@ -4260,9 +4276,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "aws-lc-rs", "log", @@ -5746,6 +5762,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "universal-hash" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 967fc38..0d6d59c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ async-imap = { version = "0.11.2", default-features = false, features = [ "compress", ] } webpki-roots = "1.0.7" -rustls = { version = "0.23.38", default-features = false, features = ["ring"] } +rustls = { version = "0.23.39", default-features = false, features = ["ring"] } rustls-pki-types = "1.14.0" tokio-io-timeout = "1.2.1" semver = "1.0.28" diff --git a/config.toml b/config.toml index 0c15d30..2bab9bc 100644 --- a/config.toml +++ b/config.toml @@ -1,2 +1,2 @@ base_url = "http://localhost:15630" -api_token = "2g2viN7zi4fKU1YgY50aTjl4" +api_token = "WuqNC0g8yNle7CVnxcvjUwjN" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7d16cf4..92e52de 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -21,4 +21,7 @@ chrono.workspace = true mail-send.workspace = true base64.workspace = true codepage-strings = "1.0.2" -hex = "0.4.3" \ No newline at end of file +hex = "0.4.3" +sysinfo.workspace = true +indicatif = "0.18.4" +serde_json.workspace = true \ No newline at end of file diff --git a/crates/cli/src/api/download.rs b/crates/cli/src/api/download.rs new file mode 100644 index 0000000..5d1cc89 --- /dev/null +++ b/crates/cli/src/api/download.rs @@ -0,0 +1,90 @@ +use bichon_core::{base64_encode, store::envelope::Envelope}; +use chrono::{TimeZone, Utc}; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; + +use crate::BichonCtlConfig; + +pub async fn download_and_export_with_json_header( + client: &Client, + config: &BichonCtlConfig, + envelope: Envelope, + file: &mut tokio::fs::File, +) -> bool { + let url = format!( + "{}/api/v1/download-message/{}/{}", + config.base_url, &envelope.account_id, &envelope.id + ); + + let response = match client + .get(&url) + .header("Authorization", format!("Bearer {}", config.api_token)) + .send() + .await + { + Ok(res) => { + if !res.status().is_success() { + eprintln!( + " ✘ HTTP Error {}: Failed for {}", + res.status(), + &envelope.id + ); + return false; + } + res + } + Err(e) => { + eprintln!(" ✘ Network error: {} for {}", e, &envelope.id); + return false; + } + }; + + let email_bytes = match response.bytes().await { + Ok(b) => b, + Err(e) => { + eprintln!( + " ✘ Failed to read response body for {}: {}", + &envelope.id, e + ); + return false; + } + }; + + let date_dt = Utc.timestamp_opt(envelope.date / 1000, 0).unwrap(); + let date_str = date_dt.format("%a %b %e %H:%M:%S %Y").to_string(); + let from_line = format!("From {} {}\n", envelope.from.clone(), date_str); + + let custom_header = build_metadata_header(BichonMetadata { + account_email: envelope.account_email, + mailbox_name: envelope.mailbox_name, + tags: envelope.tags, + }); + + let mut final_buffer = + Vec::with_capacity(from_line.len() + custom_header.len() + email_bytes.len() + 2); + final_buffer.extend_from_slice(from_line.as_bytes()); + final_buffer.extend_from_slice(custom_header.as_bytes()); + final_buffer.extend_from_slice(&email_bytes); + final_buffer.extend_from_slice(b"\n\n"); + + if let Err(e) = file.write_all(&final_buffer).await { + eprintln!(" ✘ IO Error: Failed to write to mbox: {}", e); + return false; + } + + true +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +pub struct BichonMetadata { + pub account_email: Option, + pub mailbox_name: Option, + pub tags: Option>, +} + +fn build_metadata_header(meta: BichonMetadata) -> String { + let json_str = serde_json::to_string(&meta).ok().unwrap(); + let encoded = base64_encode!(json_str); + format!("X-Bichon-Metadata: {}\r\n", encoded) +} diff --git a/crates/cli/src/api/mod.rs b/crates/cli/src/api/mod.rs new file mode 100644 index 0000000..8514321 --- /dev/null +++ b/crates/cli/src/api/mod.rs @@ -0,0 +1,4 @@ +pub mod download; +pub mod search; +pub mod sender; +pub mod stats; diff --git a/crates/cli/src/api/search.rs b/crates/cli/src/api/search.rs new file mode 100644 index 0000000..b8e74ad --- /dev/null +++ b/crates/cli/src/api/search.rs @@ -0,0 +1,54 @@ +use bichon_core::{ + common::paginated::DataPage, + message::search::{EmailSearchFilter, EmailSearchRequest, SortBy}, + store::envelope::Envelope, +}; +use reqwest::Client; + +use crate::BichonCtlConfig; + +pub async fn search_messages( + client: &Client, + config: &BichonCtlConfig, + page: u64, + page_size: u64, +) -> Option> { + let url = format!("{}/api/v1/search-messages", config.base_url); + + let payload = EmailSearchRequest { + filter: EmailSearchFilter::default(), + page, + page_size, + sort_by: Some(SortBy::DATE), + desc: Some(false), + }; + + match client + .post(&url) + .header("Authorization", format!("Bearer {}", config.api_token)) + .json(&payload) + .send() + .await + { + Ok(res) if res.status().is_success() => match res.json::>().await { + Ok(data) => Some(data), + Err(e) => { + eprintln!(" ✘ Failed to parse search response: {}", e); + None + } + }, + Ok(res) => { + let status = res.status(); + let error_body = res.text().await.unwrap_or_default(); + eprintln!( + " ✘ Failed to search messages. Status: {}\n Server error: {}", + status, error_body + ); + None + } + Err(e) => { + eprintln!(" ✘ Network error performing search: {}", e); + None + } + } +} diff --git a/crates/cli/src/sender.rs b/crates/cli/src/api/sender.rs similarity index 100% rename from crates/cli/src/sender.rs rename to crates/cli/src/api/sender.rs diff --git a/crates/cli/src/api/stats.rs b/crates/cli/src/api/stats.rs new file mode 100644 index 0000000..65f4647 --- /dev/null +++ b/crates/cli/src/api/stats.rs @@ -0,0 +1,48 @@ +use bichon_core::account::stats::AccountStats; +use reqwest::Client; + +use crate::BichonCtlConfig; + +pub async fn fetch_account_stats( + client: &Client, + config: &BichonCtlConfig, + account_id: u64, +) -> Option { + let url = format!("{}/api/v1/accounts/{}/stats", config.base_url, account_id); + + match client + .get(&url) + .header("Authorization", format!("Bearer {}", config.api_token)) + .send() + .await + { + Ok(res) if res.status().is_success() => { + match res.json::().await { + Ok(stats) => Some(stats), + Err(e) => { + eprintln!(" ✘ Failed to parse stats response: {}", e); + None + } + } + } + Ok(res) => { + let status = res.status(); + let error_body = res.text().await.unwrap_or_default(); + eprintln!( + " ✘ Failed to fetch stats for account [{}]. Status: {}\n Server error: {}", + account_id, + status, + error_body + ); + None + } + Err(e) => { + eprintln!( + " ✘ Network error fetching stats for [{}]: {}", + account_id, + e + ); + None + } + } +} \ No newline at end of file diff --git a/crates/cli/src/auth.rs b/crates/cli/src/auth.rs index 3db1afb..684c589 100644 --- a/crates/cli/src/auth.rs +++ b/crates/cli/src/auth.rs @@ -29,7 +29,11 @@ use bichon_core::{ use crate::BichonCtlConfig; -pub async fn verify_user_and_get_account(config: &BichonCtlConfig, theme: &ColorfulTheme) -> u64 { +pub async fn verify_user_and_get_account( + config: &BichonCtlConfig, + theme: &ColorfulTheme, + only_nosync: bool, +) -> MinimalAccount { let client = Client::new(); let url = format!("{}/api/v1/current-user", config.base_url); @@ -88,7 +92,7 @@ pub async fn verify_user_and_get_account(config: &BichonCtlConfig, theme: &Color println!("Welcome, {}!", style(&user.username).cyan()); let account_list_url = format!( - "{}/api/v1/minimal-account-list?only_nosync=true", + "{}/api/v1/minimal-account-list?only_nosync={only_nosync}", config.base_url ); let acc_response = client @@ -184,5 +188,5 @@ pub async fn verify_user_and_get_account(config: &BichonCtlConfig, theme: &Color style(&selected_acc.email).cyan().bold() ); - selected_acc.id + selected_acc.clone() } diff --git a/crates/cli/src/eml/mod.rs b/crates/cli/src/eml/mod.rs index 78667bc..b0f2e93 100644 --- a/crates/cli/src/eml/mod.rs +++ b/crates/cli/src/eml/mod.rs @@ -29,7 +29,7 @@ use reqwest::Client; use bichon_core::base64_encode_url_safe; -use crate::{sender::send_batch_request, BichonCtlConfig}; +use crate::{BichonCtlConfig, api::sender::send_batch_request}; pub async fn handle_eml_directory_import( config: &BichonCtlConfig, diff --git a/crates/cli/src/export/mod.rs b/crates/cli/src/export/mod.rs new file mode 100644 index 0000000..d18366b --- /dev/null +++ b/crates/cli/src/export/mod.rs @@ -0,0 +1,206 @@ +use crate::api::download::download_and_export_with_json_header; +use crate::api::search::search_messages; +use crate::api::stats::fetch_account_stats; +use crate::BichonCtlConfig; +use bichon_core::account::payload::MinimalAccount; +use console::style; +use dialoguer::Confirm; +use dialoguer::{theme::ColorfulTheme, Input}; +use indicatif::{ProgressBar, ProgressStyle}; +use reqwest::Client; +use std::path::{Path, PathBuf}; +use sysinfo::Disks; + +pub async fn handle_account_export( + config: &BichonCtlConfig, + account: MinimalAccount, + theme: &ColorfulTheme, +) { + let client = Client::new(); + + println!("Fetching account statistics..."); + let stats = match fetch_account_stats(&client, config, account.id).await { + Some(s) => s, + None => { + eprintln!("{} Failed to fetch account statistics.", style("✘").red()); + return; + } + }; + + println!("\n--- Account Statistics ---"); + println!(" Total Emails: {}", style(stats.total_count).cyan()); + println!( + " Total Size: {}", + style(format_bytes(stats.total_size)).cyan() + ); + + let path = loop { + let input: String = Input::with_theme(theme) + .with_prompt("Enter ABSOLUTE directory path for MBOX file") + .interact_text() + .unwrap(); + + let p = PathBuf::from(&input); + + if !p.is_absolute() { + eprintln!( + " {} {}", + style("✘").red(), + style("Invalid path: Must be an absolute path.").red() + ); + continue; + } + + if !p.exists() { + eprintln!( + " {} {}", + style("✘").red(), + style("Invalid path: Directory does not exist.").red() + ); + continue; + } + + if !p.is_dir() { + eprintln!( + " {} {}", + style("✘").red(), + style("Invalid path: The path provided is not a directory.").red() + ); + continue; + } + break p; + }; + + let disks = Disks::new_with_refreshed_list(); + let disk_result = disks + .list() + .iter() + .find(|d| path.starts_with(d.mount_point())) + .ok_or_else(|| "Could not identify the disk for the provided path."); + + match disk_result { + Ok(disk) => { + let free_space = disk.available_space(); + let required_space = (stats.total_size as f64 * 1.2) as u64; + + if free_space < required_space { + eprintln!( + " {} Insufficient disk space (including 10% safety buffer)!\n Required: {} (Base: {})\n Available: {}", + style("✘").red(), + style(format_bytes(required_space)).yellow(), + style(format_bytes(stats.total_size)).yellow(), + style(format_bytes(free_space)).yellow() + ); + return; + } + + println!( + " {} Disk space check passed. (Required: {}, Available: {})", + style("✔").green(), + style(format_bytes(required_space)).cyan(), + style(format_bytes(free_space)).cyan() + ); + } + Err(e) => { + eprintln!(" {} {}", style("✘").red(), style(e).red()); + return; + } + } + let mbox_file = get_unique_mbox_path(&path, account.id, &account.email); + if Confirm::with_theme(theme) + .with_prompt(format!( + "Export {} emails to '{}'?", + stats.total_count, + mbox_file.display() + )) + .default(true) + .interact() + .unwrap() + { + println!(" {} Starting export...", style("✔").green()); + + let pb = ProgressBar::new(stats.total_count as u64); + pb.set_style(ProgressStyle::with_template( + "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta}) {msg}" + ).unwrap()); + + let mut file = match tokio::fs::OpenOptions::new() + .append(true) + .create(true) + .open(&mbox_file) + .await + { + Ok(f) => f, + Err(e) => { + eprintln!(" ✘ Failed to open file '{}': {}", path.display(), e); + return; + } + }; + + let page_size = 100; + let mut current_page = 1; + let mut total_pages; + + loop { + if let Some(batch) = search_messages(&client, config, current_page, page_size).await { + total_pages = batch.total_pages.unwrap(); + + println!(" → Processing page {}/{}", current_page, total_pages); + + for envelope in batch.items { + let success = + download_and_export_with_json_header(&client, config, envelope, &mut file) + .await; + + if !success { + eprintln!(" ✘ Failed to export an email. Aborting process..."); + return; + } + pb.inc(1); + } + if current_page >= total_pages { + break; + } + current_page += 1; + } else { + eprintln!( + " ✘ Failed to fetch page {}. Aborting process...", + current_page + ); + return; + } + } + pb.finish(); + println!(" {} Export complete!", style("✔").green()); + } +} + +fn format_bytes(bytes: u64) -> String { + if bytes < 1024 { + format!("{:.2} B", bytes) + } else if bytes < 1024 * 1024 { + format!("{:.2} KB", bytes / 1024) + } else if bytes < 1024 * 1024 * 1024 { + format!("{:.2} MB", bytes / 1024 / 1024) + } else { + format!("{:.2} GB", bytes / 1024 / 1024 / 1024) + } +} + +fn get_unique_mbox_path(base_dir: &Path, account_id: u64, email: &str) -> PathBuf { + let email_part = email.replace(' ', "_"); + + let mut base_name = format!("account_{}_{}", account_id, email_part); + if base_name.starts_with('.') { + base_name = format!("_{}", base_name); + } + + let mut final_path = base_dir.join(format!("{}.mbox", base_name)); + + let mut counter = 1; + while final_path.exists() { + final_path = base_dir.join(format!("{}_{}.mbox", base_name, counter)); + counter += 1; + } + final_path +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index a558e34..a4ffb34 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -25,15 +25,16 @@ use std::fs; use crate::{ auth::verify_user_and_get_account, eml::handle_eml_directory_import, - mbox::handle_mbox_single_file_import, pst::handle_pst_import, + export::handle_account_export, mbox::handle_mbox_single_file_import, pst::handle_pst_import, thunderbird::handle_thunderbird_import, }; +pub mod api; pub mod auth; pub mod eml; +pub mod export; pub mod mbox; pub mod pst; -pub mod sender; pub mod thunderbird; #[derive(Parser, Debug)] @@ -124,27 +125,48 @@ async fn main() { } }; - let target_account_id = verify_user_and_get_account(&final_config, &theme).await; - - let import_modes = &[ - "1. EML: Scan directory recursively (Maintains folder structure)", - "2. MBOX: Single archive file (Stream from one file)", - "3. Thunderbird: Import from local profile directory", - "4. PST: Outlook Personal Storage (Single .pst file)", + let operations = &[ + "1. Import: Upload email data to Bichon", + "2. Export: Download account data as MBOX file", ]; - let mode_idx = Select::with_theme(&theme) - .with_prompt("Select import method") - .items(import_modes) + let op_idx = Select::with_theme(&theme) + .with_prompt("Select operation") + .items(operations) .default(0) .interact() .unwrap(); - match mode_idx { - 0 => handle_eml_directory_import(&final_config, target_account_id, &theme).await, - 1 => handle_mbox_single_file_import(&final_config, target_account_id, &theme).await, - 2 => handle_thunderbird_import(&final_config, target_account_id, &theme).await, - 3 => handle_pst_import(&final_config, target_account_id, &theme).await, + match op_idx { + 0 => { + let target_account = verify_user_and_get_account(&final_config, &theme, true).await; + + let import_modes = &[ + "1. EML: Scan directory recursively (Maintains folder structure)", + "2. MBOX: Single archive file (Stream from one file)", + "3. Thunderbird: Import from local profile directory", + "4. PST: Outlook Personal Storage (Single .pst file)", + ]; + + let mode_idx = Select::with_theme(&theme) + .with_prompt("Select import method") + .items(import_modes) + .default(0) + .interact() + .unwrap(); + + match mode_idx { + 0 => handle_eml_directory_import(&final_config, target_account.id, &theme).await, + 1 => handle_mbox_single_file_import(&final_config, target_account.id, &theme).await, + 2 => handle_thunderbird_import(&final_config, target_account.id, &theme).await, + 3 => handle_pst_import(&final_config, target_account.id, &theme).await, + _ => unreachable!(), + } + } + 1 => { + let target_account = verify_user_and_get_account(&final_config, &theme, false).await; + handle_account_export(&final_config, target_account, &theme).await; + } _ => unreachable!(), } } diff --git a/crates/cli/src/mbox/mod.rs b/crates/cli/src/mbox/mod.rs index a062a1b..a52f83f 100644 --- a/crates/cli/src/mbox/mod.rs +++ b/crates/cli/src/mbox/mod.rs @@ -19,9 +19,9 @@ use std::collections::HashMap; use std::path::PathBuf; +use crate::api::sender::send_batch_request; use crate::mbox::gmail::determine_folder; use crate::mbox::reader::MboxFile; -use crate::sender::send_batch_request; use crate::BichonCtlConfig; use bichon_core::base64_encode_url_safe; use console::style; diff --git a/crates/cli/src/pst/mod.rs b/crates/cli/src/pst/mod.rs index a0b5917..c11ee4a 100644 --- a/crates/cli/src/pst/mod.rs +++ b/crates/cli/src/pst/mod.rs @@ -23,9 +23,9 @@ use mail_send::mail_builder::headers::text::Text; use mail_send::mail_builder::MessageBuilder; use outlook_pst::ltp::prop_context::PropertyValue; -use crate::BichonCtlConfig; +use crate::api::sender::send_batch_request; use crate::pst::encoding::decode_subject; -use crate::sender::send_batch_request; +use crate::BichonCtlConfig; use bichon_core::base64_encode_url_safe; use dialoguer::Confirm; use outlook_pst::messaging::attachment::AttachmentProperties; diff --git a/crates/core/src/account/mod.rs b/crates/core/src/account/mod.rs index 4dad257..d5bddad 100644 --- a/crates/core/src/account/mod.rs +++ b/crates/core/src/account/mod.rs @@ -19,8 +19,9 @@ pub mod entity; pub mod grant; pub mod migration; -pub mod payload; -pub mod state; -pub mod since; pub mod old_state; +pub mod payload; +pub mod since; +pub mod state; +pub mod stats; pub mod view; diff --git a/crates/core/src/account/stats.rs b/crates/core/src/account/stats.rs new file mode 100644 index 0000000..ef006e1 --- /dev/null +++ b/crates/core/src/account/stats.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct AccountStats { + pub total_size: u64, + pub total_count: u64, +} diff --git a/crates/core/src/message/search.rs b/crates/core/src/message/search.rs index 07eff21..40afdf5 100644 --- a/crates/core/src/message/search.rs +++ b/crates/core/src/message/search.rs @@ -21,12 +21,15 @@ use serde::{Deserialize, Serialize}; use std::collections::HashSet; use crate::{ - common::paginated::DataPage, error::{BichonResult, code::ErrorCode}, raise_error, store::{ - envelope::Envelope, - tantivy::{ - attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER, model::AttachmentModel, - }, - } + common::paginated::DataPage, + error::{code::ErrorCode, BichonResult}, + raise_error, + store::{ + envelope::Envelope, + tantivy::{ + attachment::ATTACHMENT_MANAGER, envelope::ENVELOPE_MANAGER, model::AttachmentModel, + }, + }, }; #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] @@ -66,11 +69,11 @@ pub enum SortBy { #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct EmailSearchRequest { - filter: EmailSearchFilter, - page: u64, - page_size: u64, - sort_by: Option, - desc: Option, + pub filter: EmailSearchFilter, + pub page: u64, + pub page_size: u64, + pub sort_by: Option, + pub desc: Option, } impl EmailSearchRequest { pub fn validate(&self) -> BichonResult<()> { diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index 69ce4a9..b1efe3f 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -25,22 +25,30 @@ use std::{ }; use crate::{ - account::migration::AccountModel, common::{paginated::DataPage, signal::SIGNAL_MANAGER}, dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, error::{BichonResult, code::ErrorCode}, message::{ - search::{EmailSearchFilter, SortBy}, - tags::{TagAction, TagCount, TagsRequest}, - }, raise_error, settings::dir::DATA_DIR_MANAGER, store::{ - envelope::Envelope, - storage::BLOB_MANAGER, - tantivy::{ - fatal_commit, - fields::{ - F_ACCOUNT_ID, F_DATE, F_FROM, F_REGULAR_ATTACHMENT_COUNT, F_SIZE, F_TAGS, - F_THREAD_ID, F_UID, - }, - model::{EnvelopeWithAttachments, extract_contacts}, - schema::SchemaTools, + account::{migration::AccountModel, stats::AccountStats}, + common::{paginated::DataPage, signal::SIGNAL_MANAGER}, + dashboard::{DashboardStats, Group, LargestEmail, TimeBucket}, + error::{code::ErrorCode, BichonResult}, + message::{ + search::{EmailSearchFilter, SortBy}, + tags::{TagAction, TagCount, TagsRequest}, + }, + raise_error, + settings::dir::DATA_DIR_MANAGER, + store::{ + envelope::Envelope, + storage::BLOB_MANAGER, + tantivy::{ + fatal_commit, + fields::{ + F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_REGULAR_ATTACHMENT_COUNT, F_SIZE, F_TAGS, + F_THREAD_ID, F_UID, }, - }, utc_now + model::{extract_contacts, EnvelopeWithAttachments}, + schema::SchemaTools, + }, + }, + utc_now, }; use chrono::Utc; @@ -621,6 +629,51 @@ impl IndexManager { Ok(Self::extract_max_uid(&agg_res)) } + pub async fn get_account_stats(&self, account_id: u64) -> BichonResult { + let searcher = self.create_searcher()?; + let query = self.account_query(account_id); + + let agg_req: Aggregations = serde_json::from_value(json!({ + "total_count": { + "value_count": { + "field": F_ID + } + }, + "total_size": { + "sum": { + "field": F_SIZE + } + } + })) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let collector = AggregationCollector::from_aggs(agg_req, Default::default()); + let agg_res = searcher + .search(query.as_ref(), &collector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut stats = AccountStats::default(); + + stats.total_count = Self::extract_value_count(&agg_res, "total_count")?; + + let result = agg_res.0.get("total_size").ok_or_else(|| { + raise_error!( + "missing 'total_size' aggregation result".into(), + ErrorCode::InternalError + ) + })?; + + if let AggregationResult::MetricResult(MetricResult::Sum(v)) = result { + stats.total_size = v.value.map(|v| v as u64).ok_or_else(|| { + raise_error!( + "'total_size' sum metric has no value".into(), + ErrorCode::InternalError + ) + })?; + } + Ok(stats) + } + fn extract_max_uid(agg_res: &AggregationResults) -> Option { agg_res.0.get("max_uid").and_then(|result| match result { AggregationResult::MetricResult(MetricResult::Max(max)) => { @@ -1110,13 +1163,13 @@ impl IndexManager { let agg_res = searcher .search(query.as_ref(), &collector) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - Self::extract_thread_count(&agg_res) + Self::extract_value_count(&agg_res, "thread_count") } - fn extract_thread_count(agg_res: &AggregationResults) -> BichonResult { - let Some(result) = agg_res.0.get("thread_count") else { + fn extract_value_count(agg_res: &AggregationResults, name: &str) -> BichonResult { + let Some(result) = agg_res.0.get(name) else { return Err(raise_error!( - "Missing aggregation result: thread_count".into(), + format!("Missing aggregation result: '{}'", name), ErrorCode::InternalError )); }; diff --git a/crates/server/src/rest/api/account.rs b/crates/server/src/rest/api/account.rs index 99a74df..87ef359 100644 --- a/crates/server/src/rest/api/account.rs +++ b/crates/server/src/rest/api/account.rs @@ -25,8 +25,10 @@ use bichon_core::account::payload::{ filter_accessible_accounts, AccountCreateRequest, AccountUpdateRequest, MinimalAccount, }; use bichon_core::account::state::DownloadState; +use bichon_core::account::stats::AccountStats; use bichon_core::account::view::AccountResp; use bichon_core::common::paginated::{paginate_vec, DataPage}; +use bichon_core::store::tantivy::envelope::ENVELOPE_MANAGER; use bichon_core::users::permissions::Permission; use bichon_core::users::UserModel; use poem_openapi::param::{Path, Query}; @@ -182,11 +184,11 @@ impl AccountApi { /// Get the running state of an account #[oai( - path = "/account-state/:account_id", + path = "/accounts/:account_id/download-stats", method = "get", - operation_id = "account_state" + operation_id = "accounts_download_state" )] - async fn account_state( + async fn accounts_download_state( &self, /// The account ID to check state for account_id: Path, @@ -202,6 +204,27 @@ impl AccountApi { Ok(Json(state)) } + /// Get the stats of an account + #[oai( + path = "/accounts/:account_id/stats", + method = "get", + operation_id = "accounts_stats" + )] + async fn accounts_stats( + &self, + /// The account ID to check state for + account_id: Path, + context: WrappedContext, + ) -> ApiResult> { + let account_id = account_id.0; + AccountModel::check_account_exists(account_id).await?; + context + .require_permission(Some(account_id), Permission::ACCOUNT_READ_DETAILS) + .await?; + let state = ENVELOPE_MANAGER.get_account_stats(account_id).await?; + Ok(Json(state)) + } + /// Get a minimal list of active accounts for use in selectors when creating account-related resources /// /// This endpoint provides a lightweight list of accounts containing only essential information (id and name). diff --git a/web/src/api/account/api.ts b/web/src/api/account/api.ts index d8dbb63..2381d58 100644 --- a/web/src/api/account/api.ts +++ b/web/src/api/account/api.ts @@ -146,8 +146,8 @@ export interface AccountModel { auto_download_new_mailboxes?: boolean; } -export const account_state = async (account_id: number) => { - const response = await axiosInstance.get(`api/v1/account-state/${account_id}`); +export const download_state = async (account_id: number) => { + const response = await axiosInstance.get(`api/v1/accounts/${account_id}/download-stats`); return response.data; }; diff --git a/web/src/features/accounts/components/running-state-dialog.tsx b/web/src/features/accounts/components/running-state-dialog.tsx index ca4028e..df628fe 100644 --- a/web/src/features/accounts/components/running-state-dialog.tsx +++ b/web/src/features/accounts/components/running-state-dialog.tsx @@ -26,7 +26,7 @@ import { } from '@/components/ui/dialog' import { Button } from '@/components/ui/button' import { useQuery } from '@tanstack/react-query' -import { account_state, AccountModel, FolderProgress } from '@/api/account/api' +import { download_state, AccountModel, FolderProgress } from '@/api/account/api' import { format } from 'date-fns' import { ScrollArea } from '@/components/ui/scroll-area' import { Badge } from '@/components/ui/badge' @@ -134,7 +134,7 @@ export function RunningStateDialog({ currentRow, open, onOpenChange }: Props) { const { data: state, isLoading } = useQuery({ queryKey: ['running-state', currentRow.id], - queryFn: () => account_state(currentRow.id), + queryFn: () => download_state(currentRow.id), refetchInterval: 5000, enabled: open && !!currentRow.id, })