mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-03 07:48:34 +02:00
feat: web upload supports PST, configurable MBOX/PST size limits
- Make MBOX/PST upload size limits configurable via SETTINGS
(bichon_web_mbox_upload_limit_mb defaults to 1 GB,
bichon_web_pst_upload_limit_mb defaults to 2 GB)
This commit is contained in:
@@ -20,6 +20,7 @@
|
||||
//use poem_openapi::Object;
|
||||
pub mod history;
|
||||
pub mod reader;
|
||||
pub mod pst;
|
||||
pub use history::ImportHistory;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
@@ -257,9 +258,15 @@ pub fn check_temp_disk_space() -> BichonResult<u64> {
|
||||
pub enum FileFormat {
|
||||
Eml,
|
||||
Mbox,
|
||||
Pst,
|
||||
}
|
||||
|
||||
pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
|
||||
// PST files start with OLE2 compound document magic bytes
|
||||
if bytes.len() >= 8 && &bytes[..8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1" {
|
||||
return Some(FileFormat::Pst);
|
||||
}
|
||||
|
||||
// MBOX files start with "From " (note the trailing space after From)
|
||||
if bytes.starts_with(b"From ") {
|
||||
// Double-check: look for a valid date after the first "From " line
|
||||
@@ -290,6 +297,8 @@ pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
|
||||
Some(FileFormat::Eml)
|
||||
} else if lower.ends_with(".mbox") {
|
||||
Some(FileFormat::Mbox)
|
||||
} else if lower.ends_with(".pst") {
|
||||
Some(FileFormat::Pst)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
@@ -375,7 +384,7 @@ fn validate_import_account(account_id: u64) -> BichonResult<AccountModel> {
|
||||
}
|
||||
|
||||
/// Resolve or create a mailbox/folder for the given account.
|
||||
fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
|
||||
pub(super) fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
|
||||
match account.account_type {
|
||||
AccountType::IMAP => {
|
||||
// Shouldn't reach here (validated above), but handle gracefully
|
||||
@@ -412,6 +421,13 @@ fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve or create a mailbox for a given account_id and folder name.
|
||||
/// Used by PST import to create per-folder mailboxes.
|
||||
pub fn resolve_mailbox_by_account_id(account_id: u64, folder: &str) -> BichonResult<u64> {
|
||||
let account = AccountModel::check_account_exists(account_id)?;
|
||||
resolve_mailbox(&account, folder)
|
||||
}
|
||||
|
||||
/// Process an uploaded file (EML or MBOX) and import into the given account/folder.
|
||||
/// This runs synchronously and should be spawned on a background thread.
|
||||
///
|
||||
@@ -497,6 +513,7 @@ pub fn process_uploaded_file(
|
||||
match format {
|
||||
FileFormat::Eml => process_eml_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
FileFormat::Mbox => process_mbox_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
FileFormat::Pst => process_pst_upload(import_id, file_path, account_id, mailbox_id, user_id, folder),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -512,7 +529,7 @@ fn detect_format_from_file(file_path: &Path, file_name: &str) -> BichonResult<Fi
|
||||
|
||||
detect_format(&buf, file_name).ok_or_else(|| {
|
||||
raise_error!(
|
||||
"Unknown file format. Supported: .eml, .mbox".into(),
|
||||
"Unknown file format. Supported: .eml, .mbox, .pst".into(),
|
||||
ErrorCode::InvalidParameter
|
||||
)
|
||||
})
|
||||
@@ -694,6 +711,111 @@ fn process_single_eml(
|
||||
}
|
||||
}
|
||||
|
||||
/// Process a PST file uploaded via the web UI.
|
||||
/// Two-pass approach: count messages first, then process with periodic progress updates.
|
||||
fn process_pst_upload(
|
||||
import_id: &str,
|
||||
file_path: &Path,
|
||||
account_id: u64,
|
||||
_mailbox_id: u64, // ignored; PST creates its own mailboxes per folder
|
||||
user_id: u64,
|
||||
folder: &str,
|
||||
) {
|
||||
// Pass 1: count total messages
|
||||
let total = match pst::count_pst_messages(file_path) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
update_progress(import_id, ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Processing,
|
||||
format: "pst".to_string(),
|
||||
total,
|
||||
success: 0,
|
||||
duplicates: 0,
|
||||
failed: 0,
|
||||
failed_details: vec![],
|
||||
});
|
||||
|
||||
// Pass 2: process messages with progress updates
|
||||
let mut success_count: usize = 0;
|
||||
let mut failed_details: Vec<FailedItemDetail> = Vec::new();
|
||||
let mut index: usize = 0;
|
||||
|
||||
let pst_store = match outlook_pst::open_store(file_path) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Progress callback: update progress every 50 messages
|
||||
let import_id = import_id.to_string();
|
||||
let format_str = "pst".to_string();
|
||||
pst::process_folder_with_progress(
|
||||
&ipm_subtree_folder,
|
||||
"", // parent_path starts empty
|
||||
account_id,
|
||||
total, // pass pre-counted total for accurate progress
|
||||
&mut success_count,
|
||||
&mut failed_details,
|
||||
&mut index,
|
||||
&|processed, actual_failed| {
|
||||
update_progress(&import_id, ImportProgress {
|
||||
import_id: import_id.clone(),
|
||||
status: ImportStatus::Processing,
|
||||
format: format_str.clone(),
|
||||
total,
|
||||
success: processed - actual_failed,
|
||||
duplicates: 0,
|
||||
failed: actual_failed,
|
||||
failed_details: vec![],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// Clean up temp file
|
||||
let _ = std::fs::remove_file(file_path);
|
||||
|
||||
let final_progress = ImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: ImportStatus::Completed,
|
||||
format: "pst".to_string(),
|
||||
total,
|
||||
success: success_count,
|
||||
duplicates: 0,
|
||||
failed: failed_details.len(),
|
||||
failed_details,
|
||||
};
|
||||
history::save_import_history(user_id, account_id, folder, &final_progress);
|
||||
update_progress(&import_id, final_progress);
|
||||
}
|
||||
|
||||
/// Record a fatal failure and save history.
|
||||
fn fail_progress(
|
||||
import_id: &str,
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
|
||||
pub fn decode_subject(value: &PropertyValue) -> Option<String> {
|
||||
match value {
|
||||
PropertyValue::String8(value) => {
|
||||
let offset = match value.buffer().first() {
|
||||
Some(1) => 2,
|
||||
_ => 0,
|
||||
};
|
||||
let buffer: Vec<_> = value
|
||||
.buffer()
|
||||
.iter()
|
||||
.skip(offset)
|
||||
.map(|&b| u16::from(b))
|
||||
.collect();
|
||||
Some(String::from_utf16_lossy(&buffer))
|
||||
}
|
||||
PropertyValue::Unicode(value) => {
|
||||
let offset = match value.buffer().first() {
|
||||
Some(1) => 2,
|
||||
_ => 0,
|
||||
};
|
||||
Some(String::from_utf16_lossy(&value.buffer()[offset..]))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option<String> {
|
||||
match code_page {
|
||||
20127 => {
|
||||
let buffer: Vec<_> = buffer.iter().map(|&b| u16::from(b)).collect();
|
||||
Some(String::from_utf16_lossy(&buffer))
|
||||
}
|
||||
_ => {
|
||||
let coding = codepage_strings::Coding::new(code_page).ok()?;
|
||||
Some(coding.decode(buffer).ok()?.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_rtf_compressed(buffer: &[u8]) -> Option<String> {
|
||||
compressed_rtf::decompress_rtf(buffer).ok()
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
//
|
||||
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
|
||||
//
|
||||
// This file is part of the Bichon Email Archiving Project
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::base64_encode_url_safe;
|
||||
use crate::envelope::extractor::extract_envelope_from_eml;
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use mail_send::mail_builder::headers::text::Text;
|
||||
use mail_send::mail_builder::MessageBuilder;
|
||||
use outlook_pst::ltp::prop_context::PropertyValue;
|
||||
use outlook_pst::messaging::attachment::AttachmentProperties;
|
||||
use outlook_pst::messaging::folder::Folder;
|
||||
use outlook_pst::messaging::message::{Message, MessageProperties};
|
||||
use outlook_pst::ndb::node_id::NodeId;
|
||||
use std::rc::Rc;
|
||||
|
||||
mod encoding;
|
||||
|
||||
/// Convert a PST Message into a base64-encoded EML string.
|
||||
pub fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
|
||||
let properties = message.properties();
|
||||
|
||||
let mut builder = MessageBuilder::new();
|
||||
if let Some(sub) = extract_subject(properties) {
|
||||
builder = builder.subject(sub);
|
||||
}
|
||||
if let Some(mid) = extract_string_property(properties, 0x1035) {
|
||||
builder = builder.message_id(mid);
|
||||
}
|
||||
if let Some(irt) = extract_string_property(properties, 0x1042) {
|
||||
builder = builder.in_reply_to(irt);
|
||||
}
|
||||
|
||||
if let Some(refs) = extract_string_property(properties, 0x1039) {
|
||||
builder = builder.header("References", Text::new(refs));
|
||||
}
|
||||
|
||||
if let Some(cid_val) = properties.get(0x3013) {
|
||||
if let PropertyValue::Binary(bin) = cid_val {
|
||||
builder = builder.header(
|
||||
"X-Bichon-Conversation-ID",
|
||||
Text::new(hex::encode(bin.buffer())),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let from = extract_string_property(properties, 0x5D01)
|
||||
.or_else(|| extract_string_property(properties, 0x5D02))
|
||||
.or_else(|| extract_string_property(properties, 0x0C1F));
|
||||
|
||||
if let Some(f) = from {
|
||||
builder = builder.from(f);
|
||||
}
|
||||
|
||||
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
|
||||
let dt = filetime_to_datetime(filetime).timestamp();
|
||||
builder = builder.date(dt);
|
||||
}
|
||||
|
||||
let (to, cc, bcc) = extract_recipients_list(&message);
|
||||
if !to.is_empty() {
|
||||
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !cc.is_empty() {
|
||||
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
if !bcc.is_empty() {
|
||||
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
if let Some(html) = extract_html(properties) {
|
||||
builder = builder.html_body(html);
|
||||
}
|
||||
|
||||
if let Some(text) = extract_text(properties) {
|
||||
builder = builder.text_body(text);
|
||||
}
|
||||
|
||||
if let Some(attachment_table) = message.attachment_table() {
|
||||
for row in attachment_table.rows_matrix() {
|
||||
let node_id = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
|
||||
let att_props = attachment.properties();
|
||||
let name = extract_attachment_string_property(att_props, 0x3707);
|
||||
let mime = extract_attachment_string_property(att_props, 0x370E)
|
||||
.unwrap_or_else(|| "application/octet-stream".into());
|
||||
let cid = extract_attachment_string_property(att_props, 0x3712);
|
||||
let is_inline = att_props
|
||||
.get(0x3714)
|
||||
.and_then(|val| {
|
||||
if let PropertyValue::Integer32(f) = val {
|
||||
Some(f)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|flag| (flag & 0x4) != 0)
|
||||
.unwrap_or(false);
|
||||
|
||||
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
|
||||
let data = bin.buffer().to_vec();
|
||||
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
|
||||
|
||||
if is_inline && cid.is_some() {
|
||||
let content_id = cid.unwrap();
|
||||
builder = builder.inline(mime, content_id, data);
|
||||
} else {
|
||||
builder = builder.attachment(mime, file_name, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match builder.write_to_vec() {
|
||||
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to generate EML from PST message: {:?}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
|
||||
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
|
||||
let nsecs = (filetime % 10_000_000) * 100;
|
||||
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
|
||||
}
|
||||
|
||||
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
|
||||
let mut to = Vec::new();
|
||||
let mut cc = Vec::new();
|
||||
let mut bcc = Vec::new();
|
||||
|
||||
let recipient_table = message.recipient_table();
|
||||
if let Some(recipient_table) = recipient_table {
|
||||
let context = recipient_table.context();
|
||||
for row in recipient_table.rows_matrix() {
|
||||
if let Ok(cols) = row.columns(context) {
|
||||
let mut r_type = 0;
|
||||
let mut email = String::new();
|
||||
|
||||
for (col, val) in context.columns().iter().zip(cols) {
|
||||
let prop_val = val
|
||||
.as_ref()
|
||||
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
|
||||
match col.prop_id() {
|
||||
0x0C15 => {
|
||||
if let Some(PropertyValue::Integer32(t)) = prop_val {
|
||||
r_type = t;
|
||||
}
|
||||
}
|
||||
0x39FE | 0x3003 => {
|
||||
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
|
||||
email = s;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !email.is_empty() {
|
||||
match r_type {
|
||||
1 => to.push(email),
|
||||
2 => cc.push(email),
|
||||
3 => bcc.push(email),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let receiver = extract_string_property(message.properties(), 0x0076);
|
||||
if let Some(receiver) = receiver {
|
||||
to.push(receiver);
|
||||
}
|
||||
}
|
||||
(to, cc, bcc)
|
||||
}
|
||||
|
||||
fn extract_subject(props: &MessageProperties) -> Option<String> {
|
||||
props.get(0x0037).and_then(|val| encoding::decode_subject(val))
|
||||
}
|
||||
|
||||
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_attachment_string_property(
|
||||
properties: &AttachmentProperties,
|
||||
prop_id: u16,
|
||||
) -> Option<String> {
|
||||
properties
|
||||
.get(prop_id)
|
||||
.and_then(|value| extract_string(value))
|
||||
}
|
||||
|
||||
fn extract_string(value: &PropertyValue) -> Option<String> {
|
||||
match value {
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1000).and_then(extract_string).or_else(|| {
|
||||
properties.get(0x1009).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_html(properties: &MessageProperties) -> Option<String> {
|
||||
properties.get(0x1013).and_then(|value| match value {
|
||||
PropertyValue::Binary(value) => {
|
||||
let code_page = properties
|
||||
.get(0x3FDE)
|
||||
.and_then(|v| {
|
||||
if let PropertyValue::Integer32(cpid) = v {
|
||||
Some(*cpid as u16)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(65001);
|
||||
encoding::decode_html_body(value.buffer(), code_page)
|
||||
}
|
||||
PropertyValue::String8(value) => Some(value.to_string()),
|
||||
PropertyValue::Unicode(value) => Some(value.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
|
||||
for &prop_id in prop_ids {
|
||||
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
|
||||
return Some(*value);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Open a PST file and count total messages across all folders.
|
||||
/// Called from the web upload flow to get the total before processing.
|
||||
pub fn count_pst_messages(pst_path: &std::path::Path) -> crate::error::BichonResult<usize> {
|
||||
let pst_store = outlook_pst::open_store(pst_path).map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Failed to open PST file: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
let ipm_sub_tree = pst_store.properties().ipm_sub_tree_entry_id().map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Could not find IPM_SUBTREE in PST: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
let ipm_subtree_folder = pst_store.open_folder(&ipm_sub_tree).map_err(|e| {
|
||||
crate::raise_error!(
|
||||
format!("Failed to open root mailbox folder: {:?}", e),
|
||||
crate::error::code::ErrorCode::InvalidParameter
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(count_folder_messages(&ipm_subtree_folder))
|
||||
}
|
||||
|
||||
fn count_folder_messages(folder: &Rc<dyn Folder>) -> usize {
|
||||
let mut count = 0usize;
|
||||
|
||||
if let Some(contents_table) = folder.contents_table() {
|
||||
for row in contents_table.rows_matrix() {
|
||||
let store = folder.store().clone();
|
||||
let entry_id = match store
|
||||
.properties()
|
||||
.make_entry_id(NodeId::from(u32::from(row.id())))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
if store.open_message(&entry_id, None).is_ok() {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
count += count_folder_messages(&sub_folder);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
count
|
||||
}
|
||||
|
||||
/// Walk all folders and process messages, calling the progress callback
|
||||
/// every 50 messages. Used by the web upload flow.
|
||||
pub fn process_folder_with_progress<F>(
|
||||
folder: &Rc<dyn Folder>,
|
||||
parent_path: &str,
|
||||
account_id: u64,
|
||||
total: usize,
|
||||
success_count: &mut usize,
|
||||
failed_details: &mut Vec<super::FailedItemDetail>,
|
||||
index: &mut usize,
|
||||
progress_cb: &F,
|
||||
) where
|
||||
F: Fn(usize, usize), // (processed, failed)
|
||||
{
|
||||
process_folder_with_progress_inner(
|
||||
folder,
|
||||
parent_path,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
|
||||
fn process_folder_with_progress_inner<F>(
|
||||
folder: &Rc<dyn Folder>,
|
||||
parent_path: &str,
|
||||
account_id: u64,
|
||||
total: usize,
|
||||
success_count: &mut usize,
|
||||
failed_details: &mut Vec<super::FailedItemDetail>,
|
||||
index: &mut usize,
|
||||
progress_cb: &F,
|
||||
) where
|
||||
F: Fn(usize, usize),
|
||||
{
|
||||
let folder_name = folder
|
||||
.properties()
|
||||
.display_name()
|
||||
.unwrap_or_else(|_| "Unknown".to_string());
|
||||
|
||||
let mail_folder = if parent_path.is_empty() {
|
||||
folder_name
|
||||
} else {
|
||||
format!("{}/{}", parent_path, folder_name)
|
||||
};
|
||||
|
||||
tracing::debug!("Processing PST folder: {}", mail_folder);
|
||||
|
||||
let mailbox_id = match super::resolve_mailbox_by_account_id(account_id, &mail_folder) {
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to resolve mailbox '{}': {:?}", mail_folder, e);
|
||||
// Still recurse into sub-folders even if this folder's mailbox creation fails
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
process_folder_with_progress_inner(
|
||||
&sub_folder,
|
||||
&mail_folder,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut batch_size = 0usize;
|
||||
|
||||
if let Some(contents_table) = folder.contents_table() {
|
||||
for row in contents_table.rows_matrix() {
|
||||
let store = folder.store().clone();
|
||||
|
||||
let entry_id = match store
|
||||
.properties()
|
||||
.make_entry_id(NodeId::from(u32::from(row.id())))
|
||||
{
|
||||
Ok(id) => id,
|
||||
Err(e) => {
|
||||
tracing::warn!("Skip PST row {}: {:?}", row.unique(), e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match store.open_message(&entry_id, None) {
|
||||
Ok(message) => match build_eml_base64(message) {
|
||||
Some(base64_eml) => {
|
||||
let decoded = match crate::base64_decode_url_safe!(base64_eml.as_bytes()) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
failed_details.push(super::FailedItemDetail {
|
||||
index: *index,
|
||||
error_message: format!(
|
||||
"Failed to decode base64 EML at index {}: {:?}",
|
||||
*index, e
|
||||
),
|
||||
});
|
||||
*index += 1;
|
||||
batch_size += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(
|
||||
extract_envelope_from_eml(&decoded, account_id, mailbox_id)
|
||||
) {
|
||||
Ok(_) => {
|
||||
*success_count += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
failed_details.push(super::FailedItemDetail {
|
||||
index: *index,
|
||||
error_message: format!("{:?}", e),
|
||||
});
|
||||
}
|
||||
};
|
||||
*index += 1;
|
||||
batch_size += 1;
|
||||
}
|
||||
None => {}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!("Open PST message error: {:?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress every 50 messages
|
||||
if batch_size % 50 == 0 {
|
||||
progress_cb(*success_count + failed_details.len(), failed_details.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(hierarchy_table) = folder.hierarchy_table() {
|
||||
for row in hierarchy_table.rows_matrix() {
|
||||
let node = NodeId::from(u32::from(row.id()));
|
||||
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
|
||||
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
|
||||
process_folder_with_progress_inner(
|
||||
&sub_folder,
|
||||
&mail_folder,
|
||||
account_id,
|
||||
total,
|
||||
success_count,
|
||||
failed_details,
|
||||
index,
|
||||
progress_cb,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -339,6 +339,25 @@ pub struct Settings {
|
||||
help = "Maximum HTTP request body size in MB for file uploads"
|
||||
)]
|
||||
pub bichon_upload_body_limit_mb: u64,
|
||||
|
||||
/// Maximum per-file size in MB for MBOX uploads via the web UI (default: 1024 MB = 1 GB).
|
||||
/// Individual EML files are always capped at 100 MB regardless of this setting.
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "1024",
|
||||
env,
|
||||
help = "Maximum per-file size in MB for MBOX uploads via the web UI"
|
||||
)]
|
||||
pub bichon_web_mbox_upload_limit_mb: u64,
|
||||
|
||||
/// Maximum per-file size in MB for PST uploads via the web UI (default: 2048 MB = 2 GB).
|
||||
#[clap(
|
||||
long,
|
||||
default_value = "2048",
|
||||
env,
|
||||
help = "Maximum per-file size in MB for PST uploads via the web UI"
|
||||
)]
|
||||
pub bichon_web_pst_upload_limit_mb: u64,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
|
||||
@@ -67,6 +67,10 @@ pub struct SystemConfigurations {
|
||||
pub bichon_oidc_redirect_uri: Option<String>,
|
||||
|
||||
pub bichon_upload_body_limit_mb: u64,
|
||||
|
||||
pub bichon_web_mbox_upload_limit_mb: u64,
|
||||
|
||||
pub bichon_web_pst_upload_limit_mb: u64,
|
||||
}
|
||||
|
||||
impl From<&Settings> for SystemConfigurations {
|
||||
@@ -106,6 +110,8 @@ impl From<&Settings> for SystemConfigurations {
|
||||
bichon_oidc_client_id: s.bichon_oidc_client_id.clone(),
|
||||
bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(),
|
||||
bichon_upload_body_limit_mb: s.bichon_upload_body_limit_mb,
|
||||
bichon_web_mbox_upload_limit_mb: s.bichon_web_mbox_upload_limit_mb,
|
||||
bichon_web_pst_upload_limit_mb: s.bichon_web_pst_upload_limit_mb,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user