feat: web upload supports PST, configurable MBOX/PST size limits

- Make MBOX/PST upload size limits configurable via SETTINGS
    (bichon_web_mbox_upload_limit_mb defaults to 1 GB,
     bichon_web_pst_upload_limit_mb defaults to 2 GB)
This commit is contained in:
rustmailer
2026-06-28 09:43:50 +08:00
parent 5034760517
commit 40ae2d49d4
31 changed files with 882 additions and 342 deletions
+124 -2
View File
@@ -20,6 +20,7 @@
//use poem_openapi::Object;
pub mod history;
pub mod reader;
pub mod pst;
pub use history::ImportHistory;
use serde::{Deserialize, Serialize};
use std::{
@@ -257,9 +258,15 @@ pub fn check_temp_disk_space() -> BichonResult<u64> {
pub enum FileFormat {
Eml,
Mbox,
Pst,
}
pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
// PST files start with OLE2 compound document magic bytes
if bytes.len() >= 8 && &bytes[..8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1" {
return Some(FileFormat::Pst);
}
// MBOX files start with "From " (note the trailing space after From)
if bytes.starts_with(b"From ") {
// Double-check: look for a valid date after the first "From " line
@@ -290,6 +297,8 @@ pub fn detect_format(bytes: &[u8], file_name: &str) -> Option<FileFormat> {
Some(FileFormat::Eml)
} else if lower.ends_with(".mbox") {
Some(FileFormat::Mbox)
} else if lower.ends_with(".pst") {
Some(FileFormat::Pst)
} else {
None
}
@@ -375,7 +384,7 @@ fn validate_import_account(account_id: u64) -> BichonResult<AccountModel> {
}
/// Resolve or create a mailbox/folder for the given account.
fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
pub(super) fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
match account.account_type {
AccountType::IMAP => {
// Shouldn't reach here (validated above), but handle gracefully
@@ -412,6 +421,13 @@ fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult<u64> {
}
}
/// Resolve or create a mailbox for a given account_id and folder name.
/// Used by PST import to create per-folder mailboxes.
pub fn resolve_mailbox_by_account_id(account_id: u64, folder: &str) -> BichonResult<u64> {
let account = AccountModel::check_account_exists(account_id)?;
resolve_mailbox(&account, folder)
}
/// Process an uploaded file (EML or MBOX) and import into the given account/folder.
/// This runs synchronously and should be spawned on a background thread.
///
@@ -497,6 +513,7 @@ pub fn process_uploaded_file(
match format {
FileFormat::Eml => process_eml_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
FileFormat::Mbox => process_mbox_file(import_id, file_path, account_id, mailbox_id, user_id, folder),
FileFormat::Pst => process_pst_upload(import_id, file_path, account_id, mailbox_id, user_id, folder),
}
}
@@ -512,7 +529,7 @@ fn detect_format_from_file(file_path: &Path, file_name: &str) -> BichonResult<Fi
detect_format(&buf, file_name).ok_or_else(|| {
raise_error!(
"Unknown file format. Supported: .eml, .mbox".into(),
"Unknown file format. Supported: .eml, .mbox, .pst".into(),
ErrorCode::InvalidParameter
)
})
@@ -694,6 +711,111 @@ fn process_single_eml(
}
}
/// Process a PST file uploaded via the web UI.
/// Two-pass approach: count messages first, then process with periodic progress updates.
fn process_pst_upload(
import_id: &str,
file_path: &Path,
account_id: u64,
_mailbox_id: u64, // ignored; PST creates its own mailboxes per folder
user_id: u64,
folder: &str,
) {
// Pass 1: count total messages
let total = match pst::count_pst_messages(file_path) {
Ok(n) => n,
Err(e) => {
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
let _ = std::fs::remove_file(file_path);
return;
}
};
update_progress(import_id, ImportProgress {
import_id: import_id.to_string(),
status: ImportStatus::Processing,
format: "pst".to_string(),
total,
success: 0,
duplicates: 0,
failed: 0,
failed_details: vec![],
});
// Pass 2: process messages with progress updates
let mut success_count: usize = 0;
let mut failed_details: Vec<FailedItemDetail> = Vec::new();
let mut index: usize = 0;
let pst_store = match outlook_pst::open_store(file_path) {
Ok(s) => s,
Err(e) => {
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
let _ = std::fs::remove_file(file_path);
return;
}
};
let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() {
Ok(id) => id,
Err(e) => {
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
let _ = std::fs::remove_file(file_path);
return;
}
};
let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) {
Ok(f) => f,
Err(e) => {
fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder);
let _ = std::fs::remove_file(file_path);
return;
}
};
// Progress callback: update progress every 50 messages
let import_id = import_id.to_string();
let format_str = "pst".to_string();
pst::process_folder_with_progress(
&ipm_subtree_folder,
"", // parent_path starts empty
account_id,
total, // pass pre-counted total for accurate progress
&mut success_count,
&mut failed_details,
&mut index,
&|processed, actual_failed| {
update_progress(&import_id, ImportProgress {
import_id: import_id.clone(),
status: ImportStatus::Processing,
format: format_str.clone(),
total,
success: processed - actual_failed,
duplicates: 0,
failed: actual_failed,
failed_details: vec![],
});
},
);
// Clean up temp file
let _ = std::fs::remove_file(file_path);
let final_progress = ImportProgress {
import_id: import_id.to_string(),
status: ImportStatus::Completed,
format: "pst".to_string(),
total,
success: success_count,
duplicates: 0,
failed: failed_details.len(),
failed_details,
};
history::save_import_history(user_id, account_id, folder, &final_progress);
update_progress(&import_id, final_progress);
}
/// Record a fatal failure and save history.
fn fail_progress(
import_id: &str,
@@ -0,0 +1,62 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use outlook_pst::ltp::prop_context::PropertyValue;
pub fn decode_subject(value: &PropertyValue) -> Option<String> {
match value {
PropertyValue::String8(value) => {
let offset = match value.buffer().first() {
Some(1) => 2,
_ => 0,
};
let buffer: Vec<_> = value
.buffer()
.iter()
.skip(offset)
.map(|&b| u16::from(b))
.collect();
Some(String::from_utf16_lossy(&buffer))
}
PropertyValue::Unicode(value) => {
let offset = match value.buffer().first() {
Some(1) => 2,
_ => 0,
};
Some(String::from_utf16_lossy(&value.buffer()[offset..]))
}
_ => None,
}
}
pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option<String> {
match code_page {
20127 => {
let buffer: Vec<_> = buffer.iter().map(|&b| u16::from(b)).collect();
Some(String::from_utf16_lossy(&buffer))
}
_ => {
let coding = codepage_strings::Coding::new(code_page).ok()?;
Some(coding.decode(buffer).ok()?.to_string())
}
}
}
pub fn decode_rtf_compressed(buffer: &[u8]) -> Option<String> {
compressed_rtf::decompress_rtf(buffer).ok()
}
+486
View File
@@ -0,0 +1,486 @@
//
// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com)
//
// This file is part of the Bichon Email Archiving Project
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::base64_encode_url_safe;
use crate::envelope::extractor::extract_envelope_from_eml;
use chrono::{DateTime, TimeZone, Utc};
use mail_send::mail_builder::headers::text::Text;
use mail_send::mail_builder::MessageBuilder;
use outlook_pst::ltp::prop_context::PropertyValue;
use outlook_pst::messaging::attachment::AttachmentProperties;
use outlook_pst::messaging::folder::Folder;
use outlook_pst::messaging::message::{Message, MessageProperties};
use outlook_pst::ndb::node_id::NodeId;
use std::rc::Rc;
mod encoding;
/// Convert a PST Message into a base64-encoded EML string.
pub fn build_eml_base64(message: Rc<dyn Message>) -> Option<String> {
let properties = message.properties();
let mut builder = MessageBuilder::new();
if let Some(sub) = extract_subject(properties) {
builder = builder.subject(sub);
}
if let Some(mid) = extract_string_property(properties, 0x1035) {
builder = builder.message_id(mid);
}
if let Some(irt) = extract_string_property(properties, 0x1042) {
builder = builder.in_reply_to(irt);
}
if let Some(refs) = extract_string_property(properties, 0x1039) {
builder = builder.header("References", Text::new(refs));
}
if let Some(cid_val) = properties.get(0x3013) {
if let PropertyValue::Binary(bin) = cid_val {
builder = builder.header(
"X-Bichon-Conversation-ID",
Text::new(hex::encode(bin.buffer())),
);
}
}
let from = extract_string_property(properties, 0x5D01)
.or_else(|| extract_string_property(properties, 0x5D02))
.or_else(|| extract_string_property(properties, 0x0C1F));
if let Some(f) = from {
builder = builder.from(f);
}
if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) {
let dt = filetime_to_datetime(filetime).timestamp();
builder = builder.date(dt);
}
let (to, cc, bcc) = extract_recipients_list(&message);
if !to.is_empty() {
builder = builder.to(to.iter().map(|s| s.as_str()).collect::<Vec<_>>());
}
if !cc.is_empty() {
builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
}
if !bcc.is_empty() {
builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::<Vec<_>>());
}
if let Some(html) = extract_html(properties) {
builder = builder.html_body(html);
}
if let Some(text) = extract_text(properties) {
builder = builder.text_body(text);
}
if let Some(attachment_table) = message.attachment_table() {
for row in attachment_table.rows_matrix() {
let node_id = NodeId::from(u32::from(row.id()));
if let Ok(attachment) = message.clone().read_attachment(node_id, None) {
let att_props = attachment.properties();
let name = extract_attachment_string_property(att_props, 0x3707);
let mime = extract_attachment_string_property(att_props, 0x370E)
.unwrap_or_else(|| "application/octet-stream".into());
let cid = extract_attachment_string_property(att_props, 0x3712);
let is_inline = att_props
.get(0x3714)
.and_then(|val| {
if let PropertyValue::Integer32(f) = val {
Some(f)
} else {
None
}
})
.map(|flag| (flag & 0x4) != 0)
.unwrap_or(false);
if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) {
let data = bin.buffer().to_vec();
let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string());
if is_inline && cid.is_some() {
let content_id = cid.unwrap();
builder = builder.inline(mime, content_id, data);
} else {
builder = builder.attachment(mime, file_name, data);
}
}
}
}
}
match builder.write_to_vec() {
Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)),
Err(e) => {
tracing::error!("Failed to generate EML from PST message: {:?}", e);
None
}
}
}
fn filetime_to_datetime(filetime: i64) -> DateTime<Utc> {
let unix_secs = (filetime / 10_000_000) - 11_644_473_600;
let nsecs = (filetime % 10_000_000) * 100;
Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap()
}
fn extract_recipients_list(message: &Rc<dyn Message>) -> (Vec<String>, Vec<String>, Vec<String>) {
let mut to = Vec::new();
let mut cc = Vec::new();
let mut bcc = Vec::new();
let recipient_table = message.recipient_table();
if let Some(recipient_table) = recipient_table {
let context = recipient_table.context();
for row in recipient_table.rows_matrix() {
if let Ok(cols) = row.columns(context) {
let mut r_type = 0;
let mut email = String::new();
for (col, val) in context.columns().iter().zip(cols) {
let prop_val = val
.as_ref()
.and_then(|v| recipient_table.read_column(v, col.prop_type()).ok());
match col.prop_id() {
0x0C15 => {
if let Some(PropertyValue::Integer32(t)) = prop_val {
r_type = t;
}
}
0x39FE | 0x3003 => {
if let Some(s) = prop_val.and_then(|v| extract_string(&v)) {
email = s;
}
}
_ => {}
}
}
if !email.is_empty() {
match r_type {
1 => to.push(email),
2 => cc.push(email),
3 => bcc.push(email),
_ => {}
}
}
}
}
} else {
let receiver = extract_string_property(message.properties(), 0x0076);
if let Some(receiver) = receiver {
to.push(receiver);
}
}
(to, cc, bcc)
}
fn extract_subject(props: &MessageProperties) -> Option<String> {
props.get(0x0037).and_then(|val| encoding::decode_subject(val))
}
fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option<String> {
properties
.get(prop_id)
.and_then(|value| extract_string(value))
}
fn extract_attachment_string_property(
properties: &AttachmentProperties,
prop_id: u16,
) -> Option<String> {
properties
.get(prop_id)
.and_then(|value| extract_string(value))
}
fn extract_string(value: &PropertyValue) -> Option<String> {
match value {
PropertyValue::String8(value) => Some(value.to_string()),
PropertyValue::Unicode(value) => Some(value.to_string()),
_ => None,
}
}
fn extract_text(properties: &MessageProperties) -> Option<String> {
properties.get(0x1000).and_then(extract_string).or_else(|| {
properties.get(0x1009).and_then(|value| match value {
PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()),
_ => None,
})
})
}
fn extract_html(properties: &MessageProperties) -> Option<String> {
properties.get(0x1013).and_then(|value| match value {
PropertyValue::Binary(value) => {
let code_page = properties
.get(0x3FDE)
.and_then(|v| {
if let PropertyValue::Integer32(cpid) = v {
Some(*cpid as u16)
} else {
None
}
})
.unwrap_or(65001);
encoding::decode_html_body(value.buffer(), code_page)
}
PropertyValue::String8(value) => Some(value.to_string()),
PropertyValue::Unicode(value) => Some(value.to_string()),
_ => None,
})
}
fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option<i64> {
for &prop_id in prop_ids {
if let Some(PropertyValue::Time(value)) = properties.get(prop_id) {
return Some(*value);
}
}
None
}
/// Open a PST file and count total messages across all folders.
/// Called from the web upload flow to get the total before processing.
pub fn count_pst_messages(pst_path: &std::path::Path) -> crate::error::BichonResult<usize> {
let pst_store = outlook_pst::open_store(pst_path).map_err(|e| {
crate::raise_error!(
format!("Failed to open PST file: {:?}", e),
crate::error::code::ErrorCode::InvalidParameter
)
})?;
let ipm_sub_tree = pst_store.properties().ipm_sub_tree_entry_id().map_err(|e| {
crate::raise_error!(
format!("Could not find IPM_SUBTREE in PST: {:?}", e),
crate::error::code::ErrorCode::InvalidParameter
)
})?;
let ipm_subtree_folder = pst_store.open_folder(&ipm_sub_tree).map_err(|e| {
crate::raise_error!(
format!("Failed to open root mailbox folder: {:?}", e),
crate::error::code::ErrorCode::InvalidParameter
)
})?;
Ok(count_folder_messages(&ipm_subtree_folder))
}
fn count_folder_messages(folder: &Rc<dyn Folder>) -> usize {
let mut count = 0usize;
if let Some(contents_table) = folder.contents_table() {
for row in contents_table.rows_matrix() {
let store = folder.store().clone();
let entry_id = match store
.properties()
.make_entry_id(NodeId::from(u32::from(row.id())))
{
Ok(id) => id,
Err(_) => continue,
};
if store.open_message(&entry_id, None).is_ok() {
count += 1;
}
}
}
if let Some(hierarchy_table) = folder.hierarchy_table() {
for row in hierarchy_table.rows_matrix() {
let node = NodeId::from(u32::from(row.id()));
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
count += count_folder_messages(&sub_folder);
}
}
}
}
count
}
/// Walk all folders and process messages, calling the progress callback
/// every 50 messages. Used by the web upload flow.
pub fn process_folder_with_progress<F>(
folder: &Rc<dyn Folder>,
parent_path: &str,
account_id: u64,
total: usize,
success_count: &mut usize,
failed_details: &mut Vec<super::FailedItemDetail>,
index: &mut usize,
progress_cb: &F,
) where
F: Fn(usize, usize), // (processed, failed)
{
process_folder_with_progress_inner(
folder,
parent_path,
account_id,
total,
success_count,
failed_details,
index,
progress_cb,
);
}
fn process_folder_with_progress_inner<F>(
folder: &Rc<dyn Folder>,
parent_path: &str,
account_id: u64,
total: usize,
success_count: &mut usize,
failed_details: &mut Vec<super::FailedItemDetail>,
index: &mut usize,
progress_cb: &F,
) where
F: Fn(usize, usize),
{
let folder_name = folder
.properties()
.display_name()
.unwrap_or_else(|_| "Unknown".to_string());
let mail_folder = if parent_path.is_empty() {
folder_name
} else {
format!("{}/{}", parent_path, folder_name)
};
tracing::debug!("Processing PST folder: {}", mail_folder);
let mailbox_id = match super::resolve_mailbox_by_account_id(account_id, &mail_folder) {
Ok(id) => id,
Err(e) => {
tracing::error!("Failed to resolve mailbox '{}': {:?}", mail_folder, e);
// Still recurse into sub-folders even if this folder's mailbox creation fails
if let Some(hierarchy_table) = folder.hierarchy_table() {
for row in hierarchy_table.rows_matrix() {
let node = NodeId::from(u32::from(row.id()));
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
process_folder_with_progress_inner(
&sub_folder,
&mail_folder,
account_id,
total,
success_count,
failed_details,
index,
progress_cb,
);
}
}
}
}
return;
}
};
let mut batch_size = 0usize;
if let Some(contents_table) = folder.contents_table() {
for row in contents_table.rows_matrix() {
let store = folder.store().clone();
let entry_id = match store
.properties()
.make_entry_id(NodeId::from(u32::from(row.id())))
{
Ok(id) => id,
Err(e) => {
tracing::warn!("Skip PST row {}: {:?}", row.unique(), e);
continue;
}
};
match store.open_message(&entry_id, None) {
Ok(message) => match build_eml_base64(message) {
Some(base64_eml) => {
let decoded = match crate::base64_decode_url_safe!(base64_eml.as_bytes()) {
Ok(bytes) => bytes,
Err(e) => {
failed_details.push(super::FailedItemDetail {
index: *index,
error_message: format!(
"Failed to decode base64 EML at index {}: {:?}",
*index, e
),
});
*index += 1;
batch_size += 1;
continue;
}
};
match futures::executor::block_on(
extract_envelope_from_eml(&decoded, account_id, mailbox_id)
) {
Ok(_) => {
*success_count += 1;
}
Err(e) => {
failed_details.push(super::FailedItemDetail {
index: *index,
error_message: format!("{:?}", e),
});
}
};
*index += 1;
batch_size += 1;
}
None => {}
},
Err(e) => {
tracing::warn!("Open PST message error: {:?}", e);
}
}
// Report progress every 50 messages
if batch_size % 50 == 0 {
progress_cb(*success_count + failed_details.len(), failed_details.len());
}
}
}
if let Some(hierarchy_table) = folder.hierarchy_table() {
for row in hierarchy_table.rows_matrix() {
let node = NodeId::from(u32::from(row.id()));
if let Ok(entry_id) = folder.store().properties().make_entry_id(node) {
if let Ok(sub_folder) = folder.store().open_folder(&entry_id) {
process_folder_with_progress_inner(
&sub_folder,
&mail_folder,
account_id,
total,
success_count,
failed_details,
index,
progress_cb,
);
}
}
}
}
}
+19
View File
@@ -339,6 +339,25 @@ pub struct Settings {
help = "Maximum HTTP request body size in MB for file uploads"
)]
pub bichon_upload_body_limit_mb: u64,
/// Maximum per-file size in MB for MBOX uploads via the web UI (default: 1024 MB = 1 GB).
/// Individual EML files are always capped at 100 MB regardless of this setting.
#[clap(
long,
default_value = "1024",
env,
help = "Maximum per-file size in MB for MBOX uploads via the web UI"
)]
pub bichon_web_mbox_upload_limit_mb: u64,
/// Maximum per-file size in MB for PST uploads via the web UI (default: 2048 MB = 2 GB).
#[clap(
long,
default_value = "2048",
env,
help = "Maximum per-file size in MB for PST uploads via the web UI"
)]
pub bichon_web_pst_upload_limit_mb: u64,
}
impl Settings {
+6
View File
@@ -67,6 +67,10 @@ pub struct SystemConfigurations {
pub bichon_oidc_redirect_uri: Option<String>,
pub bichon_upload_body_limit_mb: u64,
pub bichon_web_mbox_upload_limit_mb: u64,
pub bichon_web_pst_upload_limit_mb: u64,
}
impl From<&Settings> for SystemConfigurations {
@@ -106,6 +110,8 @@ impl From<&Settings> for SystemConfigurations {
bichon_oidc_client_id: s.bichon_oidc_client_id.clone(),
bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(),
bichon_upload_body_limit_mb: s.bichon_upload_body_limit_mb,
bichon_web_mbox_upload_limit_mb: s.bichon_web_mbox_upload_limit_mb,
bichon_web_pst_upload_limit_mb: s.bichon_web_pst_upload_limit_mb,
}
}
}