feat: add per-account FilterRule, ExtractionRules and ArchiveRules with regex validation

This commit is contained in:
rustmailer
2026-06-24 21:46:32 +08:00
parent 9e55026f12
commit bba1ea5cc7
4 changed files with 622 additions and 7 deletions
+2
View File
@@ -263,6 +263,8 @@ impl From<AccountV3> for AccountModel {
auto_download_new_mailboxes: None,
download_schedule: None,
deleting: false,
archive_rules: None,
extraction_rules: None,
}
}
}
+519
View File
@@ -64,6 +64,218 @@ pub enum QuotaWindow {
Monthly,
}
/// Include/exclude filter rule.
///
/// - `include` non-empty: only values matching these patterns pass.
/// - `exclude` non-empty: values matching these patterns are rejected.
/// - Both empty: all values pass.
/// - Both set: include checked first, then exclude.
///
/// Extension patterns use case-insensitive exact match; all others use regex.
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
pub struct FilterRule {
#[serde(default)]
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
}
impl FilterRule {
pub fn is_empty(&self) -> bool {
self.include.is_empty() && self.exclude.is_empty()
}
fn matches_exact(&self, value: &str) -> bool {
if !self.include.is_empty()
&& !self.include.iter().any(|e| e.eq_ignore_ascii_case(value))
{
return false;
}
if !self.exclude.is_empty()
&& self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value))
{
return false;
}
true
}
fn matches_regex(&self, value: &str) -> bool {
if !self.include.is_empty() && !matches_any_regex(&self.include, value) {
return false;
}
if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) {
return false;
}
true
}
fn validate_regex(&self, field: &str) -> Result<(), String> {
validate_patterns(&self.include, &format!("{field}.include"))?;
validate_patterns(&self.exclude, &format!("{field}.exclude"))?;
Ok(())
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
pub struct ExtractionRules {
/// Type 0: Master switch.
#[serde(default)]
pub enabled: bool,
/// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`).
#[serde(default)]
pub extensions: FilterRule,
/// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`).
#[serde(default)]
pub folders: FilterRule,
/// Type 3: Attachment filename patterns (regex).
#[serde(default)]
pub attachment_names: FilterRule,
/// Type 4: Sender patterns (regex).
#[serde(default)]
pub senders: FilterRule,
}
impl ExtractionRules {
/// Returns `true` if the attachment should be extracted under these rules.
pub fn should_extract(
&self,
ext: &str,
folder: Option<&str>,
attachment_name: Option<&str>,
sender: Option<&str>,
) -> bool {
if !self.enabled {
return false;
}
if !self.extensions.matches_exact(ext) {
return false;
}
if !self.folders.is_empty() {
if let Some(folder) = folder {
if !self.folders.matches_regex(folder) {
return false;
}
}
}
if !self.attachment_names.is_empty() {
if let Some(name) = attachment_name {
if !self.attachment_names.matches_regex(name) {
return false;
}
}
}
if !self.senders.is_empty() {
if let Some(sender) = sender {
if !self.senders.matches_regex(sender) {
return false;
}
}
}
true
}
pub fn validate(&self) -> Result<(), String> {
self.folders.validate_regex("folders")?;
self.attachment_names.validate_regex("attachment_names")?;
self.senders.validate_regex("senders")?;
Ok(())
}
}
/// Archive filtering rules — skip unwanted emails before storage.
///
/// Rule types:
/// 0 — Master switch
/// 1 — Sender filter (regex)
/// 2 — Subject filter (regex)
/// 3 — Skip emails larger than this (bytes)
/// 4 — Skip emails with spam headers (X-Spam-Flag, X-Spam)
///
/// `None` = archive everything (backward compatible).
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
pub struct ArchiveRules {
/// Type 0: Master switch. `false` = archive everything.
#[serde(default)]
pub enabled: bool,
/// Type 1: Sender filter (regex, include/exclude).
#[serde(default)]
pub senders: FilterRule,
/// Type 2: Subject filter (regex, include/exclude).
#[serde(default)]
pub subjects: FilterRule,
/// Type 3: Skip emails larger than this (bytes). `None` = no size limit.
#[serde(default)]
pub skip_larger_than: Option<u64>,
/// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`).
/// When the value is `yes` or `true` (case-insensitive), the email is skipped.
/// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin),
/// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange).
#[serde(default)]
pub spam_headers: Vec<String>,
}
impl ArchiveRules {
/// Returns `true` if the email should be archived under these rules.
pub fn should_archive(
&self,
sender: Option<&str>,
subject: Option<&str>,
size: u32,
is_spam: bool,
) -> bool {
if !self.enabled {
return true;
}
if !self.senders.is_empty() {
if let Some(sender) = sender {
if !self.senders.matches_regex(sender) {
return false;
}
}
}
if !self.subjects.is_empty() {
if let Some(subject) = subject {
if !self.subjects.matches_regex(subject) {
return false;
}
}
}
if let Some(limit) = self.skip_larger_than {
if size as u64 > limit {
return false;
}
}
if !self.spam_headers.is_empty() && is_spam {
return false;
}
true
}
/// Validate all regex patterns are well-formed.
pub fn validate(&self) -> Result<(), String> {
self.senders.validate_regex("senders")?;
self.subjects.validate_regex("subjects")?;
Ok(())
}
}
fn matches_any_regex(patterns: &[String], value: &str) -> bool {
patterns
.iter()
.any(|p| regex::Regex::new(p).map(|re| re.is_match(value)).unwrap_or(false))
}
fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> {
for p in patterns {
regex::Regex::new(p)
.map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?;
}
Ok(())
}
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))]
pub struct Account {
@@ -99,6 +311,14 @@ pub struct Account {
pub download_schedule: Option<String>,
#[serde(default)]
pub deleting: bool,
/// Email-level filtering rules (Pro feature).
/// `None` = archive everything (backward compatible).
#[serde(default)]
pub archive_rules: Option<ArchiveRules>,
/// Attachment text extraction rules (Pro feature).
/// `None` = extract everything (backward compatible).
#[serde(default)]
pub extraction_rules: Option<ExtractionRules>,
}
impl MemDbModel for Account {
@@ -139,6 +359,8 @@ impl Account {
imap_quota_window: request.imap_quota_window,
download_schedule: request.download_schedule,
deleting: false,
archive_rules: request.archive_rules,
extraction_rules: request.extraction_rules,
})
}
@@ -477,7 +699,304 @@ impl Account {
if request.clear_download_schedule == Some(true) {
new.download_schedule = None;
}
if request.extraction_rules.is_some() {
new.extraction_rules = request.extraction_rules;
}
if request.archive_rules.is_some() {
new.archive_rules = request.archive_rules;
}
new.updated_at = utc_now!();
Ok(new)
}
}
#[cfg(test)]
mod tests {
use super::*;
// ── FilterRule ───────────────────────────────────────────────────
#[test]
fn filter_rule_include_only() {
let r = FilterRule {
include: vec![r"@ok\.com$".into()],
..Default::default()
};
assert!(r.matches_regex("bob@ok.com"));
assert!(!r.matches_regex("spam@bad.com"));
}
#[test]
fn filter_rule_exclude_only() {
let r = FilterRule {
exclude: vec![r"@spam\.com$".into()],
..Default::default()
};
assert!(r.matches_regex("bob@ok.com"));
assert!(!r.matches_regex("bot@spam.com"));
}
#[test]
fn filter_rule_include_then_exclude() {
let r = FilterRule {
include: vec![r"@company\.com$".into()],
exclude: vec![r"noreply@company\.com$".into()],
..Default::default()
};
assert!(r.matches_regex("bob@company.com"));
assert!(!r.matches_regex("noreply@company.com"));
assert!(!r.matches_regex("spam@other.com"));
}
#[test]
fn filter_rule_exact_match() {
let r = FilterRule {
include: vec!["pdf".into(), "docx".into()],
exclude: vec!["xlsx".into()],
..Default::default()
};
assert!(r.matches_exact("pdf"));
assert!(r.matches_exact("docx"));
assert!(r.matches_exact("DOCX")); // case-insensitive
assert!(!r.matches_exact("xlsx"));
assert!(!r.matches_exact("txt"));
}
// ── ExtractionRules ─────────────────────────────────────────────
#[test]
fn extraction_rules_master_switch() {
let rules = ExtractionRules {
enabled: false,
..Default::default()
};
assert!(!rules.should_extract("pdf", None, None, None));
}
#[test]
fn extraction_rules_extension_include() {
let rules = ExtractionRules {
enabled: true,
extensions: FilterRule {
include: vec!["pdf".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.should_extract("pdf", None, None, None));
assert!(!rules.should_extract("docx", None, None, None));
}
#[test]
fn extraction_rules_extension_exclude() {
let rules = ExtractionRules {
enabled: true,
extensions: FilterRule {
exclude: vec!["xlsx".into(), "pptx".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.should_extract("pdf", None, None, None));
assert!(!rules.should_extract("xlsx", None, None, None));
}
#[test]
fn extraction_rules_folder_regex() {
let rules = ExtractionRules {
enabled: true,
folders: FilterRule {
include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None));
assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None));
assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None));
}
#[test]
fn extraction_rules_attachment_name_regex() {
let rules = ExtractionRules {
enabled: true,
attachment_names: FilterRule {
include: vec![r"^invoice-.*\.pdf$".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None));
assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None));
}
#[test]
fn extraction_rules_sender_regex() {
let rules = ExtractionRules {
enabled: true,
senders: FilterRule {
exclude: vec![r"@noreply\.com$".into()],
..Default::default()
},
..Default::default()
};
// non-excluded sender passes
assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com")));
// excluded sender blocked
assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com")));
}
#[test]
fn extraction_rules_empty_filters_pass_everything() {
let rules = ExtractionRules {
enabled: true,
..Default::default()
};
assert!(rules.should_extract(
"anything",
Some("any/folder"),
Some("any.pdf"),
Some("any@x.com")
));
}
// ── ArchiveRules ────────────────────────────────────────────────
#[test]
fn archive_rules_disabled_archives_everything() {
let rules = ArchiveRules {
enabled: false,
..Default::default()
};
assert!(rules.should_archive(
Some("spam@x.com"),
Some("BUY NOW"),
999,
false
));
}
#[test]
fn archive_rules_sender_exclude() {
let rules = ArchiveRules {
enabled: true,
senders: FilterRule {
exclude: vec![r"@spam\.com$".into()],
..Default::default()
},
..Default::default()
};
assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false));
assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false));
}
#[test]
fn archive_rules_subject_exclude() {
let rules = ArchiveRules {
enabled: true,
subjects: FilterRule {
exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()],
..Default::default()
},
..Default::default()
};
assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false));
assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false));
assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false));
}
#[test]
fn archive_rules_sender_include() {
// Only archive emails from specific senders
let rules = ArchiveRules {
enabled: true,
senders: FilterRule {
include: vec![r"@partner\.com$".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false));
assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false));
}
#[test]
fn archive_rules_skip_larger_than() {
let rules = ArchiveRules {
enabled: true,
skip_larger_than: Some(50_000_000),
..Default::default()
};
assert!(rules.should_archive(None, None, 1_000_000, false));
assert!(!rules.should_archive(None, None, 60_000_000, false));
}
#[test]
fn archive_rules_skip_spam_headers() {
let rules = ArchiveRules {
enabled: true,
spam_headers: vec!["X-Spam-Flag".into()],
..Default::default()
};
assert!(!rules.should_archive(None, None, 100, true));
assert!(rules.should_archive(None, None, 100, false));
}
// ── Validation ──────────────────────────────────────────────────
#[test]
fn validate_extraction_rules_valid() {
let rules = ExtractionRules {
folders: FilterRule {
include: vec![r"^INBOX/.*".into()],
..Default::default()
},
senders: FilterRule {
exclude: vec![r"@spam\.com$".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.validate().is_ok());
}
#[test]
fn validate_extraction_rules_invalid_regex() {
let rules = ExtractionRules {
folders: FilterRule {
include: vec!["***bad[".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.validate().is_err());
}
#[test]
fn validate_archive_rules_valid() {
let rules = ArchiveRules {
senders: FilterRule {
exclude: vec![r"@spam\.com$".into()],
..Default::default()
},
subjects: FilterRule {
include: vec![r"(?i)invoice".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.validate().is_ok());
}
#[test]
fn validate_archive_rules_invalid_regex() {
let rules = ArchiveRules {
senders: FilterRule {
include: vec!["[unclosed".into()],
..Default::default()
},
..Default::default()
};
assert!(rules.validate().is_err());
}
}
+33 -1
View File
@@ -19,7 +19,7 @@
use std::str::FromStr;
use crate::account::entity::ImapConfig;
use crate::account::migration::{AccountModel, AccountType, QuotaWindow};
use crate::account::migration::{AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow};
use crate::account::since::{DateSince, RelativeDate};
use crate::error::code::ErrorCode;
use crate::error::BichonResult;
@@ -56,6 +56,12 @@ pub struct AccountCreateRequest {
pub imap_quota_window: Option<QuotaWindow>,
pub auto_download_new_mailboxes: Option<bool>,
pub download_schedule: Option<String>,
/// Email archive filtering rules (Pro feature).
/// `None` = archive everything (backward compatible).
pub archive_rules: Option<ArchiveRules>,
/// Attachment text extraction rules (Pro feature).
/// `None` = extract everything (backward compatible).
pub extraction_rules: Option<ExtractionRules>,
}
impl AccountCreateRequest {
@@ -106,6 +112,16 @@ impl AccountCreateRequest {
}
AccountType::NoSync => {}
}
if let Some(ref rules) = self.extraction_rules {
rules.validate().map_err(|e| {
raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter)
})?;
}
if let Some(ref rules) = self.archive_rules {
rules.validate().map_err(|e| {
raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
})?;
}
Ok(AccountModel::new(user_id, self)?)
}
@@ -180,6 +196,12 @@ pub struct AccountUpdateRequest {
pub auto_download_new_mailboxes: Option<bool>,
pub download_schedule: Option<String>,
pub clear_download_schedule: Option<bool>,
/// Email archive filtering rules (Pro feature).
/// `None` = no change. Use `Some(ArchiveRules { .. })` to set.
pub archive_rules: Option<ArchiveRules>,
/// Attachment text extraction rules (Pro feature).
/// `None` = no change. Use `Some(ExtractionRules { .. })` to set.
pub extraction_rules: Option<ExtractionRules>,
}
impl AccountUpdateRequest {
@@ -235,6 +257,16 @@ impl AccountUpdateRequest {
validate_cron_expression(schedule)?;
}
}
if let Some(ref rules) = self.extraction_rules {
rules.validate().map_err(|e| {
raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter)
})?;
}
if let Some(ref rules) = self.archive_rules {
rules.validate().map_err(|e| {
raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter)
})?;
}
Ok(())
}
}
+68 -6
View File
@@ -16,6 +16,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
use crate::account::migration::AccountModel;
use crate::cache::imap::mailbox::MailBox;
use crate::common::AddrVec;
use crate::envelope::meta::parse_bichon_metadata;
@@ -112,6 +113,34 @@ async fn extract_envelope_core(
)
})?;
if let Ok(account) = AccountModel::get(account_id) {
if let Some(ref rules) = account.archive_rules {
let sender = message.from().and_then(|addr| {
AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address)
});
let subject = message.subject().map(|s| s.to_string());
let is_spam = !rules.spam_headers.is_empty()
&& rules.spam_headers.iter().any(|h| {
message
.header_raw(h.clone())
.map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true"))
.unwrap_or(false)
});
if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) {
tracing::debug!(
account_id,
uid,
sender = sender.as_deref().unwrap_or("?"),
subject = subject.as_deref().unwrap_or("?"),
"Email filtered out by archive rules"
);
return Ok(());
}
}
}
let preview_limit = 100;
let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
text
@@ -173,7 +202,7 @@ async fn extract_envelope_core(
.and_then(|add| add.address)
.unwrap_or_else(|| "unknown".to_string());
let attachment_count = message.attachment_count();
let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await;
let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await;
let envelope_id = Uuid::new_v4().to_string();
let now = utc_now!();
@@ -399,7 +428,27 @@ pub async fn detach_and_store_attachments(
original_body: &[u8],
message: &Message<'_>,
eml_content_hash: &str,
account_id: u64,
mailbox_id: u64,
) -> Vec<AttachmentInfo> {
let rules = if account_id > 0 {
AccountModel::get(account_id)
.ok()
.and_then(|a| a.extraction_rules)
} else {
None
};
let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) {
Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name),
_ => None,
};
let sender = message
.from()
.and_then(|addr| AddrVec::from(addr).0.into_iter().next())
.and_then(|add| add.address);
let mut stripped_eml = original_body.to_vec();
let mut attachment_infos = Vec::new();
// Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity
@@ -468,19 +517,30 @@ pub async fn detach_and_store_attachments(
})
.unwrap_or_else(|| "application/octet-stream".to_string());
let has_cid = att.content_id().is_some();
let ext = att
.attachment_name()
let att_name = att.attachment_name().map(|n| n.to_string());
let ext = att_name
.as_deref()
.and_then(|n| {
std::path::Path::new(&n)
std::path::Path::new(n)
.extension()
.and_then(|e| e.to_str())
.map(|s| s.to_ascii_lowercase())
})
.unwrap_or_default();
let should_extract = rules.as_ref().map_or(true, |r| {
r.should_extract(
&ext,
mailbox_name.as_deref(),
att_name.as_deref(),
sender.as_deref(),
)
});
if !inline || !has_cid {
let decoded_len = att.contents().len();
if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
if should_extract
&& decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES
&& crate::ext::text_extractor::should_try_extract(&file_type, &ext)
{
text_candidates.push(TextCandidate {
@@ -731,7 +791,7 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult<Bytes> {
ErrorCode::InternalError
)
})?;
detach_and_store_attachments(&raw_body, &message, &fetched_hash).await;
detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await;
Ok(Bytes::from(raw_body))
}
@@ -828,6 +888,8 @@ mod test {
truncated,
&message,
"test_content_hash",
0,
0,
)
.await;