diff --git a/crates/cli/src/api/download.rs b/crates/cli/src/api/download.rs index 5d1cc89..4619c5e 100644 --- a/crates/cli/src/api/download.rs +++ b/crates/cli/src/api/download.rs @@ -1,11 +1,9 @@ -use bichon_core::{base64_encode, store::envelope::Envelope}; +use crate::BichonCtlConfig; +use bichon_core::{base64_encode, envelope::meta::BichonMetadata, store::envelope::Envelope}; use chrono::{TimeZone, Utc}; use reqwest::Client; -use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; -use crate::BichonCtlConfig; - pub async fn download_and_export_with_json_header( client: &Client, config: &BichonCtlConfig, @@ -76,13 +74,6 @@ pub async fn download_and_export_with_json_header( true } -#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] -pub struct BichonMetadata { - pub account_email: Option, - pub mailbox_name: Option, - pub tags: Option>, -} - fn build_metadata_header(meta: BichonMetadata) -> String { let json_str = serde_json::to_string(&meta).ok().unwrap(); let encoded = base64_encode!(json_str); diff --git a/crates/cli/src/export/mod.rs b/crates/cli/src/export/mod.rs index d18366b..f1d9898 100644 --- a/crates/cli/src/export/mod.rs +++ b/crates/cli/src/export/mod.rs @@ -117,7 +117,11 @@ pub async fn handle_account_export( .interact() .unwrap() { - println!(" {} Starting export...", style("✔").green()); + println!( + " {} Starting export ({} items per page)...", + style("✔").green(), + 100 + ); let pb = ProgressBar::new(stats.total_count as u64); pb.set_style(ProgressStyle::with_template( @@ -145,7 +149,7 @@ pub async fn handle_account_export( if let Some(batch) = search_messages(&client, config, current_page, page_size).await { total_pages = batch.total_pages.unwrap(); - println!(" → Processing page {}/{}", current_page, total_pages); + pb.set_message(format!("Page {}/{}", current_page, total_pages)); for envelope in batch.items { let success = @@ -153,6 +157,7 @@ pub async fn handle_account_export( .await; if !success { + pb.finish_with_message("Failed"); eprintln!(" ✘ Failed to export an email. Aborting process..."); return; } @@ -163,6 +168,7 @@ pub async fn handle_account_export( } current_page += 1; } else { + pb.finish_with_message("Error"); eprintln!( " ✘ Failed to fetch page {}. Aborting process...", current_page diff --git a/crates/cli/src/mbox/mod.rs b/crates/cli/src/mbox/mod.rs index a52f83f..e57222a 100644 --- a/crates/cli/src/mbox/mod.rs +++ b/crates/cli/src/mbox/mod.rs @@ -24,6 +24,7 @@ use crate::mbox::gmail::determine_folder; use crate::mbox::reader::MboxFile; use crate::BichonCtlConfig; use bichon_core::base64_encode_url_safe; +use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata}; use console::style; use dialoguer::{theme::ColorfulTheme, Input}; use dialoguer::{Confirm, Select}; @@ -59,6 +60,7 @@ pub async fn handle_mbox_single_file_import( let options = vec![ "Use labels from mail headers (X-Gmail-Labels)", "Specify a single target folder for all emails", + "Use X-Bichon-Metadata header (Automatic)", ]; let selection = Select::with_theme(theme) @@ -78,6 +80,7 @@ pub async fn handle_mbox_single_file_import( .unwrap(); Some(folder) } + 2 => None, _ => unreachable!(), }; @@ -150,20 +153,31 @@ pub async fn run_import( } }; - let folder_name = match target_folder { - Some(ref folder_name) => folder_name.clone(), - None => { - let gmail_labels = message.header_raw("X-Gmail-Labels").unwrap_or("INBOX"); - let text_cow = MessageStream::new(gmail_labels.as_bytes()) - .parse_unstructured() - .into_text(); - let data: &str = match &text_cow { - Some(c) => c.as_ref(), - None => "INBOX", - }; - determine_folder(data) - } + let mut metadata: Option = None; + if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") { + metadata = parse_bichon_metadata(meta_header); + } + + let get_default_folder = || { + let gmail_labels = message.header_raw("X-Gmail-Labels").unwrap_or("INBOX"); + let text_cow = MessageStream::new(gmail_labels.as_bytes()) + .parse_unstructured() + .into_text(); + let data: &str = match &text_cow { + Some(c) => c.as_ref(), + None => "INBOX", + }; + determine_folder(data) }; + + let folder_name = if let Some(ref folder) = target_folder { + folder.clone() + } else if let Some(ref meta) = metadata { + meta.mailbox_name.clone().unwrap_or_else(get_default_folder) + } else { + get_default_folder() + }; + let b64_eml = base64_encode_url_safe!(&body); let buffer = folder_buffers .entry(folder_name.clone()) diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index b28ee4c..845f895 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . use crate::common::AddrVec; +use crate::envelope::meta::parse_bichon_metadata; use crate::envelope::utils::normalize_subject; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -33,6 +34,7 @@ use async_imap::types::Fetch; use bytes::Bytes; use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders}; use tantivy::TantivyDocument; +use tantivy::schema::Facet; use tracing::error; use uuid::Uuid; @@ -159,6 +161,36 @@ async fn extract_envelope_core( let envelope_id = Uuid::new_v4().to_string(); let now = utc_now!(); + + let mut final_tags = Vec::new(); + + if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") { + if let Some(bmd) = parse_bichon_metadata(meta_header) { + if let Some(tags) = bmd.tags { + let validated_tags: Result, _> = tags + .iter() + .map(|tag| { + Facet::from_text(tag) + .map(|_| tag.clone()) + .map_err(|e| e) + }) + .collect(); + + match validated_tags { + Ok(valid_list) => { + final_tags = valid_list; + } + Err(e) => { + eprintln!( + "Tag validation failed, ignoring all tags: {:#?}", + e + ); + } + } + } + } + } + let attachment_docs: Vec = attachments .iter() .filter(|a| !a.inline || a.content_id.is_none()) @@ -210,7 +242,7 @@ async fn extract_envelope_core( thread_id, attachment_count, regular_attachment_count: attachment_docs.len(), - tags: None, + tags: (!final_tags.is_empty()).then_some(final_tags), account_email: None, mailbox_name: None, content_hash: email_content_hash, diff --git a/crates/core/src/envelope/meta.rs b/crates/core/src/envelope/meta.rs new file mode 100644 index 0000000..e70e1ed --- /dev/null +++ b/crates/core/src/envelope/meta.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Serialize}; + +use crate::base64_decode; + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +pub struct BichonMetadata { + pub account_email: Option, + pub mailbox_name: Option, + pub tags: Option>, +} + +pub fn parse_bichon_metadata(header_value: &str) -> Option { + let decoded = base64_decode!(header_value.trim()); + serde_json::from_slice(&decoded).ok() +} diff --git a/crates/core/src/envelope/mod.rs b/crates/core/src/envelope/mod.rs index 5ee0d35..6e244a8 100644 --- a/crates/core/src/envelope/mod.rs +++ b/crates/core/src/envelope/mod.rs @@ -17,4 +17,5 @@ // along with this program. If not, see . pub mod extractor; +pub mod meta; pub mod utils; diff --git a/crates/core/src/store/tantivy/model.rs b/crates/core/src/store/tantivy/model.rs index cc74e28..473fa0e 100644 --- a/crates/core/src/store/tantivy/model.rs +++ b/crates/core/src/store/tantivy/model.rs @@ -106,6 +106,12 @@ impl EnvelopeWithAttachments { } } + if let Some(tags) = &self.envelope.tags { + for tag in tags { + doc.add_facet(fields.f_tags, tag); + } + } + doc.add_u64( fields.f_attachment_count, self.envelope.attachment_count as u64, @@ -166,7 +172,7 @@ impl EnvelopeWithAttachments { fields.f_regular_attachment_count, F_REGULAR_ATTACHMENT_COUNT, )? as usize, - tags: if tags.is_empty() { None } else { Some(tags) }, + tags: (!tags.is_empty()).then_some(tags), content_hash: extract_string_field(doc, fields.f_content_hash, F_CONTENT_HASH)?, ingest_at: extract_i64_field(doc, fields.f_ingest_at, F_INGEST_AT)?, };