feat(import): support X-Bichon-Metadata and optimize CLI progress reporting

crates/cli/src/api/download.rs

@@ -1,11 +1,9 @@
-use bichon_core::{base64_encode, store::envelope::Envelope};
+use crate::BichonCtlConfig;
+use bichon_core::{base64_encode, envelope::meta::BichonMetadata, store::envelope::Envelope};
 use chrono::{TimeZone, Utc};
 use reqwest::Client;
-use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
 
-use crate::BichonCtlConfig;
-
 pub async fn download_and_export_with_json_header(
     client: &Client,
     config: &BichonCtlConfig,
@@ -76,13 +74,6 @@ pub async fn download_and_export_with_json_header(
     true
 }
 
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
-pub struct BichonMetadata {
-    pub account_email: Option<String>,
-    pub mailbox_name: Option<String>,
-    pub tags: Option<Vec<String>>,
-}
-
 fn build_metadata_header(meta: BichonMetadata) -> String {
     let json_str = serde_json::to_string(&meta).ok().unwrap();
     let encoded = base64_encode!(json_str);

crates/cli/src/export/mod.rs

@@ -117,7 +117,11 @@ pub async fn handle_account_export(
         .interact()
         .unwrap()
     {
-        println!(" {} Starting export...", style("✔").green());
+        println!(
+            " {} Starting export ({} items per page)...",
+            style("✔").green(),
+            100
+        );
 
         let pb = ProgressBar::new(stats.total_count as u64);
         pb.set_style(ProgressStyle::with_template(
@@ -145,7 +149,7 @@ pub async fn handle_account_export(
             if let Some(batch) = search_messages(&client, config, current_page, page_size).await {
                 total_pages = batch.total_pages.unwrap();
 
-                println!(" → Processing page {}/{}", current_page, total_pages);
+                pb.set_message(format!("Page {}/{}", current_page, total_pages));
 
                 for envelope in batch.items {
                     let success =
@@ -153,6 +157,7 @@ pub async fn handle_account_export(
                             .await;
 
                     if !success {
+                        pb.finish_with_message("Failed");
                         eprintln!(" ✘ Failed to export an email. Aborting process...");
                         return;
                     }
@@ -163,6 +168,7 @@ pub async fn handle_account_export(
                 }
                 current_page += 1;
             } else {
+                pb.finish_with_message("Error");
                 eprintln!(
                     " ✘ Failed to fetch page {}. Aborting process...",
                     current_page

crates/cli/src/mbox/mod.rs

@@ -24,6 +24,7 @@ use crate::mbox::gmail::determine_folder;
 use crate::mbox::reader::MboxFile;
 use crate::BichonCtlConfig;
 use bichon_core::base64_encode_url_safe;
+use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata};
 use console::style;
 use dialoguer::{theme::ColorfulTheme, Input};
 use dialoguer::{Confirm, Select};
@@ -59,6 +60,7 @@ pub async fn handle_mbox_single_file_import(
     let options = vec![
         "Use labels from mail headers (X-Gmail-Labels)",
         "Specify a single target folder for all emails",
+        "Use X-Bichon-Metadata header (Automatic)",
     ];
 
     let selection = Select::with_theme(theme)
@@ -78,6 +80,7 @@ pub async fn handle_mbox_single_file_import(
                 .unwrap();
             Some(folder)
         }
+        2 => None,
         _ => unreachable!(),
     };
 
@@ -150,20 +153,31 @@ pub async fn run_import(
             }
         };
 
-        let folder_name = match target_folder {
+        let mut metadata: Option<BichonMetadata> = None;
-            Some(ref folder_name) => folder_name.clone(),
+        if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") {
-            None => {
+            metadata = parse_bichon_metadata(meta_header);
-                let gmail_labels = message.header_raw("X-Gmail-Labels").unwrap_or("INBOX");
+        }
-                let text_cow = MessageStream::new(gmail_labels.as_bytes())
+
-                    .parse_unstructured()
+        let get_default_folder = || {
-                    .into_text();
+            let gmail_labels = message.header_raw("X-Gmail-Labels").unwrap_or("INBOX");
-                let data: &str = match &text_cow {
+            let text_cow = MessageStream::new(gmail_labels.as_bytes())
-                    Some(c) => c.as_ref(),
+                .parse_unstructured()
-                    None => "INBOX",
+                .into_text();
-                };
+            let data: &str = match &text_cow {
-                determine_folder(data)
+                Some(c) => c.as_ref(),
-            }
+                None => "INBOX",
+            };
+            determine_folder(data)
         };
+
+        let folder_name = if let Some(ref folder) = target_folder {
+            folder.clone()
+        } else if let Some(ref meta) = metadata {
+            meta.mailbox_name.clone().unwrap_or_else(get_default_folder)
+        } else {
+            get_default_folder()
+        };
+
         let b64_eml = base64_encode_url_safe!(&body);
         let buffer = folder_buffers
             .entry(folder_name.clone())

crates/core/src/envelope/extractor.rs

@@ -17,6 +17,7 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::common::AddrVec;
+use crate::envelope::meta::parse_bichon_metadata;
 use crate::envelope::utils::normalize_subject;
 use crate::error::code::ErrorCode;
 use crate::error::BichonResult;
@@ -33,6 +34,7 @@ use async_imap::types::Fetch;
 use bytes::Bytes;
 use mail_parser::{Address, HeaderName, Message, MessageParser, MimeHeaders};
 use tantivy::TantivyDocument;
+use tantivy::schema::Facet;
 use tracing::error;
 use uuid::Uuid;
 
@@ -159,6 +161,36 @@ async fn extract_envelope_core(
     let envelope_id = Uuid::new_v4().to_string();
     let now = utc_now!();
 
+
+    let mut final_tags = Vec::new();
+
+    if let Some(meta_header) = message.header_raw("X-Bichon-Metadata") {
+        if let Some(bmd) = parse_bichon_metadata(meta_header) {
+            if let Some(tags) = bmd.tags {
+                let validated_tags: Result<Vec<String>, _> = tags
+                    .iter()
+                    .map(|tag| {
+                        Facet::from_text(tag)
+                            .map(|_| tag.clone()) 
+                            .map_err(|e| e)
+                    })
+                    .collect();
+
+                match validated_tags {
+                    Ok(valid_list) => {
+                        final_tags = valid_list;
+                    }
+                    Err(e) => {
+                        eprintln!(
+                            "Tag validation failed, ignoring all tags: {:#?}",
+                            e
+                        );
+                    }
+                }
+            }
+        }
+    }
+
     let attachment_docs: Vec<TantivyDocument> = attachments
         .iter()
         .filter(|a| !a.inline || a.content_id.is_none())
@@ -210,7 +242,7 @@ async fn extract_envelope_core(
         thread_id,
         attachment_count,
         regular_attachment_count: attachment_docs.len(),
-        tags: None,
+        tags: (!final_tags.is_empty()).then_some(final_tags),
         account_email: None,
         mailbox_name: None,
         content_hash: email_content_hash,

crates/core/src/envelope/meta.rs

@@ -0,0 +1,15 @@
+use serde::{Deserialize, Serialize};
+
+use crate::base64_decode;
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
+pub struct BichonMetadata {
+    pub account_email: Option<String>,
+    pub mailbox_name: Option<String>,
+    pub tags: Option<Vec<String>>,
+}
+
+pub fn parse_bichon_metadata(header_value: &str) -> Option<BichonMetadata> {
+    let decoded = base64_decode!(header_value.trim());
+    serde_json::from_slice(&decoded).ok()
+}

crates/core/src/envelope/mod.rs

@@ -17,4 +17,5 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 pub mod extractor;
+pub mod meta;
 pub mod utils;

crates/core/src/store/tantivy/model.rs

@@ -106,6 +106,12 @@ impl EnvelopeWithAttachments {
             }
         }
 
+        if let Some(tags) = &self.envelope.tags {
+            for tag in tags {
+                doc.add_facet(fields.f_tags, tag);
+            }
+        }
+
         doc.add_u64(
             fields.f_attachment_count,
             self.envelope.attachment_count as u64,
@@ -166,7 +172,7 @@ impl EnvelopeWithAttachments {
                 fields.f_regular_attachment_count,
                 F_REGULAR_ATTACHMENT_COUNT,
             )? as usize,
-            tags: if tags.is_empty() { None } else { Some(tags) },
+            tags: (!tags.is_empty()).then_some(tags),
             content_hash: extract_string_field(doc, fields.f_content_hash, F_CONTENT_HASH)?,
             ingest_at: extract_i64_field(doc, fields.f_ingest_at, F_INGEST_AT)?,
         };