feat(import): introduce /api/v1/import endpoint to support batch EML email import

src/modules/context/executors.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
+use crate::modules::account::migration::AccountType;
 use crate::modules::context::Initialize;
 use crate::modules::error::code::ErrorCode;
 use crate::raise_error;
@@ -33,8 +33,7 @@ use dashmap::DashMap;
 use std::sync::{Arc, LazyLock};
 use tracing::info;
 
-pub static MAIL_CONTEXT: LazyLock<EmailClientExecutors> =
-    LazyLock::new(EmailClientExecutors::new);
+pub static MAIL_CONTEXT: LazyLock<EmailClientExecutors> = LazyLock::new(EmailClientExecutors::new);
 
 pub struct EmailClientExecutors {
     start_at: i64,
@@ -88,15 +87,17 @@ impl EmailClientExecutors {
 
     pub async fn start_account_syncers(&self) -> BichonResult<()> {
         let accounts = AccountModel::list_all().await?;
-        let active_accounts: Vec<AccountModel> =
-            accounts.into_iter().filter(|a| a.enabled).collect();
+        let active_accounts: Vec<AccountModel> = accounts
+            .into_iter()
+            .filter(|a| a.enabled && matches!(a.account_type, AccountType::IMAP))
+            .collect();
 
         if active_accounts.is_empty() {
             info!("No active accounts found for account initialization.");
             return Ok(());
         }
         info!(
-            "System has {} active accounts to initialize.",
+            "System has {} active IMAP accounts to initialize.",
             active_accounts.len()
         );
         for account in active_accounts {

src/modules/envelope/extractor.rs

@@ -16,7 +16,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 use crate::modules::common::AddrVec;
 use crate::modules::error::code::ErrorCode;
 use crate::modules::error::BichonResult;
@@ -33,12 +32,12 @@ pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> Bich
         .map(|d| d.timestamp_millis())
         .unwrap_or(0);
     let uid = fetch.uid.unwrap_or(0);
-    let size = fetch.size.unwrap_or(0);
 
     let body = fetch
         .body()
         .ok_or_else(|| raise_error!("No body available".into(), ErrorCode::InternalError))?;
 
+    let size = fetch.size.unwrap_or(body.len() as u32);
     let message = MessageParser::new().parse(body).ok_or_else(|| {
         raise_error!(
             "Email header parse result is not available".into(),
@@ -118,6 +117,92 @@ pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> Bich
     Ok(envelope)
 }
 
+pub fn extract_envelope_from_eml(
+    body: &[u8],
+    account_id: u64,
+    mailbox_id: u64,
+) -> BichonResult<Envelope> {
+    let uid = 0;
+    let size = body.len() as u32;
+    let message = MessageParser::new().parse(body).ok_or_else(|| {
+        raise_error!(
+            "Email header parse result is not available".into(),
+            ErrorCode::InternalError
+        )
+    })?;
+
+    let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) {
+        text
+    } else if let Some(html) = message.body_html(0).map(|cow| cow.into_owned()) {
+        from_read(html.as_bytes(), 0)
+            .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?
+    } else {
+        String::new()
+    };
+
+    let message_id = message
+        .message_id()
+        .map(String::from)
+        .unwrap_or(generate_message_id());
+    let in_reply_to = message.in_reply_to().as_text().map(String::from);
+    let references = extract_references(&message);
+    let thread_id = compute_thread_id(in_reply_to, references, &message_id);
+    let subject = message.subject().map(String::from).unwrap_or("".into());
+    let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
+    let bcc: Option<Vec<String>> = message.bcc().map(|addr| {
+        AddrVec::from(addr)
+            .0
+            .into_iter()
+            .filter_map(|a| a.address)
+            .collect()
+    });
+    let cc: Option<Vec<String>> = message.cc().map(|addr| {
+        AddrVec::from(addr)
+            .0
+            .into_iter()
+            .filter_map(|a| a.address)
+            .collect()
+    });
+    let to: Option<Vec<String>> = message.to().map(|addr| {
+        AddrVec::from(addr)
+            .0
+            .into_iter()
+            .filter_map(|a| a.address)
+            .collect()
+    });
+    let from = message
+        .from()
+        .and_then(|addr| AddrVec::from(addr).0.into_iter().next())
+        .and_then(|add| add.address)
+        .unwrap_or_else(|| "unknown".to_string());
+
+    let attachments: Vec<String> = message
+        .attachments()
+        .filter_map(|att| att.attachment_name())
+        .map(|name| name.to_string())
+        .collect();
+    let envelope = Envelope {
+        id: create_hash(account_id, &message_id),
+        message_id,
+        account_id,
+        mailbox_id,
+        uid,
+        subject,
+        text,
+        from,
+        to: to.unwrap_or_default(),
+        cc: cc.unwrap_or_default(),
+        bcc: bcc.unwrap_or_default(),
+        date,
+        internal_date: date,
+        size,
+        thread_id,
+        attachments,
+        tags: None,
+    };
+    Ok(envelope)
+}
+
 pub fn compute_thread_id(
     in_reply_to: Option<String>,
     references: Option<Vec<String>>,

src/modules/import/mod.rs

@@ -0,0 +1,162 @@
+use poem_openapi::Object;
+use serde::{Deserialize, Serialize};
+use tantivy::doc;
+
+use crate::{
+    base64_decode_url_safe,
+    modules::{
+        account::migration::{AccountModel, AccountType},
+        cache::imap::mailbox::{Attribute, AttributeEnum, MailBox},
+        envelope::extractor::extract_envelope_from_eml,
+        error::{code::ErrorCode, BichonResult},
+        indexer::{
+            manager::{EML_INDEX_MANAGER, ENVELOPE_INDEX_MANAGER},
+            schema::SchemaTools,
+        },
+        utils::create_hash,
+    },
+    raise_error,
+};
+
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Object)]
+pub struct BatchEmlRequest {
+    pub account_id: u64,
+    pub mail_folder: String,
+    /// A list of emails in base64-encoded format. Each element represents one .eml file.
+    pub emls: Vec<String>,
+}
+
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Object)]
+pub struct FailedEmlDetail {
+    /// The 0-based index of the failed EML in the request list
+    pub index: usize,
+    /// The error message that caused the import to fail
+    pub error_message: String,
+}
+
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize, Object)]
+pub struct BatchEmlResult {
+    /// Total number of emails processed
+    pub total: usize,
+    /// Number of emails successfully imported
+    pub success: usize,
+    /// Number of emails failed to import
+    pub failed: usize,
+    /// A list of details for failed imports
+    pub failed_details: Vec<FailedEmlDetail>,
+}
+
+pub struct ImportEmls;
+
+impl ImportEmls {
+    pub async fn do_import(request: BatchEmlRequest) -> BichonResult<BatchEmlResult> {
+        let account = AccountModel::check_account_exists(request.account_id).await?;
+        
+        if !account.enabled {
+            return Err(raise_error!("The account is disabled and cannot be used for this operation.".into(), ErrorCode::InvalidParameter));
+        }
+
+        let mailbox_id = match account.account_type {
+            AccountType::IMAP => {
+                let all_mailboxes = MailBox::list_all(account.id).await?;
+                let mailbox = all_mailboxes.into_iter().find(|m| m.name == request.mail_folder);
+                
+                match mailbox {
+                    Some(mailbox) => mailbox.id,
+                    None => return Err(raise_error!(
+                        format!("Mail folder '{}' not found for account ID {}. The target folder must exist before importing.", 
+                                request.mail_folder, 
+                                request.account_id).into(),
+                        ErrorCode::ResourceNotFound
+                    )),
+                }
+            },
+            AccountType::NoSync => {
+                let mailbox = MailBox {
+                    id: create_hash(request.account_id, &request.mail_folder),
+                    account_id: request.account_id,
+                    name: request.mail_folder.clone(),
+                    delimiter: Some("/".to_string()),
+                    attributes: vec![Attribute {
+                        attr: AttributeEnum::Extension,
+                        extension: Some("CreatedByBichon".into()),
+                    }],
+                    exists: 0,
+                    unseen: None,
+                    uid_next: None,
+                    uid_validity: None,
+                };
+                let mailbox_id = mailbox.id;
+                // Upsert the mailbox, creating it if it doesn't exist
+                MailBox::batch_upsert(&[mailbox]).await?;
+                mailbox_id
+            },
+        };
+
+        let fields = SchemaTools::eml_fields();
+        let account_id = account.id;
+        let mut success_count = 0;
+        let mut failed_details: Vec<FailedEmlDetail> = Vec::new(); // Store failure details
+
+        let total = request.emls.len();
+        for (index, eml_base64) in request.emls.into_iter().enumerate() {
+            // 1. Decode Base64
+            let decoded = match base64_decode_url_safe!(eml_base64.as_bytes()) {
+                Ok(bytes) => bytes,
+                Err(e) => {
+                    let error_msg =
+                        format!("Failed to decode base64 EML at index {}: {:?}", index, e);
+                    tracing::error!("{}", error_msg);
+                    failed_details.push(FailedEmlDetail {
+                        index,
+                        error_message: error_msg,
+                    });
+                    continue;
+                }
+            };
+
+            let envelope = match extract_envelope_from_eml(&decoded, account_id, mailbox_id) {
+                Ok(env) => env,
+                Err(e) => {
+                    let error_msg = format!(
+                        "Failed to extract envelope from EML at index {}: {:?}",
+                        index, e
+                    );
+                    tracing::error!("{}", error_msg);
+                    failed_details.push(FailedEmlDetail {
+                        index,
+                        error_message: error_msg,
+                    });
+                    continue;
+                }
+            };
+
+            ENVELOPE_INDEX_MANAGER
+                .add_document(envelope.id, envelope.to_document(mailbox_id).unwrap())
+                .await;
+
+            EML_INDEX_MANAGER
+                .add_document(
+                    envelope.id,
+                    doc!(
+                        fields.f_id => envelope.id,
+                        fields.f_account_id => account_id,
+                        fields.f_mailbox_id => mailbox_id,
+                        fields.f_eml => decoded
+                    ),
+                )
+                .await;
+
+            success_count += 1;
+        }
+
+        let failed_count = failed_details.len();
+
+        Ok(BatchEmlResult {
+            total,
+            success: success_count,
+            failed: failed_count,
+            failed_details, // Return the list of failure details
+        })
+    }
+}

src/modules/mod.rs

@@ -16,7 +16,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 pub mod account;
 pub mod autoconfig;
 pub mod cache;
@@ -27,6 +26,7 @@ pub mod database;
 pub mod envelope;
 pub mod error;
 pub mod imap;
+pub mod import;
 pub mod indexer;
 pub mod logger;
 pub mod mailbox;

src/modules/rest/api/import.rs

@@ -0,0 +1,49 @@
+//
+// Copyright (c) 2025 rustmailer.com (https://rustmailer.com)
+//
+// This file is part of the Bichon Email Archiving Project
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::modules::common::auth::ClientContext;
+use crate::modules::import::BatchEmlResult;
+use crate::modules::import::{BatchEmlRequest, ImportEmls};
+use crate::modules::rest::api::ApiTags;
+use crate::modules::rest::ApiResult;
+use poem_openapi::payload::Json;
+use poem_openapi::OpenApi;
+
+pub struct ImportApi;
+
+#[OpenApi(prefix_path = "/api/v1", tag = "ApiTags::Import")]
+impl ImportApi {
+    /// Batch import one or more EML files into a specified account and mail folder.
+    ///
+    /// This endpoint accepts a JSON payload containing:
+    /// - `account_id`: the target account to import emails into
+    /// - `mail_folder`: the mailbox/folder name
+    /// - `emls`: a list of base64-encoded .eml files
+    ///
+    /// Returns a summary of the import result, including total processed, successful, and failed emails.
+    #[oai(path = "/import", method = "post", operation_id = "do_batch_import")]
+    async fn do_batch_import(
+        &self,
+        /// JSON payload with account info and EML files to import
+        payload: Json<BatchEmlRequest>,
+        context: ClientContext,
+    ) -> ApiResult<Json<BatchEmlResult>> {
+        context.require_root()?;
+        Ok(Json(ImportEmls::do_import(payload.0).await?))
+    }
+}

src/modules/rest/api/mod.rs

@@ -16,7 +16,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 use access_token::AccessTokenApi;
 use account::AccountApi;
 use auto_config::AutoConfigApi;
@@ -26,11 +25,12 @@ use oauth2::OAuth2Api;
 use poem_openapi::{OpenApiService, Tags};
 use system::SystemApi;
 
-use crate::bichon_version;
+use crate::{bichon_version, modules::rest::api::import::ImportApi};
 
 pub mod access_token;
 pub mod account;
 pub mod auto_config;
+pub mod import;
 pub mod mailbox;
 pub mod message;
 pub mod oauth2;
@@ -45,6 +45,7 @@ pub enum ApiTags {
     OAuth2,
     Message,
     System,
+    Import,
 }
 
 type RustMailOpenApi = (
@@ -55,6 +56,7 @@ type RustMailOpenApi = (
     MailBoxApi,
     OAuth2Api,
     MessageApi,
+    ImportApi,
 );
 
 pub fn create_openapi_service() -> OpenApiService<RustMailOpenApi, ()> {
@@ -67,6 +69,7 @@ pub fn create_openapi_service() -> OpenApiService<RustMailOpenApi, ()> {
             MailBoxApi,
             OAuth2Api,
             MessageApi,
+            ImportApi,
         ),
         "BichonApi",
         bichon_version!(),