From 19cc807f12967f6bdbe7234fb4f6a37b433149c0 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 17 May 2026 14:37:55 +0800 Subject: [PATCH] Created SMTP Ingest Service (markdown) --- SMTP-Ingest-Service.md | 206 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 SMTP-Ingest-Service.md diff --git a/SMTP-Ingest-Service.md b/SMTP-Ingest-Service.md new file mode 100644 index 0000000..3af3672 --- /dev/null +++ b/SMTP-Ingest-Service.md @@ -0,0 +1,206 @@ +> Introduced in **v1.0.0** + +Bichon's built-in SMTP server allows external mail transfer agents (MTAs), scripts, or applications to deliver email messages directly into the archive over a standard SMTP connection. Rather than relying on IMAP polling, messages are pushed into Bichon the moment they arrive — without any intermediate mail server. + +--- + +## What It Does + +The SMTP service accepts inbound email deliveries and stores each message into the recipient's `INBOX` mailbox inside the Bichon archive. It is a **receive-only** (MX-style) service — it does not relay or forward mail to the outside world. + +When a message is delivered successfully, Bichon: + +1. Looks up the `RCPT TO` address and resolves it to a registered **Account**. +2. Creates (or reuses) that account's `INBOX` mailbox. +3. Extracts and indexes the envelope, headers, and body so the message is immediately searchable. + +--- + +## Functional Boundaries + +| Capability | Supported | +|---|---| +| Receiving inbound messages | ✅ | +| STARTTLS upgrade (opportunistic TLS) | ✅ | +| Implicit TLS / SMTPS (port 465-style) | ✅ | +| AUTH PLAIN and AUTH LOGIN | ✅ | +| Unauthenticated (open relay) mode | ✅ (configurable) | +| Sender whitelist filtering | ⚠️ Partially (logic implemented, not yet exposed via configuration) | +| Multiple recipients per transaction | ❌ (max 1 per DATA transaction) | +| Outbound mail relay | ❌ | +| SMTP extensions beyond SIZE, 8BITMIME, STARTTLS, AUTH | ❌ | +| Maximum message size | 50 MB | + +> **Note on the single-recipient limit.** Bichon enforces exactly one `RCPT TO` per transaction. If a sending client attempts to add a second recipient, it receives `452 4.5.3 Too many recipients`. Clients that need to deliver to multiple Bichon accounts must open a separate SMTP transaction for each one. + +--- + +## Typical Use Cases + +**Archive mail from an existing MTA.** Configure Postfix, Exim, or any other MTA to BCC or forward a copy of every message to Bichon's SMTP port. The MTA handles delivery to end users; Bichon silently keeps a permanent, searchable copy. + +**Ingest from scripts or applications.** Any script that can speak SMTP (e.g. Python's `smtplib`, Rust's `lettre`) can push messages directly into the archive. This is useful for automated reports, monitoring alerts, or batch imports. + +**Replace IMAP polling for high-volume accounts.** Push-based ingestion via SMTP is lower-latency and more efficient than periodic IMAP FETCH loops for accounts that receive a large volume of mail. + +--- + +## Accounts and Users + +Bichon separates two distinct concepts: + +- An **Account** represents a monitored email address (e.g. `alice@example.com`). It is the archiving target. +- A **User** is a human operator who authenticates to Bichon and manages the system. + +### How RCPT TO Resolution Works + +When Bichon receives a `RCPT TO:` command, it queries its database for an Account whose email matches `addr`. Three outcomes are possible: + +| Outcome | SMTP Response | +|---|---| +| Account found and access permitted | `250 OK` | +| Account found but authenticated user lacks the `DATA_SMTP_INGEST` permission | `554 5.7.1 Access denied: Insufficient permissions` | +| No matching Account | `550 5.1.1 : Bichon account not found` | + +Only addresses that correspond to a registered Bichon Account can receive mail through this service. The SMTP server is not a general-purpose MX. + +### Authentication and Permissions + +When `auth_required` is enabled, a connecting client must authenticate before issuing `MAIL FROM` or `RCPT TO`. Bichon validates credentials against its **Access Token** system — the password field of `AUTH PLAIN` or `AUTH LOGIN` must be a valid Bichon API access token, not a user account password. The username field is ignored entirely; you may pass an empty string or any arbitrary value. + +After successful authentication, Bichon checks whether the authenticated User holds the `DATA_SMTP_INGEST` permission for the target Account. This allows fine-grained control: an operator can be granted ingest rights for some accounts but not others. + +When `auth_required` is disabled, any client that can reach the port can deliver mail to any registered Account — use sender whitelisting to restrict this. + +--- + +## Configuration + +All settings are controlled via the Bichon CLI / environment configuration under the `SETTINGS` object. The relevant keys are: + +| Setting | Description | Default | +|---|---|---| +| `bichon_enable_smtp` | Master switch. Set to `true` to start the service. | `false` | +| `bichon_smtp_port` | TCP port the server listens on. | — | +| `bichon_bind_ip` | IP address to bind to. | `0.0.0.0` | +| `bichon_smtp_encryption` | Encryption mode: `none`, `starttls`, or `tls`. | — | +| `bichon_smtp_tls_cert_path` | Path to a PEM certificate file. If omitted, a self-signed cert is generated automatically. | — | +| `bichon_smtp_tls_key_path` | Path to the corresponding PEM private key. | — | +| `bichon_smtp_auth_required` | Require `AUTH` before `MAIL FROM`. Set to `false` for open-relay mode (trusted networks only). | — | + +### Encryption Modes + +**`none`** — Plain TCP only. No TLS is offered or accepted. Suitable only for loopback or trusted internal networks. + +**`starttls`** — Plain TCP with an optional `STARTTLS` upgrade. Clients that support it will negotiate TLS; older clients can still connect in the clear. The TLS acceptor is passed through for the upgrade handshake. + +**`tls`** — Implicit TLS from the first byte. The service starts as an SMTPS listener (equivalent to wrapping the socket with `TlsAcceptor` before any SMTP dialogue). Use this when you control both ends of the connection. + +If TLS is enabled but no certificate paths are provided, Bichon generates a self-signed certificate for `localhost` / `127.0.0.1` at startup. For production deployments, supply your own certificate. + +--- + +## Session Limits + +| Limit | Value | +|---|---| +| Global session timeout | 10 minutes (600 s) | +| Per-command idle timeout | 60 s | +| Per-line timeout during DATA | 30 s | +| Maximum message size | 50 MB | + +A session that exceeds the global timeout is dropped with a warning log entry. A message larger than 50 MB is rejected with `552 5.3.4` after the size limit is crossed mid-transfer. + +--- + +## Startup and Shutdown + +The SMTP server is started from `start_smtp_server()` during application boot, after all other subsystems (TLS, user manager, blob store, search indices) have been initialized. It subscribes to the global `SIGNAL_MANAGER` shutdown broadcast and exits cleanly when the application receives a termination signal. + +Shutdown order: + +1. HTTP server stops accepting requests. +2. Periodic background tasks shut down. +3. SMTP server drains and stops (`SmtpServer::stop()`). +4. Sync tasks, envelope indexer, attachment indexer, and blob store shut down in sequence. + +--- + +## Integration Testing + +```rust +use std::error::Error; + +use lettre::address::Envelope; +use lettre::transport::smtp::authentication::{Credentials, Mechanism}; +use lettre::transport::smtp::client::{Tls, TlsParameters}; +use lettre::{Message, SmtpTransport, Transport}; + +#[tokio::test] +async fn test_smtp_archiving_flow() { + let email = Message::builder() + .from("tester@bichon.local".parse().unwrap()) + .to("archive@bichon.local".parse().unwrap()) + .subject("Integration Test") + .body(String::from("Checking if Bichon saves this!")) + .unwrap(); + + let envelope = Envelope::new( + Some("sender@example.com".parse().unwrap()), + vec!["placeholder@example.com".parse().unwrap()], // the email of a bichon account + ) + .unwrap(); + let mailer = SmtpTransport::builder_dangerous("127.0.0.1") + .port(2525) + .tls(Tls::None) + .build(); + + let result = mailer.send_raw(&envelope, &email.formatted()); + assert!( + result.is_ok(), + "SMTP delivery should succeed, got: {:?}", + result.err() + ); +} + +#[test] +fn test_bichon_smtp_logic() -> Result<(), Box> { + let smtp_host = "127.0.0.1"; + let smtp_port = 2525; + + println!("--- Testing STARTTLS Upgrade ---"); + + let tls_parameters = TlsParameters::builder(smtp_host.to_string()) + .dangerous_accept_invalid_certs(true) + .build()?; + + let mailer = SmtpTransport::starttls_relay(smtp_host)? + .port(smtp_port) + .tls(Tls::Required(tls_parameters)) + .authentication(vec![Mechanism::Login, Mechanism::Plain]) + .credentials(Credentials::new( + "test_user".to_string(), + "hP1Z4ZBs4IjdXtjbImFoX9kM".to_string(), + )) + .build(); + + // If RCPT TO is not explicitly specified in the envelope, the addresses in the 'To' header + // will be treated as envelope recipients. Bichon enforces a single-recipient policy per + // transaction; if multiple recipients are detected, it will reject with: + // "452 4.5.3 Too many recipients, try again in a new transaction". + let email = Message::builder() + .from("sender@bichon.com".parse()?) + .to("placeholder@example.com".parse()?) + .subject("TLS Test") + .body(String::from("Hello Bichon with TLS!"))?; + + let result = mailer.send(&email); + assert!( + result.is_ok(), + "STARTTLS encryption or Auth failed: {:?}", + result.err() + ); + println!("SUCCESS: TLS upgrade and mail delivery worked."); + Ok(()) +} +```