fix: stitch adjacent RFC2047 words to prevent byte-split artifacts #79

src/modules/envelope/extractor.rs

@@ -17,13 +17,14 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::modules::common::AddrVec;
+use crate::modules::envelope::utils::normalize_subject;
 use crate::modules::error::code::ErrorCode;
 use crate::modules::error::BichonResult;
 use crate::modules::utils::create_hash;
 use crate::{calculate_hash, raise_error, utc_now};
 use crate::{id, modules::indexer::envelope::Envelope};
 use async_imap::types::Fetch;
-use mail_parser::{Message, MessageParser, MimeHeaders};
+use mail_parser::{HeaderName, Message, MessageParser, MimeHeaders};
 
 pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> BichonResult<Envelope> {
     let internal_date = fetch
@@ -62,7 +63,13 @@ pub fn extract_envelope(fetch: &Fetch, account_id: u64, mailbox_id: u64) -> Bich
     let in_reply_to = message.in_reply_to().as_text().map(String::from);
     let references = extract_references(&message);
     let thread_id = compute_thread_id(in_reply_to, references, &message_id);
-    let subject = message.subject().map(String::from).unwrap_or("".into());
+
+    let mut subject = message.subject().map(String::from).unwrap_or_default();
+
+    if subject.contains('\u{FFFD}') {
+        subject = normalize_subject(message.header_raw(HeaderName::Subject));
+    }
+
     let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
     let bcc: Option<Vec<String>> = message.bcc().map(|addr| {
         AddrVec::from(addr)
@@ -150,7 +157,12 @@ pub fn extract_envelope_from_eml(
     let in_reply_to = message.in_reply_to().as_text().map(String::from);
     let references = extract_references(&message);
     let thread_id = compute_thread_id(in_reply_to, references, &message_id);
-    let subject = message.subject().map(String::from).unwrap_or("".into());
+
+    let mut subject = message.subject().map(String::from).unwrap_or_default();
+    if subject.contains('\u{FFFD}') {
+        subject = normalize_subject(message.header_raw(HeaderName::Subject));
+    }
+
     let date = message.date().map(|d| d.to_timestamp() * 1000).unwrap_or(0);
     let bcc: Option<Vec<String>> = message.bcc().map(|addr| {
         AddrVec::from(addr)

src/modules/envelope/mod.rs

@@ -16,5 +16,5 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
 pub mod extractor;
+pub mod utils;

src/modules/envelope/utils.rs

@@ -0,0 +1,114 @@
+use mail_parser::parsers::MessageStream;
+use regex::{Captures, Regex};
+
+fn merge_contiguous_encoded_words(input: &str) -> String {
+    let block_re =
+        Regex::new(r"(?:=\?[^?]+\?[bBqQ]\?[^?]+\?=)(?:\s+(?:=\?[^?]+\?[bBqQ]\?[^?]+\?=))+")
+            .unwrap();
+
+    let word_re = Regex::new(r"=\?([^?]+)\?([bBqQ])\?([^?]+)\?=").unwrap();
+
+    block_re
+        .replace_all(input, |caps: &Captures| {
+            let whole = caps.get(0).unwrap().as_str();
+
+            let mut charset: Option<String> = None;
+            let mut encoding: Option<String> = None;
+            let mut combined = String::new();
+            let mut ok = true;
+
+            for cap in word_re.captures_iter(whole) {
+                let cs = &cap[1];
+                let enc = cap[2].to_ascii_uppercase();
+                let text = &cap[3];
+
+                if let Some(ref c) = charset {
+                    if c != cs {
+                        ok = false;
+                        break;
+                    }
+                } else {
+                    charset = Some(cs.to_string());
+                }
+
+                if let Some(ref e) = encoding {
+                    if e != &enc {
+                        ok = false;
+                        break;
+                    }
+                } else {
+                    encoding = Some(enc);
+                }
+
+                combined.push_str(text);
+            }
+
+            if ok {
+                format!(
+                    "=?{}?{}?{}?=",
+                    charset.unwrap(),
+                    encoding.unwrap(),
+                    combined
+                )
+            } else {
+                whole.to_string()
+            }
+        })
+        .to_string()
+}
+
+pub fn normalize_subject(raw_subject: Option<&str>) -> String {
+    let subject = match raw_subject {
+        Some(subject) => merge_contiguous_encoded_words(subject),
+        None => return String::new(),
+    };
+
+    MessageStream::new(subject.as_bytes())
+        .parse_unstructured()
+        .as_text()
+        .map(String::from)
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::modules::envelope::utils::merge_contiguous_encoded_words;
+
+
+    #[tokio::test]
+    async fn test3() {
+        let s = "Hello =?UTF-8?B?SGVsbG8=?= =?UTF-8?B?V29ybGQ=?= !!!";
+        assert_eq!(
+            merge_contiguous_encoded_words(s),
+            "Hello =?UTF-8?B?SGVsbG8=V29ybGQ=?= !!!"
+        );
+
+        let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?= =?UTF-8?B?Qw==?=";
+        assert_eq!(
+            merge_contiguous_encoded_words(s),
+            "=?UTF-8?B?QQ==Qg==Qw==?="
+        );
+
+        let s = "=?UTF-8?B?QQ==?= =?UTF-8?B?Qg==?= test =?UTF-8?B?Qw==?= =?UTF-8?B?RA==?=";
+        assert_eq!(
+            merge_contiguous_encoded_words(s),
+            "=?UTF-8?B?QQ==Qg==?= test =?UTF-8?B?Qw==RA==?="
+        );
+
+        let s = "=?UTF-8?B?QQ==?= =?GBK?B?Qg==?=";
+        assert_eq!(merge_contiguous_encoded_words(s), s);
+        let s = "=?UTF-8?B?QQ==?= =?UTF-8?Q?Qg?=";
+        assert_eq!(merge_contiguous_encoded_words(s), s);
+
+        let s = "=?UTF-8?b?QQ==?= =?UTF-8?B?Qg==?=";
+        assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?=");
+        let s = "Hello =?UTF-8?B?SGVsbG8=?= !!!";
+        assert_eq!(merge_contiguous_encoded_words(s), s);
+        let s = "=?UTF-8?B?QQ==?=    =?UTF-8?B?Qg==?=";
+        assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?B?QQ==Qg==?=");
+        let s = "Just a normal subject line";
+        assert_eq!(merge_contiguous_encoded_words(s), s);
+        let s = "=?UTF-8?Q?Hello_?= =?UTF-8?Q?World?=";
+        assert_eq!(merge_contiguous_encoded_words(s), "=?UTF-8?Q?Hello_World?=");
+    }
+}

src/modules/imap/tests.rs

@@ -16,11 +16,13 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use mail_parser::MessageParser;
+use mail_parser::{parsers::MessageStream, HeaderName, MessageParser};
 
 use crate::{
     base64_encode_url_safe,
-    modules::{account::entity::Encryption, imap::client::Client},
+    modules::{
+        account::entity::Encryption, envelope::utils::normalize_subject, imap::client::Client,
+    },
 };
 
 #[tokio::test]
@@ -48,3 +50,72 @@ async fn test1() {
         println!("{}", part.is_multipart());
     }
 }
+
+#[tokio::test]
+async fn test2() {
+    const MESSAGE: &str = r#"From: Art Vandelay <art@vandelay.com> (Vandelay Industries)
+To: "Colleagues": "James Smythe" <james@vandelay.com>; Friends:
+    jane@example.com, =?UTF-8?Q?John_Sm=C3=AEth?= <john@example.com>;
+Date: Sat, 20 Nov 2021 14:22:01 -0800
+Subject: =?utf-8?B?SnVzdCAxNSBkYXlzIGxlZnQgdG8gdmlzaXQgTkFSTklBISDinYTvuI/wn462?=
+Content-Type: multipart/mixed; boundary="festivus";
+
+--festivus
+Content-Type: text/html; charset="us-ascii"
+Content-Transfer-Encoding: base64
+
+PGh0bWw+PHA+SSB3YXMgdGhpbmtpbmcgYWJvdXQgcXVpdHRpbmcgdGhlICZsZHF1bztle
+HBvcnRpbmcmcmRxdW87IHRvIGZvY3VzIGp1c3Qgb24gdGhlICZsZHF1bztpbXBvcnRpbm
+cmcmRxdW87LDwvcD48cD5idXQgdGhlbiBJIHRob3VnaHQsIHdoeSBub3QgZG8gYm90aD8
+gJiN4MjYzQTs8L3A+PC9odG1sPg==
+--festivus
+Content-Type: message/rfc822
+
+From: "Cosmo Kramer" <kramer@kramerica.com>
+Subject: Exporting my book about coffee tables
+Content-Type: multipart/mixed; boundary="giddyup";
+
+--giddyup
+Content-Type: text/plain; charset="utf-16"
+Content-Transfer-Encoding: quoted-printable
+
+=FF=FE=0C!5=D8"=DD5=D8)=DD5=D8-=DD =005=D8*=DD5=D8"=DD =005=D8"=
+=DD5=D85=DD5=D8-=DD5=D8,=DD5=D8/=DD5=D81=DD =005=D8*=DD5=D86=DD =
+=005=D8=1F=DD5=D8,=DD5=D8,=DD5=D8(=DD =005=D8-=DD5=D8)=DD5=D8"=
+=DD5=D8=1E=DD5=D80=DD5=D8"=DD!=00
+--giddyup
+Content-Type: image/gif; name*1="about "; name*0="Book ";
+              name*2*=utf-8''%e2%98%95 tables.gif
+Content-Transfer-Encoding: Base64
+Content-Disposition: attachment
+
+R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7
+--giddyup--
+--festivus--
+"#;
+
+    let message = MessageParser::default().parse(MESSAGE).unwrap();
+    let raw_subject = message.header_raw("Subject").unwrap().as_bytes();
+
+    let data = MessageStream::new(raw_subject)
+        .parse_unstructured()
+        .unwrap_text()
+        .to_string();
+
+    println!("{}", data);
+    // RFC2047 support for encoded text in message readers
+    println!("{}", message.subject().unwrap());
+}
+
+#[tokio::test]
+async fn test44() {
+    let path = r"C:\Users\polly\Downloads\test222.eml";
+    let input = std::fs::read(path).unwrap();
+    let message = MessageParser::default().parse(&input).unwrap();
+    let subject = message.subject().unwrap();
+    println!("Subject: {}", subject);
+    if subject.contains('\u{FFFD}') {
+        let subject = normalize_subject(message.header_raw(HeaderName::Subject));
+        println!("Subject: {}", subject);
+    }
+}

web/src/features/users/api-tokens/index.tsx

@@ -18,7 +18,6 @@
 
 import { useState } from 'react'
 import useDialogState from '@/hooks/use-dialog-state'
-import { Button } from '@/components/ui/button'
 import { getColumns } from './components/columns'
 import { ApiTokenDeleteDialog } from './components/delete-dialog'
 import { ApiTokensTable } from './components/table'