fix : Memory usage keeps growing #167

src/modules/indexer/manager.rs

@@ -26,6 +26,7 @@ use std::{
 use crate::modules::{
     duckdb::init::duckdb,
     message::{content::AttachmentInfo, search::SortBy, tags::TagCount},
+    settings::cli::SETTINGS,
     utils::create_hash,
 };
 use crate::{
@@ -340,10 +341,13 @@ impl EmlIndexManager {
         let index = Self::open_or_create_index(&DATA_DIR_MANAGER.eml_dir);
         let index_writer = Arc::new(Mutex::new(
             index
-                .writer_with_num_threads(8, 536_870_912)
+                .writer_with_num_threads(
+                    SETTINGS.bichon_tantivy_threads,
+                    SETTINGS.bichon_tantivy_buffer_size,
+                )
                 .unwrap_or_else(|e| {
                     panic!(
-                        "Failed to create IndexWriter with 8 threads and 512MB buffer for {:?}: {}",
+                        "Failed to create IndexWriter with 8 threads and 128MB buffer for {:?}: {}",
                         DATA_DIR_MANAGER.eml_dir, e
                     )
                 }),

src/modules/settings/cli.rs

@@ -262,6 +262,25 @@ pub struct Settings {
         help = "Maximum memory DuckDB may use (e.g. 512MB, 2GB, 80%)"
     )]
     pub bichon_duckdb_max_memory: Option<String>,
+
+    /// Number of Tantivy execution threads (default: 4)
+    #[clap(
+        long,
+        env,
+        default_value = "4",
+        help = "Number of Tantivy execution threads (default: 4)",
+        value_parser = clap::value_parser!(u16).range(1..)
+    )]
+    pub bichon_tantivy_threads: usize,
+
+    /// Tantivy indexer memory budget in bytes (default: 256MB)
+    #[clap(
+        long,
+        env,
+        default_value = "268435456",
+        help = "Set the memory budget for Tantivy indexer in bytes (default: 256MB)"
+    )]
+    pub bichon_tantivy_buffer_size: usize,
 }
 
 impl Settings {