fix: open NewIndexWriter once across all migration segments

Previously, do_migrate_segment created a fresh NewIndexWriter (and
therefore a new Fjall Database) on every call, meaning the Fjall
database at bichon-storage/ was opened and closed once per segment.

This caused the migration to fail mid-way through (observed at segment
9/16) with:

  Storage(InvalidTag(("ChecksumType", 171)))

Root cause: after segment N writes email blobs via Fjall's ingestion
API (start_ingestion / write / finish), those SSTables and KV-separated
blob files are flushed to disk and the Database is dropped. When segment
N+1 calls Database::builder(storage_dir).open(), Fjall must discover and
catalog all on-disk files produced by the previous segments. During that
discovery it reads SSTable or blob-file block headers and encounters a
ChecksumType discriminant byte (171 / 0xAB) that lsm-tree 3.1.4 does
not recognise, causing the fatal error.

The first N segments succeed because the cumulative set of ingested
SSTables stays small enough that Fjall does not need to read the
offending headers during reopen. Once enough data has accumulated the
reopen triggers a manifest or compaction read that exposes the mismatch.

Fix: open NewIndexWriter once, before the segment loop, and pass a
&mut reference into each do_migrate_segment call. finish_writers() is
called a single time after all segments complete. The Fjall Database
stays open for the entire migration and is never closed and reopened,
eliminating the incompatible-reopen path entirely.
This commit is contained in:
fama
2026-06-03 15:10:03 -06:00
parent 257736a47b
commit 427f7248d2
2 changed files with 22 additions and 6 deletions
+21 -2
View File
@@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
use bichon_core::migrate::{
count_eml_segments, do_migrate_segment, is_tantivy_index_dir,
store::{LegacyDirs, NewDirs},
store::{LegacyDirs, NewDirs, NewIndexWriter},
};
use console::style;
use dialoguer::{theme::ColorfulTheme, Confirm, Input};
@@ -326,6 +326,18 @@ pub fn handle_migration(theme: &ColorfulTheme) {
.progress_chars("#>-"),
);
let mut writer = match NewIndexWriter::open(NewDirs::new(
new_index_path.clone(),
new_data_path.clone(),
)) {
Ok(w) => w,
Err(e) => {
pb.finish_with_message(format!("{}", style("Migration failed.").red()));
eprintln!("\n{} {:?}", style("").red().bold(), e);
return;
}
};
let mut grand_total_migrated: usize = 0;
let mut grand_total_skipped: usize = 0;
@@ -337,7 +349,7 @@ pub fn handle_migration(theme: &ColorfulTheme) {
match do_migrate_segment(
batch_size,
legacy,
NewDirs::new(new_index_path.clone(), new_data_path.clone()),
&mut writer,
seg_idx,
|msg| {
if let Some(data) = msg.strip_prefix("TOTAL:") {
@@ -407,6 +419,13 @@ pub fn handle_migration(theme: &ColorfulTheme) {
pb.set_position((seg_idx + 1) as u64);
}
pb.set_message(style("Finalizing indexes...").dim().to_string());
if let Err(e) = writer.finish_writers() {
pb.finish_with_message(format!("{}", style("Migration failed.").red()));
eprintln!("\n{} {:?}", style("").red().bold(), e);
return;
}
pb.finish_with_message(format!(
"Migration finished. Total: {}, Skipped: {}",
grand_total_migrated, grand_total_skipped
+1 -4
View File
@@ -121,7 +121,7 @@ fn is_dir_not_empty(path: &PathBuf) -> std::io::Result<bool> {
pub fn do_migrate_segment<F>(
batch_size: u32,
legacy: LegacyDirs,
new_dirs: NewDirs,
writer: &mut NewIndexWriter,
segment_index: usize,
mut on_progress: F,
) -> BichonResult<()>
@@ -226,8 +226,6 @@ where
drop(envelope_index);
// ── Phase 2: process EML docs, streaming one at a time ─────────────
let mut writer = NewIndexWriter::open(new_dirs)?;
let mut total_migrated = 0usize;
let mut total_skipped = 0usize;
@@ -308,7 +306,6 @@ where
chunk_start = chunk_end;
}
writer.finish_writers()?;
on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped));
Ok(())
}