External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(blob): add delete_batch, gc_if_needed, background flush, and fix bincode compat

Browse Source 
- Replace bincode 3.0.0 (empty crate) with bincode_reloaded 3.1.10
  - Add delete_batch for efficient grouped tombstone writes
  - Add gc_if_needed to skip GC when no segment exceeds threshold
  - Add flush() for lightweight fsync+meta checkpoint without compact
  - Add Config::flush_interval_secs to spawn a background flush thread
  - Remove per-put/per-delete fsync; persistence via background flush
  - Split gc_segments into gc_prepare/gc_finish to reduce write lock hold time
This commit is contained in:
rustmailer
2026-07-10 01:52:04 +08:00
parent 7dee5a7874
commit 4cdf3ee5f1
19 changed files with 1431 additions and 1797 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/blob/tests/acid_test.rs
+73 -101
View File
@@ -3,9 +3,6 @@
/// Since we can't kill the process mid-write in an inline test, we simulate crashes
/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
/// then re-opening and verifying recovery produced consistent state.
///
/// For true power-loss simulation, each test writes data, drops the engine abruptly,
/// then reopens and verifies: no corruption, no lost committed data, no partial writes.
use std::fs;
use std::path::Path;
@@ -50,16 +47,13 @@ fn test_durability_single_write_survives_crash() {
// Write
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
} // <-- Engine dropped = simulated crash
// Recover
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
}
@@ -73,21 +67,23 @@ fn test_durability_many_writes_survive_crash() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..n {
let key = make_key(i as u64);
keys.push(key);
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
} // crash
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for (i, key) in keys.iter().enumerate() {
let result = engine.read("alice", key).unwrap();
assert_eq!(result, Some(value.clone()), "missing key at index {}", i);
let result = engine.get(key).unwrap();
assert_eq!(
result,
Some(value.clone()),
"missing key at index {}",
i
);
}
}
}
@@ -100,20 +96,17 @@ fn test_durability_delete_survives_crash() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
} // crash after write
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.delete("alice", &key).unwrap();
engine.delete(&key).unwrap();
} // crash after delete
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, None, "delete should persist across crash");
}
}
@@ -129,22 +122,20 @@ fn test_atomicity_no_partial_entries_after_crash() {
// Write enough entries to fill part of a segment, then crash
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let value = make_value(50_000); // big enough to notice
let value = make_value(50_000);
for i in 0..200u64 {
engine
.write("alice", make_key(i), &value, Codec::None)
.put(make_key(i), &value, Codec::None)
.unwrap();
}
} // crash
// Recovery should clean up any partial tail entries and all committed
// entries should be readable
// Recovery should clean up any partial tail entries
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let value = make_value(50_000);
for i in 0..200u64 {
let result = engine.read("alice", &make_key(i)).unwrap();
let result = engine.get(&make_key(i)).unwrap();
assert_eq!(
result,
Some(value.clone()),
@@ -162,20 +153,18 @@ fn test_atomicity_crash_during_segment_roll() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
// Write enough to cross at least one segment boundary (256 MB)
for i in 0..140u64 {
engine
.write("alice", make_key(i), &big_value, Codec::None)
.put(make_key(i), &big_value, Codec::None)
.unwrap();
}
} // crash mid-way or after multiple segments
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
// All committed writes (that returned Ok) must be readable
for i in 0..140u64 {
let result = engine.read("alice", &make_key(i)).unwrap();
let result = engine.get(&make_key(i)).unwrap();
assert!(
result.is_some(),
"key {} should exist after segment roll recovery",
@@ -197,16 +186,12 @@ fn test_consistency_crc_detects_corruption() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
// Corrupt the segment file by flipping a byte
let seg_path = find_first_segment(dir.path(), "alice");
let seg_path = find_first_segment(dir.path());
let mut data = fs::read(&seg_path).unwrap();
// Flip a byte in the data portion, not the header
let flip_pos = data.len() - 100;
data[flip_pos] ^= 0xFF;
fs::write(&seg_path, &data).unwrap();
@@ -214,16 +199,14 @@ fn test_consistency_crc_detects_corruption() {
// Reading should detect CRC mismatch
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key);
// Either error or None is acceptable — never silently wrong data
let result = engine.get(&key);
match result {
Err(_) => {} // CRC mismatch detected — good
Err(_) => {} // CRC mismatch detected - good
Ok(None) => {} // index may point to truncated/removed data
Ok(Some(v)) => {
if v == value {
panic!("CRC corruption was NOT detected — silent data corruption!");
panic!("CRC corruption was NOT detected - silent data corruption!");
}
// If value differs, index pointed elsewhere after recovery
}
}
}
@@ -235,35 +218,39 @@ fn test_consistency_corrupt_magic_truncated_on_recovery() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..10u64 {
engine
.write("alice", make_key(i), &make_value(4096), Codec::Zstd)
.put(make_key(i), &make_value(4096), Codec::Zstd)
.unwrap();
}
}
// Append garbage to the segment file (simulating partial write from crash)
let seg_path = find_first_segment(dir.path(), "alice");
let seg_path = find_first_segment(dir.path());
let mut data = fs::read(&seg_path).unwrap();
let orig_len = data.len();
// Append garbage that doesn't start with the magic number
data.extend_from_slice(&[0xFF; 200]);
fs::write(&seg_path, &data).unwrap();
// Recovery should truncate the garbage
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
// Verify committed data is still intact
for i in 0..10u64 {
let result = engine.read("alice", &make_key(i)).unwrap();
assert!(result.is_some(), "committed key {} should survive tail truncation", i);
let result = engine.get(&make_key(i)).unwrap();
assert!(
result.is_some(),
"committed key {} should survive tail truncation",
i
);
}
}
// Verify file was actually truncated
let truncated_len = fs::metadata(&seg_path).unwrap().len();
assert!(truncated_len <= orig_len as u64, "garbage should have been truncated");
assert!(
truncated_len <= orig_len as u64,
"garbage should have been truncated"
);
}
// ---------------------------------------------------------------------------
@@ -274,22 +261,17 @@ fn test_consistency_corrupt_magic_truncated_on_recovery() {
fn test_isolation_reader_sees_snapshot_not_partial_write() {
let dir = TempDir::new().unwrap();
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
engine.create_account("alice").unwrap();
// Pre-populate a known key
let original_value = make_value(4096);
let key = make_key(100);
engine
.write("alice", key, &original_value, Codec::Zstd)
.unwrap();
engine.put(key, &original_value, Codec::Zstd).unwrap();
let running = Arc::new(AtomicBool::new(true));
let writer_done = Arc::new(AtomicBool::new(false));
// Spawn a writer that continuously overwrites the same key
let writer_engine = engine.clone();
let writer_running = running.clone();
let writer_done_flag = writer_done.clone();
let writer_key = key;
let writer = thread::spawn(move || {
@@ -299,11 +281,10 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
}
let val = make_value(4096 + (i as usize % 100));
writer_engine
.write("alice", writer_key, &val, Codec::Zstd)
.put(writer_key, &val, Codec::Zstd)
.unwrap();
thread::yield_now();
}
writer_done_flag.store(true, Ordering::SeqCst);
});
// Concurrent reader: reads should never panic or hang
@@ -315,11 +296,10 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
if !reader_running.load(Ordering::Relaxed) && reads > 0 {
break;
}
let result = reader_engine.read("alice", &key);
let result = reader_engine.get(&key);
match result {
Ok(Some(_)) | Ok(None) => {} // OK
Ok(Some(_)) | Ok(None) => {}
Err(e) => {
// Accept transient errors but report them
eprintln!("reader saw error: {:?}", e);
}
}
@@ -332,8 +312,7 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
running.store(false, Ordering::SeqCst);
writer.join().unwrap();
// Final read should see the last committed value (not partial)
let final_result = engine.read("alice", &key).unwrap();
let final_result = engine.get(&key).unwrap();
assert!(final_result.is_some(), "final read should find a value");
}
@@ -347,21 +326,18 @@ fn test_crash_during_gc_leaves_data_intact() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let value = make_value(500_000); // 500 KB each
// Write enough entries and delete some to create GC candidate
let value = make_value(500_000);
for i in 0..500u64 {
engine
.write("alice", make_key(i), &value, Codec::None)
.put(make_key(i), &value, Codec::None)
.unwrap();
}
// Delete ~40%
for i in (0..500u64).step_by(5) {
engine.delete("alice", &make_key(i)).unwrap();
engine.delete(&make_key(i)).unwrap();
}
// Single GC run (may or may not trigger)
let _ = engine.gc("alice");
let _ = engine.gc();
} // crash after GC
// All non-deleted entries must still be readable
@@ -370,9 +346,8 @@ fn test_crash_during_gc_leaves_data_intact() {
let value = make_value(500_000);
for i in 0..500u64 {
let key = make_key(i);
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
if i % 5 == 0 {
// Deleted keys
assert_eq!(result, None, "key {} should be deleted", i);
} else {
assert_eq!(
@@ -401,11 +376,8 @@ fn test_multiple_crash_reopen_cycles() {
// Populate and crash
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..50u64 {
engine
.write("alice", make_key(i), &value, Codec::Zstd)
.unwrap();
engine.put(make_key(i), &value, Codec::Zstd).unwrap();
alive.insert(i);
}
}
@@ -414,11 +386,11 @@ fn test_multiple_crash_reopen_cycles() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for &k in &alive {
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
assert!(engine.get(&make_key(k)).unwrap().is_some());
}
for i in 100..150u64 {
engine
.write("alice", make_key(i), &value, Codec::Zstd)
.put(make_key(i), &value, Codec::Zstd)
.unwrap();
alive.insert(i);
}
@@ -428,10 +400,10 @@ fn test_multiple_crash_reopen_cycles() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for &k in &alive {
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
assert!(engine.get(&make_key(k)).unwrap().is_some());
}
for i in 0..10u64 {
engine.delete("alice", &make_key(i)).unwrap();
engine.delete(&make_key(i)).unwrap();
alive.remove(&i);
}
}
@@ -440,44 +412,44 @@ fn test_multiple_crash_reopen_cycles() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for &k in &alive {
assert!(engine.read("alice", &make_key(k)).unwrap().is_some(),
"key {} should exist", k);
assert!(
engine.get(&make_key(k)).unwrap().is_some(),
"key {} should exist",
k
);
}
for i in 0..10u64 {
assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None,
"key {} should be deleted", i);
assert_eq!(
engine.get(&make_key(i)).unwrap(),
None,
"key {} should be deleted",
i
);
}
}
}
// ---------------------------------------------------------------------------
// 7. Account-level isolation
// 7. Global dedup: same content stored once
// ---------------------------------------------------------------------------
#[test]
fn test_account_isolation_crash_one_account_does_not_affect_others() {
fn test_global_dedup_after_crash() {
let dir = TempDir::new().unwrap();
let key = make_key(42);
let value = make_value(8192);
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine.create_account("bob").unwrap();
engine
.write("alice", make_key(1), &make_value(4096), Codec::Zstd)
.unwrap();
engine
.write("bob", make_key(1), &make_value(8192), Codec::Zstd)
.unwrap();
// Write same key twice (simulating two sources with same content)
engine.put(key, &value, Codec::Zstd).unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
// Delete alice's account dir partially to simulate corruption
// Then verify bob is intact
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
// Bob should be fine
let result = engine.read("bob", &make_key(1)).unwrap();
assert!(result.is_some(), "bob should be unaffected");
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
}
@@ -485,8 +457,8 @@ fn test_account_isolation_crash_one_account_does_not_affect_others() {
// Helpers
// ---------------------------------------------------------------------------
fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf {
let seg_dir = store_root.join("accounts").join(account).join("segments");
fn find_first_segment(store_root: &Path) -> std::path::PathBuf {
let seg_dir = store_root.join("segments");
for entry in fs::read_dir(&seg_dir).unwrap() {
let entry = entry.unwrap();
let name = entry.file_name().to_string_lossy().into_owned();
crates/blob/tests/integration_test.rs
+70 -207
View File
@@ -1,33 +1,17 @@
use bichon_blob::{Codec, Config, Engine};
use tempfile::TempDir;
#[test]
fn test_create_and_list_accounts() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine.create_account("bob").unwrap();
let accounts = engine.list_accounts();
assert!(accounts.contains(&"alice".to_string()));
assert!(accounts.contains(&"bob".to_string()));
}
#[test]
fn test_write_and_read() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let key = [0xAA; 32];
let value = b"Hello, this is a test email!".to_vec();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
@@ -35,10 +19,9 @@ fn test_write_and_read() {
fn test_read_missing_key() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let key = [0xFF; 32];
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, None);
}
@@ -46,45 +29,43 @@ fn test_read_missing_key() {
fn test_delete() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let key = [0xBB; 32];
let value = b"Some email content".to_vec();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.delete("alice", &key).unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
engine.delete(&key).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, None);
}
#[test]
fn test_delete_account() {
fn test_exists() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine.delete_account("alice").unwrap();
let accounts = engine.list_accounts();
assert!(!accounts.contains(&"alice".to_string()));
let key = [0xCC; 32];
assert!(!engine.exists(&key).unwrap());
engine.put(key, b"data", Codec::None).unwrap();
assert!(engine.exists(&key).unwrap());
engine.delete(&key).unwrap();
assert!(!engine.exists(&key).unwrap());
}
#[test]
fn test_small_value_not_compressed() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let key = [0xCC; 32];
let value = b"hi"; // Smaller than 4KB threshold
engine
.write("alice", key, value, Codec::Zstd)
.unwrap();
engine.put(key, value, Codec::Zstd).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value.to_vec()));
}
@@ -92,16 +73,13 @@ fn test_small_value_not_compressed() {
fn test_large_value() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let key = [0xDD; 32];
let value = vec![b'X'; 100_000]; // 100KB
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
@@ -109,22 +87,19 @@ fn test_large_value() {
fn test_multiple_keys() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let n = 100;
for i in 0..n {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
let value = format!("email number {}", i).into_bytes();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
for i in 0..n {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
}
}
@@ -133,7 +108,6 @@ fn test_multiple_keys() {
fn test_gc() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
// Write many entries
let value = vec![b'Y'; 5000];
@@ -142,26 +116,24 @@ fn test_gc() {
for i in 0..n {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
engine
.write("alice", key, &value, Codec::None)
.unwrap();
engine.put(key, &value, Codec::None).unwrap();
}
// Delete even-numbered keys
for i in (0..n).step_by(2) {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
engine.delete("alice", &key).unwrap();
engine.delete(&key).unwrap();
}
// Run GC
let _result = engine.gc("alice").unwrap();
let _result = engine.gc().unwrap();
// Verify remaining keys still readable
for i in (1..n).step_by(2) {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value.clone()));
}
@@ -169,7 +141,7 @@ fn test_gc() {
for i in (0..n).step_by(2) {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, None);
}
}
@@ -182,16 +154,13 @@ fn test_reopen_persistence() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
// Reopen
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
}
@@ -200,21 +169,18 @@ fn test_reopen_persistence() {
fn test_stats() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", [1u8; 32], b"hello", Codec::None)
.unwrap();
engine.put([1u8; 32], b"hello", Codec::None).unwrap();
let stats = engine.stats("alice").unwrap();
let stats = engine.stats().unwrap();
assert!(stats.total_bytes > 0);
assert!(stats.total_keys > 0);
}
#[test]
fn test_batch_write() {
let dir = TempDir::new().unwrap();
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let n = 50;
let entries: Vec<_> = (0..n)
@@ -226,10 +192,10 @@ fn test_batch_write() {
})
.collect();
engine.write_batch("alice", &entries).unwrap();
engine.put_batch(&entries).unwrap();
for (key, value, _) in &entries {
let result = engine.read("alice", key).unwrap();
let result = engine.get(key).unwrap();
assert_eq!(result.as_ref(), Some(value));
}
}
@@ -247,14 +213,13 @@ fn test_batch_write_persistence() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine.write_batch("alice", &entries).unwrap();
engine.put_batch(&entries).unwrap();
}
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for (key, value, _) in &entries {
let result = engine.read("alice", key).unwrap();
let result = engine.get(key).unwrap();
assert_eq!(result.as_ref(), Some(value));
}
}
@@ -264,7 +229,7 @@ fn test_batch_write_persistence() {
fn test_invalid_config_rejected() {
let dir = TempDir::new().unwrap();
let mut config = Config::default();
config.lru_bucket_count = 0;
config.compact_threshold = 0;
assert!(Engine::open(dir.path(), config).is_err());
let mut config = Config::default();
@@ -279,13 +244,14 @@ fn test_concurrent_reads() {
let dir = TempDir::new().unwrap();
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
engine.create_account("alice").unwrap();
// Write some data
for i in 0..50u32 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
engine.write("alice", key, &vec![i as u8; 1024], Codec::None).unwrap();
engine
.put(key, &vec![i as u8; 1024], Codec::None)
.unwrap();
}
// Spawn 4 threads, each reading a different subset
@@ -296,7 +262,7 @@ fn test_concurrent_reads() {
for i in (t * 12)..((t + 1) * 12) {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
let read = engine.read("alice", &key).unwrap();
let read = engine.get(&key).unwrap();
assert!(read.is_some(), "key {} should exist", i);
}
}));
@@ -307,43 +273,25 @@ fn test_concurrent_reads() {
}
#[test]
fn test_concurrent_writes_different_accounts() {
use std::sync::Arc;
use std::thread;
fn test_global_dedup() {
let dir = TempDir::new().unwrap();
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for name in &["alice", "bob", "carol"] {
engine.create_account(name).unwrap();
}
let key = [0x42; 32];
let value = b"same content across what would be accounts".to_vec();
let mut handles = vec![];
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
let engine = engine.clone();
let account_name = name.to_string();
handles.push(thread::spawn(move || {
for i in 0..20 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
let value = vec![(t * 100 + i) as u8; 512];
engine.write(&account_name, key, &value, Codec::None).unwrap();
}
}));
}
for h in handles {
h.join().unwrap();
}
// Write same key twice (simulating two accounts ingesting the same email)
engine.put(key, &value, Codec::Zstd).unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
// Verify all writes persisted
for (t, name) in ["alice", "bob", "carol"].iter().enumerate() {
for i in 0..20 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes());
let read = engine.read(name, &key).unwrap();
assert!(read.is_some(), "account {} key {} should exist", name, i);
}
}
// Should still be readable
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
// Stats should reflect dedup (not double count)
let stats = engine.stats().unwrap();
// The key appears once in the bucket store
assert!(stats.total_keys > 0);
}
#[test]
@@ -354,144 +302,59 @@ fn test_crash_recovery() {
// Phase 1: write data, then drop without shutdown (simulates crash)
{
let engine = Engine::open(&dir_path, Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..50u32 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
engine.write("alice", key, &vec![i as u8; 512], Codec::None).unwrap();
engine
.put(key, &vec![i as u8; 512], Codec::None)
.unwrap();
}
// Engine dropped here without calling shutdown()
}
// Phase 2: reopen — recovery should run, data should be intact
// Phase 2: reopen - recovery should run, data should be intact
let engine = Engine::open(&dir_path, Config::default()).unwrap();
let stats = engine.stats("alice").unwrap();
let stats = engine.stats().unwrap();
assert!(stats.total_keys > 0, "recovery should preserve data");
// Verify reads work
for i in 0..50u32 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
let read = engine.read("alice", &key).unwrap();
let read = engine.get(&key).unwrap();
assert!(read.is_some(), "key {} should survive crash recovery", i);
}
}
#[test]
fn test_meta_bin_durability() {
// Verify meta.bin has valid CRC and can be read after a write cycle.
let dir = TempDir::new().unwrap();
let dir_path = dir.path().to_path_buf();
{
let engine = Engine::open(&dir_path, Config::default()).unwrap();
engine.create_account("alice").unwrap();
let key = [0x42u8; 32];
engine.write("alice", key, b"durable", Codec::None).unwrap();
engine.put(key, b"durable", Codec::None).unwrap();
}
// Engine dropped → shutdown() called → meta saved via write_bin (with fsync)
// Engine dropped -> shutdown() called -> meta saved
// Verify meta.bin exists and has valid CRC
let meta_path = dir_path
.join("accounts")
.join("alice")
.join("meta.bin");
// Verify meta.bin exists
let meta_path = dir_path.join("meta.bin");
assert!(meta_path.exists(), "meta.bin should exist after clean shutdown");
let data = std::fs::read(&meta_path).unwrap();
assert!(data.len() >= 8, "meta.bin should have at least 8 bytes (crc + version)");
assert!(
data.len() >= 8,
"meta.bin should have at least 8 bytes"
);
let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap());
assert_ne!(stored_crc, 0, "stored CRC should be non-zero");
// Reopen and verify data is intact
let engine = Engine::open(&dir_path, Config::default()).unwrap();
let read = engine.read("alice", &[0x42u8; 32]).unwrap();
let read = engine.get(&[0x42u8; 32]).unwrap();
assert_eq!(read, Some(b"durable".to_vec()));
}
#[test]
fn test_gc_concurrent_with_writes() {
// GC should not lose entries that are written concurrently.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::thread;
let dir = TempDir::new().unwrap();
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
engine.create_account("alice").unwrap();
// Pre-fill: write enough to trigger eventual GC
let big_value = vec![b'X'; 8192];
for i in 0..500u32 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
engine.write("alice", key, &big_value, Codec::None).unwrap();
}
// Delete some to create GC candidates
for i in 0..250u32 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
engine.delete("alice", &key).unwrap();
}
let running = Arc::new(AtomicBool::new(true));
let engine_gc = engine.clone();
let running_gc = running.clone();
// Thread 1: run GC in a loop
let gc_handle = thread::spawn(move || {
while running_gc.load(Ordering::Relaxed) {
let _ = engine_gc.gc("alice");
thread::sleep(std::time::Duration::from_millis(10));
}
});
// Thread 2: keep writing new entries
let engine_write = engine.clone();
let running_write = running.clone();
let write_handle = thread::spawn(move || {
let mut counter = 10000u32;
while running_write.load(Ordering::Relaxed) {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&counter.to_le_bytes());
engine_write
.write("alice", key, &vec![counter as u8; 256], Codec::None)
.unwrap();
counter += 1;
}
counter
});
// Let them race for a bit
thread::sleep(std::time::Duration::from_millis(500));
running.store(false, Ordering::Relaxed);
gc_handle.join().unwrap();
let final_counter = write_handle.join().unwrap();
// All written entries must be readable
let mut missing = 0;
for i in 0..500u32 {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
if engine.read("alice", &key).unwrap().is_none() {
// Entries 0..250 were deleted, they should be gone
if i >= 250 {
missing += 1;
}
}
}
assert_eq!(missing, 0, "pre-existing entries should survive concurrent GC");
// Entries written during the race should be readable
for i in 10000..final_counter {
let mut key = [0u8; 32];
key[0..4].copy_from_slice(&i.to_le_bytes());
let read = engine.read("alice", &key).unwrap();
assert!(read.is_some(), "concurrently written key {} should exist after GC", i);
}
}