External/bichon
1
0
Fork 0
mirror of https://github.com/rustmailer/bichon.git synced 2026-08-03 07:48:34 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(blob): add delete_batch, gc_if_needed, background flush, and fix bincode compat

Browse Source 
- Replace bincode 3.0.0 (empty crate) with bincode_reloaded 3.1.10
  - Add delete_batch for efficient grouped tombstone writes
  - Add gc_if_needed to skip GC when no segment exceeds threshold
  - Add flush() for lightweight fsync+meta checkpoint without compact
  - Add Config::flush_interval_secs to spawn a background flush thread
  - Remove per-put/per-delete fsync; persistence via background flush
  - Split gc_segments into gc_prepare/gc_finish to reduce write lock hold time
This commit is contained in:
rustmailer
2026-07-10 01:52:04 +08:00
parent 7dee5a7874
commit 4cdf3ee5f1
19 changed files with 1431 additions and 1797 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/blob/tests/acid_test.rs
+73 -101
View File
@@ -3,9 +3,6 @@
/// Since we can't kill the process mid-write in an inline test, we simulate crashes
/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
/// then re-opening and verifying recovery produced consistent state.
///
/// For true power-loss simulation, each test writes data, drops the engine abruptly,
/// then reopens and verifies: no corruption, no lost committed data, no partial writes.
use std::fs;
use std::path::Path;
@@ -50,16 +47,13 @@ fn test_durability_single_write_survives_crash() {
// Write
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
} // <-- Engine dropped = simulated crash
// Recover
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
}
@@ -73,21 +67,23 @@ fn test_durability_many_writes_survive_crash() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..n {
let key = make_key(i as u64);
keys.push(key);
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
} // crash
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for (i, key) in keys.iter().enumerate() {
let result = engine.read("alice", key).unwrap();
assert_eq!(result, Some(value.clone()), "missing key at index {}", i);
let result = engine.get(key).unwrap();
assert_eq!(
result,
Some(value.clone()),
"missing key at index {}",
i
);
}
}
}
@@ -100,20 +96,17 @@ fn test_durability_delete_survives_crash() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
} // crash after write
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.delete("alice", &key).unwrap();
engine.delete(&key).unwrap();
} // crash after delete
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
assert_eq!(result, None, "delete should persist across crash");
}
}
@@ -129,22 +122,20 @@ fn test_atomicity_no_partial_entries_after_crash() {
// Write enough entries to fill part of a segment, then crash
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let value = make_value(50_000); // big enough to notice
let value = make_value(50_000);
for i in 0..200u64 {
engine
.write("alice", make_key(i), &value, Codec::None)
.put(make_key(i), &value, Codec::None)
.unwrap();
}
} // crash
// Recovery should clean up any partial tail entries and all committed
// entries should be readable
// Recovery should clean up any partial tail entries
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let value = make_value(50_000);
for i in 0..200u64 {
let result = engine.read("alice", &make_key(i)).unwrap();
let result = engine.get(&make_key(i)).unwrap();
assert_eq!(
result,
Some(value.clone()),
@@ -162,20 +153,18 @@ fn test_atomicity_crash_during_segment_roll() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
// Write enough to cross at least one segment boundary (256 MB)
for i in 0..140u64 {
engine
.write("alice", make_key(i), &big_value, Codec::None)
.put(make_key(i), &big_value, Codec::None)
.unwrap();
}
} // crash mid-way or after multiple segments
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
// All committed writes (that returned Ok) must be readable
for i in 0..140u64 {
let result = engine.read("alice", &make_key(i)).unwrap();
let result = engine.get(&make_key(i)).unwrap();
assert!(
result.is_some(),
"key {} should exist after segment roll recovery",
@@ -197,16 +186,12 @@ fn test_consistency_crc_detects_corruption() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine
.write("alice", key, &value, Codec::Zstd)
.unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
// Corrupt the segment file by flipping a byte
let seg_path = find_first_segment(dir.path(), "alice");
let seg_path = find_first_segment(dir.path());
let mut data = fs::read(&seg_path).unwrap();
// Flip a byte in the data portion, not the header
let flip_pos = data.len() - 100;
data[flip_pos] ^= 0xFF;
fs::write(&seg_path, &data).unwrap();
@@ -214,16 +199,14 @@ fn test_consistency_crc_detects_corruption() {
// Reading should detect CRC mismatch
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
let result = engine.read("alice", &key);
// Either error or None is acceptable — never silently wrong data
let result = engine.get(&key);
match result {
Err(_) => {} // CRC mismatch detected — good
Err(_) => {} // CRC mismatch detected - good
Ok(None) => {} // index may point to truncated/removed data
Ok(Some(v)) => {
if v == value {
panic!("CRC corruption was NOT detected — silent data corruption!");
panic!("CRC corruption was NOT detected - silent data corruption!");
}
// If value differs, index pointed elsewhere after recovery
}
}
}
@@ -235,35 +218,39 @@ fn test_consistency_corrupt_magic_truncated_on_recovery() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..10u64 {
engine
.write("alice", make_key(i), &make_value(4096), Codec::Zstd)
.put(make_key(i), &make_value(4096), Codec::Zstd)
.unwrap();
}
}
// Append garbage to the segment file (simulating partial write from crash)
let seg_path = find_first_segment(dir.path(), "alice");
let seg_path = find_first_segment(dir.path());
let mut data = fs::read(&seg_path).unwrap();
let orig_len = data.len();
// Append garbage that doesn't start with the magic number
data.extend_from_slice(&[0xFF; 200]);
fs::write(&seg_path, &data).unwrap();
// Recovery should truncate the garbage
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
// Verify committed data is still intact
for i in 0..10u64 {
let result = engine.read("alice", &make_key(i)).unwrap();
assert!(result.is_some(), "committed key {} should survive tail truncation", i);
let result = engine.get(&make_key(i)).unwrap();
assert!(
result.is_some(),
"committed key {} should survive tail truncation",
i
);
}
}
// Verify file was actually truncated
let truncated_len = fs::metadata(&seg_path).unwrap().len();
assert!(truncated_len <= orig_len as u64, "garbage should have been truncated");
assert!(
truncated_len <= orig_len as u64,
"garbage should have been truncated"
);
}
// ---------------------------------------------------------------------------
@@ -274,22 +261,17 @@ fn test_consistency_corrupt_magic_truncated_on_recovery() {
fn test_isolation_reader_sees_snapshot_not_partial_write() {
let dir = TempDir::new().unwrap();
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
engine.create_account("alice").unwrap();
// Pre-populate a known key
let original_value = make_value(4096);
let key = make_key(100);
engine
.write("alice", key, &original_value, Codec::Zstd)
.unwrap();
engine.put(key, &original_value, Codec::Zstd).unwrap();
let running = Arc::new(AtomicBool::new(true));
let writer_done = Arc::new(AtomicBool::new(false));
// Spawn a writer that continuously overwrites the same key
let writer_engine = engine.clone();
let writer_running = running.clone();
let writer_done_flag = writer_done.clone();
let writer_key = key;
let writer = thread::spawn(move || {
@@ -299,11 +281,10 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
}
let val = make_value(4096 + (i as usize % 100));
writer_engine
.write("alice", writer_key, &val, Codec::Zstd)
.put(writer_key, &val, Codec::Zstd)
.unwrap();
thread::yield_now();
}
writer_done_flag.store(true, Ordering::SeqCst);
});
// Concurrent reader: reads should never panic or hang
@@ -315,11 +296,10 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
if !reader_running.load(Ordering::Relaxed) && reads > 0 {
break;
}
let result = reader_engine.read("alice", &key);
let result = reader_engine.get(&key);
match result {
Ok(Some(_)) | Ok(None) => {} // OK
Ok(Some(_)) | Ok(None) => {}
Err(e) => {
// Accept transient errors but report them
eprintln!("reader saw error: {:?}", e);
}
}
@@ -332,8 +312,7 @@ fn test_isolation_reader_sees_snapshot_not_partial_write() {
running.store(false, Ordering::SeqCst);
writer.join().unwrap();
// Final read should see the last committed value (not partial)
let final_result = engine.read("alice", &key).unwrap();
let final_result = engine.get(&key).unwrap();
assert!(final_result.is_some(), "final read should find a value");
}
@@ -347,21 +326,18 @@ fn test_crash_during_gc_leaves_data_intact() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
let value = make_value(500_000); // 500 KB each
// Write enough entries and delete some to create GC candidate
let value = make_value(500_000);
for i in 0..500u64 {
engine
.write("alice", make_key(i), &value, Codec::None)
.put(make_key(i), &value, Codec::None)
.unwrap();
}
// Delete ~40%
for i in (0..500u64).step_by(5) {
engine.delete("alice", &make_key(i)).unwrap();
engine.delete(&make_key(i)).unwrap();
}
// Single GC run (may or may not trigger)
let _ = engine.gc("alice");
let _ = engine.gc();
} // crash after GC
// All non-deleted entries must still be readable
@@ -370,9 +346,8 @@ fn test_crash_during_gc_leaves_data_intact() {
let value = make_value(500_000);
for i in 0..500u64 {
let key = make_key(i);
let result = engine.read("alice", &key).unwrap();
let result = engine.get(&key).unwrap();
if i % 5 == 0 {
// Deleted keys
assert_eq!(result, None, "key {} should be deleted", i);
} else {
assert_eq!(
@@ -401,11 +376,8 @@ fn test_multiple_crash_reopen_cycles() {
// Populate and crash
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
for i in 0..50u64 {
engine
.write("alice", make_key(i), &value, Codec::Zstd)
.unwrap();
engine.put(make_key(i), &value, Codec::Zstd).unwrap();
alive.insert(i);
}
}
@@ -414,11 +386,11 @@ fn test_multiple_crash_reopen_cycles() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for &k in &alive {
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
assert!(engine.get(&make_key(k)).unwrap().is_some());
}
for i in 100..150u64 {
engine
.write("alice", make_key(i), &value, Codec::Zstd)
.put(make_key(i), &value, Codec::Zstd)
.unwrap();
alive.insert(i);
}
@@ -428,10 +400,10 @@ fn test_multiple_crash_reopen_cycles() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for &k in &alive {
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
assert!(engine.get(&make_key(k)).unwrap().is_some());
}
for i in 0..10u64 {
engine.delete("alice", &make_key(i)).unwrap();
engine.delete(&make_key(i)).unwrap();
alive.remove(&i);
}
}
@@ -440,44 +412,44 @@ fn test_multiple_crash_reopen_cycles() {
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
for &k in &alive {
assert!(engine.read("alice", &make_key(k)).unwrap().is_some(),
"key {} should exist", k);
assert!(
engine.get(&make_key(k)).unwrap().is_some(),
"key {} should exist",
k
);
}
for i in 0..10u64 {
assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None,
"key {} should be deleted", i);
assert_eq!(
engine.get(&make_key(i)).unwrap(),
None,
"key {} should be deleted",
i
);
}
}
}
// ---------------------------------------------------------------------------
// 7. Account-level isolation
// 7. Global dedup: same content stored once
// ---------------------------------------------------------------------------
#[test]
fn test_account_isolation_crash_one_account_does_not_affect_others() {
fn test_global_dedup_after_crash() {
let dir = TempDir::new().unwrap();
let key = make_key(42);
let value = make_value(8192);
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
engine.create_account("alice").unwrap();
engine.create_account("bob").unwrap();
engine
.write("alice", make_key(1), &make_value(4096), Codec::Zstd)
.unwrap();
engine
.write("bob", make_key(1), &make_value(8192), Codec::Zstd)
.unwrap();
// Write same key twice (simulating two sources with same content)
engine.put(key, &value, Codec::Zstd).unwrap();
engine.put(key, &value, Codec::Zstd).unwrap();
}
// Delete alice's account dir partially to simulate corruption
// Then verify bob is intact
{
let engine = Engine::open(dir.path(), Config::default()).unwrap();
// Bob should be fine
let result = engine.read("bob", &make_key(1)).unwrap();
assert!(result.is_some(), "bob should be unaffected");
let result = engine.get(&key).unwrap();
assert_eq!(result, Some(value));
}
}
@@ -485,8 +457,8 @@ fn test_account_isolation_crash_one_account_does_not_affect_others() {
// Helpers
// ---------------------------------------------------------------------------
fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf {
let seg_dir = store_root.join("accounts").join(account).join("segments");
fn find_first_segment(store_root: &Path) -> std::path::PathBuf {
let seg_dir = store_root.join("segments");
for entry in fs::read_dir(&seg_dir).unwrap() {
let entry = entry.unwrap();
let name = entry.file_name().to_string_lossy().into_owned();