mirror of
https://github.com/rustmailer/bichon.git
synced 2026-08-03 07:48:34 +02:00
feat(blob): add embedded KV storage engine for email archiving
This commit is contained in:
rustmailer
+2
-1
@@ -3,4 +3,5 @@
|
||||
.idea
|
||||
config.toml
|
||||
node_modules
|
||||
dedup_report.txt
|
||||
dedup_report.txt
|
||||
crates/*/target
|
||||
Generated
+183
-22
@@ -60,6 +60,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
@@ -296,11 +302,11 @@ name = "bichon-admin"
|
||||
version = "1.5.3"
|
||||
dependencies = [
|
||||
"bichon-core",
|
||||
"bichon-memdb",
|
||||
"console",
|
||||
"dialoguer",
|
||||
"indicatif",
|
||||
"itertools",
|
||||
"memdb",
|
||||
"itertools 0.14.0",
|
||||
"native_db",
|
||||
"native_model",
|
||||
"serde",
|
||||
@@ -309,6 +315,22 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-blob"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"criterion",
|
||||
"lz4_flex",
|
||||
"rand 0.10.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-cli"
|
||||
version = "1.5.3"
|
||||
@@ -341,6 +363,7 @@ version = "1.5.3"
|
||||
dependencies = [
|
||||
"async-imap",
|
||||
"base64 0.22.1",
|
||||
"bichon-memdb",
|
||||
"blake3",
|
||||
"bytes 1.11.1",
|
||||
"chrono",
|
||||
@@ -355,12 +378,11 @@ dependencies = [
|
||||
"governor",
|
||||
"hickory-resolver",
|
||||
"html2text",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"itoa",
|
||||
"lru 0.18.0",
|
||||
"mail-parser",
|
||||
"mail-send",
|
||||
"memdb",
|
||||
"murmur3",
|
||||
"num_cpus",
|
||||
"oauth2",
|
||||
@@ -394,6 +416,18 @@ dependencies = [
|
||||
"whichlang",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-memdb"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rand 0.9.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bichon-server"
|
||||
version = "1.5.3"
|
||||
@@ -613,6 +647,12 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.60"
|
||||
@@ -668,6 +708,33 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"ciborium-ll",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-io"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-ll"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.1"
|
||||
@@ -868,6 +935,39 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679"
|
||||
dependencies = [
|
||||
"anes",
|
||||
"cast",
|
||||
"ciborium",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"itertools 0.13.0",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
"plotters",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools 0.10.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "critical-section"
|
||||
version = "1.2.0"
|
||||
@@ -1602,6 +1702,17 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crunchy",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
@@ -2128,6 +2239,24 @@ dependencies = [
|
||||
"phf 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.10.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.14.0"
|
||||
@@ -2338,9 +2467,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.13.0"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a"
|
||||
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
|
||||
dependencies = [
|
||||
"twox-hash",
|
||||
]
|
||||
@@ -2445,18 +2574,6 @@ version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "memdb"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"rand 0.9.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.10"
|
||||
@@ -2858,6 +2975,12 @@ version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.78"
|
||||
@@ -3090,6 +3213,34 @@ version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"plotters-backend",
|
||||
"plotters-svg",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-backend"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
|
||||
|
||||
[[package]]
|
||||
name = "plotters-svg"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
|
||||
dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "poem"
|
||||
version = "3.1.12"
|
||||
@@ -3159,7 +3310,7 @@ dependencies = [
|
||||
"email_address",
|
||||
"futures-util",
|
||||
"indexmap",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"mime",
|
||||
"num-traits",
|
||||
"poem",
|
||||
@@ -4320,7 +4471,7 @@ dependencies = [
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"htmlescape",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"levenshtein_automata",
|
||||
"log",
|
||||
"lru 0.16.4",
|
||||
@@ -4371,7 +4522,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
|
||||
dependencies = [
|
||||
"downcast-rs",
|
||||
"fastdivide",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"serde",
|
||||
"tantivy-bitpacker",
|
||||
"tantivy-common",
|
||||
@@ -4423,7 +4574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"tantivy-bitpacker",
|
||||
"tantivy-common",
|
||||
"tantivy-fst",
|
||||
@@ -4574,6 +4725,16 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.11.0"
|
||||
|
||||
+3
-1
@@ -3,6 +3,7 @@
|
||||
members = [
|
||||
"crates/memdb",
|
||||
"crates/core",
|
||||
"crates/blob",
|
||||
"crates/server",
|
||||
"crates/cli",
|
||||
"crates/admin",
|
||||
@@ -18,7 +19,8 @@ edition = "2021"
|
||||
[workspace.dependencies]
|
||||
chrono = "0.4.45"
|
||||
clap = { version = "4.6.1", features = ["derive", "env"] }
|
||||
memdb = { path = "crates/memdb" }
|
||||
bichon-memdb = { path = "crates/memdb" }
|
||||
bichon-blob = { path = "crates/blob" }
|
||||
itertools = "0.14.0"
|
||||
ring = { version = "0.17.14", features = ["std"] }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
|
||||
@@ -17,4 +17,4 @@ serde_json.workspace = true
|
||||
|
||||
itertools.workspace = true
|
||||
snafu.workspace = true
|
||||
memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
@@ -20,7 +20,7 @@ use bichon_core::{
|
||||
};
|
||||
use console::style;
|
||||
use itertools::Itertools;
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
use native_db::*;
|
||||
use native_model::{native_model, Model};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
Generated
+1105
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "bichon-blob"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Embedded KV storage engine for email"
|
||||
|
||||
[dependencies]
|
||||
crc32fast = "1.4"
|
||||
zstd = "0.13"
|
||||
lz4_flex = "0.13.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
tracing = "0.1"
|
||||
thiserror = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
rand = "0.10.1"
|
||||
criterion = { version = "0.6", features = ["html_reports"] }
|
||||
|
||||
[[bench]]
|
||||
name = "benchmark"
|
||||
harness = false
|
||||
@@ -0,0 +1,316 @@
|
||||
use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput};
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
|
||||
fn make_key(seed: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&seed.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let mut v = Vec::with_capacity(size);
|
||||
// Fill with somewhat realistic text-like data so compression works
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
let n = rem.min(pattern.len());
|
||||
v.extend_from_slice(&pattern[..n]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
pub fn bench_write_small(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024); // 1 KB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("1KB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_write_medium(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Bytes(64 * 1024));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(64 * 1024); // 64 KB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("64KB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_write_large(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("write");
|
||||
group.throughput(Throughput::Bytes(1024 * 1024));
|
||||
group.measurement_time(Duration::from_secs(15));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
let mut counter = 0u64;
|
||||
|
||||
group.bench_function("1MB", |b| {
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
(make_key(counter), value.clone())
|
||||
},
|
||||
|(key, val)| {
|
||||
engine
|
||||
.write("bench", key, &val, Codec::Zstd)
|
||||
.unwrap()
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_hit(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate: 10 keys, all in same bucket → cache hit after first read
|
||||
let value = make_value(4096);
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_hit", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 10);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_cache_miss(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 8; // Small cache to force misses
|
||||
let engine = Engine::open(dir.path(), config).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
// Write 1000 keys spread across all 16 buckets — small LRU will thrash
|
||||
for i in 0..1000u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("cache_miss", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 1000);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_read_large_value(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("read");
|
||||
group.throughput(Throughput::Bytes(1024 * 1024));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(1024 * 1024); // 1 MB
|
||||
for i in 0..5u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter = 0u64;
|
||||
group.bench_function("1MB_cache_hit", |b| {
|
||||
b.iter(|| {
|
||||
let key = make_key(counter % 5);
|
||||
counter += 1;
|
||||
std::hint::black_box(engine.read("bench", &key).unwrap());
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_delete(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("delete");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(10));
|
||||
|
||||
group.bench_function("delete", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
let value = make_value(4096);
|
||||
let mut counter = 0u64;
|
||||
|
||||
b.iter_batched(
|
||||
|| {
|
||||
counter += 1;
|
||||
let key = make_key(counter);
|
||||
engine
|
||||
.write("bench", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
key
|
||||
},
|
||||
|key| {
|
||||
engine.delete("bench", &key).unwrap();
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
)
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_mixed_workload(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("mixed");
|
||||
group.throughput(Throughput::Elements(1));
|
||||
group.measurement_time(Duration::from_secs(15));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Pre-populate with 500 entries
|
||||
let value = make_value(8192);
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let mut counter: u64 = 500;
|
||||
group.bench_function("80w_15r_5d", |b| {
|
||||
b.iter(|| {
|
||||
counter += 1;
|
||||
let op = counter % 100;
|
||||
match op {
|
||||
0..=79 => {
|
||||
// 80% writes
|
||||
let key = make_key(counter);
|
||||
let val = make_value(4096);
|
||||
engine.write("bench", key, &val, Codec::Zstd).unwrap();
|
||||
}
|
||||
80..=94 => {
|
||||
// 15% reads
|
||||
std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap());
|
||||
}
|
||||
_ => {
|
||||
// 5% deletes
|
||||
if counter % 2 == 0 {
|
||||
let key = make_key(counter % 500);
|
||||
let _ = engine.delete("bench", &key);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn bench_gc(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("gc");
|
||||
group.measurement_time(Duration::from_secs(30));
|
||||
group.sample_size(10);
|
||||
|
||||
group.bench_function("gc_30pct_deleted", |b| {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("bench").unwrap();
|
||||
|
||||
// Fill a segment with ~1000 entries, then delete 30%
|
||||
let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB
|
||||
let n = 1200u64;
|
||||
for i in 0..n {
|
||||
engine
|
||||
.write("bench", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~30%
|
||||
for i in (0..n).step_by(3) {
|
||||
engine.delete("bench", &make_key(i)).unwrap();
|
||||
}
|
||||
|
||||
b.iter(|| {
|
||||
engine.gc("bench").unwrap();
|
||||
})
|
||||
});
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_write_small,
|
||||
bench_write_medium,
|
||||
bench_write_large,
|
||||
bench_read_cache_hit,
|
||||
bench_read_cache_miss,
|
||||
bench_read_large_value,
|
||||
bench_delete,
|
||||
bench_mixed_workload,
|
||||
bench_gc,
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,242 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
use crate::types::Codec;
|
||||
|
||||
pub struct Account {
|
||||
pub id: String,
|
||||
dir: PathBuf,
|
||||
meta: AccountMeta,
|
||||
active_writer: SegmentWriter,
|
||||
readers: HashMap<u32, SegmentReader>,
|
||||
write_lock: Mutex<()>,
|
||||
}
|
||||
|
||||
impl Account {
|
||||
/// Open an existing account.
|
||||
pub fn open(store_root: &Path, account_id: &str) -> Result<Self> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if !dir.exists() {
|
||||
return Err(Error::AccountNotFound(account_id.to_string()));
|
||||
}
|
||||
|
||||
let meta = AccountMeta::load(&dir)?;
|
||||
|
||||
// Open active writer
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(meta.active_segment_id));
|
||||
let active_writer = if seg_path.exists() {
|
||||
SegmentWriter::open_append(seg_path, meta.active_segment_id)?
|
||||
} else {
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
SegmentWriter::create(seg_path, meta.active_segment_id)?
|
||||
};
|
||||
|
||||
// Open readers for existing sealed segments
|
||||
let mut readers = HashMap::new();
|
||||
for (&seg_id, stats) in &meta.segments {
|
||||
if stats.sealed {
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if seg_path.exists() {
|
||||
readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
meta,
|
||||
active_writer,
|
||||
readers,
|
||||
write_lock: Mutex::new(()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new account.
|
||||
pub fn create(store_root: &Path, account_id: &str) -> Result<Self> {
|
||||
let dir = store_root.join("accounts").join(account_id);
|
||||
if dir.exists() {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
|
||||
fs::create_dir_all(dir.join("segments"))?;
|
||||
BucketFile::ensure_dir(&dir)?;
|
||||
|
||||
let meta = AccountMeta::new(account_id.to_string(), 1);
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let active_writer = SegmentWriter::create(seg_path, 1)?;
|
||||
|
||||
meta.save(&dir)?;
|
||||
|
||||
Ok(Self {
|
||||
id: account_id.to_string(),
|
||||
dir,
|
||||
meta,
|
||||
active_writer,
|
||||
readers: HashMap::new(),
|
||||
write_lock: Mutex::new(()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &AccountMeta {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// Lock for writing. Returns a guard.
|
||||
pub fn lock_write(&self) -> std::sync::MutexGuard<'_, ()> {
|
||||
self.write_lock.lock().unwrap()
|
||||
}
|
||||
|
||||
/// Mark the segment as indexed up to the given offset and persist meta.
|
||||
/// Called after append_index to enable incremental recovery.
|
||||
pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> {
|
||||
if let Some(stats) = self.meta.segments.get_mut(&segment_id) {
|
||||
if indexed_up_to_offset > stats.indexed_up_to_offset {
|
||||
stats.indexed_up_to_offset = indexed_up_to_offset;
|
||||
}
|
||||
}
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Append an entry without fsync. Returns (segment_id, offset, data_size).
|
||||
/// Caller must hold the write lock and should call `flush_active()` after.
|
||||
pub fn append_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
if self.active_writer.is_full() {
|
||||
self.seal_active()?;
|
||||
}
|
||||
|
||||
use crate::segment::Entry;
|
||||
let entry = if flags == 1 {
|
||||
Entry::tombstone(key)
|
||||
} else {
|
||||
Entry::new(key, data, flags, codec)
|
||||
};
|
||||
|
||||
let data_size = entry.data.len() as u32;
|
||||
let segment_id = self.active_writer.id();
|
||||
let offset = self.active_writer.append(&entry)?;
|
||||
|
||||
let stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(segment_id)
|
||||
.or_insert_with(|| SegmentStats::new(segment_id));
|
||||
stats.total_bytes += data_size as u64;
|
||||
if flags == 1 {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
stats.recompute_ratio();
|
||||
|
||||
Ok((segment_id, offset, data_size))
|
||||
}
|
||||
|
||||
/// Fsync the active segment and persist meta.
|
||||
pub fn flush_active(&mut self) -> Result<()> {
|
||||
self.active_writer.fsync()?;
|
||||
self.meta.save(&self.dir)
|
||||
}
|
||||
|
||||
/// Write an entry with fsync. Convenience wrapper for single writes.
|
||||
pub fn write_entry(
|
||||
&mut self,
|
||||
key: [u8; 32],
|
||||
data: &[u8],
|
||||
flags: u8,
|
||||
codec: Codec,
|
||||
) -> Result<(u32, u64, u32)> {
|
||||
let result = self.append_entry(key, data, flags, codec)?;
|
||||
self.flush_active()?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn seal_active(&mut self) -> Result<()> {
|
||||
let old_id = self.active_writer.id();
|
||||
let old_stats = self
|
||||
.meta
|
||||
.segments
|
||||
.entry(old_id)
|
||||
.or_insert_with(|| SegmentStats::new(old_id));
|
||||
old_stats.sealed = true;
|
||||
|
||||
// Open reader for the old segment
|
||||
let seg_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(old_id));
|
||||
self.readers
|
||||
.insert(old_id, SegmentReader::open(seg_path, old_id)?);
|
||||
|
||||
// Create new segment
|
||||
let new_id = old_id + 1;
|
||||
self.meta.active_segment_id = new_id;
|
||||
let new_path = self
|
||||
.dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(new_id));
|
||||
self.active_writer = SegmentWriter::create(new_path, new_id)?;
|
||||
self.meta.save(&self.dir)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the on-disk path for a segment.
|
||||
pub fn segment_path(&self, segment_id: u32) -> Result<PathBuf> {
|
||||
let filename = segment::segment_filename(segment_id);
|
||||
let path = self.dir.join("segments").join(&filename);
|
||||
if path.exists() {
|
||||
Ok(path)
|
||||
} else {
|
||||
Err(Error::SegmentNotFound(segment_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// Append index record to the appropriate bucket file. No fsync.
|
||||
pub fn append_index(&self, record: &IndexRecord) -> Result<()> {
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
let bf = BucketFile::open(&self.dir, bucket_id);
|
||||
bf.append(record)
|
||||
}
|
||||
|
||||
/// Return list of sealed segment IDs for GC consideration.
|
||||
pub fn sealed_segments(&self) -> Vec<u32> {
|
||||
self.meta
|
||||
.segments
|
||||
.iter()
|
||||
.filter(|(_, s)| s.sealed)
|
||||
.map(|(id, _)| *id)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// All segment IDs (including active).
|
||||
pub fn all_segment_ids(&self) -> Vec<u32> {
|
||||
let mut ids: Vec<u32> = self.meta.segments.keys().copied().collect();
|
||||
if !ids.contains(&self.meta.active_segment_id) {
|
||||
ids.push(self.meta.active_segment_id);
|
||||
}
|
||||
ids.sort_unstable();
|
||||
ids
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE};
|
||||
|
||||
/// On-disk format: 52 bytes per record.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct IndexRecord {
|
||||
pub key: [u8; 32],
|
||||
pub segment_id: u32,
|
||||
pub offset: u64,
|
||||
pub data_size: u32,
|
||||
pub flags: u8,
|
||||
}
|
||||
|
||||
impl IndexRecord {
|
||||
pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self {
|
||||
Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
self.flags == 1
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] {
|
||||
let mut buf = [0u8; INDEX_RECORD_SIZE];
|
||||
buf[0..32].copy_from_slice(&self.key);
|
||||
buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes());
|
||||
buf[36..44].copy_from_slice(&self.offset.to_le_bytes());
|
||||
buf[44..48].copy_from_slice(&self.data_size.to_le_bytes());
|
||||
buf[48] = self.flags;
|
||||
// bytes 49..52 are padding (keep zero)
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self {
|
||||
let mut key = [0u8; 32];
|
||||
key.copy_from_slice(&buf[0..32]);
|
||||
let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap());
|
||||
let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap());
|
||||
let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap());
|
||||
let flags = buf[48];
|
||||
Self {
|
||||
key,
|
||||
segment_id,
|
||||
offset,
|
||||
data_size,
|
||||
flags,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a loaded and deduplicated bucket in memory.
|
||||
pub struct BucketIndex {
|
||||
pub bucket_id: u16,
|
||||
/// Records sorted by key, deduplicated (one record per key, latest wins).
|
||||
pub records: Vec<IndexRecord>,
|
||||
}
|
||||
|
||||
impl BucketIndex {
|
||||
/// Build from raw records: sort by key, dedup keeping the one with max offset.
|
||||
pub fn from_records(mut records: Vec<IndexRecord>, bucket_id: u16) -> Self {
|
||||
records.sort_by_key(|a| a.key);
|
||||
// Dedup: keep last (max offset) for each key
|
||||
let mut deduped = Vec::with_capacity(records.len());
|
||||
let mut i = 0;
|
||||
while i < records.len() {
|
||||
let mut best = i;
|
||||
let mut j = i + 1;
|
||||
while j < records.len() && records[j].key == records[i].key {
|
||||
if records[j].offset > records[best].offset {
|
||||
best = j;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
deduped.push(records[best].clone());
|
||||
i = j;
|
||||
}
|
||||
Self {
|
||||
bucket_id,
|
||||
records: deduped,
|
||||
}
|
||||
}
|
||||
|
||||
/// Binary search for a key. Returns the record if found.
|
||||
pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => Some(&self.records[idx]),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a new record and maintain sorted order.
|
||||
pub fn insert(&mut self, record: IndexRecord) {
|
||||
match self.records.binary_search_by(|r| r.key.cmp(&record.key)) {
|
||||
Ok(idx) => {
|
||||
// Replace if newer (larger offset)
|
||||
if record.offset > self.records[idx].offset {
|
||||
self.records[idx] = record;
|
||||
}
|
||||
}
|
||||
Err(idx) => {
|
||||
self.records.insert(idx, record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.records.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.records.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages a bucket index file on disk.
|
||||
pub struct BucketFile {
|
||||
path: PathBuf,
|
||||
bucket_id: u16,
|
||||
}
|
||||
|
||||
impl BucketFile {
|
||||
pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf {
|
||||
account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id))
|
||||
}
|
||||
|
||||
pub fn open(account_dir: &Path, bucket_id: u16) -> Self {
|
||||
Self {
|
||||
path: Self::path_for(account_dir, bucket_id),
|
||||
bucket_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bucket_id(&self) -> u16 {
|
||||
self.bucket_id
|
||||
}
|
||||
|
||||
/// Ensure the buckets directory exists.
|
||||
pub fn ensure_dir(account_dir: &Path) -> Result<()> {
|
||||
let dir = account_dir.join("buckets");
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a single record to the bucket file.
|
||||
pub fn append(&self, record: &IndexRecord) -> Result<()> {
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
file.write_all(&record.encode())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append multiple records at once.
|
||||
pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
if records.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.path)?;
|
||||
for r in records {
|
||||
file.write_all(&r.encode())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load all records from the bucket file.
|
||||
/// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write),
|
||||
/// the trailing bytes are silently ignored.
|
||||
pub fn load_all(&self) -> Result<Vec<IndexRecord>> {
|
||||
if !self.path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let data = std::fs::read(&self.path)?;
|
||||
let remainder = data.len() % INDEX_RECORD_SIZE;
|
||||
let count = data.len() / INDEX_RECORD_SIZE;
|
||||
let mut records = Vec::with_capacity(count);
|
||||
for i in 0..count {
|
||||
let start = i * INDEX_RECORD_SIZE;
|
||||
let end = start + INDEX_RECORD_SIZE;
|
||||
let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end]
|
||||
.try_into()
|
||||
.map_err(|_| crate::error::Error::BucketIndexCorrupt {
|
||||
path: self.path.clone(),
|
||||
reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE),
|
||||
})?;
|
||||
records.push(IndexRecord::decode(buf));
|
||||
}
|
||||
if remainder > 0 {
|
||||
tracing::warn!(
|
||||
"Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring",
|
||||
self.path, remainder, INDEX_RECORD_SIZE
|
||||
);
|
||||
}
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Load all records, sort, and deduplicate into a BucketIndex.
|
||||
pub fn load_index(&self) -> Result<BucketIndex> {
|
||||
let records = self.load_all()?;
|
||||
Ok(BucketIndex::from_records(records, self.bucket_id))
|
||||
}
|
||||
|
||||
/// Rewrite the bucket file with a sorted, deduplicated set of records.
|
||||
pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> {
|
||||
let temp_path = self.path.with_extension("idx.tmp");
|
||||
{
|
||||
let mut file = File::create(&temp_path)?;
|
||||
for r in records {
|
||||
file.write_all(&r.encode())?;
|
||||
}
|
||||
file.sync_all()?;
|
||||
}
|
||||
std::fs::rename(&temp_path, &self.path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete the bucket file.
|
||||
pub fn delete(&self) -> Result<()> {
|
||||
if self.path.exists() {
|
||||
std::fs::remove_file(&self.path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute bucket_id from a key's first 2 bytes.
|
||||
pub fn bucket_id(key: &[u8; 32]) -> u16 {
|
||||
u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_index_record_encode_decode() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&[1, 2, 3, 4]);
|
||||
let rec = IndexRecord::new(key, 5, 12345, 500, 0);
|
||||
let encoded = rec.encode();
|
||||
let decoded = IndexRecord::decode(&encoded);
|
||||
assert_eq!(rec, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_id_deterministic() {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x0F;
|
||||
assert_eq!(bucket_id(&key), 15);
|
||||
key[0] = 0x00;
|
||||
key[1] = 0x10;
|
||||
assert_eq!(bucket_id(&key), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_append_and_load() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0);
|
||||
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [2u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_dedup() {
|
||||
let recs = vec![
|
||||
IndexRecord::new([1u8; 32], 1, 100, 50, 0),
|
||||
IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins
|
||||
IndexRecord::new([2u8; 32], 1, 300, 60, 0),
|
||||
];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert_eq!(idx.len(), 2);
|
||||
let found = idx.find(&[1u8; 32]).unwrap();
|
||||
assert_eq!(found.segment_id, 2);
|
||||
assert_eq!(found.offset, 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_index_find_missing() {
|
||||
let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)];
|
||||
let idx = BucketIndex::from_records(recs, 0);
|
||||
assert!(idx.find(&[99u8; 32]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_rewrite() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let bucket = BucketFile::open(dir.path(), 0);
|
||||
BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
|
||||
let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0);
|
||||
let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0);
|
||||
bucket.append(&r1).unwrap();
|
||||
bucket.append(&r2).unwrap();
|
||||
|
||||
// Rewrite sorted
|
||||
let sorted = vec![r2.clone(), r1.clone()];
|
||||
bucket.rewrite(&sorted).unwrap();
|
||||
|
||||
let loaded = bucket.load_all().unwrap();
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].key, [1u8; 32]);
|
||||
assert_eq!(loaded[1].key, [3u8; 32]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::bucket::{BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
|
||||
/// Cache key: (account_name, bucket_id)
|
||||
type CacheKey = (String, u16);
|
||||
|
||||
/// LRU bucket cache. Thread-safe.
|
||||
pub struct BucketCache {
|
||||
max_entries: usize,
|
||||
entries: Mutex<Vec<CacheEntry>>,
|
||||
index: Mutex<HashMap<CacheKey, usize>>,
|
||||
}
|
||||
|
||||
struct CacheEntry {
|
||||
key: CacheKey,
|
||||
index: BucketIndex,
|
||||
}
|
||||
|
||||
impl BucketCache {
|
||||
pub fn new(max_entries: usize) -> Self {
|
||||
Self {
|
||||
max_entries: max_entries.max(1),
|
||||
entries: Mutex::new(Vec::new()),
|
||||
index: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get or load a bucket index. Returns the sorted, deduplicated records for the bucket.
|
||||
pub fn get_or_load(
|
||||
&self,
|
||||
account: &str,
|
||||
bucket_id: u16,
|
||||
account_dir: &std::path::Path,
|
||||
) -> Result<Vec<IndexRecord>> {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
|
||||
// Check cache
|
||||
{
|
||||
let index = self.index.lock().unwrap();
|
||||
if let Some(&pos) = index.get(&key) {
|
||||
let entries = self.entries.lock().unwrap();
|
||||
return Ok(entries[pos].index.records.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Load from disk
|
||||
let bucket_file = BucketFile::open(account_dir, bucket_id);
|
||||
let bucket_index = bucket_file.load_index()?;
|
||||
let records = bucket_index.records.clone();
|
||||
|
||||
// Insert into cache
|
||||
self.insert(key, bucket_index);
|
||||
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
fn insert(&self, key: CacheKey, index: BucketIndex) {
|
||||
let mut idx_map = self.index.lock().unwrap();
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
|
||||
// If already exists, update and move to front
|
||||
if let Some(&pos) = idx_map.get(&key) {
|
||||
entries[pos].index = index;
|
||||
let entry = entries.remove(pos);
|
||||
entries.insert(0, entry);
|
||||
// Rebuild index
|
||||
idx_map.clear();
|
||||
for (i, e) in entries.iter().enumerate() {
|
||||
idx_map.insert(e.key.clone(), i);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Evict if full
|
||||
if entries.len() >= self.max_entries {
|
||||
if let Some(evicted) = entries.pop() {
|
||||
idx_map.remove(&evicted.key);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert at front (most recently used)
|
||||
entries.insert(0, CacheEntry { key: key.clone(), index });
|
||||
// Rebuild index (positions shifted)
|
||||
idx_map.clear();
|
||||
for (i, e) in entries.iter().enumerate() {
|
||||
idx_map.insert(e.key.clone(), i);
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert or update a single record in a cached bucket. If bucket not cached, no-op.
|
||||
pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut idx_map = self.index.lock().unwrap();
|
||||
|
||||
if let Some(&pos) = idx_map.get(&key) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
entries[pos].index.insert(record);
|
||||
// Move to front
|
||||
let entry = entries.remove(pos);
|
||||
entries.insert(0, entry);
|
||||
// Rebuild index
|
||||
idx_map.clear();
|
||||
for (i, e) in entries.iter().enumerate() {
|
||||
idx_map.insert(e.key.clone(), i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Invalidate a cached bucket.
|
||||
pub fn invalidate(&self, account: &str, bucket_id: u16) {
|
||||
let key: CacheKey = (account.to_string(), bucket_id);
|
||||
let mut idx_map = self.index.lock().unwrap();
|
||||
if let Some(&pos) = idx_map.get(&key) {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
entries.remove(pos);
|
||||
idx_map.clear();
|
||||
for (i, e) in entries.iter().enumerate() {
|
||||
idx_map.insert(e.key.clone(), i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.lock().unwrap().len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.lock().unwrap().is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::IndexRecord;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_cache_miss_loads_from_disk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let bf = BucketFile::open(dir.path(), 0);
|
||||
bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0))
|
||||
.unwrap();
|
||||
|
||||
let cache = BucketCache::new(10);
|
||||
let _ = cache.get_or_load("test", 0, dir.path()).unwrap();
|
||||
// Second call should hit cache
|
||||
let records = cache
|
||||
.get_or_load("test", 0, dir.path())
|
||||
.unwrap();
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(cache.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_eviction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap();
|
||||
let cache = BucketCache::new(2);
|
||||
|
||||
for b in 0..4 {
|
||||
let bf = BucketFile::open(dir.path(), b);
|
||||
bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0))
|
||||
.unwrap();
|
||||
let _ = cache.get_or_load("test", b, dir.path()).unwrap();
|
||||
}
|
||||
|
||||
assert!(cache.len() <= 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
use crc32fast::Hasher;
|
||||
|
||||
pub fn crc32(data: &[u8]) -> u32 {
|
||||
let mut h = Hasher::new();
|
||||
h.update(data);
|
||||
h.finalize()
|
||||
}
|
||||
|
||||
pub struct CrcWriter {
|
||||
hasher: Hasher,
|
||||
}
|
||||
|
||||
impl Default for CrcWriter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl CrcWriter {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
hasher: Hasher::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update(&mut self, data: &[u8]) {
|
||||
self.hasher.update(data);
|
||||
}
|
||||
|
||||
pub fn finalize(self) -> u32 {
|
||||
self.hasher.finalize()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_crc32_deterministic() {
|
||||
let a = crc32(b"hello");
|
||||
let b = crc32(b"hello");
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_different() {
|
||||
let a = crc32(b"hello");
|
||||
let b = crc32(b"world");
|
||||
assert!(a != b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc_writer_matches_crc32() {
|
||||
let mut w = CrcWriter::new();
|
||||
w.update(b"hello");
|
||||
w.update(b" world");
|
||||
assert_eq!(w.finalize(), crc32(b"hello world"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_empty() {
|
||||
assert_eq!(crc32(b""), 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
use crate::types::Codec;
|
||||
|
||||
pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec<u8>, Codec) {
|
||||
if data.len() < threshold {
|
||||
return (data.to_vec(), Codec::None);
|
||||
}
|
||||
let (compressed, actual_codec) = match codec {
|
||||
Codec::Zstd => {
|
||||
match zstd::encode_all(data, level) {
|
||||
Ok(out) => (out, Codec::Zstd),
|
||||
Err(e) => {
|
||||
tracing::warn!("zstd compression failed, storing uncompressed: {}", e);
|
||||
(data.to_vec(), Codec::None)
|
||||
}
|
||||
}
|
||||
}
|
||||
Codec::Lz4 => {
|
||||
let out = lz4_flex::compress(data);
|
||||
(out, Codec::Lz4)
|
||||
}
|
||||
Codec::None => (data.to_vec(), Codec::None),
|
||||
};
|
||||
// If compression made it larger, store uncompressed
|
||||
if compressed.len() >= data.len() {
|
||||
(data.to_vec(), Codec::None)
|
||||
} else {
|
||||
(compressed, actual_codec)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result<Vec<u8>> {
|
||||
match codec {
|
||||
Codec::None => Ok(data.to_vec()),
|
||||
Codec::Zstd => {
|
||||
zstd::decode_all(data)
|
||||
.map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e)))
|
||||
}
|
||||
Codec::Lz4 => {
|
||||
lz4_flex::decompress(data, raw_size)
|
||||
.map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_small_data_not_compressed() {
|
||||
let data = b"hi";
|
||||
let (out, codec) = compress(data, Codec::Zstd, 4096, 0);
|
||||
assert_eq!(out, b"hi");
|
||||
assert_eq!(codec, Codec::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_data_compressed_zstd() {
|
||||
let data = vec![b'A'; 5000];
|
||||
let (out, codec) = compress(&data, Codec::Zstd, 4096, 0);
|
||||
assert_eq!(codec, Codec::Zstd);
|
||||
assert!(out.len() < data.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_zstd() {
|
||||
let data = vec![b'B'; 10000];
|
||||
let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_lz4() {
|
||||
let data = vec![b'C'; 10000];
|
||||
let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_none() {
|
||||
let data = vec![b'D'; 100];
|
||||
let (compressed, codec) = compress(&data, Codec::None, 4096, 0);
|
||||
assert_eq!(codec, Codec::None);
|
||||
let decompressed = decompress(&compressed, codec, data.len()).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_threshold_zero_always_compresses() {
|
||||
let data = vec![b'E'; 100];
|
||||
let (out, codec) = compress(&data, Codec::Zstd, 0, 0);
|
||||
assert_eq!(codec, Codec::Zstd);
|
||||
assert!(out.len() < data.len());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::RwLock;
|
||||
|
||||
use crate::account::Account;
|
||||
use crate::bucket::{self, IndexRecord};
|
||||
use crate::cache::BucketCache;
|
||||
use crate::compress;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::gc::{self, GcStats};
|
||||
use crate::meta::GlobalMeta;
|
||||
use crate::recovery;
|
||||
use crate::segment::{self, SegmentReader};
|
||||
use crate::types::{Codec, Config, ENTRY_HEADER_SIZE};
|
||||
|
||||
pub struct Engine {
|
||||
root: PathBuf,
|
||||
config: Config,
|
||||
cache: BucketCache,
|
||||
accounts: RwLock<HashMap<String, Account>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountStats {
|
||||
pub account_id: String,
|
||||
pub total_keys: u64,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
pub segment_count: usize,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
/// Open or create the store at `path`. Runs recovery on startup.
|
||||
pub fn open(path: &Path, config: Config) -> Result<Self> {
|
||||
config.validate()?;
|
||||
fs::create_dir_all(path)?;
|
||||
fs::create_dir_all(path.join("accounts"))?;
|
||||
|
||||
let mut global = GlobalMeta::load(path)?;
|
||||
global.save(path)?;
|
||||
|
||||
let cache = BucketCache::new(config.lru_bucket_count);
|
||||
|
||||
// Discover accounts on disk
|
||||
let accounts_dir = path.join("accounts");
|
||||
let mut accounts = HashMap::new();
|
||||
|
||||
if accounts_dir.exists() {
|
||||
for entry in fs::read_dir(&accounts_dir)? {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_dir() {
|
||||
let account_name = entry.file_name().to_string_lossy().into_owned();
|
||||
|
||||
// Clean up temp files from interrupted GC
|
||||
let _ = recovery::cleanup_temp_files(&entry.path());
|
||||
|
||||
// Run recovery
|
||||
match recovery::recover_account(&entry.path()) {
|
||||
Ok(_meta) => {
|
||||
match Account::open(path, &account_name) {
|
||||
Ok(account) => {
|
||||
accounts.insert(account_name, account);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to open account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to recover account {}: {}",
|
||||
account_name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update global account list
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(path)?;
|
||||
|
||||
Ok(Self {
|
||||
root: path.to_path_buf(),
|
||||
config,
|
||||
cache,
|
||||
accounts: RwLock::new(accounts),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Account management ──────────────────────────────────────────────────
|
||||
|
||||
pub fn create_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
if accounts.contains_key(account_id) {
|
||||
return Err(Error::AccountAlreadyExists(account_id.to_string()));
|
||||
}
|
||||
let account = Account::create(&self.root, account_id)?;
|
||||
accounts.insert(account_id.to_string(), account);
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_account(&self, account_id: &str) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let account = accounts
|
||||
.remove(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
let account_dir = account.dir().to_path_buf();
|
||||
drop(account);
|
||||
fs::remove_dir_all(&account_dir)?;
|
||||
|
||||
let mut global = GlobalMeta::load(&self.root)?;
|
||||
global.accounts = accounts.keys().cloned().collect();
|
||||
global.save(&self.root)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn list_accounts(&self) -> Vec<String> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
accounts.keys().cloned().collect()
|
||||
}
|
||||
|
||||
// ── Read / Write / Delete ───────────────────────────────────────────────
|
||||
|
||||
pub fn write(
|
||||
&self,
|
||||
account_id: &str,
|
||||
key: [u8; 32],
|
||||
value: &[u8],
|
||||
codec: Codec,
|
||||
) -> Result<()> {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let account = accounts
|
||||
.get_mut(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
// Acquire per-account write lock
|
||||
{
|
||||
let _lock = account.lock_write();
|
||||
}
|
||||
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level);
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
account.write_entry(key, &data, 0, actual_codec)?;
|
||||
|
||||
let record = IndexRecord::new(key, segment_id, offset, data_size, 0);
|
||||
account.append_index(&record)?;
|
||||
|
||||
// Update indexed_up_to_offset for incremental recovery
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
account.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result<Option<Vec<u8>>> {
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
|
||||
// Acquire lock only to get bucket records and segment routing info.
|
||||
let record: Option<IndexRecord> = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
let account = accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
let records = self
|
||||
.cache
|
||||
.get_or_load(account_id, bucket_id, account.dir())?;
|
||||
match records.binary_search_by(|r| r.key.cmp(key)) {
|
||||
Ok(idx) => Some(records[idx].clone()),
|
||||
Err(_) => None,
|
||||
}
|
||||
}; // accounts read lock dropped here — I/O happens outside the lock
|
||||
|
||||
let record = match record {
|
||||
Some(r) => r,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
if record.is_tombstone() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// I/O outside the global lock
|
||||
let seg_path = self
|
||||
.root
|
||||
.join("accounts")
|
||||
.join(account_id)
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(record.segment_id));
|
||||
|
||||
if !seg_path.exists() {
|
||||
return Err(Error::SegmentNotFound(record.segment_id));
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path, record.segment_id)?;
|
||||
let (entry, _) = reader.read_entry_at(record.offset)?;
|
||||
|
||||
let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?;
|
||||
|
||||
Ok(Some(value))
|
||||
}
|
||||
|
||||
pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let account = accounts
|
||||
.get_mut(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
// Acquire per-account write lock
|
||||
{
|
||||
let _lock = account.lock_write();
|
||||
}
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
account.write_entry(*key, &[], 1, Codec::None)?;
|
||||
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 1);
|
||||
account.append_index(&record)?;
|
||||
|
||||
// Update indexed_up_to_offset for incremental recovery
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
account.mark_indexed(segment_id, entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(key);
|
||||
self.cache.update_record(account_id, bucket_id, record);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Batch write ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Batch-write multiple entries with a single fsync.
|
||||
/// Each element is (key, value, codec).
|
||||
pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec<u8>, Codec)]) -> Result<()> {
|
||||
if entries.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
let account = accounts
|
||||
.get_mut(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
{
|
||||
let _lock = account.lock_write();
|
||||
}
|
||||
|
||||
// Phase 1: compress and append all entries without fsync
|
||||
let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len());
|
||||
for (key, value, codec) in entries {
|
||||
if value.len() > crate::types::MAX_VALUE_SIZE {
|
||||
return Err(Error::ValueTooLarge { size: value.len() });
|
||||
}
|
||||
let (data, actual_codec) =
|
||||
compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level);
|
||||
|
||||
let (segment_id, offset, data_size) =
|
||||
account.append_entry(*key, &data, 0, actual_codec)?;
|
||||
|
||||
let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
let record = IndexRecord::new(*key, segment_id, offset, data_size, 0);
|
||||
pending.push((record, entry_end));
|
||||
}
|
||||
|
||||
// Phase 2: single fsync
|
||||
account.flush_active()?;
|
||||
|
||||
// Phase 3: append indices and update cache
|
||||
for (record, entry_end) in &pending {
|
||||
account.append_index(record)?;
|
||||
account.mark_indexed(record.segment_id, *entry_end)?;
|
||||
|
||||
let bucket_id = bucket::bucket_id(&record.key);
|
||||
self.cache.update_record(account_id, bucket_id, record.clone());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── GC ──────────────────────────────────────────────────────────────────
|
||||
|
||||
pub fn gc(&self, account_id: &str) -> Result<Option<GcStats>> {
|
||||
// Get account dir while holding read lock, then release before heavy I/O.
|
||||
let account_dir = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
let account = accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
account.dir().to_path_buf()
|
||||
}; // read lock released — GC runs without blocking reads
|
||||
|
||||
let result = gc::gc_account(&account_dir, self.config.gc_deleted_ratio)?;
|
||||
|
||||
// Invalidate all cached buckets for this account after GC rewrites them
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn compact_buckets(&self, account_id: &str) -> Result<()> {
|
||||
let account_dir = {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
let account = accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
account.dir().to_path_buf()
|
||||
};
|
||||
|
||||
gc::compact_buckets(&account_dir)?;
|
||||
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
self.cache.invalidate(account_id, bid);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Stats / Shutdown ────────────────────────────────────────────────────
|
||||
|
||||
pub fn stats(&self, account_id: &str) -> Result<AccountStats> {
|
||||
let accounts = self.accounts.read().unwrap();
|
||||
let account = accounts
|
||||
.get(account_id)
|
||||
.ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?;
|
||||
|
||||
let meta = account.meta();
|
||||
let mut total_bytes = 0u64;
|
||||
let mut deleted_bytes = 0u64;
|
||||
|
||||
for seg in meta.segments.values() {
|
||||
total_bytes += seg.total_bytes;
|
||||
deleted_bytes += seg.deleted_bytes;
|
||||
}
|
||||
|
||||
// Count total live keys from bucket indices
|
||||
let mut total_keys = 0u64;
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
if let Ok(records) =
|
||||
self.cache
|
||||
.get_or_load(account_id, bid, account.dir())
|
||||
{
|
||||
total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(AccountStats {
|
||||
account_id: account_id.to_string(),
|
||||
total_keys,
|
||||
total_bytes,
|
||||
deleted_bytes,
|
||||
segment_count: meta.segments.len(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Gracefully shut down: fsync all active segments and persist meta.
|
||||
pub fn shutdown(&self) -> Result<()> {
|
||||
let mut accounts = self.accounts.write().unwrap();
|
||||
for (_, account) in accounts.iter_mut() {
|
||||
account.flush_active()?;
|
||||
}
|
||||
let global = GlobalMeta::load(&self.root)?;
|
||||
global.save(&self.root)?;
|
||||
tracing::info!("bichon-blob shut down cleanly");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Engine {
|
||||
fn drop(&mut self) {
|
||||
if let Err(e) = self.shutdown() {
|
||||
tracing::error!("bichon-blob shutdown error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use std::{io, path::PathBuf};
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("CRC32 mismatch at {path}:{offset}")]
|
||||
CrcMismatch { path: PathBuf, offset: u64 },
|
||||
|
||||
#[error("Corrupt entry at {path}:{offset}: {reason}")]
|
||||
CorruptEntry {
|
||||
path: PathBuf,
|
||||
offset: u64,
|
||||
reason: String,
|
||||
},
|
||||
|
||||
#[error("Account not found: {0}")]
|
||||
AccountNotFound(String),
|
||||
|
||||
#[error("Account already exists: {0}")]
|
||||
AccountAlreadyExists(String),
|
||||
|
||||
#[error("Segment not found: {0}")]
|
||||
SegmentNotFound(u32),
|
||||
|
||||
#[error("Value too large: {size} bytes (max 100 MB)")]
|
||||
ValueTooLarge { size: usize },
|
||||
|
||||
#[error("Compression error: {0}")]
|
||||
Compression(String),
|
||||
|
||||
#[error("Disk full: {0}")]
|
||||
DiskFull(String),
|
||||
|
||||
#[error("Invalid config: {0}")]
|
||||
InvalidConfig(String),
|
||||
|
||||
#[error("Bucket index corrupt at {path}: {reason}")]
|
||||
BucketIndexCorrupt { path: PathBuf, reason: String },
|
||||
|
||||
#[error("Segment file truncated at {path}: expected {expected}, got {actual}")]
|
||||
SegmentTruncated {
|
||||
path: PathBuf,
|
||||
expected: u64,
|
||||
actual: u64,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord};
|
||||
use crate::error::Result;
|
||||
#[cfg(test)]
|
||||
use crate::meta::SegmentStats;
|
||||
use crate::segment::{self, SegmentReader, SegmentWriter};
|
||||
|
||||
/// Result of a GC run.
|
||||
#[derive(Debug)]
|
||||
pub struct GcStats {
|
||||
pub segment_id: u32,
|
||||
pub bytes_before: u64,
|
||||
pub bytes_after: u64,
|
||||
pub entries_kept: usize,
|
||||
pub entries_skipped: usize,
|
||||
}
|
||||
|
||||
/// Run GC on an account: pick the sealed segment with highest deleted_ratio,
|
||||
/// rewrite it without deleted/overwritten entries, then rebuild all bucket files.
|
||||
pub fn gc_account(
|
||||
account_dir: &Path,
|
||||
deleted_ratio_threshold: f64,
|
||||
) -> Result<Option<GcStats>> {
|
||||
let meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
|
||||
// Find the best candidate
|
||||
let candidate = meta
|
||||
.segments
|
||||
.values()
|
||||
.filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold)
|
||||
.max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap());
|
||||
|
||||
let target = match candidate {
|
||||
Some(s) => s.clone(),
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(target.segment_id));
|
||||
let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?;
|
||||
|
||||
// Build a global view: for each key, which entry (segment_id + offset) is the latest?
|
||||
let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new();
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let rpath = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !rpath.exists() {
|
||||
continue;
|
||||
}
|
||||
let r = SegmentReader::open(rpath, seg_id)?;
|
||||
let _ = r.scan_entries(0, |entry, offset| {
|
||||
match latest_key.get(&entry.key) {
|
||||
Some((existing_seg, existing_off)) => {
|
||||
if seg_id > *existing_seg
|
||||
|| (seg_id == *existing_seg && offset > *existing_off)
|
||||
{
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
latest_key.insert(entry.key, (seg_id, offset));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
// Create temp segment with a unique name
|
||||
let timestamp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let temp_name = format!("temp_{:016x}.seg", timestamp);
|
||||
let temp_path = account_dir.join("segments").join(&temp_name);
|
||||
let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?;
|
||||
|
||||
let mut bytes_after: u64 = 0;
|
||||
let mut entries_kept: usize = 0;
|
||||
let mut entries_skipped: usize = 0;
|
||||
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
// Skip tombstones
|
||||
if entry.is_tombstone() {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
// Skip if this key has a newer entry in another segment
|
||||
if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) {
|
||||
if *latest_seg != target.segment_id || *latest_off != offset {
|
||||
entries_skipped += 1;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
// Keep this entry
|
||||
writer.append(entry)?;
|
||||
bytes_after += entry.data.len() as u64;
|
||||
entries_kept += 1;
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
writer.fsync()?;
|
||||
|
||||
// Atomic rename: replace old segment with new one
|
||||
fs::rename(&temp_path, &seg_path)?;
|
||||
|
||||
// Rebuild all bucket files
|
||||
rebuild_buckets(account_dir, &meta)?;
|
||||
|
||||
// Update meta
|
||||
let mut meta = crate::meta::AccountMeta::load(account_dir)?;
|
||||
if let Some(stats) = meta.segments.get_mut(&target.segment_id) {
|
||||
stats.total_bytes = bytes_after;
|
||||
stats.deleted_bytes = 0;
|
||||
stats.recompute_ratio();
|
||||
}
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(Some(GcStats {
|
||||
segment_id: target.segment_id,
|
||||
bytes_before: target.total_bytes,
|
||||
bytes_after,
|
||||
entries_kept,
|
||||
entries_skipped,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Rebuild all 16 bucket files from scratch by scanning all segments.
|
||||
fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> {
|
||||
let mut bucket_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
for i in 0..crate::types::BUCKET_COUNT {
|
||||
bucket_records.insert(i, Vec::new());
|
||||
}
|
||||
|
||||
for &seg_id in meta.segments.keys() {
|
||||
let seg_path = account_dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(seg_id));
|
||||
if !seg_path.exists() {
|
||||
continue;
|
||||
}
|
||||
let reader = SegmentReader::open(seg_path, seg_id)?;
|
||||
reader.scan_entries(0, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
bucket_records.entry(bid).or_default().push(rec);
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
||||
for (bid, records) in &bucket_records {
|
||||
let index = BucketIndex::from_records(records.clone(), *bid);
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact bucket files: load, dedup, rewrite.
|
||||
pub fn compact_buckets(account_dir: &Path) -> Result<()> {
|
||||
for bid in 0..crate::types::BUCKET_COUNT {
|
||||
let bf = BucketFile::open(account_dir, bid);
|
||||
if bf.path().exists() {
|
||||
let index = bf.load_index()?;
|
||||
bf.rewrite(&index.records)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::segment::Entry;
|
||||
use crate::types::Codec;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn setup_account(dir: &Path) {
|
||||
fs::create_dir_all(dir.join("segments")).unwrap();
|
||||
crate::bucket::BucketFile::ensure_dir(dir).unwrap();
|
||||
|
||||
let seg_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let mut writer = SegmentWriter::create(seg_path, 1).unwrap();
|
||||
|
||||
// Write 5 entries
|
||||
for i in 0..5u8 {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None);
|
||||
writer.append(&entry).unwrap();
|
||||
}
|
||||
|
||||
// Tombstone entry 2
|
||||
let mut key2 = [0u8; 32];
|
||||
key2[0] = 2;
|
||||
let tomb = Entry::tombstone(key2);
|
||||
writer.append(&tomb).unwrap();
|
||||
|
||||
writer.fsync().unwrap();
|
||||
|
||||
// Save meta
|
||||
let mut meta = crate::meta::AccountMeta::new("test".into(), 2);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 6000,
|
||||
deleted_bytes: 1000,
|
||||
deleted_ratio: 1000.0 / 6000.0,
|
||||
sealed: true,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
// Make segment 2 active so segment 1 is sealed
|
||||
let seg2_path = dir
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(2));
|
||||
SegmentWriter::create(seg2_path, 2).unwrap();
|
||||
meta.save(dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc_removes_tombstones() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
|
||||
let result = gc_account(dir.path(), 0.01).unwrap();
|
||||
assert!(result.is_some());
|
||||
|
||||
// Verify segment 1 no longer has the tombstone'd entry
|
||||
let seg_path = dir
|
||||
.path()
|
||||
.join("segments")
|
||||
.join(segment::segment_filename(1));
|
||||
let reader = SegmentReader::open(seg_path, 1).unwrap();
|
||||
let mut count = 0;
|
||||
reader.scan_entries(0, |entry, _offset| {
|
||||
count += 1;
|
||||
assert!(entry.key[0] != 2);
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
assert_eq!(count, 4); // 5 original - 1 tombstoned
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compact_buckets() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
setup_account(dir.path());
|
||||
compact_buckets(dir.path()).unwrap();
|
||||
// Should not panic
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
pub mod account;
|
||||
pub mod bucket;
|
||||
pub mod cache;
|
||||
pub mod checksum;
|
||||
pub mod compress;
|
||||
pub mod engine;
|
||||
pub mod error;
|
||||
pub mod gc;
|
||||
pub mod meta;
|
||||
pub mod recovery;
|
||||
pub mod segment;
|
||||
pub mod types;
|
||||
|
||||
pub use engine::{AccountStats, Engine};
|
||||
pub use error::{Error, Result};
|
||||
pub use types::{Codec, Config};
|
||||
@@ -0,0 +1,156 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GlobalMeta {
|
||||
pub version: u32,
|
||||
pub accounts: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for GlobalMeta {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: 1,
|
||||
accounts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GlobalMeta {
|
||||
pub fn load(store_root: &Path) -> Result<Self> {
|
||||
let path = store_root.join("global_meta.json");
|
||||
if !path.exists() {
|
||||
return Ok(Self::default());
|
||||
}
|
||||
let data = std::fs::read_to_string(&path)?;
|
||||
Ok(serde_json::from_str(&data)?)
|
||||
}
|
||||
|
||||
pub fn save(&self, store_root: &Path) -> Result<()> {
|
||||
let path = store_root.join("global_meta.json");
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
let data = serde_json::to_string_pretty(self)?;
|
||||
std::fs::write(&tmp, &data)?;
|
||||
std::fs::rename(&tmp, &path)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SegmentStats {
|
||||
pub segment_id: u32,
|
||||
pub total_bytes: u64,
|
||||
pub deleted_bytes: u64,
|
||||
pub deleted_ratio: f64,
|
||||
pub sealed: bool,
|
||||
/// Byte offset up to which entries have been indexed in bucket files.
|
||||
/// Recovery starts scanning from here instead of 0.
|
||||
pub indexed_up_to_offset: u64,
|
||||
}
|
||||
|
||||
impl SegmentStats {
|
||||
pub fn new(segment_id: u32) -> Self {
|
||||
Self {
|
||||
segment_id,
|
||||
total_bytes: 0,
|
||||
deleted_bytes: 0,
|
||||
deleted_ratio: 0.0,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn recompute_ratio(&mut self) {
|
||||
if self.total_bytes > 0 {
|
||||
self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64;
|
||||
} else {
|
||||
self.deleted_ratio = 0.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AccountMeta {
|
||||
pub account_id: String,
|
||||
pub active_segment_id: u32,
|
||||
pub segments: HashMap<u32, SegmentStats>,
|
||||
}
|
||||
|
||||
impl AccountMeta {
|
||||
pub fn new(account_id: String, active_segment_id: u32) -> Self {
|
||||
Self {
|
||||
account_id,
|
||||
active_segment_id,
|
||||
segments: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(account_dir: &Path) -> Result<Self> {
|
||||
let path = account_dir.join("meta.json");
|
||||
if !path.exists() {
|
||||
return Err(crate::error::Error::AccountNotFound(
|
||||
account_dir.to_string_lossy().into(),
|
||||
));
|
||||
}
|
||||
let data = std::fs::read_to_string(&path)?;
|
||||
Ok(serde_json::from_str(&data)?)
|
||||
}
|
||||
|
||||
pub fn save(&self, account_dir: &Path) -> Result<()> {
|
||||
let path = account_dir.join("meta.json");
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
let data = serde_json::to_string_pretty(self)?;
|
||||
std::fs::write(&tmp, &data)?;
|
||||
std::fs::rename(&tmp, &path)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = GlobalMeta::default();
|
||||
meta.accounts.push("alice".into());
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.accounts, vec!["alice"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_global_meta_default_when_missing() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let meta = GlobalMeta::load(dir.path()).unwrap();
|
||||
assert!(meta.accounts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_meta_roundtrip() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut meta = AccountMeta::new("alice".into(), 1);
|
||||
meta.segments.insert(
|
||||
1,
|
||||
SegmentStats {
|
||||
segment_id: 1,
|
||||
total_bytes: 1000,
|
||||
deleted_bytes: 300,
|
||||
deleted_ratio: 0.3,
|
||||
sealed: false,
|
||||
indexed_up_to_offset: 0,
|
||||
},
|
||||
);
|
||||
meta.save(dir.path()).unwrap();
|
||||
|
||||
let loaded = AccountMeta::load(dir.path()).unwrap();
|
||||
assert_eq!(loaded.active_segment_id, 1);
|
||||
assert_eq!(loaded.segments[&1].total_bytes, 1000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::bucket::{self, BucketFile, IndexRecord};
|
||||
use crate::error::Result;
|
||||
use crate::meta::{AccountMeta, SegmentStats};
|
||||
use crate::segment::{self, SegmentReader};
|
||||
|
||||
/// Recover an account after a crash: scan segments, repair indices, update stats.
|
||||
pub fn recover_account(account_dir: &Path) -> Result<AccountMeta> {
|
||||
let meta_path = account_dir.join("meta.json");
|
||||
let mut meta = if meta_path.exists() {
|
||||
AccountMeta::load(account_dir).unwrap_or_else(|_| {
|
||||
AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
)
|
||||
})
|
||||
} else {
|
||||
return Ok(AccountMeta::new(
|
||||
account_dir
|
||||
.file_name()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
.into(),
|
||||
1,
|
||||
));
|
||||
};
|
||||
|
||||
// Discover all segment files on disk
|
||||
let seg_dir = account_dir.join("segments");
|
||||
if !seg_dir.exists() {
|
||||
fs::create_dir_all(&seg_dir)?;
|
||||
}
|
||||
|
||||
let mut disk_segments: Vec<u32> = Vec::new();
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".seg") && !name_str.contains("temp_") {
|
||||
if let Some(id_str) = name_str.strip_suffix(".seg") {
|
||||
if let Ok(id) = id_str.parse::<u32>() {
|
||||
disk_segments.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
disk_segments.sort_unstable();
|
||||
|
||||
if disk_segments.is_empty() {
|
||||
meta.active_segment_id = 1;
|
||||
} else {
|
||||
let max_id = *disk_segments.last().unwrap();
|
||||
meta.active_segment_id = max_id;
|
||||
}
|
||||
|
||||
// Ensure buckets directory exists
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
fs::create_dir_all(&buckets_dir)?;
|
||||
|
||||
// For each segment, scan only the unindexed tail and update stats incrementally
|
||||
for &seg_id in &disk_segments {
|
||||
let seg_path = seg_dir.join(segment::segment_filename(seg_id));
|
||||
let file_size = fs::metadata(&seg_path)?.len();
|
||||
|
||||
// Preserve existing stats; start fresh if this is a newly discovered segment
|
||||
let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id));
|
||||
let is_sealed = seg_id != meta.active_segment_id;
|
||||
stats.sealed = is_sealed;
|
||||
|
||||
// Scan start: from last indexed offset. Clamp defensively.
|
||||
let scan_start = if stats.indexed_up_to_offset <= file_size {
|
||||
stats.indexed_up_to_offset
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// If fully indexed, skip scanning entirely
|
||||
if scan_start >= file_size {
|
||||
meta.segments.insert(seg_id, stats);
|
||||
continue;
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(seg_path.clone(), seg_id)?;
|
||||
let mut new_records: HashMap<u16, Vec<IndexRecord>> = HashMap::new();
|
||||
|
||||
let truncation_point = reader.scan_entries(scan_start, |entry, offset| {
|
||||
let bid = bucket::bucket_id(&entry.key);
|
||||
let rec = IndexRecord::new(
|
||||
entry.key,
|
||||
seg_id,
|
||||
offset,
|
||||
entry.data.len() as u32,
|
||||
entry.flags,
|
||||
);
|
||||
new_records.entry(bid).or_default().push(rec);
|
||||
|
||||
stats.total_bytes += entry.data.len() as u64;
|
||||
if entry.is_tombstone() {
|
||||
stats.deleted_bytes += entry.raw_size as u64;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
// Merge new records into bucket files (only the newly discovered ones)
|
||||
for (bid, records) in &new_records {
|
||||
let bf = BucketFile::open(account_dir, *bid);
|
||||
bf.append_batch(records)?;
|
||||
}
|
||||
|
||||
// Truncate if tail corruption found
|
||||
if truncation_point < file_size {
|
||||
segment::truncate_segment(&seg_path, truncation_point)?;
|
||||
}
|
||||
|
||||
stats.indexed_up_to_offset = truncation_point;
|
||||
stats.recompute_ratio();
|
||||
meta.segments.insert(seg_id, stats);
|
||||
}
|
||||
|
||||
meta.save(account_dir)?;
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
/// Clean up leftover temp files from interrupted GC.
|
||||
pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> {
|
||||
let seg_dir = account_dir.join("segments");
|
||||
if seg_dir.exists() {
|
||||
for entry in fs::read_dir(&seg_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.starts_with("temp_") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also cleanup temp bucket files
|
||||
let buckets_dir = account_dir.join("buckets");
|
||||
if buckets_dir.exists() {
|
||||
for entry in fs::read_dir(&buckets_dir)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
if name_str.ends_with(".tmp") {
|
||||
let path = entry.path();
|
||||
tracing::warn!("Removing leftover temp bucket file: {:?}", path);
|
||||
fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_recover_fresh_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(&account_dir).unwrap();
|
||||
|
||||
let meta = recover_account(&account_dir).unwrap();
|
||||
assert_eq!(meta.active_segment_id, 1);
|
||||
assert!(meta.segments.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cleanup_temp_files() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let account_dir = dir.path().join("test");
|
||||
fs::create_dir_all(account_dir.join("segments")).unwrap();
|
||||
fs::create_dir_all(account_dir.join("buckets")).unwrap();
|
||||
fs::write(
|
||||
account_dir.join("segments").join("temp_ABC123.seg"),
|
||||
b"garbage",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap();
|
||||
|
||||
cleanup_temp_files(&account_dir).unwrap();
|
||||
|
||||
assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,454 @@
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::checksum;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE};
|
||||
|
||||
/// In-memory representation of a stored entry.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Entry {
|
||||
pub flags: u8,
|
||||
pub codec: Codec,
|
||||
pub key: [u8; 32],
|
||||
pub raw_size: u32,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
/// Create a normal data entry.
|
||||
pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self {
|
||||
Self {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size: raw_data.len() as u32,
|
||||
data: raw_data.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a tombstone entry.
|
||||
pub fn tombstone(key: [u8; 32]) -> Self {
|
||||
Self {
|
||||
flags: 1,
|
||||
codec: Codec::None,
|
||||
key,
|
||||
raw_size: 0,
|
||||
data: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_tombstone(&self) -> bool {
|
||||
self.flags == 1
|
||||
}
|
||||
|
||||
/// Total on-disk size: header + data
|
||||
pub fn disk_size(&self) -> usize {
|
||||
ENTRY_HEADER_SIZE + self.data.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Write entries sequentially to a segment file.
|
||||
pub struct SegmentWriter {
|
||||
file: File,
|
||||
path: PathBuf,
|
||||
id: u32,
|
||||
bytes_written: u64,
|
||||
}
|
||||
|
||||
impl SegmentWriter {
|
||||
pub fn create(path: PathBuf, id: u32) -> Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(&path)?;
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
id,
|
||||
bytes_written: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open_append(path: PathBuf, id: u32) -> Result<Self> {
|
||||
let mut file = OpenOptions::new().write(true).open(&path)?;
|
||||
file.seek(SeekFrom::End(0))?;
|
||||
let bytes_written = file.stream_position()?;
|
||||
Ok(Self {
|
||||
file,
|
||||
path,
|
||||
id,
|
||||
bytes_written,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn id(&self) -> u32 {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn bytes_written(&self) -> u64 {
|
||||
self.bytes_written
|
||||
}
|
||||
|
||||
pub fn is_full(&self) -> bool {
|
||||
self.bytes_written >= SEGMENT_MAX_SIZE
|
||||
}
|
||||
|
||||
/// Append an entry. Returns the offset where it was written.
|
||||
pub fn append(&mut self, entry: &Entry) -> Result<u64> {
|
||||
let offset = self.bytes_written;
|
||||
self.write_entry(entry)
|
||||
.map_err(|e| map_io_err(e, &self.path))?;
|
||||
Ok(offset)
|
||||
}
|
||||
|
||||
fn write_entry(&mut self, entry: &Entry) -> Result<()> {
|
||||
let data_size = entry.data.len() as u32;
|
||||
|
||||
// Write magic
|
||||
self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?;
|
||||
|
||||
// CRC32 placeholder: write zeros, remember position
|
||||
let crc_pos = self.file.stream_position()?;
|
||||
self.file.write_all(&0u32.to_le_bytes())?;
|
||||
|
||||
// Write flags, codec, key, raw_size, data_size
|
||||
self.file.write_all(&[entry.flags])?;
|
||||
self.file.write_all(&[entry.codec as u8])?;
|
||||
self.file.write_all(&entry.key)?;
|
||||
self.file.write_all(&entry.raw_size.to_le_bytes())?;
|
||||
self.file.write_all(&data_size.to_le_bytes())?;
|
||||
|
||||
// Write data
|
||||
self.file.write_all(&entry.data)?;
|
||||
|
||||
// Calculate CRC32 over everything after the crc32 field
|
||||
let crc = {
|
||||
let mut hasher = checksum::CrcWriter::new();
|
||||
hasher.update(&[entry.flags]);
|
||||
hasher.update(&[entry.codec as u8]);
|
||||
hasher.update(&entry.key);
|
||||
hasher.update(&entry.raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&entry.data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
// Seek back and write the real CRC32
|
||||
self.file.seek(SeekFrom::Start(crc_pos))?;
|
||||
self.file.write_all(&crc.to_le_bytes())?;
|
||||
|
||||
// Seek back to end
|
||||
self.file.seek(SeekFrom::End(0))?;
|
||||
|
||||
self.bytes_written += entry.disk_size() as u64;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn fsync(&self) -> Result<()> {
|
||||
self.file.sync_all().map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::StorageFull {
|
||||
Error::DiskFull(format!("{}: {}", self.path.display(), e))
|
||||
} else {
|
||||
Error::Io(e)
|
||||
}
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Read entries from a segment file.
|
||||
pub struct SegmentReader {
|
||||
path: PathBuf,
|
||||
id: u32,
|
||||
}
|
||||
|
||||
impl SegmentReader {
|
||||
pub fn open(path: PathBuf, id: u32) -> Result<Self> {
|
||||
Ok(Self { path, id })
|
||||
}
|
||||
|
||||
pub fn id(&self) -> u32 {
|
||||
self.id
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn file_size(&self) -> Result<u64> {
|
||||
Ok(fs::metadata(&self.path)?.len())
|
||||
}
|
||||
|
||||
/// Read a single entry at the given offset. Returns the entry and the offset of the next entry.
|
||||
pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> {
|
||||
let mut file = File::open(&self.path)?;
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
|
||||
// Read magic
|
||||
let mut magic_buf = [0u8; 4];
|
||||
file.read_exact(&mut magic_buf)?;
|
||||
let magic = u32::from_le_bytes(magic_buf);
|
||||
if magic != ENTRY_MAGIC {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("bad magic: 0x{:08X}", magic),
|
||||
});
|
||||
}
|
||||
|
||||
// Read CRC32
|
||||
let mut crc_buf = [0u8; 4];
|
||||
file.read_exact(&mut crc_buf)?;
|
||||
let stored_crc = u32::from_le_bytes(crc_buf);
|
||||
|
||||
// Read flags, codec
|
||||
let mut flags_buf = [0u8; 1];
|
||||
file.read_exact(&mut flags_buf)?;
|
||||
let flags = flags_buf[0];
|
||||
|
||||
let mut codec_buf = [0u8; 1];
|
||||
file.read_exact(&mut codec_buf)?;
|
||||
let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: format!("unknown codec: {}", codec_buf[0]),
|
||||
})?;
|
||||
|
||||
// Read key, raw_size, data_size
|
||||
let mut key = [0u8; 32];
|
||||
file.read_exact(&mut key)?;
|
||||
|
||||
let mut raw_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut raw_size_buf)?;
|
||||
let raw_size = u32::from_le_bytes(raw_size_buf);
|
||||
|
||||
let mut data_size_buf = [0u8; 4];
|
||||
file.read_exact(&mut data_size_buf)?;
|
||||
let data_size = u32::from_le_bytes(data_size_buf);
|
||||
|
||||
// Read data
|
||||
let mut data = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut data)?;
|
||||
|
||||
// Verify CRC32 (over everything after the crc32 field)
|
||||
let computed_crc = {
|
||||
let mut hasher = checksum::CrcWriter::new();
|
||||
hasher.update(&[flags]);
|
||||
hasher.update(&[codec as u8]);
|
||||
hasher.update(&key);
|
||||
hasher.update(&raw_size.to_le_bytes());
|
||||
hasher.update(&data_size.to_le_bytes());
|
||||
hasher.update(&data);
|
||||
hasher.finalize()
|
||||
};
|
||||
|
||||
if stored_crc != computed_crc {
|
||||
return Err(Error::CrcMismatch {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
});
|
||||
}
|
||||
|
||||
let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64;
|
||||
|
||||
Ok((
|
||||
Entry {
|
||||
flags,
|
||||
codec,
|
||||
key,
|
||||
raw_size,
|
||||
data,
|
||||
},
|
||||
next_offset,
|
||||
))
|
||||
}
|
||||
|
||||
/// Read data portion of an entry (for pread-style reads when you already know offset + data_size).
|
||||
pub fn read_data(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
|
||||
let mut file = File::open(&self.path)?;
|
||||
// Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes
|
||||
let data_start = offset + ENTRY_HEADER_SIZE as u64;
|
||||
file.seek(SeekFrom::Start(data_start))?;
|
||||
let mut buf = vec![0u8; data_size as usize];
|
||||
file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Read the full entry header + data for verification (used by recovery and GC).
|
||||
pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result<Vec<u8>> {
|
||||
let mut file = File::open(&self.path)?;
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
let total = ENTRY_HEADER_SIZE + data_size as usize;
|
||||
let mut buf = vec![0u8; total];
|
||||
file.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Iterate over all valid entries in the segment, calling f for each.
|
||||
/// Stops when hitting a corrupt/incomplete entry at the tail.
|
||||
pub fn scan_entries<F>(&self, start_offset: u64, mut f: F) -> Result<u64>
|
||||
where
|
||||
F: FnMut(&Entry, u64) -> Result<()>,
|
||||
{
|
||||
let file_size = self.file_size()?;
|
||||
let mut offset = start_offset;
|
||||
|
||||
while offset + ENTRY_HEADER_SIZE as u64 <= file_size {
|
||||
match self.read_entry_at(offset) {
|
||||
Ok((entry, next)) => {
|
||||
f(&entry, offset)?;
|
||||
offset = next;
|
||||
}
|
||||
Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => {
|
||||
// If near end of file (within one max entry), truncate
|
||||
if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 {
|
||||
// Likely a partial write at tail, stop here
|
||||
break;
|
||||
} else {
|
||||
return Err(Error::CorruptEntry {
|
||||
path: self.path.clone(),
|
||||
offset,
|
||||
reason: "mid-file corruption detected".into(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(offset) // return the truncation point
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate a segment file to the given size.
|
||||
pub fn truncate_segment(path: &Path, size: u64) -> Result<()> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
file.set_len(size)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Map an Error, converting Io(StorageFull) to DiskFull with path context.
|
||||
fn map_io_err(e: Error, path: &Path) -> Error {
|
||||
match e {
|
||||
Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => {
|
||||
Error::DiskFull(format!("{}: {}", path.display(), io))
|
||||
}
|
||||
_ => e,
|
||||
}
|
||||
}
|
||||
|
||||
/// Segment file name from id: "00000001.seg"
|
||||
pub fn segment_filename(id: u32) -> String {
|
||||
format!("{:08}.seg", id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf {
|
||||
dir.path().join(segment_filename(id))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read_entry() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let key = [0xAAu8; 32];
|
||||
let data = b"hello world".to_vec();
|
||||
|
||||
let entry = Entry::new(key, &data, 0, Codec::None);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let (read_entry, next) = reader.read_entry_at(0).unwrap();
|
||||
|
||||
assert_eq!(read_entry.key, key);
|
||||
assert_eq!(read_entry.data, data);
|
||||
assert_eq!(read_entry.flags, 0);
|
||||
assert_eq!(read_entry.raw_size, 11);
|
||||
assert!(next > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tombstone_entry() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let key = [0xBBu8; 32];
|
||||
|
||||
let entry = Entry::tombstone(key);
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
writer.append(&entry).unwrap();
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let (read_entry, _) = reader.read_entry_at(0).unwrap();
|
||||
|
||||
assert!(read_entry.is_tombstone());
|
||||
assert_eq!(read_entry.data.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_entries() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
|
||||
let entries: Vec<_> = (0..10)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0] = i;
|
||||
Entry::new(key, &vec![i; 100], 0, Codec::None)
|
||||
})
|
||||
.collect();
|
||||
|
||||
{
|
||||
let mut writer = SegmentWriter::create(path.clone(), 1).unwrap();
|
||||
for e in &entries {
|
||||
writer.append(e).unwrap();
|
||||
}
|
||||
writer.fsync().unwrap();
|
||||
}
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let mut offset = 0u64;
|
||||
for (i, expected) in entries.iter().enumerate() {
|
||||
let (entry, next) = reader.read_entry_at(offset).unwrap();
|
||||
assert_eq!(entry.key[0], i as u8);
|
||||
assert_eq!(entry.data, expected.data);
|
||||
offset = next;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bad_magic_detected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
// Write garbage
|
||||
std::fs::write(&path, vec![0xFFu8; 100]).unwrap();
|
||||
|
||||
let reader = SegmentReader::open(path, 1).unwrap();
|
||||
let result = reader.read_entry_at(0);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_full() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = temp_segment_path(&dir, 1);
|
||||
let writer = SegmentWriter::create(path, 1).unwrap();
|
||||
assert!(!writer.is_full());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Magic number for entry identification
|
||||
pub const ENTRY_MAGIC: u32 = 0xB3DB_0001;
|
||||
|
||||
/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4)
|
||||
pub const ENTRY_HEADER_SIZE: usize = 50;
|
||||
|
||||
/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3)
|
||||
pub const INDEX_RECORD_SIZE: usize = 52;
|
||||
|
||||
/// Maximum segment size (256 MB)
|
||||
pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024;
|
||||
|
||||
/// Number of hash buckets per account
|
||||
pub const BUCKET_COUNT: u16 = 16;
|
||||
|
||||
/// Maximum value size (100 MB)
|
||||
pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024;
|
||||
|
||||
/// Default compression threshold (4 KB)
|
||||
pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096;
|
||||
|
||||
/// Default LRU bucket cache size
|
||||
pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256;
|
||||
|
||||
/// Default GC deleted ratio threshold
|
||||
pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Codec {
|
||||
None = 0,
|
||||
Zstd = 1,
|
||||
Lz4 = 2,
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
0 => Some(Codec::None),
|
||||
1 => Some(Codec::Zstd),
|
||||
2 => Some(Codec::Lz4),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
pub compress_threshold: usize,
|
||||
pub default_codec: Codec,
|
||||
pub compression_level: i32,
|
||||
pub lru_bucket_count: usize,
|
||||
pub gc_deleted_ratio: f64,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
compress_threshold: DEFAULT_COMPRESS_THRESHOLD,
|
||||
default_codec: Codec::Zstd,
|
||||
compression_level: 0,
|
||||
lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT,
|
||||
gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn validate(&self) -> crate::error::Result<()> {
|
||||
if self.lru_bucket_count == 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"lru_bucket_count must be > 0".into(),
|
||||
));
|
||||
}
|
||||
if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"gc_deleted_ratio must be in (0.0, 1.0)".into(),
|
||||
));
|
||||
}
|
||||
if self.compression_level < 0 {
|
||||
return Err(crate::error::Error::InvalidConfig(
|
||||
"compression_level must be >= 0".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
/// Crash-consistency and ACID property tests for bichon-blob.
|
||||
///
|
||||
/// Since we can't kill the process mid-write in an inline test, we simulate crashes
|
||||
/// by dropping the Engine without calling any cleanup (close/drop is the "crash"),
|
||||
/// then re-opening and verifying recovery produced consistent state.
|
||||
///
|
||||
/// For true power-loss simulation, each test writes data, drops the engine abruptly,
|
||||
/// then reopens and verifies: no corruption, no lost committed data, no partial writes.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::thread;
|
||||
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_key(seed: u64) -> [u8; 32] {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&seed.to_le_bytes());
|
||||
key
|
||||
}
|
||||
|
||||
fn make_value(size: usize) -> Vec<u8> {
|
||||
let pattern = b"The quick brown fox jumps over the lazy dog. ";
|
||||
let mut v = Vec::with_capacity(size);
|
||||
while v.len() < size {
|
||||
let rem = size - v.len();
|
||||
let n = rem.min(pattern.len());
|
||||
v.extend_from_slice(&pattern[..n]);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. Durability: committed data survives crash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_durability_single_write_survives_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(42);
|
||||
let value = make_value(8192);
|
||||
|
||||
// Write
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
} // <-- Engine dropped = simulated crash
|
||||
|
||||
// Recover
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durability_many_writes_survive_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let n = 500;
|
||||
let value = make_value(2048);
|
||||
let mut keys = Vec::new();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..n {
|
||||
let key = make_key(i as u64);
|
||||
keys.push(key);
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (i, key) in keys.iter().enumerate() {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()), "missing key at index {}", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_durability_delete_survives_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(99);
|
||||
let value = make_value(4096);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
} // crash after write
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
} // crash after delete
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None, "delete should persist across crash");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 2. Atomicity: no partial writes visible after crash
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_atomicity_no_partial_entries_after_crash() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
// Write enough entries to fill part of a segment, then crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
let value = make_value(50_000); // big enough to notice
|
||||
for i in 0..200u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash
|
||||
|
||||
// Recovery should clean up any partial tail entries and all committed
|
||||
// entries should be readable
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(50_000);
|
||||
for i in 0..200u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"committed key {} should be intact",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_atomicity_crash_during_segment_roll() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
// Write enough to cross at least one segment boundary (256 MB)
|
||||
for i in 0..140u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &big_value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
} // crash mid-way or after multiple segments
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// All committed writes (that returned Ok) must be readable
|
||||
for i in 0..140u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"key {} should exist after segment roll recovery",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 3. Consistency: CRC detects corruption, no silent data loss
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_consistency_crc_detects_corruption() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = make_key(77);
|
||||
let value = make_value(8192);
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Corrupt the segment file by flipping a byte
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
// Flip a byte in the data portion, not the header
|
||||
let flip_pos = data.len() - 100;
|
||||
data[flip_pos] ^= 0xFF;
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Reading should detect CRC mismatch
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key);
|
||||
// Either error or None is acceptable — never silently wrong data
|
||||
match result {
|
||||
Err(_) => {} // CRC mismatch detected — good
|
||||
Ok(None) => {} // index may point to truncated/removed data
|
||||
Ok(Some(v)) => {
|
||||
if v == value {
|
||||
panic!("CRC corruption was NOT detected — silent data corruption!");
|
||||
}
|
||||
// If value differs, index pointed elsewhere after recovery
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_consistency_corrupt_magic_truncated_on_recovery() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..10u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Append garbage to the segment file (simulating partial write from crash)
|
||||
let seg_path = find_first_segment(dir.path(), "alice");
|
||||
let mut data = fs::read(&seg_path).unwrap();
|
||||
let orig_len = data.len();
|
||||
// Append garbage that doesn't start with the magic number
|
||||
data.extend_from_slice(&[0xFF; 200]);
|
||||
fs::write(&seg_path, &data).unwrap();
|
||||
|
||||
// Recovery should truncate the garbage
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Verify committed data is still intact
|
||||
for i in 0..10u64 {
|
||||
let result = engine.read("alice", &make_key(i)).unwrap();
|
||||
assert!(result.is_some(), "committed key {} should survive tail truncation", i);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify file was actually truncated
|
||||
let truncated_len = fs::metadata(&seg_path).unwrap().len();
|
||||
assert!(truncated_len <= orig_len as u64, "garbage should have been truncated");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 4. Isolation: concurrent reader sees consistent snapshot
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_isolation_reader_sees_snapshot_not_partial_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap());
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Pre-populate a known key
|
||||
let original_value = make_value(4096);
|
||||
let key = make_key(100);
|
||||
engine
|
||||
.write("alice", key, &original_value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let writer_done = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Spawn a writer that continuously overwrites the same key
|
||||
let writer_engine = engine.clone();
|
||||
let writer_running = running.clone();
|
||||
let writer_done_flag = writer_done.clone();
|
||||
let writer_key = key;
|
||||
|
||||
let writer = thread::spawn(move || {
|
||||
for i in 0..1000u64 {
|
||||
if !writer_running.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
let val = make_value(4096 + (i as usize % 100));
|
||||
writer_engine
|
||||
.write("alice", writer_key, &val, Codec::Zstd)
|
||||
.unwrap();
|
||||
thread::yield_now();
|
||||
}
|
||||
writer_done_flag.store(true, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
// Concurrent reader: reads should never panic or hang
|
||||
let reader_engine = engine.clone();
|
||||
let reader_running = running.clone();
|
||||
let reader = thread::spawn(move || {
|
||||
let mut reads = 0;
|
||||
while reads < 500 {
|
||||
if !reader_running.load(Ordering::Relaxed) && reads > 0 {
|
||||
break;
|
||||
}
|
||||
let result = reader_engine.read("alice", &key);
|
||||
match result {
|
||||
Ok(Some(_)) | Ok(None) => {} // OK
|
||||
Err(e) => {
|
||||
// Accept transient errors but report them
|
||||
eprintln!("reader saw error: {:?}", e);
|
||||
}
|
||||
}
|
||||
reads += 1;
|
||||
thread::yield_now();
|
||||
}
|
||||
});
|
||||
|
||||
reader.join().unwrap();
|
||||
running.store(false, Ordering::SeqCst);
|
||||
writer.join().unwrap();
|
||||
|
||||
// Final read should see the last committed value (not partial)
|
||||
let final_result = engine.read("alice", &key).unwrap();
|
||||
assert!(final_result.is_some(), "final read should find a value");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 5. Crash during GC: old data intact, no corruption
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_crash_during_gc_leaves_data_intact() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let value = make_value(500_000); // 500 KB each
|
||||
// Write enough entries and delete some to create GC candidate
|
||||
for i in 0..500u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
// Delete ~40%
|
||||
for i in (0..500u64).step_by(5) {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
}
|
||||
// Single GC run (may or may not trigger)
|
||||
let _ = engine.gc("alice");
|
||||
} // crash after GC
|
||||
|
||||
// All non-deleted entries must still be readable
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let value = make_value(500_000);
|
||||
for i in 0..500u64 {
|
||||
let key = make_key(i);
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
if i % 5 == 0 {
|
||||
// Deleted keys
|
||||
assert_eq!(result, None, "key {} should be deleted", i);
|
||||
} else {
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(value.clone()),
|
||||
"key {} should survive GC+crash",
|
||||
i
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 6. Multiple crash-reopen cycles (torture test)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_multiple_crash_reopen_cycles() {
|
||||
use std::collections::HashSet;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let value = make_value(4096);
|
||||
let mut alive: HashSet<u64> = HashSet::new();
|
||||
|
||||
// Populate and crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
for i in 0..50u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen, verify all exist, write more, crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 100..150u64 {
|
||||
engine
|
||||
.write("alice", make_key(i), &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
alive.insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reopen, verify all exist, delete some, crash
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some());
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
engine.delete("alice", &make_key(i)).unwrap();
|
||||
alive.remove(&i);
|
||||
}
|
||||
}
|
||||
|
||||
// Final reopen: survivors exist, deleted gone
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for &k in &alive {
|
||||
assert!(engine.read("alice", &make_key(k)).unwrap().is_some(),
|
||||
"key {} should exist", k);
|
||||
}
|
||||
for i in 0..10u64 {
|
||||
assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None,
|
||||
"key {} should be deleted", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 7. Account-level isolation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_account_isolation_crash_one_account_does_not_affect_others() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", make_key(1), &make_value(4096), Codec::Zstd)
|
||||
.unwrap();
|
||||
engine
|
||||
.write("bob", make_key(1), &make_value(8192), Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Delete alice's account dir partially to simulate corruption
|
||||
// Then verify bob is intact
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
// Bob should be fine
|
||||
let result = engine.read("bob", &make_key(1)).unwrap();
|
||||
assert!(result.is_some(), "bob should be unaffected");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf {
|
||||
let seg_dir = store_root.join("accounts").join(account).join("segments");
|
||||
for entry in fs::read_dir(&seg_dir).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if name.ends_with(".seg") && !name.contains("temp_") {
|
||||
return entry.path();
|
||||
}
|
||||
}
|
||||
panic!("no segment found in {:?}", seg_dir);
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
use bichon_blob::{Codec, Config, Engine};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_create_and_list_accounts() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.create_account("bob").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(accounts.contains(&"alice".to_string()));
|
||||
assert!(accounts.contains(&"bob".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xAA; 32];
|
||||
let value = b"Hello, this is a test email!".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_read_missing_key() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xFF; 32];
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xBB; 32];
|
||||
let value = b"Some email content".to_vec();
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
engine.delete("alice", &key).unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_account() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.delete_account("alice").unwrap();
|
||||
|
||||
let accounts = engine.list_accounts();
|
||||
assert!(!accounts.contains(&"alice".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_value_not_compressed() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xCC; 32];
|
||||
let value = b"hi"; // Smaller than 4KB threshold
|
||||
|
||||
engine
|
||||
.write("alice", key, value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value.to_vec()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_value() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let key = [0xDD; 32];
|
||||
let value = vec![b'X'; 100_000]; // 100KB
|
||||
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_keys() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 100;
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let value = format!("email number {}", i).into_bytes();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(format!("email number {}", i).into_bytes()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gc() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
// Write many entries
|
||||
let value = vec![b'Y'; 5000];
|
||||
let n = 100;
|
||||
|
||||
for i in 0..n {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine
|
||||
.write("alice", key, &value, Codec::None)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Delete even-numbered keys
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
engine.delete("alice", &key).unwrap();
|
||||
}
|
||||
|
||||
// Run GC
|
||||
let _result = engine.gc("alice").unwrap();
|
||||
|
||||
// Verify remaining keys still readable
|
||||
for i in (1..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value.clone()));
|
||||
}
|
||||
|
||||
// Deleted keys should not exist
|
||||
for i in (0..n).step_by(2) {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..4].copy_from_slice(&(i as u32).to_le_bytes());
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reopen_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let key = [0xEE; 32];
|
||||
let value = b"persistent data".to_vec();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine
|
||||
.write("alice", key, &value, Codec::Zstd)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Reopen
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
let result = engine.read("alice", &key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stats() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
engine
|
||||
.write("alice", [1u8; 32], b"hello", Codec::None)
|
||||
.unwrap();
|
||||
|
||||
let stats = engine.stats("alice").unwrap();
|
||||
assert!(stats.total_bytes > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
|
||||
let n = 50;
|
||||
let entries: Vec<_> = (0..n)
|
||||
.map(|i: u64| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
let value = format!("batch email {}", i).into_bytes();
|
||||
(key, value, Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_batch_write_persistence() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let entries: Vec<_> = (0..30u64)
|
||||
.map(|i| {
|
||||
let mut key = [0u8; 32];
|
||||
key[0..8].copy_from_slice(&i.to_le_bytes());
|
||||
(key, format!("persist {}", i).into_bytes(), Codec::Zstd)
|
||||
})
|
||||
.collect();
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
engine.create_account("alice").unwrap();
|
||||
engine.write_batch("alice", &entries).unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let engine = Engine::open(dir.path(), Config::default()).unwrap();
|
||||
for (key, value, _) in &entries {
|
||||
let result = engine.read("alice", key).unwrap();
|
||||
assert_eq!(result.as_ref(), Some(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_config_rejected() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut config = Config::default();
|
||||
config.lru_bucket_count = 0;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
|
||||
let mut config = Config::default();
|
||||
config.gc_deleted_ratio = 1.5;
|
||||
assert!(Engine::open(dir.path(), config).is_err());
|
||||
}
|
||||
@@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [
|
||||
], optional = true }
|
||||
chrono.workspace = true
|
||||
clap.workspace = true
|
||||
memdb.workspace = true
|
||||
bichon-memdb.workspace = true
|
||||
itertools.workspace = true
|
||||
ring.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
|
||||
use crate::{
|
||||
database::MemDbModel,
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use crate::settings::dir::DATA_DIR_MANAGER;
|
||||
use memdb::{Durability, MemDb};
|
||||
use bichon_memdb::{Durability, MemDb};
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::common::paginated::Paginated;
|
||||
use crate::error::code::ErrorCode;
|
||||
use crate::error::BichonResult;
|
||||
use crate::raise_error;
|
||||
use memdb::{MemDb, Transaction};
|
||||
use bichon_memdb::{MemDb, Transaction};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::Serialize;
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[package]
|
||||
name = "memdb"
|
||||
name = "bichon-memdb"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use memdb::{DbError, MemDb, Page};
|
||||
use bichon_memdb::{DbError, MemDb, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() {
|
||||
|
||||
// Verify WAL only contains seq=3.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 2).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].seq, 3);
|
||||
|
||||
@@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() {
|
||||
|
||||
// Verify WAL seq is strictly monotonic.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), 50);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use memdb::{Durability, MemDb, Page};
|
||||
use bichon_memdb::{Durability, MemDb, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() {
|
||||
}
|
||||
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), n);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
@@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() {
|
||||
|
||||
// Verify strict seq ordering in WAL under concurrent load.
|
||||
let wal_path = db_path.join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), total as usize);
|
||||
let mut last = 0u64;
|
||||
for e in &entries {
|
||||
@@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() {
|
||||
|
||||
// The entire transaction should be a single WAL entry.
|
||||
let wal_path = dir.path().join("wal.jsonl");
|
||||
let entries = memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap();
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert_eq!(entries[0].ops.len(), n as usize);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user