diff --git a/.gitignore b/.gitignore index f48f1f4..fc2da8d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .idea config.toml node_modules -dedup_report.txt \ No newline at end of file +dedup_report.txt +crates/*/target \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index c907763..0fc62d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -296,11 +302,11 @@ name = "bichon-admin" version = "1.5.3" dependencies = [ "bichon-core", + "bichon-memdb", "console", "dialoguer", "indicatif", - "itertools", - "memdb", + "itertools 0.14.0", "native_db", "native_model", "serde", @@ -309,6 +315,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "bichon-blob" +version = "0.1.0" +dependencies = [ + "crc32fast", + "criterion", + "lz4_flex", + "rand 0.10.1", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tracing", + "zstd", +] + [[package]] name = "bichon-cli" version = "1.5.3" @@ -341,6 +363,7 @@ version = "1.5.3" dependencies = [ "async-imap", "base64 0.22.1", + "bichon-memdb", "blake3", "bytes 1.11.1", "chrono", @@ -355,12 +378,11 @@ dependencies = [ "governor", "hickory-resolver", "html2text", - "itertools", + "itertools 0.14.0", "itoa", "lru 0.18.0", "mail-parser", "mail-send", - "memdb", "murmur3", "num_cpus", "oauth2", @@ -394,6 +416,18 @@ dependencies = [ "whichlang", ] +[[package]] +name = "bichon-memdb" +version = "0.1.0" +dependencies = [ + "rand 0.9.2", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "bichon-server" version = "1.5.3" @@ -613,6 +647,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.60" @@ -668,6 +708,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -868,6 +935,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -1602,6 +1702,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -2128,6 +2239,24 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2338,9 +2467,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] @@ -2445,18 +2574,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "memdb" -version = "0.1.0" -dependencies = [ - "rand 0.9.2", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.18", - "tokio", -] - [[package]] name = "memmap2" version = "0.9.10" @@ -2858,6 +2975,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl" version = "0.10.78" @@ -3090,6 +3213,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "poem" version = "3.1.12" @@ -3159,7 +3310,7 @@ dependencies = [ "email_address", "futures-util", "indexmap", - "itertools", + "itertools 0.14.0", "mime", "num-traits", "poem", @@ -4320,7 +4471,7 @@ dependencies = [ "futures-channel", "futures-util", "htmlescape", - "itertools", + "itertools 0.14.0", "levenshtein_automata", "log", "lru 0.16.4", @@ -4371,7 +4522,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", - "itertools", + "itertools 0.14.0", "serde", "tantivy-bitpacker", "tantivy-common", @@ -4423,7 +4574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ "futures-util", - "itertools", + "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -4574,6 +4725,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" diff --git a/Cargo.toml b/Cargo.toml index 0338994..23537dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/memdb", "crates/core", + "crates/blob", "crates/server", "crates/cli", "crates/admin", @@ -18,7 +19,8 @@ edition = "2021" [workspace.dependencies] chrono = "0.4.45" clap = { version = "4.6.1", features = ["derive", "env"] } -memdb = { path = "crates/memdb" } +bichon-memdb = { path = "crates/memdb" } +bichon-blob = { path = "crates/blob" } itertools = "0.14.0" ring = { version = "0.17.14", features = ["std"] } serde = { version = "1.0.228", features = ["derive"] } diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index eddbfb2..5184ed5 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -17,4 +17,4 @@ serde_json.workspace = true itertools.workspace = true snafu.workspace = true -memdb.workspace = true \ No newline at end of file +bichon-memdb.workspace = true \ No newline at end of file diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index d843d62..d0118be 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -20,7 +20,7 @@ use bichon_core::{ }; use console::style; use itertools::Itertools; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use native_db::*; use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; diff --git a/crates/blob/Cargo.lock b/crates/blob/Cargo.lock new file mode 100644 index 0000000..b31a3bd --- /dev/null +++ b/crates/blob/Cargo.lock @@ -0,0 +1,1105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bichon-blob" +version = "0.1.0" +dependencies = [ + "crc32fast", + "criterion", + "lz4_flex", + "rand", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tracing", + "zstd", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/blob/Cargo.toml b/crates/blob/Cargo.toml new file mode 100644 index 0000000..559396b --- /dev/null +++ b/crates/blob/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "bichon-blob" +version = "0.1.0" +edition = "2021" +description = "Embedded KV storage engine for email" + +[dependencies] +crc32fast = "1.4" +zstd = "0.13" +lz4_flex = "0.13.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tracing = "0.1" +thiserror = "2" + +[dev-dependencies] +tempfile = "3" +rand = "0.10.1" +criterion = { version = "0.6", features = ["html_reports"] } + +[[bench]] +name = "benchmark" +harness = false diff --git a/crates/blob/benches/benchmark.rs b/crates/blob/benches/benchmark.rs new file mode 100644 index 0000000..278531a --- /dev/null +++ b/crates/blob/benches/benchmark.rs @@ -0,0 +1,316 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use std::time::Duration; +use tempfile::TempDir; + +use bichon_blob::{Codec, Config, Engine}; + +fn make_key(seed: u64) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&seed.to_le_bytes()); + key +} + +fn make_value(size: usize) -> Vec { + let mut v = Vec::with_capacity(size); + // Fill with somewhat realistic text-like data so compression works + let pattern = b"The quick brown fox jumps over the lazy dog. "; + while v.len() < size { + let rem = size - v.len(); + let n = rem.min(pattern.len()); + v.extend_from_slice(&pattern[..n]); + } + v +} + +pub fn bench_write_small(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024); // 1 KB + let mut counter = 0u64; + + group.bench_function("1KB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_write_medium(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Bytes(64 * 1024)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(64 * 1024); // 64 KB + let mut counter = 0u64; + + group.bench_function("64KB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_write_large(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Bytes(1024 * 1024)); + group.measurement_time(Duration::from_secs(15)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024 * 1024); // 1 MB + let mut counter = 0u64; + + group.bench_function("1MB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_read_cache_hit(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Pre-populate: 10 keys, all in same bucket → cache hit after first read + let value = make_value(4096); + for i in 0..10u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("cache_hit", |b| { + b.iter(|| { + let key = make_key(counter % 10); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_read_cache_miss(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let mut config = Config::default(); + config.lru_bucket_count = 8; // Small cache to force misses + let engine = Engine::open(dir.path(), config).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(4096); + // Write 1000 keys spread across all 16 buckets — small LRU will thrash + for i in 0..1000u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("cache_miss", |b| { + b.iter(|| { + let key = make_key(counter % 1000); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_read_large_value(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Bytes(1024 * 1024)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024 * 1024); // 1 MB + for i in 0..5u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("1MB_cache_hit", |b| { + b.iter(|| { + let key = make_key(counter % 5); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_delete(c: &mut Criterion) { + let mut group = c.benchmark_group("delete"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + group.bench_function("delete", |b| { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(4096); + let mut counter = 0u64; + + b.iter_batched( + || { + counter += 1; + let key = make_key(counter); + engine + .write("bench", key, &value, Codec::Zstd) + .unwrap(); + key + }, + |key| { + engine.delete("bench", &key).unwrap(); + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_mixed_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("mixed"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(15)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Pre-populate with 500 entries + let value = make_value(8192); + for i in 0..500u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter: u64 = 500; + group.bench_function("80w_15r_5d", |b| { + b.iter(|| { + counter += 1; + let op = counter % 100; + match op { + 0..=79 => { + // 80% writes + let key = make_key(counter); + let val = make_value(4096); + engine.write("bench", key, &val, Codec::Zstd).unwrap(); + } + 80..=94 => { + // 15% reads + std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap()); + } + _ => { + // 5% deletes + if counter % 2 == 0 { + let key = make_key(counter % 500); + let _ = engine.delete("bench", &key); + } + } + } + }) + }); + group.finish(); +} + +pub fn bench_gc(c: &mut Criterion) { + let mut group = c.benchmark_group("gc"); + group.measurement_time(Duration::from_secs(30)); + group.sample_size(10); + + group.bench_function("gc_30pct_deleted", |b| { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Fill a segment with ~1000 entries, then delete 30% + let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB + let n = 1200u64; + for i in 0..n { + engine + .write("bench", make_key(i), &value, Codec::None) + .unwrap(); + } + // Delete ~30% + for i in (0..n).step_by(3) { + engine.delete("bench", &make_key(i)).unwrap(); + } + + b.iter(|| { + engine.gc("bench").unwrap(); + }) + }); + group.finish(); +} + +criterion_group!( + benches, + bench_write_small, + bench_write_medium, + bench_write_large, + bench_read_cache_hit, + bench_read_cache_miss, + bench_read_large_value, + bench_delete, + bench_mixed_workload, + bench_gc, +); +criterion_main!(benches); diff --git a/crates/blob/src/account.rs b/crates/blob/src/account.rs new file mode 100644 index 0000000..9dc3a14 --- /dev/null +++ b/crates/blob/src/account.rs @@ -0,0 +1,242 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use crate::bucket::{self, BucketFile, IndexRecord}; +use crate::error::{Error, Result}; +use crate::meta::{AccountMeta, SegmentStats}; +use crate::segment::{self, SegmentReader, SegmentWriter}; +use crate::types::Codec; + +pub struct Account { + pub id: String, + dir: PathBuf, + meta: AccountMeta, + active_writer: SegmentWriter, + readers: HashMap, + write_lock: Mutex<()>, +} + +impl Account { + /// Open an existing account. + pub fn open(store_root: &Path, account_id: &str) -> Result { + let dir = store_root.join("accounts").join(account_id); + if !dir.exists() { + return Err(Error::AccountNotFound(account_id.to_string())); + } + + let meta = AccountMeta::load(&dir)?; + + // Open active writer + let seg_path = dir + .join("segments") + .join(segment::segment_filename(meta.active_segment_id)); + let active_writer = if seg_path.exists() { + SegmentWriter::open_append(seg_path, meta.active_segment_id)? + } else { + fs::create_dir_all(dir.join("segments"))?; + SegmentWriter::create(seg_path, meta.active_segment_id)? + }; + + // Open readers for existing sealed segments + let mut readers = HashMap::new(); + for (&seg_id, stats) in &meta.segments { + if stats.sealed { + let seg_path = dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if seg_path.exists() { + readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?); + } + } + } + + Ok(Self { + id: account_id.to_string(), + dir, + meta, + active_writer, + readers, + write_lock: Mutex::new(()), + }) + } + + /// Create a new account. + pub fn create(store_root: &Path, account_id: &str) -> Result { + let dir = store_root.join("accounts").join(account_id); + if dir.exists() { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + + fs::create_dir_all(dir.join("segments"))?; + BucketFile::ensure_dir(&dir)?; + + let meta = AccountMeta::new(account_id.to_string(), 1); + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(1)); + let active_writer = SegmentWriter::create(seg_path, 1)?; + + meta.save(&dir)?; + + Ok(Self { + id: account_id.to_string(), + dir, + meta, + active_writer, + readers: HashMap::new(), + write_lock: Mutex::new(()), + }) + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + pub fn meta(&self) -> &AccountMeta { + &self.meta + } + + /// Lock for writing. Returns a guard. + pub fn lock_write(&self) -> std::sync::MutexGuard<'_, ()> { + self.write_lock.lock().unwrap() + } + + /// Mark the segment as indexed up to the given offset and persist meta. + /// Called after append_index to enable incremental recovery. + pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> { + if let Some(stats) = self.meta.segments.get_mut(&segment_id) { + if indexed_up_to_offset > stats.indexed_up_to_offset { + stats.indexed_up_to_offset = indexed_up_to_offset; + } + } + self.meta.save(&self.dir) + } + + /// Append an entry without fsync. Returns (segment_id, offset, data_size). + /// Caller must hold the write lock and should call `flush_active()` after. + pub fn append_entry( + &mut self, + key: [u8; 32], + data: &[u8], + flags: u8, + codec: Codec, + ) -> Result<(u32, u64, u32)> { + if self.active_writer.is_full() { + self.seal_active()?; + } + + use crate::segment::Entry; + let entry = if flags == 1 { + Entry::tombstone(key) + } else { + Entry::new(key, data, flags, codec) + }; + + let data_size = entry.data.len() as u32; + let segment_id = self.active_writer.id(); + let offset = self.active_writer.append(&entry)?; + + let stats = self + .meta + .segments + .entry(segment_id) + .or_insert_with(|| SegmentStats::new(segment_id)); + stats.total_bytes += data_size as u64; + if flags == 1 { + stats.deleted_bytes += entry.raw_size as u64; + } + stats.recompute_ratio(); + + Ok((segment_id, offset, data_size)) + } + + /// Fsync the active segment and persist meta. + pub fn flush_active(&mut self) -> Result<()> { + self.active_writer.fsync()?; + self.meta.save(&self.dir) + } + + /// Write an entry with fsync. Convenience wrapper for single writes. + pub fn write_entry( + &mut self, + key: [u8; 32], + data: &[u8], + flags: u8, + codec: Codec, + ) -> Result<(u32, u64, u32)> { + let result = self.append_entry(key, data, flags, codec)?; + self.flush_active()?; + Ok(result) + } + + fn seal_active(&mut self) -> Result<()> { + let old_id = self.active_writer.id(); + let old_stats = self + .meta + .segments + .entry(old_id) + .or_insert_with(|| SegmentStats::new(old_id)); + old_stats.sealed = true; + + // Open reader for the old segment + let seg_path = self + .dir + .join("segments") + .join(segment::segment_filename(old_id)); + self.readers + .insert(old_id, SegmentReader::open(seg_path, old_id)?); + + // Create new segment + let new_id = old_id + 1; + self.meta.active_segment_id = new_id; + let new_path = self + .dir + .join("segments") + .join(segment::segment_filename(new_id)); + self.active_writer = SegmentWriter::create(new_path, new_id)?; + self.meta.save(&self.dir)?; + + Ok(()) + } + + /// Get the on-disk path for a segment. + pub fn segment_path(&self, segment_id: u32) -> Result { + let filename = segment::segment_filename(segment_id); + let path = self.dir.join("segments").join(&filename); + if path.exists() { + Ok(path) + } else { + Err(Error::SegmentNotFound(segment_id)) + } + } + + /// Append index record to the appropriate bucket file. No fsync. + pub fn append_index(&self, record: &IndexRecord) -> Result<()> { + let bucket_id = bucket::bucket_id(&record.key); + let bf = BucketFile::open(&self.dir, bucket_id); + bf.append(record) + } + + /// Return list of sealed segment IDs for GC consideration. + pub fn sealed_segments(&self) -> Vec { + self.meta + .segments + .iter() + .filter(|(_, s)| s.sealed) + .map(|(id, _)| *id) + .collect() + } + + /// All segment IDs (including active). + pub fn all_segment_ids(&self) -> Vec { + let mut ids: Vec = self.meta.segments.keys().copied().collect(); + if !ids.contains(&self.meta.active_segment_id) { + ids.push(self.meta.active_segment_id); + } + ids.sort_unstable(); + ids + } +} diff --git a/crates/blob/src/bucket.rs b/crates/blob/src/bucket.rs new file mode 100644 index 0000000..7acaed6 --- /dev/null +++ b/crates/blob/src/bucket.rs @@ -0,0 +1,333 @@ +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::error::Result; +use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE}; + +/// On-disk format: 52 bytes per record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexRecord { + pub key: [u8; 32], + pub segment_id: u32, + pub offset: u64, + pub data_size: u32, + pub flags: u8, +} + +impl IndexRecord { + pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self { + Self { + key, + segment_id, + offset, + data_size, + flags, + } + } + + pub fn is_tombstone(&self) -> bool { + self.flags == 1 + } + + pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] { + let mut buf = [0u8; INDEX_RECORD_SIZE]; + buf[0..32].copy_from_slice(&self.key); + buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes()); + buf[36..44].copy_from_slice(&self.offset.to_le_bytes()); + buf[44..48].copy_from_slice(&self.data_size.to_le_bytes()); + buf[48] = self.flags; + // bytes 49..52 are padding (keep zero) + buf + } + + pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self { + let mut key = [0u8; 32]; + key.copy_from_slice(&buf[0..32]); + let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap()); + let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap()); + let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap()); + let flags = buf[48]; + Self { + key, + segment_id, + offset, + data_size, + flags, + } + } +} + +/// Represents a loaded and deduplicated bucket in memory. +pub struct BucketIndex { + pub bucket_id: u16, + /// Records sorted by key, deduplicated (one record per key, latest wins). + pub records: Vec, +} + +impl BucketIndex { + /// Build from raw records: sort by key, dedup keeping the one with max offset. + pub fn from_records(mut records: Vec, bucket_id: u16) -> Self { + records.sort_by_key(|a| a.key); + // Dedup: keep last (max offset) for each key + let mut deduped = Vec::with_capacity(records.len()); + let mut i = 0; + while i < records.len() { + let mut best = i; + let mut j = i + 1; + while j < records.len() && records[j].key == records[i].key { + if records[j].offset > records[best].offset { + best = j; + } + j += 1; + } + deduped.push(records[best].clone()); + i = j; + } + Self { + bucket_id, + records: deduped, + } + } + + /// Binary search for a key. Returns the record if found. + pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> { + match self.records.binary_search_by(|r| r.key.cmp(key)) { + Ok(idx) => Some(&self.records[idx]), + Err(_) => None, + } + } + + /// Append a new record and maintain sorted order. + pub fn insert(&mut self, record: IndexRecord) { + match self.records.binary_search_by(|r| r.key.cmp(&record.key)) { + Ok(idx) => { + // Replace if newer (larger offset) + if record.offset > self.records[idx].offset { + self.records[idx] = record; + } + } + Err(idx) => { + self.records.insert(idx, record); + } + } + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } +} + +/// Manages a bucket index file on disk. +pub struct BucketFile { + path: PathBuf, + bucket_id: u16, +} + +impl BucketFile { + pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf { + account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id)) + } + + pub fn open(account_dir: &Path, bucket_id: u16) -> Self { + Self { + path: Self::path_for(account_dir, bucket_id), + bucket_id, + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn bucket_id(&self) -> u16 { + self.bucket_id + } + + /// Ensure the buckets directory exists. + pub fn ensure_dir(account_dir: &Path) -> Result<()> { + let dir = account_dir.join("buckets"); + std::fs::create_dir_all(&dir)?; + Ok(()) + } + + /// Append a single record to the bucket file. + pub fn append(&self, record: &IndexRecord) -> Result<()> { + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + file.write_all(&record.encode())?; + Ok(()) + } + + /// Append multiple records at once. + pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> { + if records.is_empty() { + return Ok(()); + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + for r in records { + file.write_all(&r.encode())?; + } + Ok(()) + } + + /// Load all records from the bucket file. + /// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write), + /// the trailing bytes are silently ignored. + pub fn load_all(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let data = std::fs::read(&self.path)?; + let remainder = data.len() % INDEX_RECORD_SIZE; + let count = data.len() / INDEX_RECORD_SIZE; + let mut records = Vec::with_capacity(count); + for i in 0..count { + let start = i * INDEX_RECORD_SIZE; + let end = start + INDEX_RECORD_SIZE; + let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end] + .try_into() + .map_err(|_| crate::error::Error::BucketIndexCorrupt { + path: self.path.clone(), + reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE), + })?; + records.push(IndexRecord::decode(buf)); + } + if remainder > 0 { + tracing::warn!( + "Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring", + self.path, remainder, INDEX_RECORD_SIZE + ); + } + Ok(records) + } + + /// Load all records, sort, and deduplicate into a BucketIndex. + pub fn load_index(&self) -> Result { + let records = self.load_all()?; + Ok(BucketIndex::from_records(records, self.bucket_id)) + } + + /// Rewrite the bucket file with a sorted, deduplicated set of records. + pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> { + let temp_path = self.path.with_extension("idx.tmp"); + { + let mut file = File::create(&temp_path)?; + for r in records { + file.write_all(&r.encode())?; + } + file.sync_all()?; + } + std::fs::rename(&temp_path, &self.path)?; + Ok(()) + } + + /// Delete the bucket file. + pub fn delete(&self) -> Result<()> { + if self.path.exists() { + std::fs::remove_file(&self.path)?; + } + Ok(()) + } +} + +/// Compute bucket_id from a key's first 2 bytes. +pub fn bucket_id(key: &[u8; 32]) -> u16 { + u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_index_record_encode_decode() { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&[1, 2, 3, 4]); + let rec = IndexRecord::new(key, 5, 12345, 500, 0); + let encoded = rec.encode(); + let decoded = IndexRecord::decode(&encoded); + assert_eq!(rec, decoded); + } + + #[test] + fn test_bucket_id_deterministic() { + let mut key = [0u8; 32]; + key[0] = 0x00; + key[1] = 0x0F; + assert_eq!(bucket_id(&key), 15); + key[0] = 0x00; + key[1] = 0x10; + assert_eq!(bucket_id(&key), 0); + } + + #[test] + fn test_bucket_append_and_load() { + let dir = TempDir::new().unwrap(); + let bucket = BucketFile::open(dir.path(), 0); + BucketFile::ensure_dir(dir.path()).unwrap(); + + let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0); + let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0); + + bucket.append(&r1).unwrap(); + bucket.append(&r2).unwrap(); + + let loaded = bucket.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].key, [1u8; 32]); + assert_eq!(loaded[1].key, [2u8; 32]); + } + + #[test] + fn test_bucket_index_dedup() { + let recs = vec![ + IndexRecord::new([1u8; 32], 1, 100, 50, 0), + IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins + IndexRecord::new([2u8; 32], 1, 300, 60, 0), + ]; + let idx = BucketIndex::from_records(recs, 0); + assert_eq!(idx.len(), 2); + let found = idx.find(&[1u8; 32]).unwrap(); + assert_eq!(found.segment_id, 2); + assert_eq!(found.offset, 200); + } + + #[test] + fn test_bucket_index_find_missing() { + let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)]; + let idx = BucketIndex::from_records(recs, 0); + assert!(idx.find(&[99u8; 32]).is_none()); + } + + #[test] + fn test_bucket_rewrite() { + let dir = TempDir::new().unwrap(); + let bucket = BucketFile::open(dir.path(), 0); + BucketFile::ensure_dir(dir.path()).unwrap(); + + let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0); + let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0); + bucket.append(&r1).unwrap(); + bucket.append(&r2).unwrap(); + + // Rewrite sorted + let sorted = vec![r2.clone(), r1.clone()]; + bucket.rewrite(&sorted).unwrap(); + + let loaded = bucket.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].key, [1u8; 32]); + assert_eq!(loaded[1].key, [3u8; 32]); + } +} diff --git a/crates/blob/src/cache.rs b/crates/blob/src/cache.rs new file mode 100644 index 0000000..d33cd55 --- /dev/null +++ b/crates/blob/src/cache.rs @@ -0,0 +1,189 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use crate::bucket::{BucketFile, BucketIndex, IndexRecord}; +use crate::error::Result; + +/// Cache key: (account_name, bucket_id) +type CacheKey = (String, u16); + +/// LRU bucket cache. Thread-safe. +pub struct BucketCache { + max_entries: usize, + entries: Mutex>, + index: Mutex>, +} + +struct CacheEntry { + key: CacheKey, + index: BucketIndex, +} + +impl BucketCache { + pub fn new(max_entries: usize) -> Self { + Self { + max_entries: max_entries.max(1), + entries: Mutex::new(Vec::new()), + index: Mutex::new(HashMap::new()), + } + } + + /// Get or load a bucket index. Returns the sorted, deduplicated records for the bucket. + pub fn get_or_load( + &self, + account: &str, + bucket_id: u16, + account_dir: &std::path::Path, + ) -> Result> { + let key: CacheKey = (account.to_string(), bucket_id); + + // Check cache + { + let index = self.index.lock().unwrap(); + if let Some(&pos) = index.get(&key) { + let entries = self.entries.lock().unwrap(); + return Ok(entries[pos].index.records.clone()); + } + } + + // Load from disk + let bucket_file = BucketFile::open(account_dir, bucket_id); + let bucket_index = bucket_file.load_index()?; + let records = bucket_index.records.clone(); + + // Insert into cache + self.insert(key, bucket_index); + + Ok(records) + } + + fn insert(&self, key: CacheKey, index: BucketIndex) { + let mut idx_map = self.index.lock().unwrap(); + let mut entries = self.entries.lock().unwrap(); + + // If already exists, update and move to front + if let Some(&pos) = idx_map.get(&key) { + entries[pos].index = index; + let entry = entries.remove(pos); + entries.insert(0, entry); + // Rebuild index + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + return; + } + + // Evict if full + if entries.len() >= self.max_entries { + if let Some(evicted) = entries.pop() { + idx_map.remove(&evicted.key); + } + } + + // Insert at front (most recently used) + entries.insert(0, CacheEntry { key: key.clone(), index }); + // Rebuild index (positions shifted) + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + } + + /// Insert or update a single record in a cached bucket. If bucket not cached, no-op. + pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) { + let key: CacheKey = (account.to_string(), bucket_id); + let mut idx_map = self.index.lock().unwrap(); + + if let Some(&pos) = idx_map.get(&key) { + let mut entries = self.entries.lock().unwrap(); + entries[pos].index.insert(record); + // Move to front + let entry = entries.remove(pos); + entries.insert(0, entry); + // Rebuild index + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + } + } + + /// Invalidate a cached bucket. + pub fn invalidate(&self, account: &str, bucket_id: u16) { + let key: CacheKey = (account.to_string(), bucket_id); + let mut idx_map = self.index.lock().unwrap(); + if let Some(&pos) = idx_map.get(&key) { + let mut entries = self.entries.lock().unwrap(); + entries.remove(pos); + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + } + } + + pub fn len(&self) -> usize { + self.entries.lock().unwrap().len() + } + + pub fn is_empty(&self) -> bool { + self.entries.lock().unwrap().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::IndexRecord; + use tempfile::TempDir; + + #[test] + fn test_cache_miss_loads_from_disk() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0)) + .unwrap(); + + let cache = BucketCache::new(10); + let records = cache + .get_or_load("test", 0, dir.path()) + .unwrap(); + assert_eq!(records.len(), 1); + } + + #[test] + fn test_cache_hit() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0)) + .unwrap(); + + let cache = BucketCache::new(10); + let _ = cache.get_or_load("test", 0, dir.path()).unwrap(); + // Second call should hit cache + let records = cache + .get_or_load("test", 0, dir.path()) + .unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(cache.len(), 1); + } + + #[test] + fn test_cache_eviction() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let cache = BucketCache::new(2); + + for b in 0..4 { + let bf = BucketFile::open(dir.path(), b); + bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0)) + .unwrap(); + let _ = cache.get_or_load("test", b, dir.path()).unwrap(); + } + + assert!(cache.len() <= 2); + } +} diff --git a/crates/blob/src/checksum.rs b/crates/blob/src/checksum.rs new file mode 100644 index 0000000..b773a2c --- /dev/null +++ b/crates/blob/src/checksum.rs @@ -0,0 +1,65 @@ +use crc32fast::Hasher; + +pub fn crc32(data: &[u8]) -> u32 { + let mut h = Hasher::new(); + h.update(data); + h.finalize() +} + +pub struct CrcWriter { + hasher: Hasher, +} + +impl Default for CrcWriter { + fn default() -> Self { + Self::new() + } +} + +impl CrcWriter { + pub fn new() -> Self { + Self { + hasher: Hasher::new(), + } + } + + pub fn update(&mut self, data: &[u8]) { + self.hasher.update(data); + } + + pub fn finalize(self) -> u32 { + self.hasher.finalize() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_crc32_deterministic() { + let a = crc32(b"hello"); + let b = crc32(b"hello"); + assert_eq!(a, b); + } + + #[test] + fn test_crc32_different() { + let a = crc32(b"hello"); + let b = crc32(b"world"); + assert!(a != b); + } + + #[test] + fn test_crc_writer_matches_crc32() { + let mut w = CrcWriter::new(); + w.update(b"hello"); + w.update(b" world"); + assert_eq!(w.finalize(), crc32(b"hello world")); + } + + #[test] + fn test_crc32_empty() { + assert_eq!(crc32(b""), 0); + } +} diff --git a/crates/blob/src/compress.rs b/crates/blob/src/compress.rs new file mode 100644 index 0000000..c6e0b7f --- /dev/null +++ b/crates/blob/src/compress.rs @@ -0,0 +1,97 @@ +use crate::types::Codec; + +pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec, Codec) { + if data.len() < threshold { + return (data.to_vec(), Codec::None); + } + let (compressed, actual_codec) = match codec { + Codec::Zstd => { + match zstd::encode_all(data, level) { + Ok(out) => (out, Codec::Zstd), + Err(e) => { + tracing::warn!("zstd compression failed, storing uncompressed: {}", e); + (data.to_vec(), Codec::None) + } + } + } + Codec::Lz4 => { + let out = lz4_flex::compress(data); + (out, Codec::Lz4) + } + Codec::None => (data.to_vec(), Codec::None), + }; + // If compression made it larger, store uncompressed + if compressed.len() >= data.len() { + (data.to_vec(), Codec::None) + } else { + (compressed, actual_codec) + } +} + +pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result> { + match codec { + Codec::None => Ok(data.to_vec()), + Codec::Zstd => { + zstd::decode_all(data) + .map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e))) + } + Codec::Lz4 => { + lz4_flex::decompress(data, raw_size) + .map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_small_data_not_compressed() { + let data = b"hi"; + let (out, codec) = compress(data, Codec::Zstd, 4096, 0); + assert_eq!(out, b"hi"); + assert_eq!(codec, Codec::None); + } + + #[test] + fn test_large_data_compressed_zstd() { + let data = vec![b'A'; 5000]; + let (out, codec) = compress(&data, Codec::Zstd, 4096, 0); + assert_eq!(codec, Codec::Zstd); + assert!(out.len() < data.len()); + } + + #[test] + fn test_roundtrip_zstd() { + let data = vec![b'B'; 10000]; + let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_roundtrip_lz4() { + let data = vec![b'C'; 10000]; + let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_roundtrip_none() { + let data = vec![b'D'; 100]; + let (compressed, codec) = compress(&data, Codec::None, 4096, 0); + assert_eq!(codec, Codec::None); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_threshold_zero_always_compresses() { + let data = vec![b'E'; 100]; + let (out, codec) = compress(&data, Codec::Zstd, 0, 0); + assert_eq!(codec, Codec::Zstd); + assert!(out.len() < data.len()); + } +} diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs new file mode 100644 index 0000000..f04fc38 --- /dev/null +++ b/crates/blob/src/engine.rs @@ -0,0 +1,399 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::RwLock; + +use crate::account::Account; +use crate::bucket::{self, IndexRecord}; +use crate::cache::BucketCache; +use crate::compress; +use crate::error::{Error, Result}; +use crate::gc::{self, GcStats}; +use crate::meta::GlobalMeta; +use crate::recovery; +use crate::segment::{self, SegmentReader}; +use crate::types::{Codec, Config, ENTRY_HEADER_SIZE}; + +pub struct Engine { + root: PathBuf, + config: Config, + cache: BucketCache, + accounts: RwLock>, +} + +#[derive(Debug, Clone)] +pub struct AccountStats { + pub account_id: String, + pub total_keys: u64, + pub total_bytes: u64, + pub deleted_bytes: u64, + pub segment_count: usize, +} + +impl Engine { + /// Open or create the store at `path`. Runs recovery on startup. + pub fn open(path: &Path, config: Config) -> Result { + config.validate()?; + fs::create_dir_all(path)?; + fs::create_dir_all(path.join("accounts"))?; + + let mut global = GlobalMeta::load(path)?; + global.save(path)?; + + let cache = BucketCache::new(config.lru_bucket_count); + + // Discover accounts on disk + let accounts_dir = path.join("accounts"); + let mut accounts = HashMap::new(); + + if accounts_dir.exists() { + for entry in fs::read_dir(&accounts_dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + let account_name = entry.file_name().to_string_lossy().into_owned(); + + // Clean up temp files from interrupted GC + let _ = recovery::cleanup_temp_files(&entry.path()); + + // Run recovery + match recovery::recover_account(&entry.path()) { + Ok(_meta) => { + match Account::open(path, &account_name) { + Ok(account) => { + accounts.insert(account_name, account); + } + Err(e) => { + tracing::warn!( + "Failed to open account {}: {}", + account_name, + e + ); + } + } + } + Err(e) => { + tracing::warn!( + "Failed to recover account {}: {}", + account_name, + e + ); + } + } + } + } + } + + // Update global account list + global.accounts = accounts.keys().cloned().collect(); + global.save(path)?; + + Ok(Self { + root: path.to_path_buf(), + config, + cache, + accounts: RwLock::new(accounts), + }) + } + + // ── Account management ────────────────────────────────────────────────── + + pub fn create_account(&self, account_id: &str) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + if accounts.contains_key(account_id) { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + let account = Account::create(&self.root, account_id)?; + accounts.insert(account_id.to_string(), account); + + let mut global = GlobalMeta::load(&self.root)?; + global.accounts = accounts.keys().cloned().collect(); + global.save(&self.root)?; + + Ok(()) + } + + pub fn delete_account(&self, account_id: &str) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .remove(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + let account_dir = account.dir().to_path_buf(); + drop(account); + fs::remove_dir_all(&account_dir)?; + + let mut global = GlobalMeta::load(&self.root)?; + global.accounts = accounts.keys().cloned().collect(); + global.save(&self.root)?; + + Ok(()) + } + + pub fn list_accounts(&self) -> Vec { + let accounts = self.accounts.read().unwrap(); + accounts.keys().cloned().collect() + } + + // ── Read / Write / Delete ─────────────────────────────────────────────── + + pub fn write( + &self, + account_id: &str, + key: [u8; 32], + value: &[u8], + codec: Codec, + ) -> Result<()> { + if value.len() > crate::types::MAX_VALUE_SIZE { + return Err(Error::ValueTooLarge { size: value.len() }); + } + + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .get_mut(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + // Acquire per-account write lock + { + let _lock = account.lock_write(); + } + + let (data, actual_codec) = + compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level); + + let (segment_id, offset, data_size) = + account.write_entry(key, &data, 0, actual_codec)?; + + let record = IndexRecord::new(key, segment_id, offset, data_size, 0); + account.append_index(&record)?; + + // Update indexed_up_to_offset for incremental recovery + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + account.mark_indexed(segment_id, entry_end)?; + + let bucket_id = bucket::bucket_id(&key); + self.cache.update_record(account_id, bucket_id, record); + + Ok(()) + } + + pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result>> { + let bucket_id = bucket::bucket_id(key); + + // Acquire lock only to get bucket records and segment routing info. + let record: Option = { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + let records = self + .cache + .get_or_load(account_id, bucket_id, account.dir())?; + match records.binary_search_by(|r| r.key.cmp(key)) { + Ok(idx) => Some(records[idx].clone()), + Err(_) => None, + } + }; // accounts read lock dropped here — I/O happens outside the lock + + let record = match record { + Some(r) => r, + None => return Ok(None), + }; + + if record.is_tombstone() { + return Ok(None); + } + + // I/O outside the global lock + let seg_path = self + .root + .join("accounts") + .join(account_id) + .join("segments") + .join(segment::segment_filename(record.segment_id)); + + if !seg_path.exists() { + return Err(Error::SegmentNotFound(record.segment_id)); + } + + let reader = SegmentReader::open(seg_path, record.segment_id)?; + let (entry, _) = reader.read_entry_at(record.offset)?; + + let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?; + + Ok(Some(value)) + } + + pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .get_mut(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + // Acquire per-account write lock + { + let _lock = account.lock_write(); + } + + let (segment_id, offset, data_size) = + account.write_entry(*key, &[], 1, Codec::None)?; + + let record = IndexRecord::new(*key, segment_id, offset, data_size, 1); + account.append_index(&record)?; + + // Update indexed_up_to_offset for incremental recovery + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + account.mark_indexed(segment_id, entry_end)?; + + let bucket_id = bucket::bucket_id(key); + self.cache.update_record(account_id, bucket_id, record); + + Ok(()) + } + + // ── Batch write ───────────────────────────────────────────────────────── + + /// Batch-write multiple entries with a single fsync. + /// Each element is (key, value, codec). + pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec, Codec)]) -> Result<()> { + if entries.is_empty() { + return Ok(()); + } + + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .get_mut(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + { + let _lock = account.lock_write(); + } + + // Phase 1: compress and append all entries without fsync + let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len()); + for (key, value, codec) in entries { + if value.len() > crate::types::MAX_VALUE_SIZE { + return Err(Error::ValueTooLarge { size: value.len() }); + } + let (data, actual_codec) = + compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level); + + let (segment_id, offset, data_size) = + account.append_entry(*key, &data, 0, actual_codec)?; + + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + let record = IndexRecord::new(*key, segment_id, offset, data_size, 0); + pending.push((record, entry_end)); + } + + // Phase 2: single fsync + account.flush_active()?; + + // Phase 3: append indices and update cache + for (record, entry_end) in &pending { + account.append_index(record)?; + account.mark_indexed(record.segment_id, *entry_end)?; + + let bucket_id = bucket::bucket_id(&record.key); + self.cache.update_record(account_id, bucket_id, record.clone()); + } + + Ok(()) + } + + // ── GC ────────────────────────────────────────────────────────────────── + + pub fn gc(&self, account_id: &str) -> Result> { + // Get account dir while holding read lock, then release before heavy I/O. + let account_dir = { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + account.dir().to_path_buf() + }; // read lock released — GC runs without blocking reads + + let result = gc::gc_account(&account_dir, self.config.gc_deleted_ratio)?; + + // Invalidate all cached buckets for this account after GC rewrites them + for bid in 0..crate::types::BUCKET_COUNT { + self.cache.invalidate(account_id, bid); + } + + Ok(result) + } + + pub fn compact_buckets(&self, account_id: &str) -> Result<()> { + let account_dir = { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + account.dir().to_path_buf() + }; + + gc::compact_buckets(&account_dir)?; + + for bid in 0..crate::types::BUCKET_COUNT { + self.cache.invalidate(account_id, bid); + } + + Ok(()) + } + + // ── Stats / Shutdown ──────────────────────────────────────────────────── + + pub fn stats(&self, account_id: &str) -> Result { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + let meta = account.meta(); + let mut total_bytes = 0u64; + let mut deleted_bytes = 0u64; + + for seg in meta.segments.values() { + total_bytes += seg.total_bytes; + deleted_bytes += seg.deleted_bytes; + } + + // Count total live keys from bucket indices + let mut total_keys = 0u64; + for bid in 0..crate::types::BUCKET_COUNT { + if let Ok(records) = + self.cache + .get_or_load(account_id, bid, account.dir()) + { + total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64; + } + } + + Ok(AccountStats { + account_id: account_id.to_string(), + total_keys, + total_bytes, + deleted_bytes, + segment_count: meta.segments.len(), + }) + } + + /// Gracefully shut down: fsync all active segments and persist meta. + pub fn shutdown(&self) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + for (_, account) in accounts.iter_mut() { + account.flush_active()?; + } + let global = GlobalMeta::load(&self.root)?; + global.save(&self.root)?; + tracing::info!("bichon-blob shut down cleanly"); + Ok(()) + } +} + +impl Drop for Engine { + fn drop(&mut self) { + if let Err(e) = self.shutdown() { + tracing::error!("bichon-blob shutdown error: {}", e); + } + } +} diff --git a/crates/blob/src/error.rs b/crates/blob/src/error.rs new file mode 100644 index 0000000..a745429 --- /dev/null +++ b/crates/blob/src/error.rs @@ -0,0 +1,53 @@ +use std::{io, path::PathBuf}; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("CRC32 mismatch at {path}:{offset}")] + CrcMismatch { path: PathBuf, offset: u64 }, + + #[error("Corrupt entry at {path}:{offset}: {reason}")] + CorruptEntry { + path: PathBuf, + offset: u64, + reason: String, + }, + + #[error("Account not found: {0}")] + AccountNotFound(String), + + #[error("Account already exists: {0}")] + AccountAlreadyExists(String), + + #[error("Segment not found: {0}")] + SegmentNotFound(u32), + + #[error("Value too large: {size} bytes (max 100 MB)")] + ValueTooLarge { size: usize }, + + #[error("Compression error: {0}")] + Compression(String), + + #[error("Disk full: {0}")] + DiskFull(String), + + #[error("Invalid config: {0}")] + InvalidConfig(String), + + #[error("Bucket index corrupt at {path}: {reason}")] + BucketIndexCorrupt { path: PathBuf, reason: String }, + + #[error("Segment file truncated at {path}: expected {expected}, got {actual}")] + SegmentTruncated { + path: PathBuf, + expected: u64, + actual: u64, + }, +} diff --git a/crates/blob/src/gc.rs b/crates/blob/src/gc.rs new file mode 100644 index 0000000..619d08f --- /dev/null +++ b/crates/blob/src/gc.rs @@ -0,0 +1,267 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord}; +use crate::error::Result; +#[cfg(test)] +use crate::meta::SegmentStats; +use crate::segment::{self, SegmentReader, SegmentWriter}; + +/// Result of a GC run. +#[derive(Debug)] +pub struct GcStats { + pub segment_id: u32, + pub bytes_before: u64, + pub bytes_after: u64, + pub entries_kept: usize, + pub entries_skipped: usize, +} + +/// Run GC on an account: pick the sealed segment with highest deleted_ratio, +/// rewrite it without deleted/overwritten entries, then rebuild all bucket files. +pub fn gc_account( + account_dir: &Path, + deleted_ratio_threshold: f64, +) -> Result> { + let meta = crate::meta::AccountMeta::load(account_dir)?; + + // Find the best candidate + let candidate = meta + .segments + .values() + .filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold) + .max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap()); + + let target = match candidate { + Some(s) => s.clone(), + None => return Ok(None), + }; + + let seg_path = account_dir + .join("segments") + .join(segment::segment_filename(target.segment_id)); + let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?; + + // Build a global view: for each key, which entry (segment_id + offset) is the latest? + let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new(); + + for &seg_id in meta.segments.keys() { + let rpath = account_dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if !rpath.exists() { + continue; + } + let r = SegmentReader::open(rpath, seg_id)?; + let _ = r.scan_entries(0, |entry, offset| { + match latest_key.get(&entry.key) { + Some((existing_seg, existing_off)) => { + if seg_id > *existing_seg + || (seg_id == *existing_seg && offset > *existing_off) + { + latest_key.insert(entry.key, (seg_id, offset)); + } + } + None => { + latest_key.insert(entry.key, (seg_id, offset)); + } + } + Ok(()) + })?; + } + + // Create temp segment with a unique name + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let temp_name = format!("temp_{:016x}.seg", timestamp); + let temp_path = account_dir.join("segments").join(&temp_name); + let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?; + + let mut bytes_after: u64 = 0; + let mut entries_kept: usize = 0; + let mut entries_skipped: usize = 0; + + reader.scan_entries(0, |entry, offset| { + // Skip tombstones + if entry.is_tombstone() { + entries_skipped += 1; + return Ok(()); + } + // Skip if this key has a newer entry in another segment + if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) { + if *latest_seg != target.segment_id || *latest_off != offset { + entries_skipped += 1; + return Ok(()); + } + } + // Keep this entry + writer.append(entry)?; + bytes_after += entry.data.len() as u64; + entries_kept += 1; + Ok(()) + })?; + + writer.fsync()?; + + // Atomic rename: replace old segment with new one + fs::rename(&temp_path, &seg_path)?; + + // Rebuild all bucket files + rebuild_buckets(account_dir, &meta)?; + + // Update meta + let mut meta = crate::meta::AccountMeta::load(account_dir)?; + if let Some(stats) = meta.segments.get_mut(&target.segment_id) { + stats.total_bytes = bytes_after; + stats.deleted_bytes = 0; + stats.recompute_ratio(); + } + meta.save(account_dir)?; + + Ok(Some(GcStats { + segment_id: target.segment_id, + bytes_before: target.total_bytes, + bytes_after, + entries_kept, + entries_skipped, + })) +} + +/// Rebuild all 16 bucket files from scratch by scanning all segments. +fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> { + let mut bucket_records: HashMap> = HashMap::new(); + for i in 0..crate::types::BUCKET_COUNT { + bucket_records.insert(i, Vec::new()); + } + + for &seg_id in meta.segments.keys() { + let seg_path = account_dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if !seg_path.exists() { + continue; + } + let reader = SegmentReader::open(seg_path, seg_id)?; + reader.scan_entries(0, |entry, offset| { + let bid = bucket::bucket_id(&entry.key); + let rec = IndexRecord::new( + entry.key, + seg_id, + offset, + entry.data.len() as u32, + entry.flags, + ); + bucket_records.entry(bid).or_default().push(rec); + Ok(()) + })?; + } + + for (bid, records) in &bucket_records { + let index = BucketIndex::from_records(records.clone(), *bid); + let bf = BucketFile::open(account_dir, *bid); + bf.rewrite(&index.records)?; + } + + Ok(()) +} + +/// Compact bucket files: load, dedup, rewrite. +pub fn compact_buckets(account_dir: &Path) -> Result<()> { + for bid in 0..crate::types::BUCKET_COUNT { + let bf = BucketFile::open(account_dir, bid); + if bf.path().exists() { + let index = bf.load_index()?; + bf.rewrite(&index.records)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::segment::Entry; + use crate::types::Codec; + use tempfile::TempDir; + + fn setup_account(dir: &Path) { + fs::create_dir_all(dir.join("segments")).unwrap(); + crate::bucket::BucketFile::ensure_dir(dir).unwrap(); + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(1)); + let mut writer = SegmentWriter::create(seg_path, 1).unwrap(); + + // Write 5 entries + for i in 0..5u8 { + let mut key = [0u8; 32]; + key[0] = i; + let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None); + writer.append(&entry).unwrap(); + } + + // Tombstone entry 2 + let mut key2 = [0u8; 32]; + key2[0] = 2; + let tomb = Entry::tombstone(key2); + writer.append(&tomb).unwrap(); + + writer.fsync().unwrap(); + + // Save meta + let mut meta = crate::meta::AccountMeta::new("test".into(), 2); + meta.segments.insert( + 1, + SegmentStats { + segment_id: 1, + total_bytes: 6000, + deleted_bytes: 1000, + deleted_ratio: 1000.0 / 6000.0, + sealed: true, + indexed_up_to_offset: 0, + }, + ); + // Make segment 2 active so segment 1 is sealed + let seg2_path = dir + .join("segments") + .join(segment::segment_filename(2)); + SegmentWriter::create(seg2_path, 2).unwrap(); + meta.save(dir).unwrap(); + } + + #[test] + fn test_gc_removes_tombstones() { + let dir = TempDir::new().unwrap(); + setup_account(dir.path()); + + let result = gc_account(dir.path(), 0.01).unwrap(); + assert!(result.is_some()); + + // Verify segment 1 no longer has the tombstone'd entry + let seg_path = dir + .path() + .join("segments") + .join(segment::segment_filename(1)); + let reader = SegmentReader::open(seg_path, 1).unwrap(); + let mut count = 0; + reader.scan_entries(0, |entry, _offset| { + count += 1; + assert!(entry.key[0] != 2); + Ok(()) + }).unwrap(); + assert_eq!(count, 4); // 5 original - 1 tombstoned + } + + #[test] + fn test_compact_buckets() { + let dir = TempDir::new().unwrap(); + setup_account(dir.path()); + compact_buckets(dir.path()).unwrap(); + // Should not panic + } +} diff --git a/crates/blob/src/lib.rs b/crates/blob/src/lib.rs new file mode 100644 index 0000000..c95be02 --- /dev/null +++ b/crates/blob/src/lib.rs @@ -0,0 +1,16 @@ +pub mod account; +pub mod bucket; +pub mod cache; +pub mod checksum; +pub mod compress; +pub mod engine; +pub mod error; +pub mod gc; +pub mod meta; +pub mod recovery; +pub mod segment; +pub mod types; + +pub use engine::{AccountStats, Engine}; +pub use error::{Error, Result}; +pub use types::{Codec, Config}; diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs new file mode 100644 index 0000000..d4636e0 --- /dev/null +++ b/crates/blob/src/meta.rs @@ -0,0 +1,156 @@ +use std::collections::HashMap; +use std::path::Path; + +use crate::error::Result; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GlobalMeta { + pub version: u32, + pub accounts: Vec, +} + +impl Default for GlobalMeta { + fn default() -> Self { + Self { + version: 1, + accounts: Vec::new(), + } + } +} + +impl GlobalMeta { + pub fn load(store_root: &Path) -> Result { + let path = store_root.join("global_meta.json"); + if !path.exists() { + return Ok(Self::default()); + } + let data = std::fs::read_to_string(&path)?; + Ok(serde_json::from_str(&data)?) + } + + pub fn save(&self, store_root: &Path) -> Result<()> { + let path = store_root.join("global_meta.json"); + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_string_pretty(self)?; + std::fs::write(&tmp, &data)?; + std::fs::rename(&tmp, &path)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SegmentStats { + pub segment_id: u32, + pub total_bytes: u64, + pub deleted_bytes: u64, + pub deleted_ratio: f64, + pub sealed: bool, + /// Byte offset up to which entries have been indexed in bucket files. + /// Recovery starts scanning from here instead of 0. + pub indexed_up_to_offset: u64, +} + +impl SegmentStats { + pub fn new(segment_id: u32) -> Self { + Self { + segment_id, + total_bytes: 0, + deleted_bytes: 0, + deleted_ratio: 0.0, + sealed: false, + indexed_up_to_offset: 0, + } + } + + pub fn recompute_ratio(&mut self) { + if self.total_bytes > 0 { + self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64; + } else { + self.deleted_ratio = 0.0; + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccountMeta { + pub account_id: String, + pub active_segment_id: u32, + pub segments: HashMap, +} + +impl AccountMeta { + pub fn new(account_id: String, active_segment_id: u32) -> Self { + Self { + account_id, + active_segment_id, + segments: HashMap::new(), + } + } + + pub fn load(account_dir: &Path) -> Result { + let path = account_dir.join("meta.json"); + if !path.exists() { + return Err(crate::error::Error::AccountNotFound( + account_dir.to_string_lossy().into(), + )); + } + let data = std::fs::read_to_string(&path)?; + Ok(serde_json::from_str(&data)?) + } + + pub fn save(&self, account_dir: &Path) -> Result<()> { + let path = account_dir.join("meta.json"); + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_string_pretty(self)?; + std::fs::write(&tmp, &data)?; + std::fs::rename(&tmp, &path)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_global_meta_roundtrip() { + let dir = TempDir::new().unwrap(); + let mut meta = GlobalMeta::default(); + meta.accounts.push("alice".into()); + meta.save(dir.path()).unwrap(); + + let loaded = GlobalMeta::load(dir.path()).unwrap(); + assert_eq!(loaded.accounts, vec!["alice"]); + } + + #[test] + fn test_global_meta_default_when_missing() { + let dir = TempDir::new().unwrap(); + let meta = GlobalMeta::load(dir.path()).unwrap(); + assert!(meta.accounts.is_empty()); + } + + #[test] + fn test_account_meta_roundtrip() { + let dir = TempDir::new().unwrap(); + let mut meta = AccountMeta::new("alice".into(), 1); + meta.segments.insert( + 1, + SegmentStats { + segment_id: 1, + total_bytes: 1000, + deleted_bytes: 300, + deleted_ratio: 0.3, + sealed: false, + indexed_up_to_offset: 0, + }, + ); + meta.save(dir.path()).unwrap(); + + let loaded = AccountMeta::load(dir.path()).unwrap(); + assert_eq!(loaded.active_segment_id, 1); + assert_eq!(loaded.segments[&1].total_bytes, 1000); + } +} diff --git a/crates/blob/src/recovery.rs b/crates/blob/src/recovery.rs new file mode 100644 index 0000000..cffb902 --- /dev/null +++ b/crates/blob/src/recovery.rs @@ -0,0 +1,200 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use crate::bucket::{self, BucketFile, IndexRecord}; +use crate::error::Result; +use crate::meta::{AccountMeta, SegmentStats}; +use crate::segment::{self, SegmentReader}; + +/// Recover an account after a crash: scan segments, repair indices, update stats. +pub fn recover_account(account_dir: &Path) -> Result { + let meta_path = account_dir.join("meta.json"); + let mut meta = if meta_path.exists() { + AccountMeta::load(account_dir).unwrap_or_else(|_| { + AccountMeta::new( + account_dir + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into(), + 1, + ) + }) + } else { + return Ok(AccountMeta::new( + account_dir + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into(), + 1, + )); + }; + + // Discover all segment files on disk + let seg_dir = account_dir.join("segments"); + if !seg_dir.exists() { + fs::create_dir_all(&seg_dir)?; + } + + let mut disk_segments: Vec = Vec::new(); + if seg_dir.exists() { + for entry in fs::read_dir(&seg_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".seg") && !name_str.contains("temp_") { + if let Some(id_str) = name_str.strip_suffix(".seg") { + if let Ok(id) = id_str.parse::() { + disk_segments.push(id); + } + } + } + } + } + disk_segments.sort_unstable(); + + if disk_segments.is_empty() { + meta.active_segment_id = 1; + } else { + let max_id = *disk_segments.last().unwrap(); + meta.active_segment_id = max_id; + } + + // Ensure buckets directory exists + let buckets_dir = account_dir.join("buckets"); + fs::create_dir_all(&buckets_dir)?; + + // For each segment, scan only the unindexed tail and update stats incrementally + for &seg_id in &disk_segments { + let seg_path = seg_dir.join(segment::segment_filename(seg_id)); + let file_size = fs::metadata(&seg_path)?.len(); + + // Preserve existing stats; start fresh if this is a newly discovered segment + let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id)); + let is_sealed = seg_id != meta.active_segment_id; + stats.sealed = is_sealed; + + // Scan start: from last indexed offset. Clamp defensively. + let scan_start = if stats.indexed_up_to_offset <= file_size { + stats.indexed_up_to_offset + } else { + 0 + }; + + // If fully indexed, skip scanning entirely + if scan_start >= file_size { + meta.segments.insert(seg_id, stats); + continue; + } + + let reader = SegmentReader::open(seg_path.clone(), seg_id)?; + let mut new_records: HashMap> = HashMap::new(); + + let truncation_point = reader.scan_entries(scan_start, |entry, offset| { + let bid = bucket::bucket_id(&entry.key); + let rec = IndexRecord::new( + entry.key, + seg_id, + offset, + entry.data.len() as u32, + entry.flags, + ); + new_records.entry(bid).or_default().push(rec); + + stats.total_bytes += entry.data.len() as u64; + if entry.is_tombstone() { + stats.deleted_bytes += entry.raw_size as u64; + } + + Ok(()) + })?; + + // Merge new records into bucket files (only the newly discovered ones) + for (bid, records) in &new_records { + let bf = BucketFile::open(account_dir, *bid); + bf.append_batch(records)?; + } + + // Truncate if tail corruption found + if truncation_point < file_size { + segment::truncate_segment(&seg_path, truncation_point)?; + } + + stats.indexed_up_to_offset = truncation_point; + stats.recompute_ratio(); + meta.segments.insert(seg_id, stats); + } + + meta.save(account_dir)?; + + Ok(meta) +} + +/// Clean up leftover temp files from interrupted GC. +pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> { + let seg_dir = account_dir.join("segments"); + if seg_dir.exists() { + for entry in fs::read_dir(&seg_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with("temp_") { + let path = entry.path(); + tracing::warn!("Removing leftover temp file: {:?}", path); + fs::remove_file(&path)?; + } + } + } + // Also cleanup temp bucket files + let buckets_dir = account_dir.join("buckets"); + if buckets_dir.exists() { + for entry in fs::read_dir(&buckets_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".tmp") { + let path = entry.path(); + tracing::warn!("Removing leftover temp bucket file: {:?}", path); + fs::remove_file(&path)?; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_recover_fresh_account() { + let dir = TempDir::new().unwrap(); + let account_dir = dir.path().join("test"); + fs::create_dir_all(&account_dir).unwrap(); + + let meta = recover_account(&account_dir).unwrap(); + assert_eq!(meta.active_segment_id, 1); + assert!(meta.segments.is_empty()); + } + + #[test] + fn test_cleanup_temp_files() { + let dir = TempDir::new().unwrap(); + let account_dir = dir.path().join("test"); + fs::create_dir_all(account_dir.join("segments")).unwrap(); + fs::create_dir_all(account_dir.join("buckets")).unwrap(); + fs::write( + account_dir.join("segments").join("temp_ABC123.seg"), + b"garbage", + ) + .unwrap(); + fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap(); + + cleanup_temp_files(&account_dir).unwrap(); + + assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists()); + } +} diff --git a/crates/blob/src/segment.rs b/crates/blob/src/segment.rs new file mode 100644 index 0000000..1e3085f --- /dev/null +++ b/crates/blob/src/segment.rs @@ -0,0 +1,454 @@ +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use crate::checksum; +use crate::error::{Error, Result}; +use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE}; + +/// In-memory representation of a stored entry. +#[derive(Debug, Clone)] +pub struct Entry { + pub flags: u8, + pub codec: Codec, + pub key: [u8; 32], + pub raw_size: u32, + pub data: Vec, +} + +impl Entry { + /// Create a normal data entry. + pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self { + Self { + flags, + codec, + key, + raw_size: raw_data.len() as u32, + data: raw_data.to_vec(), + } + } + + /// Create a tombstone entry. + pub fn tombstone(key: [u8; 32]) -> Self { + Self { + flags: 1, + codec: Codec::None, + key, + raw_size: 0, + data: Vec::new(), + } + } + + pub fn is_tombstone(&self) -> bool { + self.flags == 1 + } + + /// Total on-disk size: header + data + pub fn disk_size(&self) -> usize { + ENTRY_HEADER_SIZE + self.data.len() + } +} + +/// Write entries sequentially to a segment file. +pub struct SegmentWriter { + file: File, + path: PathBuf, + id: u32, + bytes_written: u64, +} + +impl SegmentWriter { + pub fn create(path: PathBuf, id: u32) -> Result { + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&path)?; + Ok(Self { + file, + path, + id, + bytes_written: 0, + }) + } + + pub fn open_append(path: PathBuf, id: u32) -> Result { + let mut file = OpenOptions::new().write(true).open(&path)?; + file.seek(SeekFrom::End(0))?; + let bytes_written = file.stream_position()?; + Ok(Self { + file, + path, + id, + bytes_written, + }) + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn bytes_written(&self) -> u64 { + self.bytes_written + } + + pub fn is_full(&self) -> bool { + self.bytes_written >= SEGMENT_MAX_SIZE + } + + /// Append an entry. Returns the offset where it was written. + pub fn append(&mut self, entry: &Entry) -> Result { + let offset = self.bytes_written; + self.write_entry(entry) + .map_err(|e| map_io_err(e, &self.path))?; + Ok(offset) + } + + fn write_entry(&mut self, entry: &Entry) -> Result<()> { + let data_size = entry.data.len() as u32; + + // Write magic + self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?; + + // CRC32 placeholder: write zeros, remember position + let crc_pos = self.file.stream_position()?; + self.file.write_all(&0u32.to_le_bytes())?; + + // Write flags, codec, key, raw_size, data_size + self.file.write_all(&[entry.flags])?; + self.file.write_all(&[entry.codec as u8])?; + self.file.write_all(&entry.key)?; + self.file.write_all(&entry.raw_size.to_le_bytes())?; + self.file.write_all(&data_size.to_le_bytes())?; + + // Write data + self.file.write_all(&entry.data)?; + + // Calculate CRC32 over everything after the crc32 field + let crc = { + let mut hasher = checksum::CrcWriter::new(); + hasher.update(&[entry.flags]); + hasher.update(&[entry.codec as u8]); + hasher.update(&entry.key); + hasher.update(&entry.raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&entry.data); + hasher.finalize() + }; + + // Seek back and write the real CRC32 + self.file.seek(SeekFrom::Start(crc_pos))?; + self.file.write_all(&crc.to_le_bytes())?; + + // Seek back to end + self.file.seek(SeekFrom::End(0))?; + + self.bytes_written += entry.disk_size() as u64; + Ok(()) + } + + pub fn fsync(&self) -> Result<()> { + self.file.sync_all().map_err(|e| { + if e.kind() == std::io::ErrorKind::StorageFull { + Error::DiskFull(format!("{}: {}", self.path.display(), e)) + } else { + Error::Io(e) + } + })?; + Ok(()) + } +} + +/// Read entries from a segment file. +pub struct SegmentReader { + path: PathBuf, + id: u32, +} + +impl SegmentReader { + pub fn open(path: PathBuf, id: u32) -> Result { + Ok(Self { path, id }) + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn file_size(&self) -> Result { + Ok(fs::metadata(&self.path)?.len()) + } + + /// Read a single entry at the given offset. Returns the entry and the offset of the next entry. + pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> { + let mut file = File::open(&self.path)?; + file.seek(SeekFrom::Start(offset))?; + + // Read magic + let mut magic_buf = [0u8; 4]; + file.read_exact(&mut magic_buf)?; + let magic = u32::from_le_bytes(magic_buf); + if magic != ENTRY_MAGIC { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("bad magic: 0x{:08X}", magic), + }); + } + + // Read CRC32 + let mut crc_buf = [0u8; 4]; + file.read_exact(&mut crc_buf)?; + let stored_crc = u32::from_le_bytes(crc_buf); + + // Read flags, codec + let mut flags_buf = [0u8; 1]; + file.read_exact(&mut flags_buf)?; + let flags = flags_buf[0]; + + let mut codec_buf = [0u8; 1]; + file.read_exact(&mut codec_buf)?; + let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("unknown codec: {}", codec_buf[0]), + })?; + + // Read key, raw_size, data_size + let mut key = [0u8; 32]; + file.read_exact(&mut key)?; + + let mut raw_size_buf = [0u8; 4]; + file.read_exact(&mut raw_size_buf)?; + let raw_size = u32::from_le_bytes(raw_size_buf); + + let mut data_size_buf = [0u8; 4]; + file.read_exact(&mut data_size_buf)?; + let data_size = u32::from_le_bytes(data_size_buf); + + // Read data + let mut data = vec![0u8; data_size as usize]; + file.read_exact(&mut data)?; + + // Verify CRC32 (over everything after the crc32 field) + let computed_crc = { + let mut hasher = checksum::CrcWriter::new(); + hasher.update(&[flags]); + hasher.update(&[codec as u8]); + hasher.update(&key); + hasher.update(&raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&data); + hasher.finalize() + }; + + if stored_crc != computed_crc { + return Err(Error::CrcMismatch { + path: self.path.clone(), + offset, + }); + } + + let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + + Ok(( + Entry { + flags, + codec, + key, + raw_size, + data, + }, + next_offset, + )) + } + + /// Read data portion of an entry (for pread-style reads when you already know offset + data_size). + pub fn read_data(&self, offset: u64, data_size: u32) -> Result> { + let mut file = File::open(&self.path)?; + // Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes + let data_start = offset + ENTRY_HEADER_SIZE as u64; + file.seek(SeekFrom::Start(data_start))?; + let mut buf = vec![0u8; data_size as usize]; + file.read_exact(&mut buf)?; + Ok(buf) + } + + /// Read the full entry header + data for verification (used by recovery and GC). + pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result> { + let mut file = File::open(&self.path)?; + file.seek(SeekFrom::Start(offset))?; + let total = ENTRY_HEADER_SIZE + data_size as usize; + let mut buf = vec![0u8; total]; + file.read_exact(&mut buf)?; + Ok(buf) + } + + /// Iterate over all valid entries in the segment, calling f for each. + /// Stops when hitting a corrupt/incomplete entry at the tail. + pub fn scan_entries(&self, start_offset: u64, mut f: F) -> Result + where + F: FnMut(&Entry, u64) -> Result<()>, + { + let file_size = self.file_size()?; + let mut offset = start_offset; + + while offset + ENTRY_HEADER_SIZE as u64 <= file_size { + match self.read_entry_at(offset) { + Ok((entry, next)) => { + f(&entry, offset)?; + offset = next; + } + Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => { + // If near end of file (within one max entry), truncate + if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 { + // Likely a partial write at tail, stop here + break; + } else { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: "mid-file corruption detected".into(), + }); + } + } + Err(e) => return Err(e), + } + } + + Ok(offset) // return the truncation point + } +} + +/// Truncate a segment file to the given size. +pub fn truncate_segment(path: &Path, size: u64) -> Result<()> { + let file = OpenOptions::new().write(true).open(path)?; + file.set_len(size)?; + Ok(()) +} + +/// Map an Error, converting Io(StorageFull) to DiskFull with path context. +fn map_io_err(e: Error, path: &Path) -> Error { + match e { + Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => { + Error::DiskFull(format!("{}: {}", path.display(), io)) + } + _ => e, + } +} + +/// Segment file name from id: "00000001.seg" +pub fn segment_filename(id: u32) -> String { + format!("{:08}.seg", id) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf { + dir.path().join(segment_filename(id)) + } + + #[test] + fn test_write_and_read_entry() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let key = [0xAAu8; 32]; + let data = b"hello world".to_vec(); + + let entry = Entry::new(key, &data, 0, Codec::None); + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + writer.append(&entry).unwrap(); + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let (read_entry, next) = reader.read_entry_at(0).unwrap(); + + assert_eq!(read_entry.key, key); + assert_eq!(read_entry.data, data); + assert_eq!(read_entry.flags, 0); + assert_eq!(read_entry.raw_size, 11); + assert!(next > 0); + } + + #[test] + fn test_tombstone_entry() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let key = [0xBBu8; 32]; + + let entry = Entry::tombstone(key); + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + writer.append(&entry).unwrap(); + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let (read_entry, _) = reader.read_entry_at(0).unwrap(); + + assert!(read_entry.is_tombstone()); + assert_eq!(read_entry.data.len(), 0); + } + + #[test] + fn test_multiple_entries() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + + let entries: Vec<_> = (0..10) + .map(|i| { + let mut key = [0u8; 32]; + key[0] = i; + Entry::new(key, &vec![i; 100], 0, Codec::None) + }) + .collect(); + + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + for e in &entries { + writer.append(e).unwrap(); + } + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let mut offset = 0u64; + for (i, expected) in entries.iter().enumerate() { + let (entry, next) = reader.read_entry_at(offset).unwrap(); + assert_eq!(entry.key[0], i as u8); + assert_eq!(entry.data, expected.data); + offset = next; + } + } + + #[test] + fn test_bad_magic_detected() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + // Write garbage + std::fs::write(&path, vec![0xFFu8; 100]).unwrap(); + + let reader = SegmentReader::open(path, 1).unwrap(); + let result = reader.read_entry_at(0); + assert!(result.is_err()); + } + + #[test] + fn test_is_full() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let writer = SegmentWriter::create(path, 1).unwrap(); + assert!(!writer.is_full()); + } +} diff --git a/crates/blob/src/types.rs b/crates/blob/src/types.rs new file mode 100644 index 0000000..33c885c --- /dev/null +++ b/crates/blob/src/types.rs @@ -0,0 +1,88 @@ +use serde::{Deserialize, Serialize}; + +/// Magic number for entry identification +pub const ENTRY_MAGIC: u32 = 0xB3DB_0001; + +/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) +pub const ENTRY_HEADER_SIZE: usize = 50; + +/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3) +pub const INDEX_RECORD_SIZE: usize = 52; + +/// Maximum segment size (256 MB) +pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024; + +/// Number of hash buckets per account +pub const BUCKET_COUNT: u16 = 16; + +/// Maximum value size (100 MB) +pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024; + +/// Default compression threshold (4 KB) +pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096; + +/// Default LRU bucket cache size +pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256; + +/// Default GC deleted ratio threshold +pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Codec { + None = 0, + Zstd = 1, + Lz4 = 2, +} + +impl Codec { + pub fn from_u8(v: u8) -> Option { + match v { + 0 => Some(Codec::None), + 1 => Some(Codec::Zstd), + 2 => Some(Codec::Lz4), + _ => None, + } + } +} + +#[derive(Debug, Clone)] +pub struct Config { + pub compress_threshold: usize, + pub default_codec: Codec, + pub compression_level: i32, + pub lru_bucket_count: usize, + pub gc_deleted_ratio: f64, +} + +impl Default for Config { + fn default() -> Self { + Self { + compress_threshold: DEFAULT_COMPRESS_THRESHOLD, + default_codec: Codec::Zstd, + compression_level: 0, + lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT, + gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO, + } + } +} + +impl Config { + pub fn validate(&self) -> crate::error::Result<()> { + if self.lru_bucket_count == 0 { + return Err(crate::error::Error::InvalidConfig( + "lru_bucket_count must be > 0".into(), + )); + } + if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 { + return Err(crate::error::Error::InvalidConfig( + "gc_deleted_ratio must be in (0.0, 1.0)".into(), + )); + } + if self.compression_level < 0 { + return Err(crate::error::Error::InvalidConfig( + "compression_level must be >= 0".into(), + )); + } + Ok(()) + } +} diff --git a/crates/blob/tests/acid_test.rs b/crates/blob/tests/acid_test.rs new file mode 100644 index 0000000..e04dae1 --- /dev/null +++ b/crates/blob/tests/acid_test.rs @@ -0,0 +1,498 @@ +/// Crash-consistency and ACID property tests for bichon-blob. +/// +/// Since we can't kill the process mid-write in an inline test, we simulate crashes +/// by dropping the Engine without calling any cleanup (close/drop is the "crash"), +/// then re-opening and verifying recovery produced consistent state. +/// +/// For true power-loss simulation, each test writes data, drops the engine abruptly, +/// then reopens and verifies: no corruption, no lost committed data, no partial writes. + +use std::fs; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +use bichon_blob::{Codec, Config, Engine}; +use tempfile::TempDir; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn make_key(seed: u64) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&seed.to_le_bytes()); + key +} + +fn make_value(size: usize) -> Vec { + let pattern = b"The quick brown fox jumps over the lazy dog. "; + let mut v = Vec::with_capacity(size); + while v.len() < size { + let rem = size - v.len(); + let n = rem.min(pattern.len()); + v.extend_from_slice(&pattern[..n]); + } + v +} + +// --------------------------------------------------------------------------- +// 1. Durability: committed data survives crash +// --------------------------------------------------------------------------- + +#[test] +fn test_durability_single_write_survives_crash() { + let dir = TempDir::new().unwrap(); + let key = make_key(42); + let value = make_value(8192); + + // Write + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } // <-- Engine dropped = simulated crash + + // Recover + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); + } +} + +#[test] +fn test_durability_many_writes_survive_crash() { + let dir = TempDir::new().unwrap(); + let n = 500; + let value = make_value(2048); + let mut keys = Vec::new(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..n { + let key = make_key(i as u64); + keys.push(key); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + } // crash + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for (i, key) in keys.iter().enumerate() { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result, Some(value.clone()), "missing key at index {}", i); + } + } +} + +#[test] +fn test_durability_delete_survives_crash() { + let dir = TempDir::new().unwrap(); + let key = make_key(99); + let value = make_value(4096); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } // crash after write + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.delete("alice", &key).unwrap(); + } // crash after delete + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None, "delete should persist across crash"); + } +} + +// --------------------------------------------------------------------------- +// 2. Atomicity: no partial writes visible after crash +// --------------------------------------------------------------------------- + +#[test] +fn test_atomicity_no_partial_entries_after_crash() { + let dir = TempDir::new().unwrap(); + + // Write enough entries to fill part of a segment, then crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + let value = make_value(50_000); // big enough to notice + for i in 0..200u64 { + engine + .write("alice", make_key(i), &value, Codec::None) + .unwrap(); + } + } // crash + + // Recovery should clean up any partial tail entries and all committed + // entries should be readable + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let value = make_value(50_000); + for i in 0..200u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert_eq!( + result, + Some(value.clone()), + "committed key {} should be intact", + i + ); + } + } +} + +#[test] +fn test_atomicity_crash_during_segment_roll() { + let dir = TempDir::new().unwrap(); + let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + // Write enough to cross at least one segment boundary (256 MB) + for i in 0..140u64 { + engine + .write("alice", make_key(i), &big_value, Codec::None) + .unwrap(); + } + } // crash mid-way or after multiple segments + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // All committed writes (that returned Ok) must be readable + for i in 0..140u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert!( + result.is_some(), + "key {} should exist after segment roll recovery", + i + ); + } + } +} + +// --------------------------------------------------------------------------- +// 3. Consistency: CRC detects corruption, no silent data loss +// --------------------------------------------------------------------------- + +#[test] +fn test_consistency_crc_detects_corruption() { + let dir = TempDir::new().unwrap(); + let key = make_key(77); + let value = make_value(8192); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + // Corrupt the segment file by flipping a byte + let seg_path = find_first_segment(dir.path(), "alice"); + let mut data = fs::read(&seg_path).unwrap(); + // Flip a byte in the data portion, not the header + let flip_pos = data.len() - 100; + data[flip_pos] ^= 0xFF; + fs::write(&seg_path, &data).unwrap(); + + // Reading should detect CRC mismatch + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key); + // Either error or None is acceptable — never silently wrong data + match result { + Err(_) => {} // CRC mismatch detected — good + Ok(None) => {} // index may point to truncated/removed data + Ok(Some(v)) => { + if v == value { + panic!("CRC corruption was NOT detected — silent data corruption!"); + } + // If value differs, index pointed elsewhere after recovery + } + } + } +} + +#[test] +fn test_consistency_corrupt_magic_truncated_on_recovery() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..10u64 { + engine + .write("alice", make_key(i), &make_value(4096), Codec::Zstd) + .unwrap(); + } + } + + // Append garbage to the segment file (simulating partial write from crash) + let seg_path = find_first_segment(dir.path(), "alice"); + let mut data = fs::read(&seg_path).unwrap(); + let orig_len = data.len(); + // Append garbage that doesn't start with the magic number + data.extend_from_slice(&[0xFF; 200]); + fs::write(&seg_path, &data).unwrap(); + + // Recovery should truncate the garbage + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // Verify committed data is still intact + for i in 0..10u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert!(result.is_some(), "committed key {} should survive tail truncation", i); + } + } + + // Verify file was actually truncated + let truncated_len = fs::metadata(&seg_path).unwrap().len(); + assert!(truncated_len <= orig_len as u64, "garbage should have been truncated"); +} + +// --------------------------------------------------------------------------- +// 4. Isolation: concurrent reader sees consistent snapshot +// --------------------------------------------------------------------------- + +#[test] +fn test_isolation_reader_sees_snapshot_not_partial_write() { + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Pre-populate a known key + let original_value = make_value(4096); + let key = make_key(100); + engine + .write("alice", key, &original_value, Codec::Zstd) + .unwrap(); + + let running = Arc::new(AtomicBool::new(true)); + let writer_done = Arc::new(AtomicBool::new(false)); + + // Spawn a writer that continuously overwrites the same key + let writer_engine = engine.clone(); + let writer_running = running.clone(); + let writer_done_flag = writer_done.clone(); + let writer_key = key; + + let writer = thread::spawn(move || { + for i in 0..1000u64 { + if !writer_running.load(Ordering::Relaxed) { + break; + } + let val = make_value(4096 + (i as usize % 100)); + writer_engine + .write("alice", writer_key, &val, Codec::Zstd) + .unwrap(); + thread::yield_now(); + } + writer_done_flag.store(true, Ordering::SeqCst); + }); + + // Concurrent reader: reads should never panic or hang + let reader_engine = engine.clone(); + let reader_running = running.clone(); + let reader = thread::spawn(move || { + let mut reads = 0; + while reads < 500 { + if !reader_running.load(Ordering::Relaxed) && reads > 0 { + break; + } + let result = reader_engine.read("alice", &key); + match result { + Ok(Some(_)) | Ok(None) => {} // OK + Err(e) => { + // Accept transient errors but report them + eprintln!("reader saw error: {:?}", e); + } + } + reads += 1; + thread::yield_now(); + } + }); + + reader.join().unwrap(); + running.store(false, Ordering::SeqCst); + writer.join().unwrap(); + + // Final read should see the last committed value (not partial) + let final_result = engine.read("alice", &key).unwrap(); + assert!(final_result.is_some(), "final read should find a value"); +} + +// --------------------------------------------------------------------------- +// 5. Crash during GC: old data intact, no corruption +// --------------------------------------------------------------------------- + +#[test] +fn test_crash_during_gc_leaves_data_intact() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let value = make_value(500_000); // 500 KB each + // Write enough entries and delete some to create GC candidate + for i in 0..500u64 { + engine + .write("alice", make_key(i), &value, Codec::None) + .unwrap(); + } + // Delete ~40% + for i in (0..500u64).step_by(5) { + engine.delete("alice", &make_key(i)).unwrap(); + } + // Single GC run (may or may not trigger) + let _ = engine.gc("alice"); + } // crash after GC + + // All non-deleted entries must still be readable + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let value = make_value(500_000); + for i in 0..500u64 { + let key = make_key(i); + let result = engine.read("alice", &key).unwrap(); + if i % 5 == 0 { + // Deleted keys + assert_eq!(result, None, "key {} should be deleted", i); + } else { + assert_eq!( + result, + Some(value.clone()), + "key {} should survive GC+crash", + i + ); + } + } + } +} + +// --------------------------------------------------------------------------- +// 6. Multiple crash-reopen cycles (torture test) +// --------------------------------------------------------------------------- + +#[test] +fn test_multiple_crash_reopen_cycles() { + use std::collections::HashSet; + + let dir = TempDir::new().unwrap(); + let value = make_value(4096); + let mut alive: HashSet = HashSet::new(); + + // Populate and crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..50u64 { + engine + .write("alice", make_key(i), &value, Codec::Zstd) + .unwrap(); + alive.insert(i); + } + } + + // Reopen, verify all exist, write more, crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some()); + } + for i in 100..150u64 { + engine + .write("alice", make_key(i), &value, Codec::Zstd) + .unwrap(); + alive.insert(i); + } + } + + // Reopen, verify all exist, delete some, crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some()); + } + for i in 0..10u64 { + engine.delete("alice", &make_key(i)).unwrap(); + alive.remove(&i); + } + } + + // Final reopen: survivors exist, deleted gone + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some(), + "key {} should exist", k); + } + for i in 0..10u64 { + assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None, + "key {} should be deleted", i); + } + } +} + +// --------------------------------------------------------------------------- +// 7. Account-level isolation +// --------------------------------------------------------------------------- + +#[test] +fn test_account_isolation_crash_one_account_does_not_affect_others() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.create_account("bob").unwrap(); + + engine + .write("alice", make_key(1), &make_value(4096), Codec::Zstd) + .unwrap(); + engine + .write("bob", make_key(1), &make_value(8192), Codec::Zstd) + .unwrap(); + } + + // Delete alice's account dir partially to simulate corruption + // Then verify bob is intact + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // Bob should be fine + let result = engine.read("bob", &make_key(1)).unwrap(); + assert!(result.is_some(), "bob should be unaffected"); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf { + let seg_dir = store_root.join("accounts").join(account).join("segments"); + for entry in fs::read_dir(&seg_dir).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".seg") && !name.contains("temp_") { + return entry.path(); + } + } + panic!("no segment found in {:?}", seg_dir); +} diff --git a/crates/blob/tests/integration_test.rs b/crates/blob/tests/integration_test.rs new file mode 100644 index 0000000..2f6d19c --- /dev/null +++ b/crates/blob/tests/integration_test.rs @@ -0,0 +1,273 @@ +use bichon_blob::{Codec, Config, Engine}; +use tempfile::TempDir; + +#[test] +fn test_create_and_list_accounts() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + + engine.create_account("alice").unwrap(); + engine.create_account("bob").unwrap(); + + let accounts = engine.list_accounts(); + assert!(accounts.contains(&"alice".to_string())); + assert!(accounts.contains(&"bob".to_string())); +} + +#[test] +fn test_write_and_read() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xAA; 32]; + let value = b"Hello, this is a test email!".to_vec(); + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); +} + +#[test] +fn test_read_missing_key() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xFF; 32]; + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); +} + +#[test] +fn test_delete() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xBB; 32]; + let value = b"Some email content".to_vec(); + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + engine.delete("alice", &key).unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); +} + +#[test] +fn test_delete_account() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.delete_account("alice").unwrap(); + + let accounts = engine.list_accounts(); + assert!(!accounts.contains(&"alice".to_string())); +} + +#[test] +fn test_small_value_not_compressed() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xCC; 32]; + let value = b"hi"; // Smaller than 4KB threshold + + engine + .write("alice", key, value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value.to_vec())); +} + +#[test] +fn test_large_value() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xDD; 32]; + let value = vec![b'X'; 100_000]; // 100KB + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); +} + +#[test] +fn test_multiple_keys() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let n = 100; + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let value = format!("email number {}", i).into_bytes(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(format!("email number {}", i).into_bytes())); + } +} + +#[test] +fn test_gc() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + // Write many entries + let value = vec![b'Y'; 5000]; + let n = 100; + + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + engine + .write("alice", key, &value, Codec::None) + .unwrap(); + } + + // Delete even-numbered keys + for i in (0..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + engine.delete("alice", &key).unwrap(); + } + + // Run GC + let _result = engine.gc("alice").unwrap(); + + // Verify remaining keys still readable + for i in (1..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value.clone())); + } + + // Deleted keys should not exist + for i in (0..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); + } +} + +#[test] +fn test_reopen_persistence() { + let dir = TempDir::new().unwrap(); + let key = [0xEE; 32]; + let value = b"persistent data".to_vec(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + // Reopen + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); + } +} + +#[test] +fn test_stats() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + engine + .write("alice", [1u8; 32], b"hello", Codec::None) + .unwrap(); + + let stats = engine.stats("alice").unwrap(); + assert!(stats.total_bytes > 0); +} + +#[test] +fn test_batch_write() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let n = 50; + let entries: Vec<_> = (0..n) + .map(|i: u64| { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&i.to_le_bytes()); + let value = format!("batch email {}", i).into_bytes(); + (key, value, Codec::Zstd) + }) + .collect(); + + engine.write_batch("alice", &entries).unwrap(); + + for (key, value, _) in &entries { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result.as_ref(), Some(value)); + } +} + +#[test] +fn test_batch_write_persistence() { + let dir = TempDir::new().unwrap(); + let entries: Vec<_> = (0..30u64) + .map(|i| { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&i.to_le_bytes()); + (key, format!("persist {}", i).into_bytes(), Codec::Zstd) + }) + .collect(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.write_batch("alice", &entries).unwrap(); + } + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for (key, value, _) in &entries { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result.as_ref(), Some(value)); + } + } +} + +#[test] +fn test_invalid_config_rejected() { + let dir = TempDir::new().unwrap(); + let mut config = Config::default(); + config.lru_bucket_count = 0; + assert!(Engine::open(dir.path(), config).is_err()); + + let mut config = Config::default(); + config.gc_deleted_ratio = 1.5; + assert!(Engine::open(dir.path(), config).is_err()); +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 83bfc9c..5e76afd 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [ ], optional = true } chrono.workspace = true clap.workspace = true -memdb.workspace = true +bichon-memdb.workspace = true itertools.workspace = true ring.workspace = true serde.workspace = true diff --git a/crates/core/src/admin/meta.rs b/crates/core/src/admin/meta.rs index b7f1439..e1858d0 100644 --- a/crates/core/src/admin/meta.rs +++ b/crates/core/src/admin/meta.rs @@ -18,7 +18,7 @@ use std::path::Path; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use crate::{ database::MemDbModel, diff --git a/crates/core/src/database/manager.rs b/crates/core/src/database/manager.rs index afd0b98..6e6f258 100644 --- a/crates/core/src/database/manager.rs +++ b/crates/core/src/database/manager.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use crate::settings::dir::DATA_DIR_MANAGER; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use std::sync::LazyLock; use std::time::Duration; diff --git a/crates/core/src/database/mod.rs b/crates/core/src/database/mod.rs index c7a6cc8..778fca1 100644 --- a/crates/core/src/database/mod.rs +++ b/crates/core/src/database/mod.rs @@ -20,7 +20,7 @@ use crate::common::paginated::Paginated; use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::raise_error; -use memdb::{MemDb, Transaction}; +use bichon_memdb::{MemDb, Transaction}; use serde::de::DeserializeOwned; use serde::Serialize; diff --git a/crates/memdb/Cargo.toml b/crates/memdb/Cargo.toml index c0700e2..b3c6876 100644 --- a/crates/memdb/Cargo.toml +++ b/crates/memdb/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "memdb" +name = "bichon-memdb" version = "0.1.0" edition = "2021" diff --git a/crates/memdb/tests/integration.rs b/crates/memdb/tests/integration.rs index ef0d586..3058795 100644 --- a/crates/memdb/tests/integration.rs +++ b/crates/memdb/tests/integration.rs @@ -1,4 +1,4 @@ -use memdb::{DbError, MemDb, Page}; +use bichon_memdb::{DbError, MemDb, Page}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; @@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() { // Verify WAL only contains seq=3. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 2).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].seq, 3); @@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() { // Verify WAL seq is strictly monotonic. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), 50); let mut last = 0u64; for e in &entries { diff --git a/crates/memdb/tests/stress.rs b/crates/memdb/tests/stress.rs index 31702c8..803b1a8 100644 --- a/crates/memdb/tests/stress.rs +++ b/crates/memdb/tests/stress.rs @@ -1,4 +1,4 @@ -use memdb::{Durability, MemDb, Page}; +use bichon_memdb::{Durability, MemDb, Page}; use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() { } let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), n); let mut last = 0u64; for e in &entries { @@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() { // Verify strict seq ordering in WAL under concurrent load. let wal_path = db_path.join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), total as usize); let mut last = 0u64; for e in &entries { @@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() { // The entire transaction should be a single WAL entry. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].ops.len(), n as usize);