From 2ce2d71cc38a9657eaf344c10e07f155b8a18615 Mon Sep 17 00:00:00 2001 From: Tyler <109685178+tlongwell-block@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:17:43 -0400 Subject: [PATCH] feat(relay): make Postgres pool size configurable, default 50 (#3191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Raise the relay's Postgres pool cap from the `buzz-db` default of 20 to 50 per pool, and expose `BUZZ_DB_POOL_SIZE` for per-deploy tuning - Applies to the writer pool and, when `READ_DATABASE_URL` is set, the reader pool; zero/unparsable values fall back to the default - The `buzz-db` library default is unchanged — only the relay opts into the larger cap ## Why During the 2026-07-27 18:40–19:05Z traffic burst on bb-public, per-pod PG pools pinned at 20 fleet-wide and ~380 requests failed on the 3s acquire timeout — membership checks, channel access lookups, and historical queries returning errors to users. The database was nowhere near a limit: Aurora (db.r8g.8xlarge, ~5,000 max connections) sat at 19% CPU, 201 connections (~4% of capacity), commit latency flat at 0.01ms. The 20-connection default was sized for "four relay pods against PG max_connections=100" (the comment in `buzz-db` says exactly that). Production now runs 12–15 pods against Aurora — the per-pod cap is the binding constraint, not the DB. Budget at the new default: 15 pods × (50 writer + 50 reader + 5 audit) ≈ 1,575 potential connections, ~30% of Aurora's ceiling — and actual usage stays demand-driven (`min_connections` stays 2, connections only open under load). Same shape as #2521 (`BUZZ_REDIS_POOL_SIZE`), which fixed the identical class of ceiling on the Redis side. ## Testing - `cargo test -p buzz-relay`: 762 passed, 1 failed — the lone red is `api::mesh_demo::tests::demo_join_forwarded_arm_round_trips_echo`, the known pre-existing flake; it fails identically on clean `main` at the same SHA (verified via `git stash` / rerun) - New test `db_pool_size_env_override_and_invalid_fallback` covers override, zero, and unparsable fallback - `defaults_are_valid` extended to pin the new default - `cargo clippy -p buzz-relay --all-targets -- -D warnings` and `cargo fmt --check` clean Signed-off-by: Tyler Longwell Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Tyler Longwell --- .env.example | 4 ++++ crates/buzz-relay/src/config.rs | 41 +++++++++++++++++++++++++++++++++ crates/buzz-relay/src/main.rs | 1 + 3 files changed, 46 insertions(+) diff --git a/.env.example b/.env.example index 024175bff..3dc54856e 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,10 @@ REDIS_URL=redis://localhost:6379 # Max connections in the relay's shared Redis pool (default 16). # BUZZ_REDIS_POOL_SIZE=16 +# Max connections in each of the relay's Postgres pools — writer and, when +# READ_DATABASE_URL is set, reader (default 50). +# BUZZ_DB_POOL_SIZE=50 + # ----------------------------------------------------------------------------- # Typesense (search) # ----------------------------------------------------------------------------- diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 47030dcf3..a1691349d 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -64,6 +64,14 @@ pub struct Config { /// pod is only 4 — small enough that rate-limit checks, presence, and /// pub/sub publishes queue behind each other under load. pub redis_pool_size: usize, + /// Maximum connections in the Postgres writer/reader pools. Defaults to 50. + /// + /// The `buzz-db` default of 20 was sized for a handful of pods against + /// `max_connections=100`. Against Aurora (~5,000 connections) that cap + /// is the binding constraint: a burst of concurrent handlers exhausts + /// the per-pod pool and requests fail on acquire timeout while the + /// database sits idle. + pub db_pool_size: u32, /// Public WebSocket URL of this relay, advertised in NIP-11. pub relay_url: String, /// Public WebSocket URL of the dedicated device-pairing relay, when configured. @@ -424,6 +432,12 @@ impl Config { .filter(|&v| v > 0) .unwrap_or(16); + let db_pool_size = std::env::var("BUZZ_DB_POOL_SIZE") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v > 0) + .unwrap_or(50); + let relay_url = std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string()); @@ -875,6 +889,7 @@ impl Config { read_database_url, redis_url, redis_pool_size, + db_pool_size, relay_url, pairing_relay_url, max_connections, @@ -942,6 +957,7 @@ mod tests { assert!(!config.database_url.is_empty()); assert!(!config.redis_url.is_empty()); assert_eq!(config.redis_pool_size, 16); + assert_eq!(config.db_pool_size, 50); assert!(config.max_connections > 0); assert!(config.send_buffer_size > 0); assert_eq!(config.max_frame_bytes, DEFAULT_MAX_FRAME_BYTES); @@ -1009,6 +1025,31 @@ mod tests { assert_eq!(junk, 16, "unparsable value must fall back to the default"); } + #[test] + fn db_pool_size_env_override_and_invalid_fallback() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous = std::env::var_os("BUZZ_DB_POOL_SIZE"); + + std::env::set_var("BUZZ_DB_POOL_SIZE", "80"); + let overridden = Config::from_env().expect("config").db_pool_size; + + std::env::set_var("BUZZ_DB_POOL_SIZE", "0"); + let zero = Config::from_env().expect("config").db_pool_size; + + std::env::set_var("BUZZ_DB_POOL_SIZE", "not-a-number"); + let junk = Config::from_env().expect("config").db_pool_size; + + if let Some(value) = previous { + std::env::set_var("BUZZ_DB_POOL_SIZE", value); + } else { + std::env::remove_var("BUZZ_DB_POOL_SIZE"); + } + + assert_eq!(overridden, 80); + assert_eq!(zero, 50, "zero must fall back to the default"); + assert_eq!(junk, 50, "unparsable value must fall back to the default"); + } + #[test] fn read_database_url_unset_or_blank_is_none() { let _guard = ENV_MUTEX.lock().unwrap(); diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index c1a127d8e..3ed820d3c 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -158,6 +158,7 @@ async fn main() -> anyhow::Result<()> { let db_config = DbConfig { database_url: config.database_url.clone(), read_database_url: config.read_database_url.clone(), + max_connections: config.db_pool_size, ..DbConfig::default() }; let db = Db::new(&db_config).await.map_err(|e| {