mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67dandTyler Longwell
b03bf2d8bd
The relay logs JSON lines synchronously to stdout. In Kubernetes stdout is a pipe drained by the container runtime's log copier; when that copier stalls (node disk-IO pressure, rotation), the ~64KB pipe fills and the next write(2) blocks while holding Rust's global stdout lock. Every worker that logs then queues behind it: the whole Tokio runtime parks, liveness probes time out, the SIGTERM handler (itself an async task) can never run, and kubelet SIGKILLs the pod after the full 60s grace (exit 137). This is the fleet-wide liveness-kill signature (1 -> 41 kills/day scaling with load). Reproduced deterministically by SIGSTOP-ing a pipe consumer; worker-thread count does not mitigate (measured identical at 2 and 8 workers). Fix: route the fmt layer through tracing_appender::non_blocking in lossy mode - the only blocking syscall moves to one dedicated OS thread behind a bounded queue. A stalled copier now costs dropped log lines instead of a dead relay. - buffered_lines_limit(4096): derived from measured prod line sizes (p50=248B, p99=641B, max=641B over 26k lines) -> ~2.6MB worst-case queue memory, minutes of steady-state absorption. - Dropped lines exported as monotonic counter buzz_log_lines_dropped_total (delta-polled from the appender's cumulative saturating ErrorCounter); doubles as a permanent copier-stall detector. The poller never logs - the log channel is exactly what has failed when it fires. - WorkerGuard held as named _log_guard in main for the process lifetime (a bare _ binding would kill logging silently). - Tests pin the motivating invariant: blocked sink + queue overrun leaves the runtime responsive, bounds memory, counts drops, and recovers output on unblock; plus end-to-end delivery through a real fmt layer, whole-line atomicity for oversized lines, and delta computation. Prod-reachable direct println!/eprintln! sites: none (all 12 are inside cfg(test) modules). Panic output goes to stderr (a separate pipe) on terminal paths only - out of scope, cannot recreate the hot-path wedge. Co-authored-by: Tyler Longwell <tlongwell@block.xyz> Signed-off-by: Tyler Longwell <tlongwell@block.xyz>
174 lines
5.8 KiB
TOML
174 lines
5.8 KiB
TOML
[workspace]
|
|
members = [
|
|
"crates/buzz-relay",
|
|
"crates/buzz-core",
|
|
"crates/buzz-conformance",
|
|
"crates/buzz-push-gateway",
|
|
"crates/buzz-db",
|
|
"crates/buzz-pubsub",
|
|
"crates/buzz-auth",
|
|
"crates/buzz-search",
|
|
"crates/buzz-audit",
|
|
"crates/buzz-acp",
|
|
"crates/buzz-agent",
|
|
"crates/sprig",
|
|
"crates/buzz-test-client",
|
|
"crates/buzz-ws-client",
|
|
"crates/buzz-admin",
|
|
"crates/buzz-workflow",
|
|
"crates/buzz-media",
|
|
"crates/buzz-cli",
|
|
"crates/buzz-pairing-cli",
|
|
"crates/buzz-sdk",
|
|
"crates/buzz-persona",
|
|
"crates/git-credential-nostr",
|
|
"crates/git-sign-nostr",
|
|
"crates/buzz-pair-relay",
|
|
"crates/buzz-relay-mesh",
|
|
"crates/buzz-dev-mcp",
|
|
"examples/countdown-bot",
|
|
]
|
|
exclude = ["desktop/src-tauri"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
rust-version = "1.88.0"
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/block/sprout"
|
|
|
|
[workspace.dependencies]
|
|
# Runtime
|
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "io-util", "signal", "process"] }
|
|
tokio-util = { version = "0.7", features = ["rt", "codec"] }
|
|
|
|
# HTTP + WebSocket
|
|
axum = { version = "0.8", features = ["ws", "macros"] }
|
|
tower = { version = "0.5", features = ["timeout", "util", "limit"] }
|
|
tower-http = { version = "0.6", features = ["trace", "cors", "compression-gzip", "limit", "timeout", "fs"] }
|
|
|
|
# Database
|
|
sqlx = { version = "0.9", features = [
|
|
"runtime-tokio", "tls-rustls", "postgres", "uuid", "chrono", "json"
|
|
] }
|
|
|
|
# Redis
|
|
redis = { version = "1.0", features = ["tokio-comp", "connection-manager", "tokio-rustls-comp"] }
|
|
deadpool-redis = { version = "0.23", features = ["rt_tokio_1"] }
|
|
|
|
# Nostr
|
|
nostr = { version = "0.44", features = ["nip44", "nip98"] }
|
|
|
|
# Serialization
|
|
serde = { version = "1", features = ["derive"] }
|
|
postcard = { version = "1", default-features = false, features = ["use-std"] }
|
|
|
|
# Inter-relay mesh transport (buzz-relay-mesh)
|
|
iroh = { version = "1.0.0-rc.0", default-features = false, features = ["tls-ring"] }
|
|
serde_json = "1"
|
|
serde_yaml = "0.9"
|
|
evalexpr = "11"
|
|
cron = "0.16"
|
|
# Observability
|
|
tracing = "0.1"
|
|
tracing-appender = "0.2"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
|
tracing-opentelemetry = { version = "0.33" }
|
|
opentelemetry = { version = "0.32", features = ["trace"] }
|
|
opentelemetry_sdk = { version = "0.32", features = ["trace", "rt-tokio"] }
|
|
opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "grpc-tonic", "tls-ring"] }
|
|
metrics = "0.24"
|
|
metrics-exporter-prometheus = "0.18"
|
|
metrics-util = "0.20"
|
|
|
|
# Error handling
|
|
thiserror = "2"
|
|
anyhow = "1"
|
|
|
|
# Utilities
|
|
uuid = { version = "1", features = ["v4", "serde"] }
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
|
|
# HTTP client (webhook delivery)
|
|
reqwest = { version = "0.13", features = ["json", "rustls"], default-features = false }
|
|
|
|
# Cryptography
|
|
sha2 = "0.11"
|
|
hex = "0.4"
|
|
hmac = "0.13"
|
|
base64 = "0.22"
|
|
|
|
# Randomness
|
|
rand = "0.10"
|
|
subtle = "2.6"
|
|
zeroize = "1.8"
|
|
|
|
# Concurrent data structures
|
|
dashmap = "6"
|
|
moka = { version = "0.12", features = ["sync"] }
|
|
|
|
# Async stream utilities
|
|
futures-util = "0.3"
|
|
|
|
# WebSocket client (test client)
|
|
tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] }
|
|
url = "2"
|
|
|
|
# Property-based testing (dev-only)
|
|
proptest = "1"
|
|
|
|
# MCP SDK (used by buzz-dev-mcp and buzz-agent)
|
|
rmcp = { version = "1.1.0", features = ["server", "transport-io", "macros"] }
|
|
schemars = { version = "1", default-features = false }
|
|
|
|
# Internal crates
|
|
buzz-core = { path = "crates/buzz-core" }
|
|
buzz-conformance = { path = "crates/buzz-conformance" }
|
|
buzz-db = { path = "crates/buzz-db" }
|
|
buzz-auth = { path = "crates/buzz-auth" }
|
|
buzz-pubsub = { path = "crates/buzz-pubsub" }
|
|
buzz-search = { path = "crates/buzz-search" }
|
|
buzz-audit = { path = "crates/buzz-audit" }
|
|
buzz-workflow = { path = "crates/buzz-workflow" }
|
|
buzz-media = { path = "crates/buzz-media" }
|
|
buzz-sdk = { path = "crates/buzz-sdk" }
|
|
buzz-ws-client = { path = "crates/buzz-ws-client" }
|
|
buzz-relay-mesh = { path = "crates/buzz-relay-mesh" }
|
|
|
|
# CI profile — builds the relay for desktop e2e. Dependencies keep full
|
|
# release optimization (warm from main's cache; they carry the runtime hot
|
|
# path: tokio/sqlx/axum). Workspace crates build at opt-level 1 — enough for
|
|
# stable e2e timing (PR #307 flakiness was opt-0 + debug-assertions) at
|
|
# roughly half the codegen cost. `incremental` is irrelevant in CI:
|
|
# rust-cache exports CARGO_INCREMENTAL=0 and never caches member artifacts.
|
|
[profile.ci]
|
|
inherits = "release"
|
|
lto = false
|
|
opt-level = 1
|
|
|
|
[profile.ci.package."*"]
|
|
opt-level = 3
|
|
|
|
# Sprig profile — optimized for deploy-anywhere Sprig release artifacts.
|
|
# Sprig is distributed over the network and installed on fresh hosts, so binary
|
|
# size matters more than compile speed here. Keep this separate from the normal
|
|
# `release` profile so desktop/dev release builds do not inherit the slower
|
|
# size-focused settings unless they opt in explicitly.
|
|
[profile.sprig]
|
|
inherits = "release"
|
|
opt-level = "z"
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
panic = "abort"
|
|
strip = true
|
|
|
|
# Temporary fork pin: aws-creds 0.39.1 (via rust-s3) cannot read EKS Pod Identity
|
|
# credentials (AWS_CONTAINER_CREDENTIALS_FULL_URI + AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE),
|
|
# which the relay pod on bb-block requires for S3 media + git storage. This pins
|
|
# aws-creds to a fork that adopts the aws-creds portion of durch/rust-s3#449
|
|
# (FULL_URI + token-file + Authorization header, refresh-safe, with a loopback
|
|
# allowlist for the auth token). Revert to crates.io once #449 lands upstream.
|
|
[patch.crates-io]
|
|
aws-creds = { git = "https://github.com/tlongwell-block/rust-s3", rev = "c9fce3620dd434c1f810101d672cf384268dbb0f" }
|