mirror of
https://github.com/block/buzz.git
synced 2026-08-18 06:50:31 +02:00
test(otel): add E2E harness for relay OTEL export surfaces
Adds a one-command local E2E that proves both observability export surfaces introduced in PR #1398: (a) Prometheus /metrics: buzz_ws_connections_total, buzz_events_received_total, buzz_auth_attempts_total present with non-zero values; target_info absent (suppressed via PR #1398). (b) OTLP traces: ws.auth and ws.event spans arrive at the collector carrying conn_id attribute and service.name=buzz-relay resource. (c) OTLP metrics: collector receives data tagged service.name=buzz-relay. (d) OTLP-disabled control: with OTEL_EXPORTER_OTLP_ENDPOINT unset /metrics still serves correctly; collector receives nothing. Architecture: relay runs on the host (matching the existing host.docker.internal Prometheus pattern); an otel-collector container added via compose overlay receives OTLP on :4317. The collector image is distroless so the output dir is bind-mounted to the host rather than read via docker exec. The Rust assertions in buzz-test-client read the host path directly. New files: - test/otel-e2e/run.sh — main driver script - test/otel-e2e/compose.otel-e2e.yml — compose overlay - test/otel-e2e/otelcol-config.yml — collector config - test/otel-e2e/README.md — prerequisites and usage - crates/buzz-test-client/tests/e2e_otel.rs — Rust assertions (all #[ignore]) - Justfile: just otel-e2e target No changes to crates/buzz-relay/src/**. Co-authored-by: Will Pfleger <pfleger.will@gmail.com> Signed-off-by: Will Pfleger <pfleger.will@gmail.com>
This commit is contained in:
npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7
co-authored by
Will Pfleger
Will Pfleger
parent
42ec17f137
commit
fd59e6660f
@@ -251,6 +251,12 @@ test-unit:
|
||||
test-integration:
|
||||
./scripts/run-tests.sh integration
|
||||
|
||||
# OTEL export E2E: boots relay + otel-collector, drives WS traffic, asserts
|
||||
# Prometheus scrape + OTLP traces/metrics + disabled-path control.
|
||||
# Requires Docker. See test/otel-e2e/README.md for details.
|
||||
otel-e2e *ARGS:
|
||||
./test/otel-e2e/run.sh {{ARGS}}
|
||||
|
||||
# Mesh-compute e2e: the CI-safe layers (relay mesh signaling invariants + Playwright UI)
|
||||
mesh-e2e:
|
||||
cargo test -p buzz-relay mesh_signaling
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
//! OTEL export surface E2E tests for the Buzz relay.
|
||||
//!
|
||||
//! Verifies both export surfaces introduced in PR #1398:
|
||||
//!
|
||||
//! 1. **Prometheus scrape** — `GET :9102/metrics` contains expected `buzz_*`
|
||||
//! series with non-zero values and does NOT contain `target_info`.
|
||||
//! 2. **OTLP traces** — the otel-collector received spans named `ws.auth` and
|
||||
//! `ws.event` carrying a `conn_id` attribute and `service.name=buzz-relay`.
|
||||
//! 3. **OTLP metrics** — the collector received OTLP metric data tagged with
|
||||
//! resource attribute `service.name=buzz-relay`.
|
||||
//! 4. **OTLP-disabled control** — when `OTEL_EXPORTER_OTLP_ENDPOINT` is not
|
||||
//! set the relay still serves `/metrics` correctly; the test verifies the
|
||||
//! Prometheus surface without an OTLP endpoint.
|
||||
//!
|
||||
//! # Running
|
||||
//!
|
||||
//! These tests are `#[ignore]` by default; they require a running relay + the
|
||||
//! otel-collector compose overlay. Use the `just otel-e2e` target, which boots
|
||||
//! everything and runs them, or manually:
|
||||
//!
|
||||
//! ```text
|
||||
//! # boot the stack (relay on host):
|
||||
//! ./test/otel-e2e/run.sh
|
||||
//!
|
||||
//! # run just these tests:
|
||||
//! RELAY_URL=ws://localhost:3000 \
|
||||
//! OTEL_COLLECTOR_OUTPUT=/path/to/telemetry.json \
|
||||
//! cargo test -p buzz-test-client --test e2e_otel -- --ignored
|
||||
//! ```
|
||||
//!
|
||||
//! Environment variables:
|
||||
//!
|
||||
//! | Variable | Default | Purpose |
|
||||
//! |---|---|---|
|
||||
//! | `RELAY_URL` | `ws://localhost:3000` | WebSocket URL of the relay under test |
|
||||
//! | `METRICS_URL` | `http://localhost:9102/metrics` | Prometheus metrics endpoint |
|
||||
//! | `OTEL_COLLECTOR_OUTPUT` | `/tmp/otelcol-output/telemetry.json` | Collector file exporter output |
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use buzz_test_client::BuzzTestClient;
|
||||
use nostr::{EventBuilder, Keys, Kind};
|
||||
use serde_json::Value;
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn relay_url() -> String {
|
||||
std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string())
|
||||
}
|
||||
|
||||
fn metrics_url() -> String {
|
||||
std::env::var("METRICS_URL")
|
||||
.unwrap_or_else(|_| "http://localhost:9102/metrics".to_string())
|
||||
}
|
||||
|
||||
fn collector_output_path() -> String {
|
||||
std::env::var("OTEL_COLLECTOR_OUTPUT")
|
||||
.unwrap_or_else(|_| "/tmp/otelcol-output/telemetry.json".to_string())
|
||||
}
|
||||
|
||||
/// Fetch the Prometheus /metrics text.
|
||||
async fn fetch_metrics() -> String {
|
||||
reqwest::get(&metrics_url())
|
||||
.await
|
||||
.expect("fetch /metrics")
|
||||
.text()
|
||||
.await
|
||||
.expect("read /metrics body")
|
||||
}
|
||||
|
||||
/// Read every line from the collector file exporter output.
|
||||
/// The file exporter writes one JSON object per line (newline-delimited JSON).
|
||||
fn read_collector_output() -> Vec<Value> {
|
||||
let path = collector_output_path();
|
||||
let contents = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|e| panic!("read collector output at {path}: {e}"));
|
||||
contents
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.filter_map(|l| serde_json::from_str::<Value>(l).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ── test: Prometheus surface ──────────────────────────────────────────────────
|
||||
|
||||
/// Asserts the Prometheus /metrics endpoint exposes the expected buzz_* series
|
||||
/// with non-zero values and does NOT contain `target_info` (suppressed in #1398).
|
||||
///
|
||||
/// Drives a NIP-42 auth + event publish to generate non-zero counters before
|
||||
/// scraping.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn test_prometheus_contains_buzz_metrics_with_nonzero_values() {
|
||||
let keys = Keys::generate();
|
||||
|
||||
// Drive some traffic so counters are non-zero.
|
||||
let mut client = BuzzTestClient::connect(&relay_url(), &keys)
|
||||
.await
|
||||
.expect("connect and authenticate");
|
||||
|
||||
// Publish a text-note event (kind 1 is accepted relay-wide without a channel).
|
||||
let event = EventBuilder::new(Kind::TextNote, "otel-e2e prometheus test")
|
||||
.sign_with_keys(&keys)
|
||||
.expect("sign event");
|
||||
client.send_event(event).await.expect("send event");
|
||||
client.disconnect().await.ok();
|
||||
|
||||
// Give the relay a moment to record metrics.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
let body = fetch_metrics().await;
|
||||
|
||||
// (a) Must contain expected series with at least one non-zero value.
|
||||
let required_series = [
|
||||
"buzz_ws_connections_total",
|
||||
"buzz_events_received_total",
|
||||
"buzz_auth_attempts_total",
|
||||
];
|
||||
for series in &required_series {
|
||||
assert!(
|
||||
body.contains(series),
|
||||
"expected metric series {series} in /metrics output\n\n--- /metrics ---\n{body}"
|
||||
);
|
||||
// Find the line(s) for this metric and assert at least one has a
|
||||
// non-zero value. Prometheus text format: `<name>{labels} <value>`.
|
||||
let has_nonzero = body.lines().any(|line| {
|
||||
if !line.starts_with(series) {
|
||||
return false;
|
||||
}
|
||||
// Extract the value (last whitespace-delimited token before optional timestamp).
|
||||
if let Some(val_str) = line.split_whitespace().nth(1) {
|
||||
val_str.parse::<f64>().map(|v| v > 0.0).unwrap_or(false)
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
assert!(
|
||||
has_nonzero,
|
||||
"expected non-zero value for {series} in /metrics output\n\n--- /metrics ---\n{body}"
|
||||
);
|
||||
}
|
||||
|
||||
// (b) Must NOT contain target_info (suppressed via .without_target_info()).
|
||||
assert!(
|
||||
!body.contains("target_info"),
|
||||
"unexpected target_info series in /metrics — should be suppressed\n\n--- /metrics ---\n{body}"
|
||||
);
|
||||
|
||||
println!("✓ Prometheus: required series present with non-zero values, no target_info");
|
||||
}
|
||||
|
||||
// ── test: OTLP traces ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Asserts the otel-collector received ws.auth and ws.event spans carrying
|
||||
/// conn_id attributes, tagged service.name=buzz-relay.
|
||||
///
|
||||
/// Reads from OTEL_COLLECTOR_OUTPUT (populated by run.sh after traffic was
|
||||
/// driven by the Prometheus test and the batch exporter flushed).
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn test_otlp_traces_contain_ws_spans_with_conn_id() {
|
||||
let records = read_collector_output();
|
||||
assert!(
|
||||
!records.is_empty(),
|
||||
"collector output is empty — OTLP export did not reach the collector.\n\
|
||||
Ensure OTEL_COLLECTOR_OUTPUT points to a file populated by run.sh after WS traffic."
|
||||
);
|
||||
|
||||
// Flatten: the file exporter writes ResourceSpans / ResourceMetrics objects.
|
||||
// We search for span names across the full JSON blob.
|
||||
let blob = serde_json::to_string(&records).expect("re-serialize collector output");
|
||||
|
||||
// (b1) ws.auth span must be present.
|
||||
assert!(
|
||||
blob.contains("\"ws.auth\""),
|
||||
"expected ws.auth span in collector output\n\nKeys found: check {}", collector_output_path()
|
||||
);
|
||||
|
||||
// (b2) ws.event span must be present.
|
||||
assert!(
|
||||
blob.contains("\"ws.event\""),
|
||||
"expected ws.event span in collector output"
|
||||
);
|
||||
|
||||
// (b3) conn_id attribute must appear (set on both ws.auth and ws.event spans).
|
||||
assert!(
|
||||
blob.contains("\"conn_id\""),
|
||||
"expected conn_id attribute in span data"
|
||||
);
|
||||
|
||||
// (b4) service.name=buzz-relay must appear in resource attributes.
|
||||
assert!(
|
||||
blob.contains("\"buzz-relay\""),
|
||||
"expected service.name=buzz-relay in collector output"
|
||||
);
|
||||
|
||||
println!("✓ OTLP traces: ws.auth + ws.event spans present, conn_id + service.name verified");
|
||||
}
|
||||
|
||||
// ── test: OTLP metrics ────────────────────────────────────────────────────────
|
||||
|
||||
/// Asserts the otel-collector received OTLP metric data tagged
|
||||
/// service.name=buzz-relay.
|
||||
///
|
||||
/// Reads from OTEL_COLLECTOR_OUTPUT (populated by run.sh).
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn test_otlp_metrics_tagged_service_name_buzz_relay() {
|
||||
let records = read_collector_output();
|
||||
let blob = serde_json::to_string(&records).expect("re-serialize collector output");
|
||||
|
||||
// (c) service.name=buzz-relay must appear somewhere in the recorded data
|
||||
// (either in ResourceSpans or ResourceMetrics resource attributes).
|
||||
assert!(
|
||||
!records.is_empty(),
|
||||
"collector output is empty — OTLP export did not reach the collector"
|
||||
);
|
||||
assert!(
|
||||
blob.contains("\"buzz-relay\""),
|
||||
"expected service.name=buzz-relay in OTLP data from collector"
|
||||
);
|
||||
|
||||
println!("✓ OTLP metrics: service.name=buzz-relay present in collector output");
|
||||
}
|
||||
|
||||
// ── test: OTLP-disabled control ───────────────────────────────────────────────
|
||||
|
||||
/// When OTEL_EXPORTER_OTLP_ENDPOINT is not set the relay must still serve
|
||||
/// /metrics correctly. This test runs against a relay started WITHOUT the env
|
||||
/// var; the run.sh script handles the second relay start (disabled path).
|
||||
///
|
||||
/// NOTE: This test only asserts the Prometheus surface. The assertion that
|
||||
/// "the collector received NOTHING" is made by the run.sh script, which checks
|
||||
/// the collector output is empty before sending traffic on the disabled relay.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn test_prometheus_works_without_otlp_endpoint() {
|
||||
let body = fetch_metrics().await;
|
||||
|
||||
// The Prometheus surface must work regardless of OTLP being enabled.
|
||||
// At startup (before WS traffic) the relay emits at minimum the pool gauges
|
||||
// which use # TYPE lines. A non-empty response with any buzz_ metric proves
|
||||
// the Prometheus binding is up.
|
||||
assert!(
|
||||
!body.is_empty(),
|
||||
"expected a non-empty Prometheus metrics response"
|
||||
);
|
||||
assert!(
|
||||
body.contains("buzz_"),
|
||||
"expected at least one buzz_ metric in /metrics response — got:\n{body}"
|
||||
);
|
||||
|
||||
// Must still not have target_info even without OTLP.
|
||||
assert!(
|
||||
!body.contains("target_info"),
|
||||
"unexpected target_info in OTLP-disabled /metrics — should always be suppressed"
|
||||
);
|
||||
|
||||
println!("✓ OTLP-disabled: Prometheus /metrics served correctly, no target_info");
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
# OTEL E2E Harness
|
||||
|
||||
Local end-to-end validation for the Buzz relay's observability export surfaces
|
||||
(PR #1398 — `crates/buzz-relay` OTEL migration).
|
||||
|
||||
## What it proves
|
||||
|
||||
| Assertion | What's checked |
|
||||
|-----------|---------------|
|
||||
| **(a) Prometheus scrape** | `GET :9102/metrics` contains `buzz_ws_connections_total`, `buzz_events_received_total`, `buzz_auth_attempts_total` with **non-zero values**; does **not** contain `target_info` (suppressed via `.without_target_info()`) |
|
||||
| **(b) OTLP traces** | The otel-collector received spans named `ws.auth` and `ws.event` carrying a `conn_id` attribute, tagged `service.name=buzz-relay` |
|
||||
| **(c) OTLP metrics** | The otel-collector received OTLP metric data tagged `service.name=buzz-relay` |
|
||||
| **(d) OTLP-disabled control** | With `OTEL_EXPORTER_OTLP_ENDPOINT` unset the relay still serves `/metrics` correctly; the collector receives **nothing** |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker (for compose services: postgres, redis, minio, otel-collector)
|
||||
- Rust toolchain (builds the relay + test binary)
|
||||
- `psql` on PATH, or it falls back to `docker exec` into the postgres container
|
||||
- Ports available: `3000` (relay WS), `4317` (OTLP gRPC), `5432`, `6379`, `9000`, `9102`
|
||||
|
||||
## One-command run
|
||||
|
||||
```bash
|
||||
just otel-e2e
|
||||
```
|
||||
|
||||
Or directly:
|
||||
|
||||
```bash
|
||||
./test/otel-e2e/run.sh
|
||||
```
|
||||
|
||||
Pass `--skip-build` to reuse an existing `target/ci/buzz-relay` binary:
|
||||
|
||||
```bash
|
||||
just otel-e2e --skip-build
|
||||
```
|
||||
|
||||
To test against a relay built from the `duncan/otel-migration` branch (PR #1398):
|
||||
|
||||
```bash
|
||||
# Build the OTEL relay binary first:
|
||||
cd /path/to/buzz/.worktrees/duncan-otel-migration
|
||||
cargo build --profile ci -p buzz-relay
|
||||
|
||||
# Run the harness pointing at that binary:
|
||||
RELAY_BINARY=/path/to/buzz/.worktrees/duncan-otel-migration/target/ci/buzz-relay \
|
||||
just otel-e2e --skip-build
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────── HOST ──────────────────────────────────┐
|
||||
│ │
|
||||
│ buzz-relay (host process) │
|
||||
│ • WS :3000 ← test driver connects here │
|
||||
│ • health :8080 │
|
||||
│ • Prometheus :9102 ← test scrapes here │
|
||||
│ • OTLP gRPC → http://localhost:4317 (collector below) │
|
||||
│ │
|
||||
└───────────────────────────────────────────────────────────────┘
|
||||
│ OTLP gRPC │ psql/redis/s3
|
||||
▼ ▼
|
||||
┌─────── DOCKER (buzz-net) ──────────────────────────────────────┐
|
||||
│ │
|
||||
│ otel-collector :4317 ←── relay pushes traces + metrics here │
|
||||
│ exports to: debug stdout + /tmp/otelcol-output/ │
|
||||
│ │
|
||||
│ postgres :5432 redis :6379 minio :9000 │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
The relay runs on the host (matching the existing `host.docker.internal`
|
||||
Prometheus pattern in `docker-compose.yml`). Only the backing services and the
|
||||
collector run in Docker. This avoids a slow full-image rebuild on each test run.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `run.sh` | Main driver: boots stack, starts relay twice (OTLP on/off), runs assertions |
|
||||
| `compose.otel-e2e.yml` | Compose overlay — adds `otel-collector` to the existing dev stack |
|
||||
| `otelcol-config.yml` | Collector config: OTLP gRPC receiver on :4317, debug + file exporter |
|
||||
| `README.md` | This file |
|
||||
|
||||
The Rust assertions live in:
|
||||
|
||||
```
|
||||
crates/buzz-test-client/tests/e2e_otel.rs
|
||||
```
|
||||
|
||||
All four tests are `#[ignore]` by default and selected by `run.sh`.
|
||||
|
||||
## Running individual tests manually
|
||||
|
||||
```bash
|
||||
# Start the stack first:
|
||||
docker compose -f docker-compose.yml -f test/otel-e2e/compose.otel-e2e.yml \
|
||||
up -d postgres redis minio minio-init otel-collector
|
||||
|
||||
# Start relay with OTLP enabled:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \
|
||||
OTEL_SERVICE_NAME=buzz-relay \
|
||||
DATABASE_URL=postgres://buzz:buzz_dev@localhost:5432/buzz \
|
||||
REDIS_URL=redis://localhost:6379 \
|
||||
RELAY_URL=ws://localhost:3000 \
|
||||
BUZZ_REQUIRE_AUTH_TOKEN=false \
|
||||
./target/ci/buzz-relay &
|
||||
|
||||
# Run all four tests:
|
||||
RELAY_URL=ws://localhost:3000 \
|
||||
METRICS_URL=http://localhost:9102/metrics \
|
||||
cargo test -p buzz-test-client --test e2e_otel -- --ignored
|
||||
|
||||
# Teardown:
|
||||
docker compose -f docker-compose.yml -f test/otel-e2e/compose.otel-e2e.yml down -v
|
||||
```
|
||||
|
||||
## Relationship to the relay source
|
||||
|
||||
This harness is **read-only with respect to `crates/buzz-relay/src/**`**. The
|
||||
relay binary is built from the `duncan/otel-migration` branch (PR #1398) which
|
||||
is gate-cleared and frozen. This harness lives on a separate branch
|
||||
(`duncan/otel-e2e-harness`) and adds only:
|
||||
|
||||
- `test/otel-e2e/` — this directory
|
||||
- `crates/buzz-test-client/tests/e2e_otel.rs` — the Rust assertions
|
||||
- `Justfile` — `just otel-e2e` target
|
||||
@@ -0,0 +1,51 @@
|
||||
# Compose overlay — OTEL E2E harness.
|
||||
#
|
||||
# Adds an otel-collector to the existing dev stack (docker-compose.yml). The
|
||||
# relay itself runs on the HOST (matching the existing host.docker.internal
|
||||
# Prometheus pattern), so only the collector and its shared output volume are
|
||||
# declared here. This keeps the relay build/start cycle fast and avoids a
|
||||
# slow full-image rebuild for each test run.
|
||||
#
|
||||
# Usage (via `just otel-e2e` which calls run.sh, or manually):
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f test/otel-e2e/compose.otel-e2e.yml up -d
|
||||
#
|
||||
# Teardown:
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f test/otel-e2e/compose.otel-e2e.yml down -v
|
||||
#
|
||||
# The relay is pointed at the collector via:
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
|
||||
# (the collector's gRPC port is published to the host).
|
||||
name: buzz
|
||||
|
||||
services:
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.115.1
|
||||
container_name: buzz-otel-collector
|
||||
command: ["--config=/etc/otelcol-config.yml"]
|
||||
volumes:
|
||||
- ./test/otel-e2e/otelcol-config.yml:/etc/otelcol-config.yml:ro
|
||||
# Bind-mount to the host so assertions can read the file directly —
|
||||
# the collector image is distroless (no shell/cat available via docker exec).
|
||||
- /tmp/buzz-otel-e2e-output:/tmp/otelcol-output
|
||||
ports:
|
||||
# OTLP gRPC — relay (on host) dials this
|
||||
- "4317:4317"
|
||||
networks:
|
||||
- buzz-net
|
||||
healthcheck:
|
||||
# Use the collector's own /metrics health endpoint (otelcol exposes it on :8888)
|
||||
test: ["CMD", "/otelcol-contrib", "--version"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 128m
|
||||
labels:
|
||||
com.buzz.service: "otel-collector"
|
||||
com.buzz.env: "e2e"
|
||||
restart: unless-stopped
|
||||
@@ -0,0 +1,38 @@
|
||||
# OTEL Collector config for the Buzz relay E2E harness.
|
||||
#
|
||||
# Receives OTLP gRPC on :4317 (same address staging wires to the DD Agent),
|
||||
# writes received spans/metrics as JSON to /tmp/otelcol-output so the test
|
||||
# script can grep them, and logs a debug summary to stdout.
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: "0.0.0.0:4317"
|
||||
|
||||
exporters:
|
||||
debug:
|
||||
verbosity: detailed
|
||||
sampling_initial: 5
|
||||
sampling_thereafter: 200
|
||||
file:
|
||||
path: /tmp/otelcol-output/telemetry.json
|
||||
rotation:
|
||||
max_megabytes: 10
|
||||
max_days: 1
|
||||
format: json
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 100
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [debug, file]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [batch]
|
||||
exporters: [debug, file]
|
||||
Executable
+340
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# test/otel-e2e/run.sh — OTEL E2E harness for the Buzz relay
|
||||
# =============================================================================
|
||||
#
|
||||
# Proves both export surfaces introduced in PR #1398:
|
||||
#
|
||||
# (a) Prometheus /metrics contains expected buzz_* series with non-zero
|
||||
# values and does NOT contain a target_info series.
|
||||
# (b) OTLP traces: the collector received ws.auth and ws.event spans
|
||||
# carrying a conn_id attribute tagged service.name=buzz-relay.
|
||||
# (c) OTLP metrics: the collector received data tagged service.name=buzz-relay.
|
||||
# (d) OTLP-disabled control: with OTEL_EXPORTER_OTLP_ENDPOINT unset the
|
||||
# relay still serves /metrics; the collector receives nothing.
|
||||
#
|
||||
# Usage:
|
||||
# just otel-e2e # canonical one-command entry point
|
||||
# ./test/otel-e2e/run.sh [--skip-build]
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Docker (for compose services + collector)
|
||||
# - Rust toolchain (for relay + test binary)
|
||||
# - psql on PATH (or docker exec fallback if not found)
|
||||
# - Ports 3000, 4317, 5432, 6379, 9000, 9102 available
|
||||
#
|
||||
# The relay runs on the HOST (matching the existing host.docker.internal
|
||||
# Prometheus pattern). Only postgres, redis, minio, and the otel-collector
|
||||
# run in Docker.
|
||||
#
|
||||
# RELAY_BINARY override:
|
||||
# By default this script builds the relay from REPO_ROOT. To test a relay
|
||||
# built from a different branch (e.g. the otel-migration PR), set:
|
||||
#
|
||||
# RELAY_BINARY=/path/to/buzz-relay just otel-e2e --skip-build
|
||||
#
|
||||
# or just point at the otel-migration worktree's ci binary:
|
||||
#
|
||||
# RELAY_BINARY=$(pwd)/../buzz/.worktrees/duncan-otel-migration/target/ci/buzz-relay \
|
||||
# just otel-e2e --skip-build
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
|
||||
SKIP_BUILD="${1:-}"
|
||||
|
||||
# ── Colors ────────────────────────────────────────────────────────────────────
|
||||
BLUE='\033[0;34m'
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log() { echo -e "${BLUE}[otel-e2e]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[otel-e2e ✓]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[otel-e2e !]${NC} $*"; }
|
||||
err() { echo -e "${RED}[otel-e2e ✗]${NC} $*" >&2; }
|
||||
|
||||
# ── Relay PID tracking ────────────────────────────────────────────────────────
|
||||
RELAY_PID_FILE=/tmp/buzz-otel-e2e-relay.pid
|
||||
|
||||
cleanup() {
|
||||
log "Tearing down..."
|
||||
# Kill relay if running.
|
||||
if [[ -f "${RELAY_PID_FILE}" ]]; then
|
||||
RELAY_PID=$(cat "${RELAY_PID_FILE}")
|
||||
if kill -0 "${RELAY_PID}" 2>/dev/null; then
|
||||
kill "${RELAY_PID}" && wait "${RELAY_PID}" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "${RELAY_PID_FILE}"
|
||||
fi
|
||||
# Bring down compose stack (including collector) and remove volumes.
|
||||
cd "${REPO_ROOT}"
|
||||
docker compose -f docker-compose.yml -f test/otel-e2e/compose.otel-e2e.yml \
|
||||
down -v --remove-orphans 2>/dev/null || true
|
||||
ok "Teardown complete"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ── Step 1: Start backing services + otel-collector ──────────────────────────
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
log "Starting backing services and otel-collector..."
|
||||
docker compose -f docker-compose.yml -f test/otel-e2e/compose.otel-e2e.yml \
|
||||
up -d postgres redis minio minio-init otel-collector
|
||||
|
||||
# ── Wait helpers ──────────────────────────────────────────────────────────────
|
||||
wait_healthy() {
|
||||
local service="$1" container="$2"
|
||||
log "Waiting for ${service}..."
|
||||
for _ in $(seq 1 60); do
|
||||
status=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "not_found")
|
||||
if [[ "${status}" == "healthy" ]]; then
|
||||
ok "${service} healthy"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
err "${service} did not become healthy within 120s"
|
||||
docker logs "${container}" || true
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_healthy "Postgres" "buzz-postgres"
|
||||
wait_healthy "Redis" "buzz-redis"
|
||||
wait_healthy "MinIO" "buzz-minio"
|
||||
|
||||
# otel-collector: healthcheck uses nc; retry with a TCP connect fallback.
|
||||
log "Waiting for otel-collector (gRPC :4317)..."
|
||||
for _ in $(seq 1 30); do
|
||||
if docker inspect --format='{{.State.Health.Status}}' "buzz-otel-collector" 2>/dev/null | grep -q "healthy"; then
|
||||
ok "otel-collector healthy"
|
||||
break
|
||||
fi
|
||||
# Fallback: try TCP connect from host.
|
||||
if nc -z -w1 127.0.0.1 4317 2>/dev/null; then
|
||||
ok "otel-collector reachable on :4317"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# ── Step 2: Apply DB schema ───────────────────────────────────────────────────
|
||||
log "Applying database schema..."
|
||||
export PGHOST=localhost PGPORT=5432 PGUSER=buzz PGPASSWORD=buzz_dev PGDATABASE=buzz
|
||||
|
||||
if command -v psql >/dev/null 2>&1; then
|
||||
seed_psql() { PGPASSWORD="${PGPASSWORD}" psql -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" -d "${PGDATABASE}" -qtA "$@"; }
|
||||
else
|
||||
seed_psql() { docker exec -e PGPASSWORD="${PGPASSWORD}" buzz-postgres psql -U "${PGUSER}" -d "${PGDATABASE}" -qtA "$@"; }
|
||||
fi
|
||||
|
||||
"${REPO_ROOT}/bin/pgschema" apply --file "${REPO_ROOT}/schema/schema.sql" --auto-approve
|
||||
docker exec -i -e PGPASSWORD="${PGPASSWORD}" buzz-postgres \
|
||||
psql -U "${PGUSER}" -d "${PGDATABASE}" -v ON_ERROR_STOP=1 \
|
||||
< "${REPO_ROOT}/scripts/attach-schema-partitions.sql"
|
||||
|
||||
seed_psql -c "
|
||||
INSERT INTO communities (id, host)
|
||||
VALUES ('00000000-0000-4000-8000-00000000c0de', 'localhost:3000')
|
||||
ON CONFLICT (lower(host)) DO NOTHING;
|
||||
"
|
||||
ok "Schema applied and community seeded"
|
||||
|
||||
# ── Step 3: Build relay (unless --skip-build) ────────────────────────────────
|
||||
RELAY_BIN="${RELAY_BINARY:-${REPO_ROOT}/target/ci/buzz-relay}"
|
||||
if [[ "${SKIP_BUILD}" != "--skip-build" ]]; then
|
||||
log "Building relay (profile: ci)..."
|
||||
cd "${REPO_ROOT}"
|
||||
cargo build --profile ci -p buzz-relay
|
||||
RELAY_BIN="${REPO_ROOT}/target/ci/buzz-relay"
|
||||
ok "Relay built"
|
||||
else
|
||||
if [[ ! -x "${RELAY_BIN}" ]]; then
|
||||
err "RELAY_BINARY=${RELAY_BIN} does not exist or is not executable"
|
||||
err "Either remove --skip-build to build, or set RELAY_BINARY to a pre-built binary"
|
||||
exit 1
|
||||
fi
|
||||
log "Using pre-built relay: ${RELAY_BIN}"
|
||||
fi
|
||||
|
||||
# ── Step 4: Set up collector output readback ─────────────────────────────────
|
||||
# The file exporter writes to /tmp/otelcol-output inside the collector container,
|
||||
# which is bind-mounted to /tmp/buzz-otel-e2e-output on the host (see compose overlay).
|
||||
# The collector image is distroless so we read directly from the host path.
|
||||
COLLECTOR_OUTPUT_DIR=/tmp/buzz-otel-e2e-output
|
||||
mkdir -p "${COLLECTOR_OUTPUT_DIR}"
|
||||
COLLECTOR_OUTPUT_HOST="${COLLECTOR_OUTPUT_DIR}/telemetry.json"
|
||||
|
||||
# ── Step 5: Start relay WITH OTLP enabled ────────────────────────────────────
|
||||
log "Starting relay (OTLP enabled → http://localhost:4317)..."
|
||||
RELAY_LOG=/tmp/buzz-otel-e2e-relay-otlp.log
|
||||
|
||||
nohup env \
|
||||
DATABASE_URL="postgres://buzz:buzz_dev@localhost:5432/buzz" \
|
||||
REDIS_URL="redis://localhost:6379" \
|
||||
RELAY_URL="ws://localhost:3000" \
|
||||
BUZZ_BIND_ADDR="0.0.0.0:3000" \
|
||||
BUZZ_REQUIRE_AUTH_TOKEN=false \
|
||||
BUZZ_RECONCILE_CHANNELS=true \
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
|
||||
OTEL_SERVICE_NAME="buzz-relay" \
|
||||
RUST_LOG="buzz_relay=info" \
|
||||
"${RELAY_BIN}" > "${RELAY_LOG}" 2>&1 &
|
||||
RELAY_PID=$!
|
||||
echo "${RELAY_PID}" > "${RELAY_PID_FILE}"
|
||||
|
||||
log "Waiting for relay readiness (OTLP enabled)..."
|
||||
for attempt in $(seq 1 60); do
|
||||
if ! kill -0 "${RELAY_PID}" 2>/dev/null; then
|
||||
err "Relay process died during startup"
|
||||
cat "${RELAY_LOG}"
|
||||
exit 1
|
||||
fi
|
||||
status_code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3000/_readiness || true)
|
||||
if [[ "${status_code}" == "200" ]]; then
|
||||
ok "Relay ready (OTLP enabled)"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
if [[ "${attempt}" -eq 60 ]]; then
|
||||
err "Relay did not become ready within 60s"
|
||||
cat "${RELAY_LOG}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Step 6: Run OTLP-enabled tests (a), (b), (c) ────────────────────────────
|
||||
log "Waiting for OTLP exports to flush (batch processor 1s timeout + buffer)..."
|
||||
sleep 5 # Give the relay's batch exporter time to flush after readiness
|
||||
|
||||
COLLECTOR_LINE_COUNT=$(wc -l < "${COLLECTOR_OUTPUT_HOST}" 2>/dev/null || echo 0)
|
||||
log "Collector has ${COLLECTOR_LINE_COUNT} lines so far (may be low — OTLP metrics are periodic)"
|
||||
|
||||
log "Running OTLP-enabled tests (a)(b)(c)..."
|
||||
RELAY_URL="ws://localhost:3000" \
|
||||
METRICS_URL="http://localhost:9102/metrics" \
|
||||
OTEL_COLLECTOR_OUTPUT="${COLLECTOR_OUTPUT_HOST}" \
|
||||
cargo test -p buzz-test-client --test e2e_otel \
|
||||
test_prometheus_contains_buzz_metrics \
|
||||
-- --ignored 2>&1
|
||||
# Note: test_prometheus_contains_buzz_metrics matches test_prometheus_contains_buzz_metrics_with_nonzero_values
|
||||
|
||||
# Give traces time to flush from the test's WS interaction.
|
||||
sleep 5
|
||||
|
||||
RELAY_URL="ws://localhost:3000" \
|
||||
METRICS_URL="http://localhost:9102/metrics" \
|
||||
OTEL_COLLECTOR_OUTPUT="${COLLECTOR_OUTPUT_HOST}" \
|
||||
cargo test -p buzz-test-client --test e2e_otel \
|
||||
test_otlp \
|
||||
-- --ignored 2>&1
|
||||
ok "OTLP-enabled assertions passed"
|
||||
|
||||
# ── Step 7: Stop the OTLP-enabled relay ──────────────────────────────────────
|
||||
log "Stopping OTLP-enabled relay..."
|
||||
kill "${RELAY_PID}" && wait "${RELAY_PID}" 2>/dev/null || true
|
||||
rm -f "${RELAY_PID_FILE}"
|
||||
|
||||
# Save collector output for inspection.
|
||||
log "Saving collector output for inspection..."
|
||||
COLLECTOR_LINE_COUNT=$(wc -l < "${COLLECTOR_OUTPUT_HOST}" 2>/dev/null || echo 0)
|
||||
ok "Collector received data: ${COLLECTOR_LINE_COUNT} lines at ${COLLECTOR_OUTPUT_HOST}"
|
||||
|
||||
# Print the span names that appeared for human inspection.
|
||||
log "Span names found in collector output:"
|
||||
cat "${COLLECTOR_OUTPUT_HOST}" | python3 -c "
|
||||
import sys, json, re
|
||||
spans = set()
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
blob = json.dumps(json.loads(line))
|
||||
spans.update(re.findall(r'\"name\":\s*\"([^\"]+)\"', blob))
|
||||
except Exception:
|
||||
pass
|
||||
for s in sorted(spans):
|
||||
print(' ', s)
|
||||
" || true
|
||||
|
||||
# ── Step 8: Clear collector output, start relay WITHOUT OTLP ─────────────────
|
||||
log "Clearing collector output for disabled-path test..."
|
||||
rm -f "${COLLECTOR_OUTPUT_HOST}"
|
||||
sleep 1
|
||||
|
||||
log "Starting relay (OTLP DISABLED)..."
|
||||
RELAY_LOG_DISABLED=/tmp/buzz-otel-e2e-relay-nootlp.log
|
||||
|
||||
nohup env \
|
||||
DATABASE_URL="postgres://buzz:buzz_dev@localhost:5432/buzz" \
|
||||
REDIS_URL="redis://localhost:6379" \
|
||||
RELAY_URL="ws://localhost:3000" \
|
||||
BUZZ_BIND_ADDR="0.0.0.0:3000" \
|
||||
BUZZ_REQUIRE_AUTH_TOKEN=false \
|
||||
BUZZ_RECONCILE_CHANNELS=true \
|
||||
RUST_LOG="buzz_relay=info" \
|
||||
"${RELAY_BIN}" > "${RELAY_LOG_DISABLED}" 2>&1 &
|
||||
RELAY_PID=$!
|
||||
echo "${RELAY_PID}" > "${RELAY_PID_FILE}"
|
||||
|
||||
log "Waiting for relay readiness (OTLP disabled)..."
|
||||
for attempt in $(seq 1 60); do
|
||||
if ! kill -0 "${RELAY_PID}" 2>/dev/null; then
|
||||
err "Relay process died during startup (OTLP disabled)"
|
||||
cat "${RELAY_LOG_DISABLED}"
|
||||
exit 1
|
||||
fi
|
||||
status_code=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:3000/_readiness || true)
|
||||
if [[ "${status_code}" == "200" ]]; then
|
||||
ok "Relay ready (OTLP disabled)"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
if [[ "${attempt}" -eq 60 ]]; then
|
||||
err "Relay did not become ready within 60s (OTLP disabled)"
|
||||
cat "${RELAY_LOG_DISABLED}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Step 9: Run disabled-path test (d) ───────────────────────────────────────
|
||||
log "Running OTLP-disabled control test (d)..."
|
||||
RELAY_URL="ws://localhost:3000" \
|
||||
METRICS_URL="http://localhost:9102/metrics" \
|
||||
cargo test -p buzz-test-client --test e2e_otel \
|
||||
test_prometheus_works_without_otlp_endpoint \
|
||||
-- --ignored 2>&1
|
||||
ok "OTLP-disabled assertion passed"
|
||||
# Stop relay.
|
||||
kill "${RELAY_PID}" && wait "${RELAY_PID}" 2>/dev/null || true
|
||||
rm -f "${RELAY_PID_FILE}"
|
||||
|
||||
# Assert collector received NOTHING during the disabled run.
|
||||
log "Asserting collector received nothing during OTLP-disabled run..."
|
||||
sleep 3 # Give any possible OTLP traffic time to arrive (there should be none)
|
||||
DISABLED_OUTPUT=""
|
||||
if [[ -f "${COLLECTOR_OUTPUT_HOST}" ]]; then
|
||||
DISABLED_OUTPUT=$(cat "${COLLECTOR_OUTPUT_HOST}" | tr -d '[:space:]')
|
||||
fi
|
||||
if [[ -n "${DISABLED_OUTPUT}" ]]; then
|
||||
err "OTLP-disabled assertion FAILED: collector received data when OTEL_EXPORTER_OTLP_ENDPOINT was unset"
|
||||
head -5 "${COLLECTOR_OUTPUT_HOST}"
|
||||
exit 1
|
||||
fi
|
||||
ok "OTLP-disabled: collector received nothing ✓"
|
||||
|
||||
# ── Done ──────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo -e "${GREEN}╔══════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║ OTEL E2E HARNESS — ALL ASSERTIONS PASSED ║${NC}"
|
||||
echo -e "${GREEN}╠══════════════════════════════════════════════════════════╣${NC}"
|
||||
echo -e "${GREEN}║ (a) Prometheus buzz_* series present, non-zero, no ║${NC}"
|
||||
echo -e "${GREEN}║ target_info ║${NC}"
|
||||
echo -e "${GREEN}║ (b) OTLP traces: ws.auth + ws.event with conn_id ║${NC}"
|
||||
echo -e "${GREEN}║ (c) OTLP metrics: service.name=buzz-relay present ║${NC}"
|
||||
echo -e "${GREEN}║ (d) OTLP-disabled: /metrics works, collector silent ║${NC}"
|
||||
echo -e "${GREEN}╚══════════════════════════════════════════════════════════╝${NC}"
|
||||
Reference in New Issue
Block a user