mirror of
https://github.com/devkitshq/notifkit.git
synced 2026-10-19 18:24:52 +02:00
feat(profiling): add containerized load, CPU scaling, and distributed budget test suite
- Provide Docker Compose test harness with CPU and memory resource limits\n- Add single-node CPU scaling benchmark suite (0.5 to 4.0 vCPU)\n- Add distributed multi-node scaling and budget benchmarks (, , /mo)\n- Support partitioned service topologies (dedicated API, pipeline worker, and delivery worker instances)\n- Implement configurable provider latency simulation with randomized network jitter\n- Support dynamic client and worker concurrency tuning
This commit is contained in:
coo1estguy
@@ -0,0 +1,24 @@
|
||||
FROM node:22-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy root build configurations, sources, packages, and prebuilt dist
|
||||
COPY package*.json tsconfig.json tsdown.config.ts ./
|
||||
COPY packages/ ./packages/
|
||||
COPY src/ ./src/
|
||||
COPY dist/ ./dist/
|
||||
|
||||
RUN npm install --ignore-scripts
|
||||
|
||||
# Setup profiling app
|
||||
WORKDIR /app/profiling
|
||||
COPY profiling/package*.json profiling/tsconfig.json ./
|
||||
RUN npm install
|
||||
|
||||
COPY profiling/src/ ./src/
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["npm", "start"]
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# Notifkit Profiling & Benchmarking Suite
|
||||
|
||||
This folder provides a realistic, container-isolated profiling and benchmarking environment for Notifkit.
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Architecture & Resource Constraints
|
||||
|
||||
The environment simulates a standard small production node (e.g. $15/mo cloud VM):
|
||||
|
||||
- **Server**: 1.0 vCPU, 1 GB RAM (`notifkit` standalone orchestrator)
|
||||
- **PostgreSQL 15**: 0.5 vCPU, 512 MB RAM
|
||||
- **Redis 7**: 0.5 vCPU, 256 MB RAM
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### 1. Install Dependencies
|
||||
|
||||
```bash
|
||||
cd profiling
|
||||
npm install
|
||||
```
|
||||
|
||||
### 2. Run the Load Test
|
||||
|
||||
```bash
|
||||
npm run test:load
|
||||
```
|
||||
|
||||
The load test will:
|
||||
|
||||
1. Boot the isolated 3-container stack (`postgres`, `redis`, `server`) via `testcontainers`.
|
||||
2. Run database schema migrations and seed project/template data.
|
||||
3. Fire 50 concurrent virtual users generating notifications against `/v1/notify`.
|
||||
4. Stream and buffer container logs until 100% of messages are delivered to the transport.
|
||||
5. Output detailed throughput, p95, and p99 percentiles for both API ingestion and background delivery.
|
||||
6. Automatically clean up and shut down the containers.
|
||||
|
||||
---
|
||||
|
||||
### 3. Run the CPU Scaling Benchmark
|
||||
|
||||
Evaluate server performance scaling from 0.5 vCPU up to 4.0 vCPUs:
|
||||
|
||||
```bash
|
||||
npm run test:scaling
|
||||
```
|
||||
|
||||
#### CLI Flags & Customization:
|
||||
|
||||
```bash
|
||||
# Rapid test (5s duration per tier)
|
||||
npm run test:scaling -- --quick
|
||||
|
||||
# Specify custom CPU tiers
|
||||
npm run test:scaling -- --cpus=0.5,1.0,2.0,4.0
|
||||
|
||||
# Customize duration and concurrency
|
||||
npm run test:scaling -- --duration=15 --concurrency=100
|
||||
```
|
||||
|
||||
The scaling benchmark produces an aggregated comparative table with:
|
||||
|
||||
- Ingestion Throughput (`req/sec`)
|
||||
- Active Delivery Throughput (`msgs/sec`)
|
||||
- Wall Drain Throughput (`msgs/sec`)
|
||||
- API & Delivery Latencies (`p50`, `p95`, `p99`)
|
||||
- Speedup vs Baseline & Diminishing Returns Analysis
|
||||
|
||||
---
|
||||
|
||||
## ➕ Adding New Profiling Tests
|
||||
|
||||
To add a new benchmark (e.g. batch delivery, quiet hours queuing, multi-channel fallback):
|
||||
|
||||
1. Create a new test file under `tests/` (e.g. `tests/batch.test.ts`).
|
||||
2. Add a corresponding script to `package.json`:
|
||||
```json
|
||||
"test:batch": "tsx tests/batch.test.ts"
|
||||
```
|
||||
3. Run `npm run test:batch`.
|
||||
@@ -0,0 +1,99 @@
|
||||
services:
|
||||
db:
|
||||
image: postgres:15-alpine
|
||||
environment:
|
||||
POSTGRES_DB: notifkit
|
||||
POSTGRES_USER: notifkit
|
||||
POSTGRES_PASSWORD: password
|
||||
ports:
|
||||
- "35432:5432"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${DB_CPUS:-1.0}"
|
||||
memory: "${DB_MEMORY:-1G}"
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "36379:6379"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${REDIS_CPUS:-1.5}"
|
||||
memory: "${REDIS_MEMORY:-1G}"
|
||||
|
||||
server1:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: profiling/Dockerfile
|
||||
environment:
|
||||
DATABASE_URL: postgres://notifkit:password@db:5432/notifkit
|
||||
REDIS_URL: redis://redis:6379
|
||||
ADMIN_API_KEY: perf-admin-key
|
||||
LOG_LEVEL: warn
|
||||
HOST: 0.0.0.0
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-100}
|
||||
DB_MAX_CONNECTIONS: ${DB_MAX_CONNECTIONS:-5}
|
||||
PROVIDER_LATENCY_MS: ${PROVIDER_LATENCY_MS:-0}
|
||||
SERVICES: ${SERVER1_SERVICES:-all}
|
||||
ports:
|
||||
- "35678:3000"
|
||||
depends_on:
|
||||
- db
|
||||
- redis
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${SERVER_CPUS:-1.0}"
|
||||
memory: "${SERVER_MEMORY:-1G}"
|
||||
|
||||
server2:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: profiling/Dockerfile
|
||||
environment:
|
||||
DATABASE_URL: postgres://notifkit:password@db:5432/notifkit
|
||||
REDIS_URL: redis://redis:6379
|
||||
ADMIN_API_KEY: perf-admin-key
|
||||
LOG_LEVEL: warn
|
||||
HOST: 0.0.0.0
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-100}
|
||||
DB_MAX_CONNECTIONS: ${DB_MAX_CONNECTIONS:-5}
|
||||
PROVIDER_LATENCY_MS: ${PROVIDER_LATENCY_MS:-0}
|
||||
SERVICES: ${SERVER2_SERVICES:-all}
|
||||
ports:
|
||||
- "35679:3000"
|
||||
depends_on:
|
||||
- db
|
||||
- redis
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${SERVER_CPUS:-1.0}"
|
||||
memory: "${SERVER_MEMORY:-1G}"
|
||||
|
||||
server3:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: profiling/Dockerfile
|
||||
environment:
|
||||
DATABASE_URL: postgres://notifkit:password@db:5432/notifkit
|
||||
REDIS_URL: redis://redis:6379
|
||||
ADMIN_API_KEY: perf-admin-key
|
||||
LOG_LEVEL: warn
|
||||
HOST: 0.0.0.0
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-100}
|
||||
DB_MAX_CONNECTIONS: ${DB_MAX_CONNECTIONS:-5}
|
||||
PROVIDER_LATENCY_MS: ${PROVIDER_LATENCY_MS:-0}
|
||||
SERVICES: ${SERVER3_SERVICES:-all}
|
||||
ports:
|
||||
- "35680:3000"
|
||||
depends_on:
|
||||
- db
|
||||
- redis
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${SERVER_CPUS:-1.0}"
|
||||
memory: "${SERVER_MEMORY:-1G}"
|
||||
@@ -0,0 +1,48 @@
|
||||
services:
|
||||
db:
|
||||
image: postgres:15-alpine
|
||||
environment:
|
||||
POSTGRES_DB: notifkit
|
||||
POSTGRES_USER: notifkit
|
||||
POSTGRES_PASSWORD: password
|
||||
ports:
|
||||
- "35432:5432"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${DB_CPUS:-0.5}"
|
||||
memory: "${DB_MEMORY:-512M}"
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "36379:6379"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${REDIS_CPUS:-0.5}"
|
||||
memory: "${REDIS_MEMORY:-256M}"
|
||||
|
||||
server:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: profiling/Dockerfile
|
||||
environment:
|
||||
DATABASE_URL: postgres://notifkit:password@db:5432/notifkit
|
||||
REDIS_URL: redis://redis:6379
|
||||
ADMIN_API_KEY: perf-admin-key
|
||||
LOG_LEVEL: warn
|
||||
HOST: 0.0.0.0
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-50}
|
||||
DB_MAX_CONNECTIONS: ${DB_MAX_CONNECTIONS:-20}
|
||||
PROVIDER_LATENCY_MS: ${PROVIDER_LATENCY_MS:-0}
|
||||
ports:
|
||||
- "35678:3000"
|
||||
depends_on:
|
||||
- db
|
||||
- redis
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${SERVER_CPUS:-1.0}"
|
||||
memory: "${SERVER_MEMORY:-1G}"
|
||||
Generated
+2720
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "notifkit-profiling",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "tsx src/server.ts",
|
||||
"test:load": "tsx tests/load.test.ts",
|
||||
"test:scaling": "tsx tests/scaling.test.ts",
|
||||
"test:distributed": "tsx tests/distributed-scaling.test.ts",
|
||||
"test:money": "tsx tests/money-scaling.test.ts",
|
||||
"test:cpu": "tsx tests/scaling.test.ts",
|
||||
"test": "tsx tests/load.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@notifkit/provider-console": "file:../packages/provider-console",
|
||||
"notifkit": "file:.."
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"testcontainers": "^10.18.0",
|
||||
"tsx": "^4.19.3",
|
||||
"typescript": "^5.8.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import {
|
||||
NotifkitServer,
|
||||
type Transport,
|
||||
type NotificationDispatchedPayload,
|
||||
type DeliveryResult,
|
||||
} from "notifkit";
|
||||
|
||||
const PORT = process.env.PORT ? parseInt(process.env.PORT, 10) : 3000;
|
||||
const workerConcurrency = process.env.WORKER_CONCURRENCY
|
||||
? parseInt(process.env.WORKER_CONCURRENCY, 10)
|
||||
: 50;
|
||||
function parseLatency(val?: string): { min: number; max: number } {
|
||||
if (!val) return { min: 0, max: 0 };
|
||||
if (val.includes("-")) {
|
||||
const [minStr, maxStr] = val.split("-");
|
||||
const min = parseInt(minStr!, 10) || 0;
|
||||
const max = parseInt(maxStr!, 10) || min;
|
||||
return { min, max };
|
||||
}
|
||||
const exact = parseInt(val, 10) || 0;
|
||||
return { min: exact, max: exact };
|
||||
}
|
||||
|
||||
const latencyRange = parseLatency(process.env.PROVIDER_LATENCY_MS);
|
||||
|
||||
class ProfilingTransport implements Transport {
|
||||
readonly channel = "email";
|
||||
readonly limits = { limit: 1_000_000, windowSeconds: 1 };
|
||||
|
||||
async send(task: NotificationDispatchedPayload): Promise<DeliveryResult> {
|
||||
if (latencyRange.max > 0) {
|
||||
const delay =
|
||||
latencyRange.min === latencyRange.max
|
||||
? latencyRange.min
|
||||
: Math.floor(Math.random() * (latencyRange.max - latencyRange.min + 1)) +
|
||||
latencyRange.min;
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
|
||||
const providerMessageId = `prof-${Date.now()}`;
|
||||
const requestTime = (task.templateVariables as Record<string, unknown> | undefined)
|
||||
?.requestTime;
|
||||
const latency = typeof requestTime === "number" ? Date.now() - requestTime : 0;
|
||||
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
taskId: task.taskId,
|
||||
channel: this.channel,
|
||||
destination: task.destination,
|
||||
providerMessageId,
|
||||
latency,
|
||||
msg: "push delivered (console transport)",
|
||||
}),
|
||||
);
|
||||
|
||||
return { success: true, providerMessageId };
|
||||
}
|
||||
}
|
||||
|
||||
import http from "node:http";
|
||||
|
||||
const rawServices = process.env.SERVICES?.trim();
|
||||
const services: (
|
||||
"api" | "enricher" | "engine" | "delivery" | "scheduler" | "ai" | "workflow" | "events"
|
||||
)[] =
|
||||
rawServices && rawServices !== "all"
|
||||
? (rawServices.split(",").map((s) => s.trim()) as (
|
||||
"api" | "enricher" | "engine" | "delivery" | "scheduler"
|
||||
)[])
|
||||
: ["api", "enricher", "engine", "delivery", "scheduler"];
|
||||
|
||||
const server = new NotifkitServer({
|
||||
services,
|
||||
redisUrl: process.env.REDIS_URL || "redis://localhost:6379",
|
||||
databaseUrl: process.env.DATABASE_URL || "postgres://notifkit:password@localhost:5432/notifkit",
|
||||
logLevel: (process.env.LOG_LEVEL as "info" | "warn" | "error" | "debug") || "info",
|
||||
autoMigrate: false,
|
||||
port: PORT,
|
||||
workerConcurrency,
|
||||
providers: [new ProfilingTransport()],
|
||||
});
|
||||
|
||||
server
|
||||
.start()
|
||||
.then(() => {
|
||||
console.log(
|
||||
`🚀 Profiling Server started on port ${PORT} [services: ${services.join(",")}] (concurrency: ${workerConcurrency})`,
|
||||
);
|
||||
|
||||
// If API is not enabled on this worker node, expose a lightweight health responder so orchestration succeeds
|
||||
if (!services.includes("api")) {
|
||||
const healthServer = http.createServer((req, res) => {
|
||||
if (req.url === "/health") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ status: "ok", services, isWorkerOnly: true }));
|
||||
} else {
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
}
|
||||
});
|
||||
healthServer.listen(PORT, "0.0.0.0");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error("Failed to start profiling server:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,376 @@
|
||||
import { DockerComposeEnvironment } from "testcontainers";
|
||||
import { createDatabase, runMigrations } from "notifkit";
|
||||
import { setGlobalDispatcher, Agent } from "undici";
|
||||
|
||||
// Configure high-concurrency connection pool for client benchmark
|
||||
setGlobalDispatcher(
|
||||
new Agent({
|
||||
connections: 300,
|
||||
pipelining: 1,
|
||||
keepAliveTimeout: 30000,
|
||||
keepAliveMaxTimeout: 60000,
|
||||
}),
|
||||
);
|
||||
|
||||
export interface DistributedLoadTestOptions {
|
||||
serverCount?: number;
|
||||
durationSeconds?: number;
|
||||
concurrency?: number;
|
||||
adminApiKey?: string;
|
||||
dbPort?: number;
|
||||
serverCpus?: string;
|
||||
serverMemory?: string;
|
||||
workerConcurrency?: number;
|
||||
dbCpus?: string;
|
||||
dbMemory?: string;
|
||||
redisCpus?: string;
|
||||
redisMemory?: string;
|
||||
dbMaxConnections?: number;
|
||||
providerLatencyMs?: number | string;
|
||||
serverServices?: string[];
|
||||
quiet?: boolean;
|
||||
}
|
||||
|
||||
export interface DistributedBenchmarkResult {
|
||||
serverCount: number;
|
||||
serverCpus: string;
|
||||
serverMemory: string;
|
||||
workerConcurrency: number;
|
||||
durationSec: number;
|
||||
concurrency: number;
|
||||
requestsSent: number;
|
||||
successCount: number;
|
||||
failCount: number;
|
||||
ingestionThroughput: number;
|
||||
firstDelivOffset: number;
|
||||
lastDelivOffset: number;
|
||||
deliverySpanSec: number;
|
||||
activeDeliveryRate: number;
|
||||
totalDeliveryRate: number;
|
||||
apiP50: number;
|
||||
apiP95: number;
|
||||
apiP99: number;
|
||||
deliveryP50: number;
|
||||
deliveryP95: number;
|
||||
deliveryP99: number;
|
||||
totalDelivered: number;
|
||||
}
|
||||
|
||||
export async function runDistributedBenchmark(
|
||||
options: DistributedLoadTestOptions = {},
|
||||
): Promise<DistributedBenchmarkResult> {
|
||||
const serverCount = options.serverCount ?? 2;
|
||||
const durationSec = options.durationSeconds ?? 10;
|
||||
const concurrency = options.concurrency ?? serverCount * 75;
|
||||
const adminApiKey = options.adminApiKey ?? "perf-admin-key";
|
||||
const dbPort = options.dbPort ?? 35432;
|
||||
const serverCpus = options.serverCpus ?? "1.0";
|
||||
const serverMemory = options.serverMemory ?? "1G";
|
||||
const workerConcurrency = options.workerConcurrency ?? 100;
|
||||
const providerLatency = options.providerLatencyMs ?? process.env.PROVIDER_LATENCY_MS ?? "200-300";
|
||||
|
||||
const composeEnv: Record<string, string> = {
|
||||
SERVER_CPUS: serverCpus,
|
||||
SERVER_MEMORY: serverMemory,
|
||||
WORKER_CONCURRENCY: String(workerConcurrency),
|
||||
PROVIDER_LATENCY_MS: String(providerLatency),
|
||||
};
|
||||
if (options.dbCpus) composeEnv.DB_CPUS = options.dbCpus;
|
||||
if (options.dbMemory) composeEnv.DB_MEMORY = options.dbMemory;
|
||||
if (options.redisCpus) composeEnv.REDIS_CPUS = options.redisCpus;
|
||||
if (options.redisMemory) composeEnv.REDIS_MEMORY = options.redisMemory;
|
||||
if (options.dbMaxConnections) composeEnv.DB_MAX_CONNECTIONS = String(options.dbMaxConnections);
|
||||
|
||||
if (options.serverServices) {
|
||||
options.serverServices.forEach((svc, idx) => {
|
||||
composeEnv[`SERVER${idx + 1}_SERVICES`] = svc;
|
||||
});
|
||||
}
|
||||
|
||||
const services = [
|
||||
"db",
|
||||
"redis",
|
||||
...Array.from({ length: serverCount }, (_, i) => `server${i + 1}`),
|
||||
];
|
||||
|
||||
console.log(
|
||||
`🚀 Booting distributed profiling environment (${serverCount} Server Instances: ${serverCpus} vCPU each, ${serverMemory} RAM | DB: ${composeEnv.DB_CPUS ?? "2.0"} vCPU | Redis: ${composeEnv.REDIS_CPUS ?? "1.5"} vCPU | Provider Latency: ${providerLatency}ms)...`,
|
||||
);
|
||||
|
||||
const environment = await new DockerComposeEnvironment(".", "docker-compose.distributed.yml")
|
||||
.withEnvironment(composeEnv)
|
||||
.withBuild()
|
||||
.up(services);
|
||||
|
||||
const serverPorts = Array.from({ length: serverCount }, (_, i) => 35678 + i);
|
||||
const serverUrls = serverPorts.map((p) => `http://localhost:${p}`);
|
||||
const dbUrl = `postgres://notifkit:password@localhost:${dbPort}/notifkit`;
|
||||
|
||||
const apiLatencies: number[] = [];
|
||||
const deliveryLatencies: number[] = [];
|
||||
|
||||
console.log(`⏳ Waiting for ${serverCount} server instances health checks...`);
|
||||
const apiUrls: string[] = [];
|
||||
for (const url of serverUrls) {
|
||||
let ready = false;
|
||||
while (!ready) {
|
||||
try {
|
||||
const res = await fetch(`${url}/health`);
|
||||
if (res.ok) {
|
||||
ready = true;
|
||||
const body = (await res.json().catch(() => ({}))) as { isWorkerOnly?: boolean };
|
||||
if (!body.isWorkerOnly) {
|
||||
apiUrls.push(url);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const targetApiUrls = apiUrls.length > 0 ? apiUrls : [serverUrls[0]!];
|
||||
|
||||
let firstDeliveryTimestamp: number | null = null;
|
||||
let lastDeliveryTimestamp: number | null = null;
|
||||
|
||||
// Hook into container logs of all active servers
|
||||
for (let i = 1; i <= serverCount; i++) {
|
||||
const serverContainer = environment.getContainer(`server${i}-1`);
|
||||
const stream = await serverContainer.logs();
|
||||
let logBuffer = "";
|
||||
stream.on("data", (chunk) => {
|
||||
logBuffer += chunk.toString();
|
||||
const lines = logBuffer.split("\n");
|
||||
logBuffer = lines.pop() || "";
|
||||
for (const line of lines) {
|
||||
if (line.includes("push delivered (console transport)")) {
|
||||
const now = Date.now();
|
||||
if (firstDeliveryTimestamp === null) {
|
||||
firstDeliveryTimestamp = now;
|
||||
}
|
||||
lastDeliveryTimestamp = now;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
deliveryLatencies.push(typeof parsed.latency === "number" ? parsed.latency : 0);
|
||||
} catch {
|
||||
deliveryLatencies.push(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
console.log("🛠️ Running database migrations...");
|
||||
const dbData = createDatabase({ url: dbUrl });
|
||||
await runMigrations(dbData.db);
|
||||
|
||||
console.log("🏗️ Provisioning benchmark project and templates...");
|
||||
const projectRes = await fetch(`${targetApiUrls[0]}/v1/projects`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${adminApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ name: "Profiling Project" }),
|
||||
});
|
||||
const project = await projectRes.json();
|
||||
const projectApiKey = project.apiKey;
|
||||
|
||||
await dbData.sql`UPDATE projects SET rate_limit_rpm = 10000000`;
|
||||
|
||||
await fetch(`${targetApiUrls[0]}/v1/templates`, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
Authorization: `Bearer ${projectApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
templates: [
|
||||
{
|
||||
id: "perf-email",
|
||||
channel: "email",
|
||||
topic: ["transactional"],
|
||||
content: {
|
||||
subject: "Perf Test Email",
|
||||
html: "<p>Hello {{name}}</p>",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
await fetch(`${targetApiUrls[0]}/v1/users`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${projectApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
id: "perf-user-1",
|
||||
email: ["perf@example.com"],
|
||||
}),
|
||||
});
|
||||
|
||||
console.log(
|
||||
`\n🔥 Starting Distributed Load Generation: ${concurrency} virtual users across ${targetApiUrls.length} API ingress node(s) for ${durationSec}s`,
|
||||
);
|
||||
|
||||
let requestsSent = 0;
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
|
||||
const startTime = Date.now();
|
||||
const endTime = startTime + durationSec * 1000;
|
||||
|
||||
async function worker(workerIndex: number) {
|
||||
const targetUrl = targetApiUrls[workerIndex % targetApiUrls.length]!;
|
||||
while (Date.now() < endTime) {
|
||||
requestsSent++;
|
||||
try {
|
||||
const reqStart = Date.now();
|
||||
const res = await fetch(`${targetUrl}/v1/notify`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${projectApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
user: "perf-user-1",
|
||||
template: "perf-email",
|
||||
channels: ["email"],
|
||||
priority: "critical",
|
||||
data: { name: "Load Tester", requestTime: reqStart },
|
||||
}),
|
||||
});
|
||||
|
||||
apiLatencies.push(Date.now() - reqStart);
|
||||
|
||||
if (res.status === 202) {
|
||||
successCount++;
|
||||
} else {
|
||||
failCount++;
|
||||
}
|
||||
} catch {
|
||||
failCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: concurrency }).map((_, i) => worker(i));
|
||||
await Promise.all(workers);
|
||||
|
||||
const activeWindowSec = durationSec;
|
||||
const ingestionThroughput = (requestsSent / activeWindowSec).toFixed(2);
|
||||
|
||||
console.log(
|
||||
`\n⏳ Load generation finished (${requestsSent} sent in ${activeWindowSec}s window, ${successCount} accepted). Awaiting queue flush...`,
|
||||
);
|
||||
|
||||
const flushStart = Date.now();
|
||||
let lastLogged = 0;
|
||||
|
||||
while (deliveryLatencies.length < successCount) {
|
||||
if (Date.now() - flushStart > 600000) {
|
||||
console.log(
|
||||
`⚠️ Flush timeout! Delivered ${deliveryLatencies.length}/${successCount} messages.`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
if (Date.now() - lastLogged > 2000) {
|
||||
lastLogged = Date.now();
|
||||
const pct = ((deliveryLatencies.length / successCount) * 100).toFixed(1);
|
||||
const rate = (deliveryLatencies.length / ((Date.now() - startTime) / 1000)).toFixed(0);
|
||||
console.log(
|
||||
`⏳ Flushing: [${deliveryLatencies.length} / ${successCount}] (${pct}%) ~ ${rate} delivered/sec`,
|
||||
);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
}
|
||||
|
||||
const totalElapsed = (Date.now() - startTime) / 1000;
|
||||
const firstDelivOffset = firstDeliveryTimestamp ? (firstDeliveryTimestamp - startTime) / 1000 : 0;
|
||||
const lastDelivOffset = lastDeliveryTimestamp
|
||||
? (lastDeliveryTimestamp - startTime) / 1000
|
||||
: totalElapsed;
|
||||
const deliverySpanSec =
|
||||
firstDeliveryTimestamp &&
|
||||
lastDeliveryTimestamp &&
|
||||
lastDeliveryTimestamp > firstDeliveryTimestamp
|
||||
? (lastDeliveryTimestamp - firstDeliveryTimestamp) / 1000
|
||||
: totalElapsed;
|
||||
|
||||
const activeDeliveryRate = (deliveryLatencies.length / deliverySpanSec).toFixed(2);
|
||||
const totalDeliveryRate = (deliveryLatencies.length / totalElapsed).toFixed(2);
|
||||
|
||||
function percentile(arr: number[], p: number) {
|
||||
if (arr.length === 0) return 0;
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const idx = Math.ceil((p / 100) * sorted.length) - 1;
|
||||
return sorted[idx];
|
||||
}
|
||||
|
||||
console.log("\n=================================");
|
||||
console.log(" 📊 LOAD TEST RESULTS 📊 ");
|
||||
console.log("=================================");
|
||||
console.log(`Topology: ${serverCount} Distributed Servers`);
|
||||
console.log(`Generation Window: ${activeWindowSec}s (${concurrency} virtual users)`);
|
||||
console.log(`Total Ingestion: ${requestsSent} requests (${ingestionThroughput} req/sec)`);
|
||||
console.log(
|
||||
`Accepted (202): ${successCount} (${((successCount / requestsSent) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
console.log(`Failed / Dropped: ${failCount}`);
|
||||
console.log("---------------------------------");
|
||||
console.log(`First Msg Delivered: +${firstDelivOffset.toFixed(2)}s from test start`);
|
||||
console.log(`Last Msg Delivered: +${lastDelivOffset.toFixed(2)}s from test start`);
|
||||
console.log(`Delivery Span (1->N): ${deliverySpanSec.toFixed(2)}s`);
|
||||
console.log(`Active Delivery Rate: ${activeDeliveryRate} msgs/sec`);
|
||||
console.log(
|
||||
`Total Wall Drain Rate:${totalDeliveryRate} msgs/sec (${totalElapsed.toFixed(2)}s total)`,
|
||||
);
|
||||
console.log("---------------------------------");
|
||||
const apiP50 = percentile(apiLatencies, 50);
|
||||
const apiP95 = percentile(apiLatencies, 95);
|
||||
const apiP99 = percentile(apiLatencies, 99);
|
||||
const deliveryP50 = percentile(deliveryLatencies, 50);
|
||||
const deliveryP95 = percentile(deliveryLatencies, 95);
|
||||
const deliveryP99 = percentile(deliveryLatencies, 99);
|
||||
|
||||
console.log(`API Latency: p50: ${apiP50}ms | p95: ${apiP95}ms | p99: ${apiP99}ms`);
|
||||
console.log(
|
||||
`Delivery Latency: p50: ${deliveryP50}ms | p95: ${deliveryP95}ms | p99: ${deliveryP99}ms`,
|
||||
);
|
||||
console.log(`Total Delivered: ${deliveryLatencies.length} / ${successCount}`);
|
||||
console.log("=================================\n");
|
||||
|
||||
console.log("🧹 Tearing down test environment...");
|
||||
await dbData.sql.end();
|
||||
await environment.down();
|
||||
console.log("✅ Benchmark completed successfully.");
|
||||
|
||||
return {
|
||||
serverCount,
|
||||
serverCpus,
|
||||
serverMemory,
|
||||
workerConcurrency,
|
||||
durationSec: activeWindowSec,
|
||||
concurrency,
|
||||
requestsSent,
|
||||
successCount,
|
||||
failCount,
|
||||
ingestionThroughput: Number(ingestionThroughput),
|
||||
firstDelivOffset: Number(firstDelivOffset.toFixed(2)),
|
||||
lastDelivOffset: Number(lastDelivOffset.toFixed(2)),
|
||||
deliverySpanSec: Number(deliverySpanSec.toFixed(2)),
|
||||
activeDeliveryRate: Number(activeDeliveryRate),
|
||||
totalDeliveryRate: Number(totalDeliveryRate),
|
||||
apiP50,
|
||||
apiP95,
|
||||
apiP99,
|
||||
deliveryP50,
|
||||
deliveryP95,
|
||||
deliveryP99,
|
||||
totalDelivered: deliveryLatencies.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import {
|
||||
runDistributedBenchmark,
|
||||
type DistributedBenchmarkResult,
|
||||
} from "./distributed-load.test.js";
|
||||
|
||||
export interface DistributedTier {
|
||||
serverCount: number;
|
||||
serverCpus: string;
|
||||
serverMemory: string;
|
||||
dbCpus: string;
|
||||
dbMemory: string;
|
||||
redisCpus: string;
|
||||
redisMemory: string;
|
||||
workerConcurrency: number;
|
||||
label: string;
|
||||
}
|
||||
|
||||
const DEFAULT_DISTRIBUTED_TIERS: DistributedTier[] = [
|
||||
{
|
||||
serverCount: 1,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "1.0",
|
||||
dbMemory: "1G",
|
||||
redisCpus: "0.5",
|
||||
redisMemory: "512M",
|
||||
workerConcurrency: 50,
|
||||
label: "1 Server (1.0 vCPU)",
|
||||
},
|
||||
{
|
||||
serverCount: 2,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "1.0",
|
||||
dbMemory: "1G",
|
||||
redisCpus: "0.5",
|
||||
redisMemory: "512M",
|
||||
workerConcurrency: 50,
|
||||
label: "2 Distributed Servers (2x 1.0 vCPU)",
|
||||
},
|
||||
{
|
||||
serverCount: 3,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "1.0",
|
||||
dbMemory: "1G",
|
||||
redisCpus: "0.5",
|
||||
redisMemory: "512M",
|
||||
workerConcurrency: 50,
|
||||
label: "3 Distributed Servers (3x 1.0 vCPU)",
|
||||
},
|
||||
];
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let durationSeconds = 10;
|
||||
let customConcurrency: number | null = null;
|
||||
let tiers = DEFAULT_DISTRIBUTED_TIERS;
|
||||
|
||||
for (const arg of args) {
|
||||
if (arg.startsWith("--duration=")) {
|
||||
durationSeconds = parseInt(arg.split("=")[1]!, 10);
|
||||
} else if (arg.startsWith("--concurrency=")) {
|
||||
customConcurrency = parseInt(arg.split("=")[1]!, 10);
|
||||
} else if (arg === "--quick") {
|
||||
durationSeconds = 5;
|
||||
} else if (arg.startsWith("--servers=")) {
|
||||
const counts = arg
|
||||
.split("=")[1]!
|
||||
.split(",")
|
||||
.map((s) => parseInt(s.trim(), 10));
|
||||
tiers = counts.map((count) => ({
|
||||
serverCount: count,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "2.0",
|
||||
dbMemory: "2G",
|
||||
redisCpus: "1.5",
|
||||
redisMemory: "1G",
|
||||
workerConcurrency: 100,
|
||||
label: `${count} Server${count > 1 ? "s" : ""} (${count}x 1.0 vCPU)`,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
return { durationSeconds, customConcurrency, tiers };
|
||||
}
|
||||
|
||||
function printSummaryTable(
|
||||
results: Array<{ tier: DistributedTier; res: DistributedBenchmarkResult }>,
|
||||
) {
|
||||
console.log("\n" + "=".repeat(140));
|
||||
console.log(" 📊 NOTIFKIT DISTRIBUTED SCALING BENCHMARK SUMMARY 📊");
|
||||
console.log("=".repeat(140));
|
||||
|
||||
const headers = [
|
||||
"Cluster Topology",
|
||||
"Clients",
|
||||
"Total vCPU",
|
||||
"Ingestion",
|
||||
"Active Deliv",
|
||||
"Wall Drain",
|
||||
"API p95",
|
||||
"Deliv p95",
|
||||
"Speedup",
|
||||
];
|
||||
|
||||
const colWidths = [36, 9, 12, 14, 15, 14, 11, 12, 10];
|
||||
|
||||
const formatRow = (cols: string[]) =>
|
||||
cols.map((col, idx) => col.padEnd(colWidths[idx]!)).join(" | ");
|
||||
|
||||
console.log(formatRow(headers));
|
||||
console.log("-".repeat(140));
|
||||
|
||||
const baselineDelivery = results[0]?.res.activeDeliveryRate || 1;
|
||||
|
||||
for (const { tier, res } of results) {
|
||||
const totalCpu = `${(tier.serverCount * parseFloat(tier.serverCpus)).toFixed(1)} vCPU`;
|
||||
const speedup = (res.activeDeliveryRate / baselineDelivery).toFixed(2) + "x";
|
||||
|
||||
const row = [
|
||||
tier.label,
|
||||
`${res.concurrency} VUs`,
|
||||
totalCpu,
|
||||
`${res.ingestionThroughput} req/s`,
|
||||
`${res.activeDeliveryRate} msg/s`,
|
||||
`${res.totalDeliveryRate} msg/s`,
|
||||
`${res.apiP95}ms`,
|
||||
`${res.deliveryP95}ms`,
|
||||
speedup,
|
||||
];
|
||||
console.log(formatRow(row));
|
||||
}
|
||||
|
||||
console.log("=".repeat(140) + "\n");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { durationSeconds, customConcurrency, tiers } = parseArgs();
|
||||
|
||||
console.log("================================================================================");
|
||||
console.log(" 🚀 STARTING NOTIFKIT DISTRIBUTED HORIZONTAL SCALING BENCHMARK ");
|
||||
console.log("================================================================================");
|
||||
console.log(`Window: ${durationSeconds}s per tier | Auto Client Concurrency: 150 VUs/server`);
|
||||
console.log(`Tiers to test: ${tiers.map((t) => t.label).join(" ➜ ")}\n`);
|
||||
|
||||
const results: Array<{ tier: DistributedTier; res: DistributedBenchmarkResult }> = [];
|
||||
|
||||
for (let i = 0; i < tiers.length; i++) {
|
||||
const tier = tiers[i]!;
|
||||
const tierConcurrency = customConcurrency ?? tier.serverCount * 150;
|
||||
console.log(
|
||||
`\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`,
|
||||
);
|
||||
console.log(
|
||||
`[Tier ${i + 1}/${tiers.length}] Benchmarking ${tier.label} (${tierConcurrency} Virtual Users)`,
|
||||
);
|
||||
console.log(
|
||||
`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n`,
|
||||
);
|
||||
|
||||
const res = await runDistributedBenchmark({
|
||||
serverCount: tier.serverCount,
|
||||
serverCpus: tier.serverCpus,
|
||||
serverMemory: tier.serverMemory,
|
||||
dbCpus: tier.dbCpus,
|
||||
dbMemory: tier.dbMemory,
|
||||
redisCpus: tier.redisCpus,
|
||||
redisMemory: tier.redisMemory,
|
||||
workerConcurrency: tier.workerConcurrency,
|
||||
durationSeconds,
|
||||
concurrency: tierConcurrency,
|
||||
});
|
||||
|
||||
results.push({ tier, res });
|
||||
|
||||
if (i < tiers.length - 1) {
|
||||
console.log("⏳ Cooling down for 4s before next tier...");
|
||||
await new Promise((r) => setTimeout(r, 4000));
|
||||
}
|
||||
}
|
||||
|
||||
printSummaryTable(results);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Distributed benchmark failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,340 @@
|
||||
import { DockerComposeEnvironment } from "testcontainers";
|
||||
import { createDatabase, runMigrations } from "notifkit";
|
||||
|
||||
export interface LoadTestOptions {
|
||||
durationSeconds?: number;
|
||||
concurrency?: number;
|
||||
adminApiKey?: string;
|
||||
port?: number;
|
||||
dbPort?: number;
|
||||
serverCpus?: string;
|
||||
serverMemory?: string;
|
||||
workerConcurrency?: number;
|
||||
dbCpus?: string;
|
||||
dbMemory?: string;
|
||||
redisCpus?: string;
|
||||
redisMemory?: string;
|
||||
providerLatencyMs?: number | string;
|
||||
quiet?: boolean;
|
||||
}
|
||||
|
||||
export interface BenchmarkResult {
|
||||
serverCpus: string;
|
||||
serverMemory: string;
|
||||
workerConcurrency: number;
|
||||
durationSec: number;
|
||||
concurrency: number;
|
||||
requestsSent: number;
|
||||
successCount: number;
|
||||
failCount: number;
|
||||
ingestionThroughput: number;
|
||||
firstDelivOffset: number;
|
||||
lastDelivOffset: number;
|
||||
deliverySpanSec: number;
|
||||
activeDeliveryRate: number;
|
||||
totalDeliveryRate: number;
|
||||
apiP50: number;
|
||||
apiP95: number;
|
||||
apiP99: number;
|
||||
deliveryP50: number;
|
||||
deliveryP95: number;
|
||||
deliveryP99: number;
|
||||
totalDelivered: number;
|
||||
}
|
||||
|
||||
export async function runLoadBenchmark(options: LoadTestOptions = {}): Promise<BenchmarkResult> {
|
||||
const durationSec = options.durationSeconds ?? 10;
|
||||
const concurrency = options.concurrency ?? 50;
|
||||
const adminApiKey = options.adminApiKey ?? "perf-admin-key";
|
||||
const port = options.port ?? 35678;
|
||||
const dbPort = options.dbPort ?? 35432;
|
||||
const serverCpus = options.serverCpus ?? "1.0";
|
||||
const serverMemory = options.serverMemory ?? "1G";
|
||||
const workerConcurrency = options.workerConcurrency ?? 50;
|
||||
const providerLatency = options.providerLatencyMs ?? process.env.PROVIDER_LATENCY_MS ?? 0;
|
||||
|
||||
const composeEnv: Record<string, string> = {
|
||||
SERVER_CPUS: serverCpus,
|
||||
SERVER_MEMORY: serverMemory,
|
||||
WORKER_CONCURRENCY: String(workerConcurrency),
|
||||
PROVIDER_LATENCY_MS: String(providerLatency),
|
||||
};
|
||||
if (options.dbCpus) composeEnv.DB_CPUS = options.dbCpus;
|
||||
if (options.dbMemory) composeEnv.DB_MEMORY = options.dbMemory;
|
||||
if (options.redisCpus) composeEnv.REDIS_CPUS = options.redisCpus;
|
||||
if (options.redisMemory) composeEnv.REDIS_MEMORY = options.redisMemory;
|
||||
|
||||
console.log(
|
||||
`🚀 Booting profiling environment (Server: ${serverCpus} vCPU, ${serverMemory} RAM | DB: ${composeEnv.DB_CPUS ?? "0.5"} vCPU, ${composeEnv.DB_MEMORY ?? "512M"} | Redis: ${composeEnv.REDIS_CPUS ?? "0.5"} vCPU, ${composeEnv.REDIS_MEMORY ?? "256M"}${providerLatency && providerLatency !== "0" ? ` | Provider Latency: ${providerLatency}ms` : ""})...`,
|
||||
);
|
||||
const environment = await new DockerComposeEnvironment(".", "docker-compose.yml")
|
||||
.withEnvironment(composeEnv)
|
||||
.withBuild()
|
||||
.up();
|
||||
|
||||
const baseUrl = `http://localhost:${port}`;
|
||||
const dbUrl = `postgres://notifkit:password@localhost:${dbPort}/notifkit`;
|
||||
|
||||
const apiLatencies: number[] = [];
|
||||
const deliveryLatencies: number[] = [];
|
||||
|
||||
console.log("⏳ Waiting for API health check...");
|
||||
let ready = false;
|
||||
while (!ready) {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/health`);
|
||||
if (res.ok) ready = true;
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
}
|
||||
|
||||
let firstDeliveryTimestamp: number | null = null;
|
||||
let lastDeliveryTimestamp: number | null = null;
|
||||
|
||||
// Hook into container logs to capture delivery latency
|
||||
const serverContainer = environment.getContainer("server-1");
|
||||
const stream = await serverContainer.logs();
|
||||
let logBuffer = "";
|
||||
stream.on("data", (chunk) => {
|
||||
logBuffer += chunk.toString();
|
||||
const lines = logBuffer.split("\n");
|
||||
logBuffer = lines.pop() || "";
|
||||
for (const line of lines) {
|
||||
if (line.includes("push delivered (console transport)")) {
|
||||
const now = Date.now();
|
||||
if (firstDeliveryTimestamp === null) {
|
||||
firstDeliveryTimestamp = now;
|
||||
}
|
||||
lastDeliveryTimestamp = now;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
deliveryLatencies.push(typeof parsed.latency === "number" ? parsed.latency : 0);
|
||||
} catch {
|
||||
deliveryLatencies.push(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log("🛠️ Running database migrations...");
|
||||
const dbData = createDatabase({ url: dbUrl });
|
||||
await runMigrations(dbData.db);
|
||||
|
||||
console.log("🏗️ Provisioning benchmark project and templates...");
|
||||
const projectRes = await fetch(`${baseUrl}/v1/projects`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${adminApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ name: "Profiling Project" }),
|
||||
});
|
||||
const project = await projectRes.json();
|
||||
const projectApiKey = project.apiKey;
|
||||
|
||||
// Uncap API rate limits for the project
|
||||
await dbData.sql`UPDATE projects SET rate_limit_rpm = 1000000`;
|
||||
|
||||
await fetch(`${baseUrl}/v1/templates`, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
Authorization: `Bearer ${projectApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
templates: [
|
||||
{
|
||||
id: "perf-email",
|
||||
channel: "email",
|
||||
topic: ["transactional"],
|
||||
content: {
|
||||
subject: "Perf Test Email",
|
||||
html: "<p>Hello {{name}}</p>",
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
await fetch(`${baseUrl}/v1/users`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${projectApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
id: "perf-user-1",
|
||||
email: ["perf@example.com"],
|
||||
}),
|
||||
});
|
||||
|
||||
console.log(`\n🔥 Starting Load Generation: ${concurrency} virtual users for ${durationSec}s`);
|
||||
|
||||
let requestsSent = 0;
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
|
||||
const startTime = Date.now();
|
||||
const endTime = startTime + durationSec * 1000;
|
||||
|
||||
async function worker() {
|
||||
while (Date.now() < endTime) {
|
||||
requestsSent++;
|
||||
try {
|
||||
const reqStart = Date.now();
|
||||
const res = await fetch(`${baseUrl}/v1/notify`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${projectApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
user: "perf-user-1",
|
||||
template: "perf-email",
|
||||
channels: ["email"],
|
||||
priority: "critical",
|
||||
data: { name: "Load Tester", requestTime: reqStart },
|
||||
}),
|
||||
});
|
||||
|
||||
apiLatencies.push(Date.now() - reqStart);
|
||||
|
||||
if (res.status === 202) {
|
||||
successCount++;
|
||||
} else {
|
||||
failCount++;
|
||||
}
|
||||
} catch {
|
||||
failCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: concurrency }).map(() => worker());
|
||||
await Promise.all(workers);
|
||||
|
||||
const _genWallTime = (Date.now() - startTime) / 1000;
|
||||
const activeWindowSec = durationSec;
|
||||
const ingestionThroughput = (requestsSent / activeWindowSec).toFixed(2);
|
||||
|
||||
console.log(
|
||||
`\n⏳ Load generation finished (${requestsSent} sent in ${activeWindowSec}s window, ${successCount} accepted). Awaiting queue flush...`,
|
||||
);
|
||||
const flushStart = Date.now();
|
||||
let lastLogged = 0;
|
||||
|
||||
while (deliveryLatencies.length < successCount) {
|
||||
if (Date.now() - flushStart > 600000) {
|
||||
console.log(
|
||||
`⚠️ Flush timeout! Delivered ${deliveryLatencies.length}/${successCount} messages.`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
if (Date.now() - lastLogged > 2000) {
|
||||
lastLogged = Date.now();
|
||||
const pct = ((deliveryLatencies.length / successCount) * 100).toFixed(1);
|
||||
const rate = (deliveryLatencies.length / ((Date.now() - startTime) / 1000)).toFixed(0);
|
||||
console.log(
|
||||
`⏳ Flushing: [${deliveryLatencies.length} / ${successCount}] (${pct}%) ~ ${rate} delivered/sec`,
|
||||
);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
}
|
||||
|
||||
const totalElapsed = (Date.now() - startTime) / 1000;
|
||||
const firstDelivOffset = firstDeliveryTimestamp ? (firstDeliveryTimestamp - startTime) / 1000 : 0;
|
||||
const lastDelivOffset = lastDeliveryTimestamp
|
||||
? (lastDeliveryTimestamp - startTime) / 1000
|
||||
: totalElapsed;
|
||||
const deliverySpanSec =
|
||||
firstDeliveryTimestamp &&
|
||||
lastDeliveryTimestamp &&
|
||||
lastDeliveryTimestamp > firstDeliveryTimestamp
|
||||
? (lastDeliveryTimestamp - firstDeliveryTimestamp) / 1000
|
||||
: totalElapsed;
|
||||
|
||||
const activeDeliveryRate = (deliveryLatencies.length / deliverySpanSec).toFixed(2);
|
||||
const totalDeliveryRate = (deliveryLatencies.length / totalElapsed).toFixed(2);
|
||||
|
||||
function percentile(arr: number[], p: number) {
|
||||
if (arr.length === 0) return 0;
|
||||
const sorted = [...arr].sort((a, b) => a - b);
|
||||
const idx = Math.ceil((p / 100) * sorted.length) - 1;
|
||||
return sorted[idx];
|
||||
}
|
||||
|
||||
console.log("\n=================================");
|
||||
console.log(" 📊 LOAD TEST RESULTS 📊 ");
|
||||
console.log("=================================");
|
||||
console.log(`Generation Window: ${activeWindowSec}s (${concurrency} virtual users)`);
|
||||
console.log(`Total Ingestion: ${requestsSent} requests (${ingestionThroughput} req/sec)`);
|
||||
console.log(
|
||||
`Accepted (202): ${successCount} (${((successCount / requestsSent) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
console.log(`Failed / Dropped: ${failCount}`);
|
||||
console.log("---------------------------------");
|
||||
console.log(`First Msg Delivered: +${firstDelivOffset.toFixed(2)}s from test start`);
|
||||
console.log(`Last Msg Delivered: +${lastDelivOffset.toFixed(2)}s from test start`);
|
||||
console.log(`Delivery Span (1->N): ${deliverySpanSec.toFixed(2)}s`);
|
||||
console.log(`Active Delivery Rate: ${activeDeliveryRate} msgs/sec`);
|
||||
console.log(
|
||||
`Total Wall Drain Rate:${totalDeliveryRate} msgs/sec (${totalElapsed.toFixed(2)}s total)`,
|
||||
);
|
||||
console.log("---------------------------------");
|
||||
const apiP50 = percentile(apiLatencies, 50);
|
||||
const apiP95 = percentile(apiLatencies, 95);
|
||||
const apiP99 = percentile(apiLatencies, 99);
|
||||
const deliveryP50 = percentile(deliveryLatencies, 50);
|
||||
const deliveryP95 = percentile(deliveryLatencies, 95);
|
||||
const deliveryP99 = percentile(deliveryLatencies, 99);
|
||||
|
||||
console.log(`API Latency: p50: ${apiP50}ms | p95: ${apiP95}ms | p99: ${apiP99}ms`);
|
||||
console.log(
|
||||
`Delivery Latency: p50: ${deliveryP50}ms | p95: ${deliveryP95}ms | p99: ${deliveryP99}ms`,
|
||||
);
|
||||
console.log(`Total Delivered: ${deliveryLatencies.length} / ${successCount}`);
|
||||
console.log("=================================\n");
|
||||
|
||||
console.log("🧹 Tearing down test environment...");
|
||||
await dbData.sql.end();
|
||||
await environment.down();
|
||||
console.log("✅ Benchmark completed successfully.");
|
||||
|
||||
return {
|
||||
serverCpus,
|
||||
serverMemory,
|
||||
workerConcurrency,
|
||||
durationSec: activeWindowSec,
|
||||
concurrency,
|
||||
requestsSent,
|
||||
successCount,
|
||||
failCount,
|
||||
ingestionThroughput: Number(ingestionThroughput),
|
||||
firstDelivOffset: Number(firstDelivOffset.toFixed(2)),
|
||||
lastDelivOffset: Number(lastDelivOffset.toFixed(2)),
|
||||
deliverySpanSec: Number(deliverySpanSec.toFixed(2)),
|
||||
activeDeliveryRate: Number(activeDeliveryRate),
|
||||
totalDeliveryRate: Number(totalDeliveryRate),
|
||||
apiP50,
|
||||
apiP95,
|
||||
apiP99,
|
||||
deliveryP50,
|
||||
deliveryP95,
|
||||
deliveryP99,
|
||||
totalDelivered: deliveryLatencies.length,
|
||||
};
|
||||
}
|
||||
|
||||
// Execute when run directly via CLI
|
||||
const isDirectRun =
|
||||
process.argv[1] &&
|
||||
(process.argv[1].endsWith("load.test.ts") || process.argv[1].endsWith("load.test.js"));
|
||||
|
||||
if (isDirectRun) {
|
||||
runLoadBenchmark().catch((err) => {
|
||||
console.error("Benchmark failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
import {
|
||||
runDistributedBenchmark,
|
||||
type DistributedBenchmarkResult,
|
||||
} from "./distributed-load.test.js";
|
||||
|
||||
export interface BudgetTier {
|
||||
budget: string;
|
||||
monthlyCost: number;
|
||||
serverCount: number;
|
||||
serverCpus: string;
|
||||
serverMemory: string;
|
||||
dbCpus: string;
|
||||
dbMemory: string;
|
||||
redisCpus: string;
|
||||
redisMemory: string;
|
||||
workerConcurrency: number;
|
||||
dbMaxConnections: number;
|
||||
concurrency: number;
|
||||
description: string;
|
||||
serverServices?: string[];
|
||||
}
|
||||
|
||||
const DEFAULT_BUDGET_TIERS: BudgetTier[] = [
|
||||
{
|
||||
budget: "$20/mo",
|
||||
monthlyCost: 20,
|
||||
serverCount: 1,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "1.0",
|
||||
dbMemory: "1G",
|
||||
redisCpus: "0.5",
|
||||
redisMemory: "512M",
|
||||
workerConcurrency: 100,
|
||||
dbMaxConnections: 5,
|
||||
concurrency: 150,
|
||||
description: "1x Monolith (API + Pipeline + Delivery)",
|
||||
serverServices: ["api,enricher,engine,delivery,scheduler"],
|
||||
},
|
||||
{
|
||||
budget: "$40/mo",
|
||||
monthlyCost: 40,
|
||||
serverCount: 2,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "2.0",
|
||||
dbMemory: "2G",
|
||||
redisCpus: "1.0",
|
||||
redisMemory: "1G",
|
||||
workerConcurrency: 200,
|
||||
dbMaxConnections: 8,
|
||||
concurrency: 300,
|
||||
description: "1x API + 1x Worker (Enricher/Engine/Delivery)",
|
||||
serverServices: ["api", "enricher,engine,delivery,scheduler"],
|
||||
},
|
||||
{
|
||||
budget: "$60/mo",
|
||||
monthlyCost: 60,
|
||||
serverCount: 3,
|
||||
serverCpus: "1.0",
|
||||
serverMemory: "1G",
|
||||
dbCpus: "3.0",
|
||||
dbMemory: "3G",
|
||||
redisCpus: "1.5",
|
||||
redisMemory: "1G",
|
||||
workerConcurrency: 300,
|
||||
dbMaxConnections: 10,
|
||||
concurrency: 300,
|
||||
description: "1x API + 1x Pipeline (Engine/Enrich) + 1x Delivery",
|
||||
serverServices: ["api", "enricher,engine,scheduler", "delivery"],
|
||||
},
|
||||
];
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2);
|
||||
let durationSeconds = 10;
|
||||
let providerLatencyMs: number | string | undefined = process.env.PROVIDER_LATENCY_MS || "60-150";
|
||||
let tiers = DEFAULT_BUDGET_TIERS;
|
||||
|
||||
for (const arg of args) {
|
||||
if (arg.startsWith("--duration=")) {
|
||||
durationSeconds = parseInt(arg.split("=")[1]!, 10);
|
||||
} else if (arg === "--quick") {
|
||||
durationSeconds = 5;
|
||||
} else if (arg.startsWith("--latency=")) {
|
||||
const val = arg.split("=")[1]!;
|
||||
providerLatencyMs = val.includes("-") ? val : parseInt(val, 10);
|
||||
} else if (arg.startsWith("--workers=")) {
|
||||
const val = parseInt(arg.split("=")[1]!, 10);
|
||||
tiers = tiers.map((t) => ({ ...t, workerConcurrency: val }));
|
||||
} else if (arg.startsWith("--tiers=")) {
|
||||
const requested = arg
|
||||
.split("=")[1]!
|
||||
.split(",")
|
||||
.map((s) => s.trim());
|
||||
tiers = DEFAULT_BUDGET_TIERS.filter((t) => requested.includes(t.budget.replace("/mo", "")));
|
||||
}
|
||||
}
|
||||
|
||||
return { durationSeconds, providerLatencyMs, tiers };
|
||||
}
|
||||
|
||||
function printSummaryTable(results: Array<{ tier: BudgetTier; res: DistributedBenchmarkResult }>) {
|
||||
console.log("\n" + "=".repeat(150));
|
||||
console.log(
|
||||
" 💰 NOTIFKIT MONEY / BUDGET SCALING BENCHMARK SUMMARY 💰",
|
||||
);
|
||||
console.log("=".repeat(150));
|
||||
|
||||
const headers = [
|
||||
"Budget Tier",
|
||||
"Architecture Topology",
|
||||
"Clients",
|
||||
"Ingestion",
|
||||
"Active Deliv",
|
||||
"Wall Drain",
|
||||
"API p95",
|
||||
"Deliv p95",
|
||||
"Cost Efficiency",
|
||||
];
|
||||
|
||||
const colWidths = [13, 44, 9, 14, 15, 14, 10, 11, 15];
|
||||
|
||||
const formatRow = (cols: string[]) =>
|
||||
cols.map((col, idx) => col.padEnd(colWidths[idx]!)).join(" | ");
|
||||
|
||||
console.log(formatRow(headers));
|
||||
console.log("-".repeat(150));
|
||||
|
||||
for (const { tier, res } of results) {
|
||||
const msgsPerDollar = (res.activeDeliveryRate / tier.monthlyCost).toFixed(1) + " msg/s/$";
|
||||
|
||||
const row = [
|
||||
tier.budget,
|
||||
tier.description,
|
||||
`${res.concurrency} VUs`,
|
||||
`${res.ingestionThroughput} req/s`,
|
||||
`${res.activeDeliveryRate} msg/s`,
|
||||
`${res.totalDeliveryRate} msg/s`,
|
||||
`${res.apiP95}ms`,
|
||||
`${res.deliveryP95}ms`,
|
||||
msgsPerDollar,
|
||||
];
|
||||
console.log(formatRow(row));
|
||||
}
|
||||
|
||||
console.log("=".repeat(150) + "\n");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { durationSeconds, providerLatencyMs, tiers } = parseArgs();
|
||||
|
||||
console.log(
|
||||
"==========================================================================================",
|
||||
);
|
||||
console.log(
|
||||
" 💰 STARTING NOTIFKIT MONEY / BUDGET SCALING BENCHMARK ",
|
||||
);
|
||||
console.log(
|
||||
"==========================================================================================",
|
||||
);
|
||||
console.log(`Window: ${durationSeconds}s per tier | Measuring: $20/mo ➜ $40/mo ➜ $60/mo`);
|
||||
console.log(`Tiers to test: ${tiers.map((t) => t.budget).join(" ➜ ")}`);
|
||||
if (providerLatencyMs && providerLatencyMs !== "0") {
|
||||
console.log(`Mock Provider Latency: ${providerLatencyMs}ms`);
|
||||
}
|
||||
console.log(
|
||||
"==========================================================================================\n",
|
||||
);
|
||||
|
||||
const results: Array<{ tier: BudgetTier; res: DistributedBenchmarkResult }> = [];
|
||||
|
||||
for (let i = 0; i < tiers.length; i++) {
|
||||
const tier = tiers[i]!;
|
||||
console.log(
|
||||
`\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`,
|
||||
);
|
||||
console.log(`[Tier ${i + 1}/${tiers.length}] Benchmarking Budget Tier: ${tier.budget}`);
|
||||
console.log(`Topology: ${tier.description} (${tier.concurrency} Virtual Users)`);
|
||||
console.log(
|
||||
`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n`,
|
||||
);
|
||||
|
||||
const res = await runDistributedBenchmark({
|
||||
serverCount: tier.serverCount,
|
||||
serverCpus: tier.serverCpus,
|
||||
serverMemory: tier.serverMemory,
|
||||
dbCpus: tier.dbCpus,
|
||||
dbMemory: tier.dbMemory,
|
||||
redisCpus: tier.redisCpus,
|
||||
redisMemory: tier.redisMemory,
|
||||
workerConcurrency: tier.workerConcurrency,
|
||||
dbMaxConnections: tier.dbMaxConnections,
|
||||
serverServices: tier.serverServices,
|
||||
providerLatencyMs,
|
||||
durationSeconds,
|
||||
concurrency: tier.concurrency,
|
||||
});
|
||||
|
||||
results.push({ tier, res });
|
||||
|
||||
if (i < tiers.length - 1) {
|
||||
console.log("⏳ Cooling down for 4s before next tier...");
|
||||
await new Promise((r) => setTimeout(r, 4000));
|
||||
}
|
||||
}
|
||||
|
||||
printSummaryTable(results);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Money scaling benchmark failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,312 @@
|
||||
import { runLoadBenchmark, type BenchmarkResult } from "./load.test.js";
|
||||
|
||||
export interface CpuScalingTier {
|
||||
cpus: string;
|
||||
memory: string;
|
||||
dbCpus: string;
|
||||
dbMemory: string;
|
||||
redisCpus: string;
|
||||
redisMemory: string;
|
||||
workerConcurrency?: number;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ScalingBenchmarkOptions {
|
||||
tiers?: CpuScalingTier[];
|
||||
durationSeconds?: number;
|
||||
concurrency?: number;
|
||||
providerLatencyMs?: number | string;
|
||||
}
|
||||
|
||||
const DEFAULT_TIERS: CpuScalingTier[] = [
|
||||
{
|
||||
cpus: "0.5",
|
||||
memory: "1G",
|
||||
dbCpus: "0.5",
|
||||
dbMemory: "512M",
|
||||
redisCpus: "0.5",
|
||||
redisMemory: "256M",
|
||||
workerConcurrency: 25,
|
||||
label: "0.5 vCPU",
|
||||
},
|
||||
{
|
||||
cpus: "1.0",
|
||||
memory: "2G",
|
||||
dbCpus: "1.0",
|
||||
dbMemory: "1G",
|
||||
redisCpus: "0.5",
|
||||
redisMemory: "512M",
|
||||
workerConcurrency: 50,
|
||||
label: "1.0 vCPU",
|
||||
},
|
||||
{
|
||||
cpus: "2.0",
|
||||
memory: "4G",
|
||||
dbCpus: "1.5",
|
||||
dbMemory: "2G",
|
||||
redisCpus: "1.0",
|
||||
redisMemory: "1G",
|
||||
workerConcurrency: 100,
|
||||
label: "2.0 vCPU",
|
||||
},
|
||||
{
|
||||
cpus: "4.0",
|
||||
memory: "6G",
|
||||
dbCpus: "2.0",
|
||||
dbMemory: "3G",
|
||||
redisCpus: "1.0",
|
||||
redisMemory: "1G",
|
||||
workerConcurrency: 200,
|
||||
label: "4.0 vCPU",
|
||||
},
|
||||
];
|
||||
|
||||
function parseArgs(): ScalingBenchmarkOptions {
|
||||
const args = process.argv.slice(2);
|
||||
let durationSeconds = 10;
|
||||
let concurrency = 50;
|
||||
let tiers = DEFAULT_TIERS;
|
||||
let providerLatencyMs: number | string | undefined = process.env.PROVIDER_LATENCY_MS || "200-300";
|
||||
|
||||
for (const arg of args) {
|
||||
if (arg.startsWith("--duration=")) {
|
||||
durationSeconds = parseInt(arg.split("=")[1]!, 10);
|
||||
} else if (arg.startsWith("--concurrency=")) {
|
||||
concurrency = parseInt(arg.split("=")[1]!, 10);
|
||||
} else if (arg.startsWith("--latency=")) {
|
||||
const val = arg.split("=")[1]!;
|
||||
providerLatencyMs = val.includes("-") ? val : parseInt(val, 10);
|
||||
} else if (arg === "--quick") {
|
||||
durationSeconds = 5;
|
||||
} else if (arg.startsWith("--cpus=")) {
|
||||
const cpusList = arg
|
||||
.split("=")[1]!
|
||||
.split(",")
|
||||
.map((s) => s.trim());
|
||||
tiers = cpusList.map((c) => {
|
||||
const num = parseFloat(c);
|
||||
let mem = "1G";
|
||||
let dbC = "0.5";
|
||||
let dbM = "512M";
|
||||
let redC = "0.5";
|
||||
let redM = "256M";
|
||||
|
||||
if (num >= 4) {
|
||||
mem = "6G";
|
||||
dbC = "2.0";
|
||||
dbM = "3G";
|
||||
redC = "1.0";
|
||||
redM = "1G";
|
||||
} else if (num >= 2) {
|
||||
mem = "4G";
|
||||
dbC = "1.5";
|
||||
dbM = "2G";
|
||||
redC = "1.0";
|
||||
redM = "1G";
|
||||
} else if (num >= 1) {
|
||||
mem = "2G";
|
||||
dbC = "1.0";
|
||||
dbM = "1G";
|
||||
redC = "0.5";
|
||||
redM = "512M";
|
||||
}
|
||||
|
||||
return {
|
||||
cpus: c,
|
||||
memory: mem,
|
||||
dbCpus: dbC,
|
||||
dbMemory: dbM,
|
||||
redisCpus: redC,
|
||||
redisMemory: redM,
|
||||
label: `${c} vCPU`,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { tiers, durationSeconds, concurrency, providerLatencyMs };
|
||||
}
|
||||
|
||||
export async function runCpuScalingBenchmark(options: ScalingBenchmarkOptions = {}) {
|
||||
const tiers = options.tiers ?? DEFAULT_TIERS;
|
||||
const durationSeconds = options.durationSeconds ?? 10;
|
||||
const concurrency = options.concurrency ?? 50;
|
||||
const providerLatencyMs = options.providerLatencyMs ?? "200-300";
|
||||
|
||||
console.log(
|
||||
"\n==========================================================================================",
|
||||
);
|
||||
console.log(" 🚀 STARTING NOTIFKIT CPU SCALING BENCHMARK SUITE");
|
||||
console.log(
|
||||
"==========================================================================================",
|
||||
);
|
||||
console.log(`Evaluated Tiers: ${tiers.map((t) => t.label).join(", ")}`);
|
||||
console.log(`Test Duration/Tier: ${durationSeconds}s`);
|
||||
console.log(`Virtual Users: ${concurrency}`);
|
||||
if (providerLatencyMs && providerLatencyMs !== "0" && providerLatencyMs !== 0) {
|
||||
console.log(`Mock Provider Latency: ${providerLatencyMs}ms`);
|
||||
}
|
||||
console.log(
|
||||
"==========================================================================================\n",
|
||||
);
|
||||
|
||||
const results: { tier: CpuScalingTier; result: BenchmarkResult }[] = [];
|
||||
|
||||
for (let i = 0; i < tiers.length; i++) {
|
||||
const tier = tiers[i]!;
|
||||
console.log(
|
||||
`\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`,
|
||||
);
|
||||
console.log(
|
||||
`[Tier ${i + 1}/${tiers.length}] Benchmarking ${tier.label} (${tier.memory} RAM | DB: ${tier.dbCpus} CPU, ${tier.dbMemory} | Redis: ${tier.redisCpus} CPU, ${tier.redisMemory})`,
|
||||
);
|
||||
console.log(
|
||||
`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n`,
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await runLoadBenchmark({
|
||||
serverCpus: tier.cpus,
|
||||
serverMemory: tier.memory,
|
||||
dbCpus: tier.dbCpus,
|
||||
dbMemory: tier.dbMemory,
|
||||
redisCpus: tier.redisCpus,
|
||||
redisMemory: tier.redisMemory,
|
||||
workerConcurrency: tier.workerConcurrency,
|
||||
providerLatencyMs,
|
||||
durationSeconds,
|
||||
concurrency,
|
||||
});
|
||||
results.push({ tier, result });
|
||||
} catch (err) {
|
||||
console.error(`❌ Benchmark failed for tier ${tier.label}:`, err);
|
||||
}
|
||||
|
||||
if (i < tiers.length - 1) {
|
||||
console.log(`\n⏳ Cooling down for 3s before next tier...\n`);
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
}
|
||||
}
|
||||
|
||||
// Print comparative summary table
|
||||
printScalingSummary(results);
|
||||
}
|
||||
|
||||
function printScalingSummary(results: { tier: CpuScalingTier; result: BenchmarkResult }[]) {
|
||||
if (results.length === 0) {
|
||||
console.log("\n❌ No benchmark results collected.");
|
||||
return;
|
||||
}
|
||||
|
||||
const baselineDelivery = results[0]?.result.activeDeliveryRate || 1;
|
||||
|
||||
console.log("\n");
|
||||
console.log(
|
||||
"===============================================================================================================================================================",
|
||||
);
|
||||
console.log(
|
||||
" 📊 NOTIFKIT SERVER CPU SCALING BENCHMARK SUMMARY 📊",
|
||||
);
|
||||
console.log(
|
||||
"===============================================================================================================================================================",
|
||||
);
|
||||
console.log(
|
||||
[
|
||||
"Server Tier".padEnd(12),
|
||||
"Server RAM".padEnd(10),
|
||||
"DB (CPU/RAM)".padEnd(14),
|
||||
"Redis (CPU/RAM)".padEnd(15),
|
||||
"Ingestion".padStart(13),
|
||||
"Active Deliv".padStart(13),
|
||||
"Wall Drain".padStart(12),
|
||||
"API p95".padStart(9),
|
||||
"Deliv p95".padStart(11),
|
||||
"1st Msg".padStart(9),
|
||||
"Speedup".padStart(10),
|
||||
].join(" | "),
|
||||
);
|
||||
console.log("-".repeat(159));
|
||||
|
||||
for (const { tier, result } of results) {
|
||||
const speedup = (result.activeDeliveryRate / baselineDelivery).toFixed(2) + "x";
|
||||
const dbSpec = `${tier.dbCpus}c / ${tier.dbMemory}`;
|
||||
const redisSpec = `${tier.redisCpus}c / ${tier.redisMemory}`;
|
||||
|
||||
console.log(
|
||||
[
|
||||
tier.label.padEnd(12),
|
||||
tier.memory.padEnd(10),
|
||||
dbSpec.padEnd(14),
|
||||
redisSpec.padEnd(15),
|
||||
`${result.ingestionThroughput.toFixed(1)} req/s`.padStart(13),
|
||||
`${result.activeDeliveryRate.toFixed(1)} msg/s`.padStart(13),
|
||||
`${result.totalDeliveryRate.toFixed(1)} msg/s`.padStart(12),
|
||||
`${result.apiP95}ms`.padStart(9),
|
||||
`${result.deliveryP95}ms`.padStart(11),
|
||||
`+${result.firstDelivOffset.toFixed(2)}s`.padStart(9),
|
||||
speedup.padStart(10),
|
||||
].join(" | "),
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
"===============================================================================================================================================================\n",
|
||||
);
|
||||
|
||||
// Summary takeaways
|
||||
console.log("📌 SCALING ANALYSIS & TAKEAWAYS:");
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const curr = results[i]!;
|
||||
if (i === 0) {
|
||||
console.log(
|
||||
` • Baseline (${curr.tier.label}): ${curr.result.activeDeliveryRate.toFixed(1)} msgs/sec delivery, ${curr.result.ingestionThroughput.toFixed(1)} req/sec ingestion.`,
|
||||
);
|
||||
} else {
|
||||
const prev = results[i - 1]!;
|
||||
const ingGain = (
|
||||
((curr.result.ingestionThroughput - prev.result.ingestionThroughput) /
|
||||
prev.result.ingestionThroughput) *
|
||||
100
|
||||
).toFixed(1);
|
||||
const delivGain = (
|
||||
((curr.result.activeDeliveryRate - prev.result.activeDeliveryRate) /
|
||||
prev.result.activeDeliveryRate) *
|
||||
100
|
||||
).toFixed(1);
|
||||
|
||||
console.log(
|
||||
` • ${prev.tier.label} ➔ ${curr.tier.label}: Ingestion ${Number(ingGain) >= 0 ? "+" : ""}${ingGain}%, Active Delivery ${Number(delivGain) >= 0 ? "+" : ""}${delivGain}%.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Detect plateauing
|
||||
if (results.length >= 3) {
|
||||
const last = results[results.length - 1]!.result.activeDeliveryRate;
|
||||
const secondLast = results[results.length - 2]!.result.activeDeliveryRate;
|
||||
const diff = ((last - secondLast) / secondLast) * 100;
|
||||
if (diff < 15) {
|
||||
console.log(
|
||||
`\n⚠️ Noticeable throughput plateau between ${results[results.length - 2]!.tier.label} and ${results[results.length - 1]!.tier.label} (+${diff.toFixed(1)}%).`,
|
||||
);
|
||||
console.log(
|
||||
` Because Notifkit runs as a single Node.js event loop, adding vCPUs beyond 1-2 cores primarily benefits V8 GC/crypto threads. Core delivery remains limited by single-thread concurrency and batch timer intervals (e.g. 100ms batch flush).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n");
|
||||
}
|
||||
|
||||
// Auto-execute if run from CLI
|
||||
const isDirectRun =
|
||||
process.argv[1] &&
|
||||
(process.argv[1].endsWith("scaling.test.ts") || process.argv[1].endsWith("scaling.test.js"));
|
||||
|
||||
if (isDirectRun) {
|
||||
const options = parseArgs();
|
||||
runCpuScalingBenchmark(options).catch((err) => {
|
||||
console.error("Scaling benchmark failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true
|
||||
},
|
||||
"include": ["src/**/*", "tests/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user