#!/usr/bin/env bash set -euo pipefail SYSTEM="${1:?Usage: bench.sh [port]}" FIXTURE_DIR="${2:?Usage: bench.sh [port]}" PORT="${3:-1349}" BASE_URL="http://localhost:${PORT}" RESULTS_FILE="bench-results-${SYSTEM}.jsonl" CONTAINER_NAME="ashim" log() { echo "[$(date +%H:%M:%S)] $*" >&2; } get_token() { curl -sf -X POST "${BASE_URL}/api/auth/login" \ -H 'Content-Type: application/json' \ -d '{"username":"admin","password":"admin"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])" } get_container_id() { docker ps -q -f name="${CONTAINER_NAME}" | head -1 } docker_mem_mb() { local cid="$1" docker stats "$cid" --no-stream --format "{{.MemUsage}}" 2>/dev/null | awk -F/ '{gsub(/[^0-9.]/, "", $1); if($1+0 > 0) print $1; else print 0}' } docker_cpu_pct() { local cid="$1" docker stats "$cid" --no-stream --format "{{.CPUPerc}}" 2>/dev/null | tr -d '%' } record() { local tier="$1" tool="$2" variant="$3" time_s="$4" pass="$5" output_size="${6:-0}" mem_mb="${7:-0}" cpu_pct="${8:-0}" extra="${9:-}" printf '{"system":"%s","tier":"%s","tool":"%s","variant":"%s","time_s":%s,"pass":%s,"output_size":%s,"mem_mb":%s,"cpu_pct":%s%s}\n' \ "$SYSTEM" "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_mb" "$cpu_pct" "$extra" >> "$RESULTS_FILE" } bench_tool() { local tier="$1" tool="$2" variant="$3" file="$4" settings="${5:-}" extra_args="${6:-}" local cid time_s http_code mem_before mem_after cpu output_file pass output_size cid=$(get_container_id) mem_before=$(docker_mem_mb "$cid" 2>/dev/null || echo "0") output_file=$(mktemp) local timing_file=$(mktemp) local curl_args=(-s -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}") if [ -n "$file" ] && [ "$file" != "NONE" ]; then curl_args+=(-F "file=@${file}") fi if [ -n "$settings" ]; then curl_args+=(-F "settings=${settings}") fi if [ -n "$extra_args" ]; then eval "curl_args+=($extra_args)" fi curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}") local result result=$(curl "${curl_args[@]}" 2>/dev/null) || result="000 0.000" http_code=$(echo "$result" | awk '{print $1}') time_s=$(echo "$result" | awk '{print $2}') mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0") cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0") output_size=$(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null || echo "0") if [ "$http_code" = "200" ]; then pass="true" else pass="false" fi record "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu" log "$tier/$tool/$variant: ${time_s}s HTTP:${http_code} mem:${mem_after}MB pass:${pass}" rm -f "$output_file" "$timing_file" } bench_tool_multifile() { local tier="$1" tool="$2" variant="$3" settings="${4:-}" shift 4 local files=("$@") local cid time_s http_code mem_after cpu output_file pass output_size cid=$(get_container_id) output_file=$(mktemp) local curl_args=(-s -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}") for f in "${files[@]}"; do curl_args+=(-F "file=@${f}") done if [ -n "$settings" ]; then curl_args+=(-F "settings=${settings}") fi curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}") local result result=$(curl "${curl_args[@]}" 2>/dev/null) || result="000 0.000" http_code=$(echo "$result" | awk '{print $1}') time_s=$(echo "$result" | awk '{print $2}') mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0") cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0") output_size=$(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null || echo "0") if [ "$http_code" = "200" ]; then pass="true" else pass="false" fi record "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu" log "$tier/$tool/$variant: ${time_s}s HTTP:${http_code} pass:${pass}" rm -f "$output_file" } run_n_times() { local n="$1" shift local times=() for i in $(seq 1 "$n"); do "$@" done } F="${FIXTURE_DIR}" S="${F}/test-200x150.png" J="${F}/test-100x100.jpg" L="${F}/content/stress-large.jpg" P="${F}/content/portrait-color.jpg" BW="${F}/content/portrait-bw.jpeg" SVG="${F}/content/svg-logo.svg" GIF="${F}/content/animated-simpsons.gif" PDF="${F}/test-3page.pdf" EXIF="${F}/test-with-exif.jpg" FACE="${F}/content/multi-face.webp" OCR="${F}/content/ocr-chat.jpeg" OCRJP="${F}/content/ocr-japanese.png" ISO="${F}/content/portrait-isolated.png" HEAD="${F}/content/portrait-headshot.heic" REDEYE="${F}/content/red-eye.jpg" BARCODE="${F}/content/barcode.avif" echo "[]" > /dev/null > "$RESULTS_FILE" log "=== Starting benchmarks on ${SYSTEM} ===" TOKEN=$(get_token) log "Auth token obtained" log "=== TIER 1: Core Tool Benchmarks ===" for run in 1 2 3; do log "--- Run $run of 3 ---" bench_tool "core" "resize" "small-r${run}" "$S" '{"width":100,"fit":"cover"}' bench_tool "core" "resize" "large-r${run}" "$L" '{"width":800,"fit":"cover"}' bench_tool "core" "crop" "small-r${run}" "$S" '{"left":10,"top":10,"width":100,"height":100}' bench_tool "core" "crop" "large-r${run}" "$L" '{"left":10,"top":10,"width":100,"height":100}' bench_tool "core" "rotate" "small-r${run}" "$S" '{"angle":90}' bench_tool "core" "rotate" "large-r${run}" "$L" '{"angle":90}' bench_tool "core" "convert" "jpg-webp-small-r${run}" "$J" '{"format":"webp","quality":80}' bench_tool "core" "convert" "jpg-webp-large-r${run}" "$L" '{"format":"webp","quality":80}' bench_tool "core" "convert" "jpg-avif-small-r${run}" "$J" '{"format":"avif","quality":50}' bench_tool "core" "convert" "jpg-avif-large-r${run}" "$L" '{"format":"avif","quality":50}' bench_tool "core" "compress" "quality-small-r${run}" "$S" '{"mode":"quality","quality":60}' bench_tool "core" "compress" "quality-large-r${run}" "$L" '{"mode":"quality","quality":60}' bench_tool "core" "compress" "targetSize-large-r${run}" "$L" '{"mode":"targetSize","targetSizeKb":500}' bench_tool "core" "strip-metadata" "small-r${run}" "$EXIF" '{"stripAll":true}' bench_tool "core" "strip-metadata" "large-r${run}" "$L" '{"stripAll":true}' bench_tool "core" "edit-metadata" "small-r${run}" "$EXIF" '{"title":"Bench","clearGps":true}' bench_tool "core" "edit-metadata" "large-r${run}" "$L" '{"title":"Bench","clearGps":true}' bench_tool "core" "color-adjustments" "small-r${run}" "$S" '{"brightness":20,"contrast":10,"effect":"grayscale"}' bench_tool "core" "color-adjustments" "large-r${run}" "$L" '{"brightness":20,"contrast":10,"effect":"grayscale"}' bench_tool "core" "sharpening" "small-r${run}" "$S" '{"method":"adaptive","sigma":1.5}' bench_tool "core" "sharpening" "large-r${run}" "$L" '{"method":"adaptive","sigma":1.5}' bench_tool "core" "watermark-text" "small-r${run}" "$S" '{"text":"BENCHMARK","position":"tiled"}' bench_tool "core" "watermark-text" "large-r${run}" "$L" '{"text":"BENCHMARK","position":"tiled"}' bench_tool_multifile "core" "compose" "small-r${run}" '{"blendMode":"overlay"}' "$S" "$J" bench_tool_multifile "core" "compose" "large-r${run}" '{"blendMode":"overlay"}' "$L" "$P" bench_tool_multifile "core" "collage" "4img-small-r${run}" '{"templateId":"4-grid"}' "$S" "$J" "${F}/test-50x50.webp" "${F}/test-100x100.svg" bench_tool_multifile "core" "collage" "4img-large-r${run}" '{"templateId":"4-grid"}' "$L" "$P" "$BW" "${F}/content/watermark.jpg" bench_tool_multifile "core" "stitch" "3img-small-r${run}" '{"direction":"horizontal"}' "$S" "$J" "${F}/test-50x50.webp" bench_tool_multifile "core" "stitch" "3img-large-r${run}" '{"direction":"horizontal"}' "$L" "$P" "$BW" bench_tool "core" "split" "small-r${run}" "$S" '{"columns":2,"rows":2}' bench_tool "core" "split" "large-r${run}" "$L" '{"columns":2,"rows":2}' bench_tool "core" "border" "small-r${run}" "$S" '{"borderWidth":20,"cornerRadius":10,"shadow":true}' bench_tool "core" "border" "large-r${run}" "$L" '{"borderWidth":20,"cornerRadius":10,"shadow":true}' bench_tool "core" "svg-to-raster" "small-r${run}" "${F}/test-100x100.svg" '{"width":2000,"dpi":300}' bench_tool "core" "vectorize" "small-r${run}" "$S" '{"colorMode":"color"}' bench_tool "core" "vectorize" "large-r${run}" "$P" '{"colorMode":"color"}' bench_tool "core" "gif-tools" "optimize-r${run}" "$GIF" '{"mode":"optimize","colors":64}' bench_tool "core" "pdf-to-image" "r${run}" "$PDF" '{"format":"png","dpi":300}' bench_tool "core" "optimize-for-web" "large-r${run}" "$L" '{"format":"webp","quality":80,"maxWidth":1920}' bench_tool "core" "favicon" "small-r${run}" "$S" '' bench_tool "core" "favicon" "large-r${run}" "$P" '' bench_tool_multifile "core" "image-to-pdf" "3img-r${run}" '{"pageSize":"A4"}' "$S" "$J" "${F}/test-50x50.webp" bench_tool "core" "replace-color" "small-r${run}" "$S" '{"sourceColor":"#FFFFFF","targetColor":"#FF0000","tolerance":30}' bench_tool "core" "replace-color" "large-r${run}" "$L" '{"sourceColor":"#FFFFFF","targetColor":"#FF0000","tolerance":30}' bench_tool "core" "info" "small-r${run}" "$S" '' bench_tool "core" "info" "large-r${run}" "$L" '' bench_tool_multifile "core" "compare" "small-r${run}" '' "$S" "$S" bench_tool_multifile "core" "compare" "large-r${run}" '' "$L" "$L" bench_tool_multifile "core" "find-duplicates" "5img-r${run}" '{"threshold":5}' "$S" "$J" "${F}/test-50x50.webp" "$S" "$J" bench_tool "core" "color-palette" "small-r${run}" "$P" '' bench_tool "core" "color-palette" "large-r${run}" "$L" '' bench_tool "core" "barcode-read" "r${run}" "$BARCODE" '{"tryHarder":true}' bench_tool "core" "image-to-base64" "small-r${run}" "$S" '{"outputFormat":"webp"}' bench_tool "core" "image-to-base64" "large-r${run}" "$L" '{"outputFormat":"webp"}' bench_tool "core" "content-aware-resize" "small-r${run}" "$S" '{"width":100}' bench_tool "core" "content-aware-resize" "large-r${run}" "$L" '{"width":100}' bench_tool "core" "image-enhancement" "small-r${run}" "$S" '{"mode":"auto","intensity":50}' bench_tool "core" "image-enhancement" "large-r${run}" "$L" '{"mode":"auto","intensity":50}' done log "=== TIER 5: Format Decode Benchmarks ===" for fmt_file in "${F}/formats/"*; do fname=$(basename "$fmt_file") for run in 1 2 3; do bench_tool "format" "resize" "fmt-${fname}-r${run}" "$fmt_file" '{"width":200}' done done log "=== TIER 6: Concurrent Load Benchmarks ===" for concurrency in 1 3 5 10 20; do log "Concurrency: ${concurrency}" local_results=$(mktemp) for i in $(seq 1 "$concurrency"); do ( result=$(curl -s -X POST "${BASE_URL}/api/v1/tools/resize" \ -H "Authorization: Bearer ${TOKEN}" \ -F "file=@${L}" \ -F 'settings={"width":800}' \ -o /dev/null -w "%{http_code} %{time_total}") echo "$result" ) >> "$local_results" & done wait cid=$(get_container_id) mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0") times=() errors=0 while IFS= read -r line; do code=$(echo "$line" | awk '{print $1}') t=$(echo "$line" | awk '{print $2}') times+=("$t") if [ "$code" != "200" ]; then errors=$((errors + 1)) fi done < "$local_results" sorted=$(printf '%s\n' "${times[@]}" | sort -n) count=${#times[@]} avg=$(printf '%s\n' "${times[@]}" | awk '{s+=$1} END {printf "%.3f", s/NR}') p95_idx=$(( (count * 95 / 100) )) [ "$p95_idx" -ge "$count" ] && p95_idx=$((count - 1)) p95=$(echo "$sorted" | sed -n "$((p95_idx + 1))p") max=$(echo "$sorted" | tail -1) min=$(echo "$sorted" | head -1) printf '{"system":"%s","tier":"concurrent","tool":"resize","variant":"c%d","concurrency":%d,"avg_s":%s,"p95_s":%s,"max_s":%s,"min_s":%s,"errors":%d,"mem_mb":%s}\n' \ "$SYSTEM" "$concurrency" "$concurrency" "$avg" "${p95:-0}" "${max:-0}" "${min:-0}" "$errors" "$mem_after" >> "$RESULTS_FILE" log "concurrent/c${concurrency}: avg=${avg}s p95=${p95}s max=${max}s errors=${errors} mem=${mem_after}MB" rm -f "$local_results" done log "=== TIER 7: Sustained Load (50 sequential resizes) ===" cid=$(get_container_id) mem_start=$(docker_mem_mb "$cid" 2>/dev/null || echo "0") for i in $(seq 1 50); do bench_tool "sustained" "resize" "seq-${i}" "$L" '{"width":800}' done mem_end=$(docker_mem_mb "$cid" 2>/dev/null || echo "0") mem_delta=$(echo "$mem_end - $mem_start" | bc 2>/dev/null || echo "0") printf '{"system":"%s","tier":"sustained-summary","test":"50-resizes","mem_start_mb":%s,"mem_end_mb":%s,"mem_delta_mb":%s}\n' \ "$SYSTEM" "$mem_start" "$mem_end" "$mem_delta" >> "$RESULTS_FILE" log "Sustained: start=${mem_start}MB end=${mem_end}MB delta=${mem_delta}MB" log "=== TIER 3: Batch Processing ===" for batch_size in 3 5 10; do files=() for i in $(seq 1 "$batch_size"); do files+=("$S") done bench_tool_multifile "batch" "resize" "small-b${batch_size}" '{"width":100}' "${files[@]}" done for batch_size in 3 5; do files=() for i in $(seq 1 "$batch_size"); do files+=("$L") done bench_tool_multifile "batch" "resize" "large-b${batch_size}" '{"width":800}' "${files[@]}" done bench_tool_multifile "batch" "convert" "mixed-5" '{"format":"webp","quality":80}' \ "$J" "$S" "${F}/test-50x50.webp" "${F}/test-200x150.heic" "${F}/formats/sample.avif" log "=== TIER 4: Pipeline Benchmarks ===" bench_tool "pipeline" "resize" "1step" "$L" '{"width":800}' log "=== Container Cold Start ===" # Record current time as baseline log "Cold start measurement requires container restart - skipping in automated run" log "=== ALL BENCHMARKS COMPLETE for ${SYSTEM} ===" log "Results in: ${RESULTS_FILE}" wc -l "$RESULTS_FILE" | awk '{print $1 " benchmark records written"}'