SnapOtter/tests/benchmark/bench.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
source "${SCRIPT_DIR}/lib/job-aware.sh"
source "${SCRIPT_DIR}/lib/metrics.sh"

SYSTEM="${1:?Usage: bench.sh <system-name> <fixture-dir> [port] [container]}"
FIXTURE_DIR="${2:?Usage: bench.sh <system-name> <fixture-dir> [port] [container]}"
PORT="${3:-1349}"
CONTAINER_REF="${4:-${SNAPOTTER_BENCH_CONTAINER:-}}"
RUN_ID="${SNAPOTTER_BENCH_RUN_ID:-$$_${RANDOM}_${RANDOM}}"
if [[ ! "$SYSTEM" =~ ^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$ ]]; then
  echo "system-name must be 1-64 letters, digits, underscores, or hyphens" >&2
  exit 2
fi
if [[ ! "$RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$ ]]; then
  echo "SNAPOTTER_BENCH_RUN_ID must be 1-64 letters, digits, underscores, or hyphens" >&2
  exit 2
fi
BASE_URL="http://localhost:${PORT}"
RESULTS_FILE="bench-results-${SYSTEM}-${RUN_ID}.jsonl"

log() { echo "[$(date +%H:%M:%S)] $*" >&2; }

get_token() {
  curl -sf -X POST "${BASE_URL}/api/auth/login" \
    -H 'Content-Type: application/json' \
    -d '{"username":"admin","password":"admin"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])"
}

get_container_id() {
  if [ -z "$CONTAINER_REF" ]; then
    return 0
  fi

  local cid running
  cid=$(docker inspect --type container --format '{{.Id}}' "$CONTAINER_REF" 2>/dev/null) || return 1
  running=$(docker inspect --type container --format '{{.State.Running}}' "$cid" 2>/dev/null) || return 1
  [ "$running" = "true" ] || return 1
  printf '%s\n' "$cid"
}

record() {
  local tier="$1" tool="$2" variant="$3" time_s="$4" pass="$5" output_size="${6:-0}" mem_mb="${7:-0}" cpu_pct="${8:-0}"
  local admission_status="${9:-0}" completion_status="${10:-failed}" completion_latency_s="${11:-0}" output_mime="${12:-unknown}"
  printf '{"system":"%s","tier":"%s","tool":"%s","variant":"%s","time_s":%s,"pass":%s,"output_size":%s,"mem_mb":%s,"cpu_pct":%s,"admission_status":%s,"completion_status":"%s","completion_latency_s":%s,"output_mime":"%s"}\n' \
    "$SYSTEM" "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_mb" "$cpu_pct" "$admission_status" "$completion_status" "$completion_latency_s" "$output_mime" >> "$RESULTS_FILE"
}

bench_tool() {
  local tier="$1" tool="$2" variant="$3" file="$4" settings="${5:-}" oracle="${6:-}"
  local cid time_s http_code response_mime mem_after cpu admission_file artifact_file pass output_size

  cid="$BENCH_CONTAINER_ID"
  admission_file=$(mktemp)
  artifact_file=$(mktemp)

  local curl_args=(-sS --max-time 300 -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}")

  if [ -n "$file" ] && [ "$file" != "NONE" ]; then
    curl_args+=(-F "file=@${file}")
  fi

  if [ -n "$settings" ]; then
    curl_args+=(-F "settings=${settings}")
  fi

  curl_args+=(-o "$admission_file" -w $'%{http_code}\t%{time_total}\t%{content_type}')

  local result
  result=$(curl "${curl_args[@]}" 2>/dev/null) || result=$'000\t0.000\tapplication/octet-stream'

  IFS=$'\t' read -r http_code time_s response_mime <<< "$result"
  resolve_benchmark_response "$BASE_URL" "$TOKEN" "$http_code" "$response_mime" \
    "$admission_file" "$artifact_file" "$time_s" "$BENCH_JOB_TIMEOUT_MS" "" "" "$oracle" || true
  pass="$BENCH_PASS"
  time_s="$BENCH_COMPLETION_LATENCY_S"
  output_size="$BENCH_OUTPUT_SIZE"

  mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
  cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0")

  record "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu" \
    "$BENCH_ADMISSION_STATUS" "$BENCH_COMPLETION_STATUS" "$BENCH_COMPLETION_LATENCY_S" "$BENCH_OUTPUT_MIME"

  log "$tier/$tool/$variant: ${time_s}s admission:${BENCH_ADMISSION_STATUS} completion:${BENCH_COMPLETION_STATUS} mime:${BENCH_OUTPUT_MIME} mem:${mem_after}MB pass:${pass}"

  rm -f "$admission_file" "$artifact_file"
}

bench_tool_multifile() {
  local tier="$1" tool="$2" variant="$3" settings="${4:-}" oracle="${5:-}"
  shift 5
  local files=("$@")

  local cid time_s http_code response_mime mem_after cpu admission_file artifact_file pass output_size

  cid="$BENCH_CONTAINER_ID"
  admission_file=$(mktemp)
  artifact_file=$(mktemp)

  local curl_args=(-sS --max-time 300 -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}")

  for f in "${files[@]}"; do
    curl_args+=(-F "file=@${f}")
  done

  if [ -n "$settings" ]; then
    curl_args+=(-F "settings=${settings}")
  fi

  curl_args+=(-o "$admission_file" -w $'%{http_code}\t%{time_total}\t%{content_type}')

  local result
  result=$(curl "${curl_args[@]}" 2>/dev/null) || result=$'000\t0.000\tapplication/octet-stream'

  IFS=$'\t' read -r http_code time_s response_mime <<< "$result"
  resolve_benchmark_response "$BASE_URL" "$TOKEN" "$http_code" "$response_mime" \
    "$admission_file" "$artifact_file" "$time_s" "$BENCH_JOB_TIMEOUT_MS" "" "" "$oracle" || true
  pass="$BENCH_PASS"
  time_s="$BENCH_COMPLETION_LATENCY_S"
  output_size="$BENCH_OUTPUT_SIZE"

  mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
  cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0")
  record "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu" \
    "$BENCH_ADMISSION_STATUS" "$BENCH_COMPLETION_STATUS" "$BENCH_COMPLETION_LATENCY_S" "$BENCH_OUTPUT_MIME"
  log "$tier/$tool/$variant: ${time_s}s admission:${BENCH_ADMISSION_STATUS} completion:${BENCH_COMPLETION_STATUS} mime:${BENCH_OUTPUT_MIME} pass:${pass}"

  rm -f "$admission_file" "$artifact_file"
}

run_n_times() {
  local n="$1"
  shift
  local times=()
  for i in $(seq 1 "$n"); do
    "$@"
  done
}

F="${FIXTURE_DIR}"
S="${F}/image/valid/test-200x150.png"
J="${F}/image/valid/test-100x100.jpg"
L="${F}/image/valid/stress-large.jpg"
P="${F}/image/valid/portrait-color.jpg"
BW="${F}/image/valid/portrait-bw.jpeg"
SVG="${F}/image/valid/svg-logo.svg"
GIF="${F}/image/valid/animated-simpsons.gif"
PDF="${F}/document/valid/test-3page.pdf"
EXIF="${F}/image/valid/test-with-exif.jpg"
FACE="${F}/image/valid/multi-face.webp"
OCR="${F}/image/valid/ocr-chat.jpeg"
OCRJP="${F}/image/valid/ocr-japanese.png"
ISO="${F}/image/valid/portrait-isolated.png"
HEAD="${F}/image/valid/portrait-headshot.heic"
REDEYE="${F}/image/valid/red-eye.jpg"
BARCODE="${F}/image/valid/barcode.avif"

echo "[]" > /dev/null
if [ -e "$RESULTS_FILE" ]; then
  log "Refusing to overwrite benchmark results: ${RESULTS_FILE}"
  exit 2
fi
: > "$RESULTS_FILE"

log "=== Starting benchmarks on ${SYSTEM} ==="
TOKEN=$(get_token)
log "Auth token obtained"
BENCH_CONTAINER_ID=$(get_container_id) || {
  log "Container '$CONTAINER_REF' does not resolve to one running Docker container"
  exit 1
}
if [ -z "$BENCH_CONTAINER_ID" ]; then
  log "Docker metrics disabled; pass a container argument or SNAPOTTER_BENCH_CONTAINER"
fi

log "=== TIER 1: Core Tool Benchmarks ==="

for run in 1 2 3; do
  log "--- Run $run of 3 ---"

  bench_tool "core" "image/resize" "small-r${run}" "$S" '{"width":100,"fit":"cover"}' '{"width":100}'
  bench_tool "core" "image/resize" "large-r${run}" "$L" '{"width":800,"fit":"cover"}' '{"width":800}'

  bench_tool "core" "image/crop" "small-r${run}" "$S" '{"left":10,"top":10,"width":100,"height":100}' '{"width":100,"height":100}'
  bench_tool "core" "image/crop" "large-r${run}" "$L" '{"left":10,"top":10,"width":100,"height":100}' '{"width":100,"height":100}'

  bench_tool "core" "image/rotate" "small-r${run}" "$S" '{"angle":90}' '{"width":150,"height":200}'
  bench_tool "core" "image/rotate" "large-r${run}" "$L" '{"angle":90}' '{"width":3000,"height":4000}'

  bench_tool "core" "image/convert" "jpg-webp-small-r${run}" "$J" '{"format":"webp","quality":80}'
  bench_tool "core" "image/convert" "jpg-webp-large-r${run}" "$L" '{"format":"webp","quality":80}'
  bench_tool "core" "image/convert" "jpg-avif-small-r${run}" "$J" '{"format":"avif","quality":50}'
  bench_tool "core" "image/convert" "jpg-avif-large-r${run}" "$L" '{"format":"avif","quality":50}'

  bench_tool "core" "image/compress" "quality-small-r${run}" "$S" '{"mode":"quality","quality":60}'
  bench_tool "core" "image/compress" "quality-large-r${run}" "$L" '{"mode":"quality","quality":60}'
  bench_tool "core" "image/compress" "targetSize-large-r${run}" "$L" '{"mode":"targetSize","targetSizeKb":500}'

  bench_tool "core" "image/strip-metadata" "small-r${run}" "$EXIF" '{"stripAll":true}'
  bench_tool "core" "image/strip-metadata" "large-r${run}" "$L" '{"stripAll":true}'

  bench_tool "core" "image/edit-metadata" "small-r${run}" "$EXIF" '{"title":"Bench","clearGps":true}'
  bench_tool "core" "image/edit-metadata" "large-r${run}" "$L" '{"title":"Bench","clearGps":true}'

  bench_tool "core" "image/adjust-colors" "small-r${run}" "$S" '{"brightness":20,"contrast":10,"effect":"grayscale"}'
  bench_tool "core" "image/adjust-colors" "large-r${run}" "$L" '{"brightness":20,"contrast":10,"effect":"grayscale"}'

  bench_tool "core" "image/sharpening" "small-r${run}" "$S" '{"method":"adaptive","sigma":1.5}'
  bench_tool "core" "image/sharpening" "large-r${run}" "$L" '{"method":"adaptive","sigma":1.5}'

  bench_tool "core" "image/watermark-text" "small-r${run}" "$S" '{"text":"BENCHMARK","position":"tiled"}'
  bench_tool "core" "image/watermark-text" "large-r${run}" "$L" '{"text":"BENCHMARK","position":"tiled"}'

  bench_tool_multifile "core" "image/compose" "small-r${run}" '{"blendMode":"overlay"}' '' "$S" "$J"
  bench_tool_multifile "core" "image/compose" "large-r${run}" '{"blendMode":"overlay"}' '' "$L" "$P"

  bench_tool_multifile "core" "image/collage" "4img-small-r${run}" '{"templateId":"4-grid"}' '' "$S" "$J" "${F}/image/valid/test-50x50.webp" "${F}/image/valid/test-100x100.svg"
  bench_tool_multifile "core" "image/collage" "4img-large-r${run}" '{"templateId":"4-grid"}' '' "$L" "$P" "$BW" "${F}/image/valid/watermark.jpg"

  bench_tool_multifile "core" "image/stitch" "3img-small-r${run}" '{"direction":"horizontal"}' '' "$S" "$J" "${F}/image/valid/test-50x50.webp"
  bench_tool_multifile "core" "image/stitch" "3img-large-r${run}" '{"direction":"horizontal"}' '' "$L" "$P" "$BW"

  bench_tool "core" "image/split" "small-r${run}" "$S" '{"columns":2,"rows":2}'
  bench_tool "core" "image/split" "large-r${run}" "$L" '{"columns":2,"rows":2}'

  bench_tool "core" "image/border" "small-r${run}" "$S" '{"borderWidth":20,"cornerRadius":10,"shadow":true}'
  bench_tool "core" "image/border" "large-r${run}" "$L" '{"borderWidth":20,"cornerRadius":10,"shadow":true}'

  bench_tool "core" "image/svg-to-raster" "small-r${run}" "${F}/image/valid/test-100x100.svg" '{"width":2000,"dpi":300}' '{"width":2000}'

  bench_tool "core" "image/vectorize" "small-r${run}" "$S" '{"colorMode":"color"}'
  bench_tool "core" "image/vectorize" "large-r${run}" "$P" '{"colorMode":"color"}'

  bench_tool "core" "image/gif-tools" "optimize-r${run}" "$GIF" '{"mode":"optimize","colors":64}'

  bench_tool "core" "pdf/pdf-to-image" "r${run}" "$PDF" '{"format":"png","dpi":300}'

  bench_tool "core" "image/optimize-for-web" "large-r${run}" "$L" '{"format":"webp","quality":80,"maxWidth":1920}' '{"width":1920}'

  bench_tool "core" "image/favicon" "small-r${run}" "$S" ''
  bench_tool "core" "image/favicon" "large-r${run}" "$P" ''

  bench_tool_multifile "core" "image/image-to-pdf" "3img-r${run}" '{"pageSize":"A4"}' '{"pages":3}' "$S" "$J" "${F}/image/valid/test-50x50.webp"

  bench_tool "core" "image/replace-color" "small-r${run}" "$S" '{"sourceColor":"#FFFFFF","targetColor":"#FF0000","tolerance":30}'
  bench_tool "core" "image/replace-color" "large-r${run}" "$L" '{"sourceColor":"#FFFFFF","targetColor":"#FF0000","tolerance":30}'

  bench_tool "core" "image/info" "small-r${run}" "$S" ''
  bench_tool "core" "image/info" "large-r${run}" "$L" ''

  bench_tool_multifile "core" "image/compare" "small-r${run}" '' '' "$S" "$S"
  bench_tool_multifile "core" "image/compare" "large-r${run}" '' '' "$L" "$L"

  bench_tool_multifile "core" "image/find-duplicates" "5img-r${run}" '{"threshold":5}' '' "$S" "$J" "${F}/image/valid/test-50x50.webp" "$S" "$J"

  bench_tool "core" "image/color-palette" "small-r${run}" "$P" ''
  bench_tool "core" "image/color-palette" "large-r${run}" "$L" ''

  bench_tool "core" "image/barcode-read" "r${run}" "$BARCODE" '{"tryHarder":true}'

  bench_tool "core" "image/image-to-base64" "small-r${run}" "$S" '{"outputFormat":"webp"}'
  bench_tool "core" "image/image-to-base64" "large-r${run}" "$L" '{"outputFormat":"webp"}'

  bench_tool "core" "image/content-aware-resize" "small-r${run}" "$S" '{"width":100}' '{"width":100}'
  bench_tool "core" "image/content-aware-resize" "large-r${run}" "$L" '{"width":100}' '{"width":100}'

  bench_tool "core" "image/image-enhancement" "small-r${run}" "$S" '{"mode":"auto","intensity":50}'
  bench_tool "core" "image/image-enhancement" "large-r${run}" "$L" '{"mode":"auto","intensity":50}'
done

log "=== TIER 5: Format Decode Benchmarks ==="

for fmt_file in "${F}/image/formats/"*; do
  fname=$(basename "$fmt_file")
  for run in 1 2 3; do
    bench_tool "format" "image/resize" "fmt-${fname}-r${run}" "$fmt_file" '{"width":200}'
  done
done

log "=== TIER 6: Concurrent Load Benchmarks ==="

for concurrency in 1 3 5 10 20; do
  log "Concurrency: ${concurrency}"
  local_results=$(mktemp)
  pids=()

  for i in $(seq 1 "$concurrency"); do
    (
      admission_file=$(mktemp)
      artifact_file=$(mktemp)
      result=$(curl -s -X POST "${BASE_URL}/api/v1/tools/image/resize" \
        -H "Authorization: Bearer ${TOKEN}" \
        -F "file=@${L}" \
        -F 'settings={"width":800}' \
        -o "$admission_file" -w $'%{http_code}\t%{time_total}\t%{content_type}') \
        || result=$'000\t0.000\tapplication/octet-stream'
      IFS=$'\t' read -r http_code admission_time response_mime <<< "$result"
      resolve_benchmark_response "$BASE_URL" "$TOKEN" "$http_code" "$response_mime" \
        "$admission_file" "$artifact_file" "$admission_time" "$BENCH_JOB_TIMEOUT_MS" "" "" \
        '{"width":800}' || true
      printf '%s\t%s\t%s\n' "$BENCH_PASS" "$BENCH_COMPLETION_LATENCY_S" "$BENCH_ADMISSION_STATUS"
      rm -f "$admission_file" "$artifact_file"
      [ "$BENCH_PASS" = "true" ]
    ) >> "$local_results" &
    pids+=("$!")
  done
  child_wait_failed=0
  wait_for_benchmark_children "$concurrency" "$local_results" "${pids[@]}" || child_wait_failed=1

  cid="$BENCH_CONTAINER_ID"
  mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")

  times=()
  errors=0
  while IFS= read -r line; do
    IFS=$'\t' read -r request_pass t code <<< "$line"
    times+=("$t")
    if [ "$request_pass" != "true" ]; then
      errors=$((errors + 1))
    fi
  done < "$local_results"
  BENCHMARK_FAILURES=$((BENCHMARK_FAILURES + errors))
  if [ "$child_wait_failed" -ne 0 ]; then
    BENCHMARK_FAILURES=$((BENCHMARK_FAILURES + BENCH_CHILD_EXIT_FAILURES))
    if [ "$BENCH_OBSERVED_ROWS" -ne "$BENCH_EXPECTED_ROWS" ]; then
      BENCHMARK_FAILURES=$((BENCHMARK_FAILURES + 1))
    fi
  fi

  sorted=$(printf '%s\n' "${times[@]}" | sort -n)
  count=${#times[@]}
  avg=$(printf '%s\n' "${times[@]}" | awk '{s+=$1} END {printf "%.3f", s/NR}')
  p95_idx=$(( (count * 95 / 100) ))
  [ "$p95_idx" -ge "$count" ] && p95_idx=$((count - 1))
  p95=$(echo "$sorted" | sed -n "$((p95_idx + 1))p")
  max=$(echo "$sorted" | tail -1)
  min=$(echo "$sorted" | head -1)
  completion_status="completed"
  if [ "$errors" -gt 0 ]; then completion_status="failed"; fi

  printf '{"system":"%s","tier":"concurrent","tool":"resize","variant":"c%d","concurrency":%d,"avg_s":%s,"p95_s":%s,"max_s":%s,"min_s":%s,"errors":%d,"mem_mb":%s,"admission_status":"mixed","completion_status":"%s","completion_latency_s":%s,"output_mime":"mixed"}\n' \
    "$SYSTEM" "$concurrency" "$concurrency" "$avg" "${p95:-0}" "${max:-0}" "${min:-0}" "$errors" "$mem_after" "$completion_status" "$avg" >> "$RESULTS_FILE"

  log "concurrent/c${concurrency}: avg=${avg}s p95=${p95}s max=${max}s errors=${errors} mem=${mem_after}MB"
  rm -f "$local_results"
done

log "=== TIER 7: Sustained Load (50 sequential resizes) ==="

cid="$BENCH_CONTAINER_ID"
mem_start=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")

for i in $(seq 1 50); do
  bench_tool "sustained" "image/resize" "seq-${i}" "$L" '{"width":800}' '{"width":800}'
done

mem_end=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
mem_delta=$(echo "$mem_end - $mem_start" | bc 2>/dev/null || echo "0")
printf '{"system":"%s","tier":"sustained-summary","test":"50-resizes","mem_start_mb":%s,"mem_end_mb":%s,"mem_delta_mb":%s}\n' \
  "$SYSTEM" "$mem_start" "$mem_end" "$mem_delta" >> "$RESULTS_FILE"
log "Sustained: start=${mem_start}MB end=${mem_end}MB delta=${mem_delta}MB"

log "=== TIER 3: Batch Processing ==="

for batch_size in 3 5 10; do
  files=()
  for i in $(seq 1 "$batch_size"); do
    files+=("$S")
  done
  bench_tool_multifile "batch" "image/resize" "small-b${batch_size}" '{"width":100}' '{"zipEach":{"width":100}}' "${files[@]}"
done

for batch_size in 3 5; do
  files=()
  for i in $(seq 1 "$batch_size"); do
    files+=("$L")
  done
  bench_tool_multifile "batch" "image/resize" "large-b${batch_size}" '{"width":800}' '{"zipEach":{"width":800}}' "${files[@]}"
done

bench_tool_multifile "batch" "image/convert" "mixed-5" '{"format":"webp","quality":80}' '' \
  "$J" "$S" "${F}/image/valid/test-50x50.webp" "${F}/image/valid/test-200x150.heic" "${F}/image/formats/sample.avif"

log "=== TIER 4: Pipeline Benchmarks ==="

bench_tool "pipeline" "image/resize" "1step" "$L" '{"width":800}' '{"width":800}'

log "=== Container Cold Start ==="
# Record current time as baseline
log "Cold start measurement requires container restart - skipping in automated run"

log "=== ALL BENCHMARKS COMPLETE for ${SYSTEM} ==="
log "Results in: ${RESULTS_FILE}"
wc -l "$RESULTS_FILE" | awk '{print $1 " benchmark records written"}'
benchmark_assert_success