mirror of
https://github.com/snapotter-hq/SnapOtter.git
synced 2026-08-03 07:46:42 +02:00
test: add hardware benchmark scripts and Mac results
Includes bench.sh, bench-ai.sh, bench-limits.sh and corresponding result files for Mac hardware baseline.
This commit is contained in:
ashim-hq
Executable
+188
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SYSTEM="${1:?Usage: bench-ai.sh <system-name> <fixture-dir> [port] [gpu-mode]}"
|
||||
FIXTURE_DIR="${2:?Usage: bench-ai.sh <system-name> <fixture-dir> [port] [gpu-mode]}"
|
||||
PORT="${3:-1349}"
|
||||
GPU_MODE="${4:-gpu}"
|
||||
BASE_URL="http://localhost:${PORT}"
|
||||
RESULTS_FILE="bench-ai-results-${SYSTEM}-${GPU_MODE}.jsonl"
|
||||
|
||||
CONTAINER_NAME="ashim"
|
||||
|
||||
log() { echo "[$(date +%H:%M:%S)] $*" >&2; }
|
||||
|
||||
get_token() {
|
||||
curl -sf -X POST "${BASE_URL}/api/auth/login" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"username":"admin","password":"admin"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])"
|
||||
}
|
||||
|
||||
get_container_id() {
|
||||
docker ps -q -f name="${CONTAINER_NAME}" | head -1
|
||||
}
|
||||
|
||||
docker_mem_mb() {
|
||||
local cid="$1"
|
||||
docker stats "$cid" --no-stream --format "{{.MemUsage}}" 2>/dev/null | awk -F/ '{gsub(/[^0-9.]/, "", $1); if($1+0 > 0) print $1; else print 0}'
|
||||
}
|
||||
|
||||
docker_cpu_pct() {
|
||||
local cid="$1"
|
||||
docker stats "$cid" --no-stream --format "{{.CPUPerc}}" 2>/dev/null | tr -d '%'
|
||||
}
|
||||
|
||||
gpu_vram_mb() {
|
||||
nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null || echo "0"
|
||||
}
|
||||
|
||||
record() {
|
||||
local tier="$1" tool="$2" variant="$3" time_s="$4" pass="$5" output_size="${6:-0}" mem_mb="${7:-0}" cpu_pct="${8:-0}" vram_mb="${9:-0}"
|
||||
printf '{"system":"%s","tier":"%s","tool":"%s","variant":"%s","time_s":%s,"pass":%s,"output_size":%s,"mem_mb":%s,"cpu_pct":%s,"vram_mb":%s,"gpu_mode":"%s"}\n' \
|
||||
"$SYSTEM" "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_mb" "$cpu_pct" "$vram_mb" "$GPU_MODE" >> "$RESULTS_FILE"
|
||||
}
|
||||
|
||||
bench_ai_tool() {
|
||||
local tool="$1" variant="$2" file="$3" settings="${4:-}" extra_args="${5:-}"
|
||||
local cid time_s http_code mem_after cpu vram output_file pass output_size
|
||||
|
||||
cid=$(get_container_id)
|
||||
output_file=$(mktemp)
|
||||
|
||||
local curl_args=(-s -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}")
|
||||
|
||||
if [ -n "$file" ] && [ "$file" != "NONE" ]; then
|
||||
curl_args+=(-F "file=@${file}")
|
||||
fi
|
||||
|
||||
if [ -n "$settings" ]; then
|
||||
curl_args+=(-F "settings=${settings}")
|
||||
fi
|
||||
|
||||
if [ -n "$extra_args" ]; then
|
||||
eval "curl_args+=($extra_args)"
|
||||
fi
|
||||
|
||||
curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}")
|
||||
|
||||
local result
|
||||
result=$(curl --max-time 300 "${curl_args[@]}" 2>/dev/null) || result="000 0.000"
|
||||
|
||||
http_code=$(echo "$result" | awk '{print $1}')
|
||||
time_s=$(echo "$result" | awk '{print $2}')
|
||||
|
||||
mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0")
|
||||
vram=$(gpu_vram_mb)
|
||||
|
||||
output_size=$(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
pass="true"
|
||||
else
|
||||
pass="false"
|
||||
fi
|
||||
|
||||
record "ai" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu" "$vram"
|
||||
log "ai/$tool/$variant: ${time_s}s HTTP:${http_code} mem:${mem_after}MB vram:${vram}MB pass:${pass}"
|
||||
|
||||
rm -f "$output_file"
|
||||
}
|
||||
|
||||
F="${FIXTURE_DIR}"
|
||||
P="${F}/content/portrait-color.jpg"
|
||||
ISO="${F}/content/portrait-isolated.png"
|
||||
J="${F}/test-100x100.jpg"
|
||||
BW="${F}/content/portrait-bw.jpeg"
|
||||
OCR="${F}/content/ocr-chat.jpeg"
|
||||
OCRJP="${F}/content/ocr-japanese.png"
|
||||
FACE="${F}/content/multi-face.webp"
|
||||
HEAD="${F}/content/portrait-headshot.heic"
|
||||
REDEYE="${F}/content/red-eye.jpg"
|
||||
S="${F}/test-200x150.png"
|
||||
L="${F}/content/stress-large.jpg"
|
||||
|
||||
> "$RESULTS_FILE"
|
||||
|
||||
log "=== Starting AI benchmarks on ${SYSTEM} (${GPU_MODE}) ==="
|
||||
TOKEN=$(get_token)
|
||||
log "Auth token obtained"
|
||||
|
||||
for run in 1 2 3; do
|
||||
log "--- AI Run $run of 3 ---"
|
||||
|
||||
bench_ai_tool "remove-background" "portrait-r${run}" "$P" '{"backgroundType":"transparent"}'
|
||||
bench_ai_tool "remove-background" "isolated-r${run}" "$ISO" '{"backgroundType":"color","backgroundColor":"#0000FF"}'
|
||||
|
||||
bench_ai_tool "upscale" "2x-small-r${run}" "$J" '{"scale":2}'
|
||||
bench_ai_tool "upscale" "2x-large-r${run}" "$P" '{"scale":2}'
|
||||
bench_ai_tool "upscale" "face-r${run}" "$P" '{"scale":2,"faceEnhance":true}'
|
||||
|
||||
bench_ai_tool "ocr" "fast-r${run}" "$OCR" '{"quality":"fast","language":"en"}'
|
||||
bench_ai_tool "ocr" "best-r${run}" "$OCR" '{"quality":"best","language":"en"}'
|
||||
bench_ai_tool "ocr" "japanese-r${run}" "$OCRJP" '{"quality":"balanced","language":"ja"}'
|
||||
|
||||
bench_ai_tool "blur-faces" "r${run}" "$FACE" '{"blurRadius":30,"sensitivity":0.5}'
|
||||
|
||||
bench_ai_tool "smart-crop" "face-r${run}" "$P" '{"mode":"face","width":400,"height":400}'
|
||||
|
||||
bench_ai_tool "colorize" "r${run}" "$BW" '{"intensity":1.0}'
|
||||
|
||||
bench_ai_tool "enhance-faces" "gfpgan-r${run}" "$P" '{"model":"gfpgan","strength":0.8}'
|
||||
bench_ai_tool "enhance-faces" "codeformer-r${run}" "$P" '{"model":"codeformer","strength":0.7}'
|
||||
|
||||
bench_ai_tool "noise-removal" "quick-r${run}" "$S" '{"tier":"quick"}'
|
||||
bench_ai_tool "noise-removal" "quality-r${run}" "$L" '{"tier":"quality"}'
|
||||
|
||||
bench_ai_tool "red-eye-removal" "r${run}" "$REDEYE" '{"sensitivity":50,"strength":80}'
|
||||
|
||||
bench_ai_tool "restore-photo" "full-r${run}" "$BW" '{"mode":"auto","scratchRemoval":true,"faceEnhancement":true,"colorize":true}'
|
||||
|
||||
bench_ai_tool "passport-photo" "r${run}" "$HEAD" ''
|
||||
|
||||
bench_ai_tool "content-aware-resize" "face-r${run}" "$P" '{"width":300,"protectFaces":true}'
|
||||
done
|
||||
|
||||
log "=== TIER 3: AI Batch Processing ==="
|
||||
|
||||
for batch_size in 3 5; do
|
||||
log "AI Batch ${batch_size} - remove-background"
|
||||
output_file=$(mktemp)
|
||||
cid=$(get_container_id)
|
||||
|
||||
curl_args=(-s -X POST "${BASE_URL}/api/v1/tools/remove-background" -H "Authorization: Bearer ${TOKEN}")
|
||||
for i in $(seq 1 "$batch_size"); do
|
||||
curl_args+=(-F "file=@${P}")
|
||||
done
|
||||
curl_args+=(-F 'settings={"backgroundType":"transparent"}')
|
||||
curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}")
|
||||
|
||||
result=$(curl --max-time 600 "${curl_args[@]}" 2>/dev/null) || result="000 0.000"
|
||||
http_code=$(echo "$result" | awk '{print $1}')
|
||||
time_s=$(echo "$result" | awk '{print $2}')
|
||||
mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
vram=$(gpu_vram_mb)
|
||||
pass=$( [ "$http_code" = "200" ] && echo "true" || echo "false" )
|
||||
|
||||
record "ai-batch" "remove-background" "b${batch_size}" "$time_s" "$pass" "0" "$mem_after" "0" "$vram"
|
||||
log "ai-batch/remove-background/b${batch_size}: ${time_s}s HTTP:${http_code}"
|
||||
rm -f "$output_file"
|
||||
done
|
||||
|
||||
log "=== Sustained AI Load (10 cycles) ==="
|
||||
cid=$(get_container_id)
|
||||
mem_start=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
vram_start=$(gpu_vram_mb)
|
||||
|
||||
for i in $(seq 1 10); do
|
||||
bench_ai_tool "remove-background" "sustained-${i}" "$P" '{"backgroundType":"transparent"}'
|
||||
done
|
||||
|
||||
mem_end=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
vram_end=$(gpu_vram_mb)
|
||||
printf '{"system":"%s","tier":"ai-sustained-summary","gpu_mode":"%s","mem_start_mb":%s,"mem_end_mb":%s,"vram_start_mb":%s,"vram_end_mb":%s}\n' \
|
||||
"$SYSTEM" "$GPU_MODE" "$mem_start" "$mem_end" "$vram_start" "$vram_end" >> "$RESULTS_FILE"
|
||||
|
||||
log "=== ALL AI BENCHMARKS COMPLETE for ${SYSTEM} (${GPU_MODE}) ==="
|
||||
log "Results in: ${RESULTS_FILE}"
|
||||
wc -l "$RESULTS_FILE" | awk '{print $1 " AI benchmark records written"}'
|
||||
Executable
+191
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SYSTEM="${1:?Usage: bench-limits.sh <system-name> <fixture-dir> <docker-image>}"
|
||||
FIXTURE_DIR="${2:?}"
|
||||
DOCKER_IMAGE="${3:-ashim:latest}"
|
||||
PORT=13491
|
||||
BASE_URL="http://localhost:${PORT}"
|
||||
RESULTS_FILE="bench-limits-results-${SYSTEM}.jsonl"
|
||||
CONTAINER_NAME="ashim-bench-limits"
|
||||
|
||||
log() { echo "[$(date +%H:%M:%S)] $*" >&2; }
|
||||
|
||||
> "$RESULTS_FILE"
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
|
||||
}
|
||||
|
||||
start_container() {
|
||||
local cpus="$1" memory="$2"
|
||||
cleanup
|
||||
log "Starting container: --cpus=${cpus} --memory=${memory}"
|
||||
|
||||
docker run --rm -d \
|
||||
--cpus="$cpus" --memory="$memory" \
|
||||
-p "${PORT}:1349" \
|
||||
-e AUTH_ENABLED=false \
|
||||
-e SKIP_MUST_CHANGE_PASSWORD=true \
|
||||
--name "$CONTAINER_NAME" \
|
||||
"$DOCKER_IMAGE" >/dev/null
|
||||
|
||||
log "Waiting for health..."
|
||||
local attempts=0
|
||||
while ! curl -sf "http://localhost:${PORT}/api/v1/health" >/dev/null 2>&1; do
|
||||
sleep 2
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -gt 60 ]; then
|
||||
log "Container failed to become healthy after 120s"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
log "Container healthy"
|
||||
}
|
||||
|
||||
run_bench() {
|
||||
local cpus="$1" memory="$2" tool="$3" variant="$4" file="$5" settings="${6:-}"
|
||||
local output_file time_s http_code pass output_size mem_after
|
||||
|
||||
output_file=$(mktemp)
|
||||
|
||||
local curl_args=(-s --max-time 120 -X POST "${BASE_URL}/api/v1/tools/${tool}")
|
||||
|
||||
if [ -n "$file" ] && [ "$file" != "NONE" ]; then
|
||||
curl_args+=(-F "file=@${file}")
|
||||
fi
|
||||
|
||||
if [ -n "$settings" ]; then
|
||||
curl_args+=(-F "settings=${settings}")
|
||||
fi
|
||||
|
||||
curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}")
|
||||
|
||||
local result
|
||||
result=$(curl "${curl_args[@]}" 2>/dev/null) || result="000 0.000"
|
||||
|
||||
http_code=$(echo "$result" | awk '{print $1}')
|
||||
time_s=$(echo "$result" | awk '{print $2}')
|
||||
|
||||
mem_after=$(docker stats "$CONTAINER_NAME" --no-stream --format "{{.MemUsage}}" 2>/dev/null | awk -F/ '{gsub(/[^0-9.]/, "", $1); if($1+0 > 0) print $1; else print 0}' || echo "0")
|
||||
|
||||
output_size=$(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
pass="true"
|
||||
else
|
||||
pass="false"
|
||||
fi
|
||||
|
||||
printf '{"system":"%s","tier":"resource-limit","cpus":"%s","memory":"%s","tool":"%s","variant":"%s","time_s":%s,"pass":%s,"output_size":%s,"mem_mb":%s}\n' \
|
||||
"$SYSTEM" "$cpus" "$memory" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" >> "$RESULTS_FILE"
|
||||
|
||||
log "cpus=${cpus} mem=${memory} ${tool}/${variant}: ${time_s}s HTTP:${http_code} pass:${pass}"
|
||||
rm -f "$output_file"
|
||||
}
|
||||
|
||||
run_batch_bench() {
|
||||
local cpus="$1" memory="$2" count="$3"
|
||||
local output_file time_s http_code pass mem_after
|
||||
|
||||
output_file=$(mktemp)
|
||||
|
||||
local curl_args=(-s --max-time 180 -X POST "${BASE_URL}/api/v1/tools/resize")
|
||||
|
||||
for i in $(seq 1 "$count"); do
|
||||
curl_args+=(-F "file=@${F}/test-200x150.png")
|
||||
done
|
||||
|
||||
curl_args+=(-F 'settings={"width":100}')
|
||||
curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}")
|
||||
|
||||
local result
|
||||
result=$(curl "${curl_args[@]}" 2>/dev/null) || result="000 0.000"
|
||||
|
||||
http_code=$(echo "$result" | awk '{print $1}')
|
||||
time_s=$(echo "$result" | awk '{print $2}')
|
||||
|
||||
mem_after=$(docker stats "$CONTAINER_NAME" --no-stream --format "{{.MemUsage}}" 2>/dev/null | awk -F/ '{gsub(/[^0-9.]/, "", $1); if($1+0 > 0) print $1; else print 0}' || echo "0")
|
||||
|
||||
if [ "$http_code" = "200" ]; then pass="true"; else pass="false"; fi
|
||||
|
||||
printf '{"system":"%s","tier":"resource-limit","cpus":"%s","memory":"%s","tool":"batch-resize","variant":"b%d","time_s":%s,"pass":%s,"output_size":0,"mem_mb":%s}\n' \
|
||||
"$SYSTEM" "$cpus" "$memory" "$count" "$time_s" "$pass" "$mem_after" >> "$RESULTS_FILE"
|
||||
|
||||
log "cpus=${cpus} mem=${memory} batch-resize/b${count}: ${time_s}s HTTP:${http_code} pass:${pass}"
|
||||
rm -f "$output_file"
|
||||
}
|
||||
|
||||
F="${FIXTURE_DIR}"
|
||||
L="${F}/content/stress-large.jpg"
|
||||
|
||||
log "=== Resource Limit Sweep on ${SYSTEM} ==="
|
||||
|
||||
configs=(
|
||||
"1:512m"
|
||||
"1:1g"
|
||||
"1:2g"
|
||||
"2:1g"
|
||||
"2:2g"
|
||||
"2:4g"
|
||||
"4:2g"
|
||||
"4:4g"
|
||||
)
|
||||
|
||||
for config in "${configs[@]}"; do
|
||||
cpus="${config%%:*}"
|
||||
memory="${config##*:}"
|
||||
|
||||
if start_container "$cpus" "$memory"; then
|
||||
sleep 2
|
||||
|
||||
run_bench "$cpus" "$memory" "resize" "large" "$L" '{"width":800,"fit":"cover"}'
|
||||
run_bench "$cpus" "$memory" "compress" "targetSize" "$L" '{"mode":"targetSize","targetSizeKb":500}'
|
||||
run_bench "$cpus" "$memory" "convert" "avif" "$L" '{"format":"avif","quality":50}'
|
||||
|
||||
run_batch_bench "$cpus" "$memory" 5
|
||||
|
||||
run_bench "$cpus" "$memory" "collage" "4img" "NONE" '' \
|
||||
|| log "Collage test skipped (needs multi-file support)"
|
||||
|
||||
cleanup
|
||||
else
|
||||
log "FAILED to start container at cpus=${cpus} mem=${memory}"
|
||||
printf '{"system":"%s","tier":"resource-limit","cpus":"%s","memory":"%s","tool":"startup","variant":"failed","time_s":0,"pass":false,"output_size":0,"mem_mb":0}\n' \
|
||||
"$SYSTEM" "$cpus" "$memory" >> "$RESULTS_FILE"
|
||||
cleanup
|
||||
fi
|
||||
done
|
||||
|
||||
log "=== Cold Start Timing ==="
|
||||
|
||||
for config in "1:512m" "2:2g" "4:4g"; do
|
||||
cpus="${config%%:*}"
|
||||
memory="${config##*:}"
|
||||
cleanup
|
||||
|
||||
start_time=$(date +%s%N)
|
||||
docker run --rm -d --cpus="$cpus" --memory="$memory" -p "${PORT}:1349" \
|
||||
-e AUTH_ENABLED=false -e SKIP_MUST_CHANGE_PASSWORD=true \
|
||||
--name "$CONTAINER_NAME" "$DOCKER_IMAGE" >/dev/null
|
||||
|
||||
attempts=0
|
||||
while ! curl -sf "http://localhost:${PORT}/api/v1/health" >/dev/null 2>&1; do
|
||||
sleep 0.5
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -gt 120 ]; then break; fi
|
||||
done
|
||||
end_time=$(date +%s%N)
|
||||
|
||||
startup_s=$(echo "scale=3; ($end_time - $start_time) / 1000000000" | bc 2>/dev/null || echo "0")
|
||||
|
||||
printf '{"system":"%s","tier":"cold-start","cpus":"%s","memory":"%s","startup_s":%s}\n' \
|
||||
"$SYSTEM" "$cpus" "$memory" "$startup_s" >> "$RESULTS_FILE"
|
||||
|
||||
log "Cold start cpus=${cpus} mem=${memory}: ${startup_s}s"
|
||||
cleanup
|
||||
done
|
||||
|
||||
log "=== Resource Limit Sweep COMPLETE ==="
|
||||
log "Results in: ${RESULTS_FILE}"
|
||||
wc -l "$RESULTS_FILE" | awk '{print $1 " records written"}'
|
||||
@@ -0,0 +1,94 @@
|
||||
{"system":"mac","tier":"core","tool":"resize","variant":"small-r1","time_s":0.014626,"pass":true,"output_size":181,"mem_mb":279.6,"cpu_pct":0.11}
|
||||
{"system":"mac","tier":"core","tool":"resize","variant":"large-r1","time_s":0.301508,"pass":true,"output_size":188,"mem_mb":356.1,"cpu_pct":0.11}
|
||||
{"system":"mac","tier":"core","tool":"crop","variant":"small-r1","time_s":0.008267,"pass":true,"output_size":179,"mem_mb":356.1,"cpu_pct":0.04}
|
||||
{"system":"mac","tier":"core","tool":"crop","variant":"large-r1","time_s":0.273358,"pass":true,"output_size":184,"mem_mb":425.7,"cpu_pct":0.10}
|
||||
{"system":"mac","tier":"core","tool":"rotate","variant":"small-r1","time_s":0.014258,"pass":true,"output_size":181,"mem_mb":426,"cpu_pct":0.03}
|
||||
{"system":"mac","tier":"core","tool":"rotate","variant":"large-r1","time_s":0.376651,"pass":true,"output_size":189,"mem_mb":545.7,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-webp-small-r1","time_s":0.006595,"pass":true,"output_size":175,"mem_mb":545.5,"cpu_pct":0.28}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-webp-large-r1","time_s":1.328183,"pass":true,"output_size":182,"mem_mb":532.2,"cpu_pct":0.14}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-avif-small-r1","time_s":0.010060,"pass":true,"output_size":175,"mem_mb":532.6,"cpu_pct":0.08}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-avif-large-r1","time_s":2.741221,"pass":true,"output_size":182,"mem_mb":603.7,"cpu_pct":0.11}
|
||||
{"system":"mac","tier":"core","tool":"compress","variant":"quality-small-r1","time_s":0.020368,"pass":true,"output_size":183,"mem_mb":603.2,"cpu_pct":0.05}
|
||||
{"system":"mac","tier":"core","tool":"compress","variant":"quality-large-r1","time_s":0.331297,"pass":true,"output_size":190,"mem_mb":615,"cpu_pct":0.08}
|
||||
{"system":"mac","tier":"core","tool":"compress","variant":"targetSize-large-r1","time_s":0.427988,"pass":true,"output_size":190,"mem_mb":695.1,"cpu_pct":0.18}
|
||||
{"system":"mac","tier":"core","tool":"strip-metadata","variant":"small-r1","time_s":0.009445,"pass":true,"output_size":191,"mem_mb":695.3,"cpu_pct":0.11}
|
||||
{"system":"mac","tier":"core","tool":"strip-metadata","variant":"large-r1","time_s":0.972625,"pass":true,"output_size":197,"mem_mb":718.3,"cpu_pct":0.10}
|
||||
{"system":"mac","tier":"core","tool":"edit-metadata","variant":"small-r1","time_s":0.182409,"pass":true,"output_size":177,"mem_mb":718.5,"cpu_pct":0.08}
|
||||
{"system":"mac","tier":"core","tool":"edit-metadata","variant":"large-r1","time_s":0.218451,"pass":true,"output_size":182,"mem_mb":718.7,"cpu_pct":0.84}
|
||||
{"system":"mac","tier":"core","tool":"color-adjustments","variant":"small-r1","time_s":0.007756,"pass":false,"output_size":40,"mem_mb":671.7,"cpu_pct":0.04}
|
||||
{"system":"mac","tier":"core","tool":"color-adjustments","variant":"large-r1","time_s":0.003707,"pass":false,"output_size":40,"mem_mb":266.7,"cpu_pct":0.33}
|
||||
{"system":"mac","tier":"core","tool":"sharpening","variant":"small-r1","time_s":0.018643,"pass":true,"output_size":185,"mem_mb":510.6,"cpu_pct":99.66}
|
||||
{"system":"mac","tier":"core","tool":"sharpening","variant":"large-r1","time_s":0.744501,"pass":true,"output_size":193,"mem_mb":511,"cpu_pct":98.44}
|
||||
{"system":"mac","tier":"core","tool":"watermark-text","variant":"small-r1","time_s":0.045615,"pass":true,"output_size":189,"mem_mb":511.8,"cpu_pct":99.62}
|
||||
{"system":"mac","tier":"core","tool":"watermark-text","variant":"large-r1","time_s":0.597063,"pass":true,"output_size":197,"mem_mb":509.4,"cpu_pct":97.67}
|
||||
{"system":"mac","tier":"core","tool":"compose","variant":"small-r1","time_s":0.002649,"pass":false,"output_size":37,"mem_mb":496.5,"cpu_pct":96.50}
|
||||
{"system":"mac","tier":"core","tool":"compose","variant":"large-r1","time_s":0.073550,"pass":false,"output_size":37,"mem_mb":156.5,"cpu_pct":0.05}
|
||||
{"system":"mac","tier":"core","tool":"collage","variant":"4img-small-r1","time_s":0.228542,"pass":true,"output_size":172,"mem_mb":157.3,"cpu_pct":0}
|
||||
{"system":"mac","tier":"core","tool":"collage","variant":"4img-large-r1","time_s":0.803269,"pass":true,"output_size":177,"mem_mb":778.5,"cpu_pct":0.06}
|
||||
{"system":"mac","tier":"core","tool":"stitch","variant":"3img-small-r1","time_s":0.015616,"pass":true,"output_size":169,"mem_mb":288.5,"cpu_pct":0.21}
|
||||
{"system":"mac","tier":"core","tool":"stitch","variant":"3img-large-r1","time_s":0.643335,"pass":true,"output_size":176,"mem_mb":549.8,"cpu_pct":101.54}
|
||||
{"system":"mac","tier":"core","tool":"split","variant":"small-r1","time_s":0.023203,"pass":true,"output_size":1134,"mem_mb":746.8,"cpu_pct":96.29}
|
||||
{"system":"mac","tier":"core","tool":"split","variant":"large-r1","time_s":0.383536,"pass":true,"output_size":1101452,"mem_mb":772,"cpu_pct":102.62}
|
||||
{"system":"mac","tier":"core","tool":"border","variant":"small-r1","time_s":0.032606,"pass":true,"output_size":182,"mem_mb":805.2,"cpu_pct":99.91}
|
||||
{"system":"mac","tier":"core","tool":"border","variant":"large-r1","time_s":3.942523,"pass":true,"output_size":183,"mem_mb":519.7,"cpu_pct":0.53}
|
||||
{"system":"mac","tier":"core","tool":"svg-to-raster","variant":"small-r1","time_s":0.152959,"pass":true,"output_size":176,"mem_mb":0,"cpu_pct":0}
|
||||
{"system":"mac","tier":"core","tool":"vectorize","variant":"small-r1","time_s":0.010541,"pass":true,"output_size":174,"mem_mb":303.4,"cpu_pct":2.73}
|
||||
{"system":"mac","tier":"core","tool":"vectorize","variant":"large-r1","time_s":0.457126,"pass":true,"output_size":182,"mem_mb":574.3,"cpu_pct":99.77}
|
||||
{"system":"mac","tier":"core","tool":"gif-tools","variant":"optimize-r1","time_s":0.088603,"pass":true,"output_size":194,"mem_mb":827.6,"cpu_pct":103.66}
|
||||
{"system":"mac","tier":"core","tool":"pdf-to-image","variant":"r1","time_s":0.372870,"pass":true,"output_size":521,"mem_mb":802.9,"cpu_pct":99.53}
|
||||
{"system":"mac","tier":"core","tool":"optimize-for-web","variant":"large-r1","time_s":0.818123,"pass":true,"output_size":182,"mem_mb":835.9,"cpu_pct":100.16}
|
||||
{"system":"mac","tier":"core","tool":"favicon","variant":"small-r1","time_s":0.034346,"pass":true,"output_size":2468,"mem_mb":861.8,"cpu_pct":98.31}
|
||||
{"system":"mac","tier":"core","tool":"favicon","variant":"large-r1","time_s":0.051474,"pass":true,"output_size":736200,"mem_mb":490.1,"cpu_pct":6.14}
|
||||
{"system":"mac","tier":"core","tool":"image-to-pdf","variant":"3img-r1","time_s":0.029216,"pass":true,"output_size":180,"mem_mb":0,"cpu_pct":0.00}
|
||||
{"system":"mac","tier":"core","tool":"replace-color","variant":"small-r1","time_s":0.007426,"pass":true,"output_size":188,"mem_mb":834.7,"cpu_pct":0.11}
|
||||
{"system":"mac","tier":"core","tool":"replace-color","variant":"large-r1","time_s":1.004593,"pass":true,"output_size":197,"mem_mb":839.8,"cpu_pct":198.06}
|
||||
{"system":"mac","tier":"core","tool":"info","variant":"small-r1","time_s":0.014790,"pass":true,"output_size":537,"mem_mb":916.4,"cpu_pct":198.94}
|
||||
{"system":"mac","tier":"core","tool":"info","variant":"large-r1","time_s":0.367496,"pass":true,"output_size":490,"mem_mb":616.5,"cpu_pct":0.65}
|
||||
{"system":"mac","tier":"core","tool":"compare","variant":"small-r1","time_s":0.019049,"pass":true,"output_size":224,"mem_mb":0,"cpu_pct":0.00}
|
||||
{"system":"mac","tier":"core","tool":"compare","variant":"large-r1","time_s":1.237066,"pass":true,"output_size":234,"mem_mb":888.1,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"find-duplicates","variant":"5img-r1","time_s":0.031787,"pass":true,"output_size":3427,"mem_mb":378.2,"cpu_pct":0.14}
|
||||
{"system":"mac","tier":"core","tool":"color-palette","variant":"small-r1","time_s":0.016019,"pass":true,"output_size":134,"mem_mb":860.2,"cpu_pct":198.97}
|
||||
{"system":"mac","tier":"core","tool":"color-palette","variant":"large-r1","time_s":0.116914,"pass":true,"output_size":132,"mem_mb":936.1,"cpu_pct":0.05}
|
||||
{"system":"mac","tier":"core","tool":"barcode-read","variant":"r1","time_s":0.136906,"pass":true,"output_size":79,"mem_mb":609.7,"cpu_pct":0}
|
||||
{"system":"mac","tier":"core","tool":"image-to-base64","variant":"small-r1","time_s":0.005899,"pass":true,"output_size":615,"mem_mb":809.6,"cpu_pct":1.25}
|
||||
{"system":"mac","tier":"core","tool":"image-to-base64","variant":"large-r1","time_s":1.250363,"pass":true,"output_size":1298304,"mem_mb":887.9,"cpu_pct":198.25}
|
||||
{"system":"mac","tier":"core","tool":"content-aware-resize","variant":"small-r1","time_s":0.274548,"pass":true,"output_size":204,"mem_mb":979.4,"cpu_pct":171.83}
|
||||
{"system":"mac","tier":"core","tool":"content-aware-resize","variant":"large-r1","time_s":120.532070,"pass":false,"output_size":188727,"mem_mb":0,"cpu_pct":0}
|
||||
{"system":"mac","tier":"core","tool":"image-enhancement","variant":"small-r1","time_s":0.060936,"pass":true,"output_size":192,"mem_mb":839.1,"cpu_pct":0.16}
|
||||
{"system":"mac","tier":"core","tool":"image-enhancement","variant":"large-r1","time_s":1.162208,"pass":true,"output_size":200,"mem_mb":871,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"resize","variant":"small-r2","time_s":0.008022,"pass":true,"output_size":181,"mem_mb":870.5,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"resize","variant":"large-r2","time_s":0.319399,"pass":true,"output_size":188,"mem_mb":836.4,"cpu_pct":0.06}
|
||||
{"system":"mac","tier":"core","tool":"crop","variant":"small-r2","time_s":0.011819,"pass":true,"output_size":179,"mem_mb":836.4,"cpu_pct":0.05}
|
||||
{"system":"mac","tier":"core","tool":"crop","variant":"large-r2","time_s":0.258094,"pass":true,"output_size":184,"mem_mb":836.6,"cpu_pct":0.15}
|
||||
{"system":"mac","tier":"core","tool":"rotate","variant":"small-r2","time_s":0.009025,"pass":true,"output_size":181,"mem_mb":836.7,"cpu_pct":0.13}
|
||||
{"system":"mac","tier":"core","tool":"rotate","variant":"large-r2","time_s":0.376644,"pass":true,"output_size":189,"mem_mb":836.9,"cpu_pct":0.13}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-webp-small-r2","time_s":0.009957,"pass":true,"output_size":175,"mem_mb":836.9,"cpu_pct":0.11}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-webp-large-r2","time_s":1.274019,"pass":true,"output_size":182,"mem_mb":851.5,"cpu_pct":0.15}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-avif-small-r2","time_s":0.011801,"pass":true,"output_size":175,"mem_mb":851.4,"cpu_pct":0.16}
|
||||
{"system":"mac","tier":"core","tool":"convert","variant":"jpg-avif-large-r2","time_s":2.436089,"pass":true,"output_size":182,"mem_mb":889.2,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"compress","variant":"quality-small-r2","time_s":0.012862,"pass":true,"output_size":183,"mem_mb":889.2,"cpu_pct":0.14}
|
||||
{"system":"mac","tier":"core","tool":"compress","variant":"quality-large-r2","time_s":0.313211,"pass":true,"output_size":190,"mem_mb":854.5,"cpu_pct":0.13}
|
||||
{"system":"mac","tier":"core","tool":"compress","variant":"targetSize-large-r2","time_s":0.421803,"pass":true,"output_size":190,"mem_mb":878.8,"cpu_pct":0.22}
|
||||
{"system":"mac","tier":"core","tool":"strip-metadata","variant":"small-r2","time_s":0.008415,"pass":true,"output_size":191,"mem_mb":878.8,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"strip-metadata","variant":"large-r2","time_s":0.965809,"pass":true,"output_size":197,"mem_mb":879.2,"cpu_pct":1.59}
|
||||
{"system":"mac","tier":"core","tool":"edit-metadata","variant":"small-r2","time_s":0.160852,"pass":true,"output_size":177,"mem_mb":878.7,"cpu_pct":0.06}
|
||||
{"system":"mac","tier":"core","tool":"edit-metadata","variant":"large-r2","time_s":0.220078,"pass":true,"output_size":182,"mem_mb":878.9,"cpu_pct":0.53}
|
||||
{"system":"mac","tier":"core","tool":"color-adjustments","variant":"small-r2","time_s":0.004969,"pass":false,"output_size":40,"mem_mb":878.8,"cpu_pct":0.16}
|
||||
{"system":"mac","tier":"core","tool":"color-adjustments","variant":"large-r2","time_s":0.002368,"pass":false,"output_size":40,"mem_mb":879.4,"cpu_pct":0.09}
|
||||
{"system":"mac","tier":"core","tool":"sharpening","variant":"small-r2","time_s":0.020579,"pass":true,"output_size":185,"mem_mb":880,"cpu_pct":2.11}
|
||||
{"system":"mac","tier":"core","tool":"sharpening","variant":"large-r2","time_s":0.715239,"pass":true,"output_size":193,"mem_mb":880.1,"cpu_pct":0.14}
|
||||
{"system":"mac","tier":"core","tool":"watermark-text","variant":"small-r2","time_s":0.012788,"pass":true,"output_size":189,"mem_mb":880.1,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"watermark-text","variant":"large-r2","time_s":0.578214,"pass":true,"output_size":197,"mem_mb":880.1,"cpu_pct":0.14}
|
||||
{"system":"mac","tier":"core","tool":"compose","variant":"small-r2","time_s":0.002695,"pass":false,"output_size":37,"mem_mb":880.1,"cpu_pct":0.42}
|
||||
{"system":"mac","tier":"core","tool":"compose","variant":"large-r2","time_s":0.072563,"pass":false,"output_size":37,"mem_mb":880.6,"cpu_pct":0.12}
|
||||
{"system":"mac","tier":"core","tool":"collage","variant":"4img-small-r2","time_s":0.248801,"pass":true,"output_size":172,"mem_mb":886.2,"cpu_pct":1.74}
|
||||
{"system":"mac","tier":"core","tool":"collage","variant":"4img-large-r2","time_s":0.763333,"pass":true,"output_size":177,"mem_mb":926.1,"cpu_pct":0.08}
|
||||
{"system":"mac","tier":"core","tool":"stitch","variant":"3img-small-r2","time_s":0.012169,"pass":true,"output_size":169,"mem_mb":886.1,"cpu_pct":0.09}
|
||||
{"system":"mac","tier":"core","tool":"stitch","variant":"3img-large-r2","time_s":0.542565,"pass":true,"output_size":176,"mem_mb":936.2,"cpu_pct":0.13}
|
||||
{"system":"mac","tier":"core","tool":"split","variant":"small-r2","time_s":0.010531,"pass":true,"output_size":1134,"mem_mb":936.2,"cpu_pct":0.18}
|
||||
{"system":"mac","tier":"core","tool":"split","variant":"large-r2","time_s":0.436432,"pass":true,"output_size":1101452,"mem_mb":936.2,"cpu_pct":0.07}
|
||||
{"system":"mac","tier":"core","tool":"border","variant":"small-r2","time_s":0.037580,"pass":true,"output_size":182,"mem_mb":895.3,"cpu_pct":0.19}
|
||||
{"system":"mac","tier":"core","tool":"border","variant":"large-r2","time_s":3.915338,"pass":true,"output_size":183,"mem_mb":898.5,"cpu_pct":0.07}
|
||||
{"system":"mac","tier":"core","tool":"svg-to-raster","variant":"small-r2","time_s":0.166333,"pass":true,"output_size":176,"mem_mb":897.8,"cpu_pct":0.13}
|
||||
{"system":"mac","tier":"core","tool":"vectorize","variant":"small-r2","time_s":0.010872,"pass":true,"output_size":174,"mem_mb":897.8,"cpu_pct":0.28}
|
||||
{"system":"mac","tier":"core","tool":"vectorize","variant":"large-r2","time_s":0.412589,"pass":true,"output_size":182,"mem_mb":900.2,"cpu_pct":0.72}
|
||||
Executable
+366
@@ -0,0 +1,366 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SYSTEM="${1:?Usage: bench.sh <system-name> <fixture-dir> [port]}"
|
||||
FIXTURE_DIR="${2:?Usage: bench.sh <system-name> <fixture-dir> [port]}"
|
||||
PORT="${3:-1349}"
|
||||
BASE_URL="http://localhost:${PORT}"
|
||||
RESULTS_FILE="bench-results-${SYSTEM}.jsonl"
|
||||
|
||||
CONTAINER_NAME="ashim"
|
||||
|
||||
log() { echo "[$(date +%H:%M:%S)] $*" >&2; }
|
||||
|
||||
get_token() {
|
||||
curl -sf -X POST "${BASE_URL}/api/auth/login" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"username":"admin","password":"admin"}' | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])"
|
||||
}
|
||||
|
||||
get_container_id() {
|
||||
docker ps -q -f name="${CONTAINER_NAME}" | head -1
|
||||
}
|
||||
|
||||
docker_mem_mb() {
|
||||
local cid="$1"
|
||||
docker stats "$cid" --no-stream --format "{{.MemUsage}}" 2>/dev/null | awk -F/ '{gsub(/[^0-9.]/, "", $1); if($1+0 > 0) print $1; else print 0}'
|
||||
}
|
||||
|
||||
docker_cpu_pct() {
|
||||
local cid="$1"
|
||||
docker stats "$cid" --no-stream --format "{{.CPUPerc}}" 2>/dev/null | tr -d '%'
|
||||
}
|
||||
|
||||
record() {
|
||||
local tier="$1" tool="$2" variant="$3" time_s="$4" pass="$5" output_size="${6:-0}" mem_mb="${7:-0}" cpu_pct="${8:-0}" extra="${9:-}"
|
||||
printf '{"system":"%s","tier":"%s","tool":"%s","variant":"%s","time_s":%s,"pass":%s,"output_size":%s,"mem_mb":%s,"cpu_pct":%s%s}\n' \
|
||||
"$SYSTEM" "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_mb" "$cpu_pct" "$extra" >> "$RESULTS_FILE"
|
||||
}
|
||||
|
||||
bench_tool() {
|
||||
local tier="$1" tool="$2" variant="$3" file="$4" settings="${5:-}" extra_args="${6:-}"
|
||||
local cid time_s http_code mem_before mem_after cpu output_file pass output_size
|
||||
|
||||
cid=$(get_container_id)
|
||||
mem_before=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
|
||||
output_file=$(mktemp)
|
||||
local timing_file=$(mktemp)
|
||||
|
||||
local curl_args=(-s -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}")
|
||||
|
||||
if [ -n "$file" ] && [ "$file" != "NONE" ]; then
|
||||
curl_args+=(-F "file=@${file}")
|
||||
fi
|
||||
|
||||
if [ -n "$settings" ]; then
|
||||
curl_args+=(-F "settings=${settings}")
|
||||
fi
|
||||
|
||||
if [ -n "$extra_args" ]; then
|
||||
eval "curl_args+=($extra_args)"
|
||||
fi
|
||||
|
||||
curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}")
|
||||
|
||||
local result
|
||||
result=$(curl "${curl_args[@]}" 2>/dev/null) || result="000 0.000"
|
||||
|
||||
http_code=$(echo "$result" | awk '{print $1}')
|
||||
time_s=$(echo "$result" | awk '{print $2}')
|
||||
|
||||
mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0")
|
||||
|
||||
output_size=$(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
pass="true"
|
||||
else
|
||||
pass="false"
|
||||
fi
|
||||
|
||||
record "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu"
|
||||
|
||||
log "$tier/$tool/$variant: ${time_s}s HTTP:${http_code} mem:${mem_after}MB pass:${pass}"
|
||||
|
||||
rm -f "$output_file" "$timing_file"
|
||||
}
|
||||
|
||||
bench_tool_multifile() {
|
||||
local tier="$1" tool="$2" variant="$3" settings="${4:-}"
|
||||
shift 4
|
||||
local files=("$@")
|
||||
|
||||
local cid time_s http_code mem_after cpu output_file pass output_size
|
||||
|
||||
cid=$(get_container_id)
|
||||
output_file=$(mktemp)
|
||||
|
||||
local curl_args=(-s -X POST "${BASE_URL}/api/v1/tools/${tool}" -H "Authorization: Bearer ${TOKEN}")
|
||||
|
||||
for f in "${files[@]}"; do
|
||||
curl_args+=(-F "file=@${f}")
|
||||
done
|
||||
|
||||
if [ -n "$settings" ]; then
|
||||
curl_args+=(-F "settings=${settings}")
|
||||
fi
|
||||
|
||||
curl_args+=(-o "$output_file" -w "%{http_code} %{time_total}")
|
||||
|
||||
local result
|
||||
result=$(curl "${curl_args[@]}" 2>/dev/null) || result="000 0.000"
|
||||
|
||||
http_code=$(echo "$result" | awk '{print $1}')
|
||||
time_s=$(echo "$result" | awk '{print $2}')
|
||||
|
||||
mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
cpu=$(docker_cpu_pct "$cid" 2>/dev/null || echo "0")
|
||||
output_size=$(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
pass="true"
|
||||
else
|
||||
pass="false"
|
||||
fi
|
||||
|
||||
record "$tier" "$tool" "$variant" "$time_s" "$pass" "$output_size" "$mem_after" "$cpu"
|
||||
log "$tier/$tool/$variant: ${time_s}s HTTP:${http_code} pass:${pass}"
|
||||
|
||||
rm -f "$output_file"
|
||||
}
|
||||
|
||||
run_n_times() {
|
||||
local n="$1"
|
||||
shift
|
||||
local times=()
|
||||
for i in $(seq 1 "$n"); do
|
||||
"$@"
|
||||
done
|
||||
}
|
||||
|
||||
F="${FIXTURE_DIR}"
|
||||
S="${F}/test-200x150.png"
|
||||
J="${F}/test-100x100.jpg"
|
||||
L="${F}/content/stress-large.jpg"
|
||||
P="${F}/content/portrait-color.jpg"
|
||||
BW="${F}/content/portrait-bw.jpeg"
|
||||
SVG="${F}/content/svg-logo.svg"
|
||||
GIF="${F}/content/animated-simpsons.gif"
|
||||
PDF="${F}/test-3page.pdf"
|
||||
EXIF="${F}/test-with-exif.jpg"
|
||||
FACE="${F}/content/multi-face.webp"
|
||||
OCR="${F}/content/ocr-chat.jpeg"
|
||||
OCRJP="${F}/content/ocr-japanese.png"
|
||||
ISO="${F}/content/portrait-isolated.png"
|
||||
HEAD="${F}/content/portrait-headshot.heic"
|
||||
REDEYE="${F}/content/red-eye.jpg"
|
||||
BARCODE="${F}/content/barcode.avif"
|
||||
|
||||
echo "[]" > /dev/null
|
||||
> "$RESULTS_FILE"
|
||||
|
||||
log "=== Starting benchmarks on ${SYSTEM} ==="
|
||||
TOKEN=$(get_token)
|
||||
log "Auth token obtained"
|
||||
|
||||
log "=== TIER 1: Core Tool Benchmarks ==="
|
||||
|
||||
for run in 1 2 3; do
|
||||
log "--- Run $run of 3 ---"
|
||||
|
||||
bench_tool "core" "resize" "small-r${run}" "$S" '{"width":100,"fit":"cover"}'
|
||||
bench_tool "core" "resize" "large-r${run}" "$L" '{"width":800,"fit":"cover"}'
|
||||
|
||||
bench_tool "core" "crop" "small-r${run}" "$S" '{"left":10,"top":10,"width":100,"height":100}'
|
||||
bench_tool "core" "crop" "large-r${run}" "$L" '{"left":10,"top":10,"width":100,"height":100}'
|
||||
|
||||
bench_tool "core" "rotate" "small-r${run}" "$S" '{"angle":90}'
|
||||
bench_tool "core" "rotate" "large-r${run}" "$L" '{"angle":90}'
|
||||
|
||||
bench_tool "core" "convert" "jpg-webp-small-r${run}" "$J" '{"format":"webp","quality":80}'
|
||||
bench_tool "core" "convert" "jpg-webp-large-r${run}" "$L" '{"format":"webp","quality":80}'
|
||||
bench_tool "core" "convert" "jpg-avif-small-r${run}" "$J" '{"format":"avif","quality":50}'
|
||||
bench_tool "core" "convert" "jpg-avif-large-r${run}" "$L" '{"format":"avif","quality":50}'
|
||||
|
||||
bench_tool "core" "compress" "quality-small-r${run}" "$S" '{"mode":"quality","quality":60}'
|
||||
bench_tool "core" "compress" "quality-large-r${run}" "$L" '{"mode":"quality","quality":60}'
|
||||
bench_tool "core" "compress" "targetSize-large-r${run}" "$L" '{"mode":"targetSize","targetSizeKb":500}'
|
||||
|
||||
bench_tool "core" "strip-metadata" "small-r${run}" "$EXIF" '{"stripAll":true}'
|
||||
bench_tool "core" "strip-metadata" "large-r${run}" "$L" '{"stripAll":true}'
|
||||
|
||||
bench_tool "core" "edit-metadata" "small-r${run}" "$EXIF" '{"title":"Bench","clearGps":true}'
|
||||
bench_tool "core" "edit-metadata" "large-r${run}" "$L" '{"title":"Bench","clearGps":true}'
|
||||
|
||||
bench_tool "core" "color-adjustments" "small-r${run}" "$S" '{"brightness":20,"contrast":10,"effect":"grayscale"}'
|
||||
bench_tool "core" "color-adjustments" "large-r${run}" "$L" '{"brightness":20,"contrast":10,"effect":"grayscale"}'
|
||||
|
||||
bench_tool "core" "sharpening" "small-r${run}" "$S" '{"method":"adaptive","sigma":1.5}'
|
||||
bench_tool "core" "sharpening" "large-r${run}" "$L" '{"method":"adaptive","sigma":1.5}'
|
||||
|
||||
bench_tool "core" "watermark-text" "small-r${run}" "$S" '{"text":"BENCHMARK","position":"tiled"}'
|
||||
bench_tool "core" "watermark-text" "large-r${run}" "$L" '{"text":"BENCHMARK","position":"tiled"}'
|
||||
|
||||
bench_tool_multifile "core" "compose" "small-r${run}" '{"blendMode":"overlay"}' "$S" "$J"
|
||||
bench_tool_multifile "core" "compose" "large-r${run}" '{"blendMode":"overlay"}' "$L" "$P"
|
||||
|
||||
bench_tool_multifile "core" "collage" "4img-small-r${run}" '{"templateId":"4-grid"}' "$S" "$J" "${F}/test-50x50.webp" "${F}/test-100x100.svg"
|
||||
bench_tool_multifile "core" "collage" "4img-large-r${run}" '{"templateId":"4-grid"}' "$L" "$P" "$BW" "${F}/content/watermark.jpg"
|
||||
|
||||
bench_tool_multifile "core" "stitch" "3img-small-r${run}" '{"direction":"horizontal"}' "$S" "$J" "${F}/test-50x50.webp"
|
||||
bench_tool_multifile "core" "stitch" "3img-large-r${run}" '{"direction":"horizontal"}' "$L" "$P" "$BW"
|
||||
|
||||
bench_tool "core" "split" "small-r${run}" "$S" '{"columns":2,"rows":2}'
|
||||
bench_tool "core" "split" "large-r${run}" "$L" '{"columns":2,"rows":2}'
|
||||
|
||||
bench_tool "core" "border" "small-r${run}" "$S" '{"borderWidth":20,"cornerRadius":10,"shadow":true}'
|
||||
bench_tool "core" "border" "large-r${run}" "$L" '{"borderWidth":20,"cornerRadius":10,"shadow":true}'
|
||||
|
||||
bench_tool "core" "svg-to-raster" "small-r${run}" "${F}/test-100x100.svg" '{"width":2000,"dpi":300}'
|
||||
|
||||
bench_tool "core" "vectorize" "small-r${run}" "$S" '{"colorMode":"color"}'
|
||||
bench_tool "core" "vectorize" "large-r${run}" "$P" '{"colorMode":"color"}'
|
||||
|
||||
bench_tool "core" "gif-tools" "optimize-r${run}" "$GIF" '{"mode":"optimize","colors":64}'
|
||||
|
||||
bench_tool "core" "pdf-to-image" "r${run}" "$PDF" '{"format":"png","dpi":300}'
|
||||
|
||||
bench_tool "core" "optimize-for-web" "large-r${run}" "$L" '{"format":"webp","quality":80,"maxWidth":1920}'
|
||||
|
||||
bench_tool "core" "favicon" "small-r${run}" "$S" ''
|
||||
bench_tool "core" "favicon" "large-r${run}" "$P" ''
|
||||
|
||||
bench_tool_multifile "core" "image-to-pdf" "3img-r${run}" '{"pageSize":"A4"}' "$S" "$J" "${F}/test-50x50.webp"
|
||||
|
||||
bench_tool "core" "replace-color" "small-r${run}" "$S" '{"sourceColor":"#FFFFFF","targetColor":"#FF0000","tolerance":30}'
|
||||
bench_tool "core" "replace-color" "large-r${run}" "$L" '{"sourceColor":"#FFFFFF","targetColor":"#FF0000","tolerance":30}'
|
||||
|
||||
bench_tool "core" "info" "small-r${run}" "$S" ''
|
||||
bench_tool "core" "info" "large-r${run}" "$L" ''
|
||||
|
||||
bench_tool_multifile "core" "compare" "small-r${run}" '' "$S" "$S"
|
||||
bench_tool_multifile "core" "compare" "large-r${run}" '' "$L" "$L"
|
||||
|
||||
bench_tool_multifile "core" "find-duplicates" "5img-r${run}" '{"threshold":5}' "$S" "$J" "${F}/test-50x50.webp" "$S" "$J"
|
||||
|
||||
bench_tool "core" "color-palette" "small-r${run}" "$P" ''
|
||||
bench_tool "core" "color-palette" "large-r${run}" "$L" ''
|
||||
|
||||
bench_tool "core" "barcode-read" "r${run}" "$BARCODE" '{"tryHarder":true}'
|
||||
|
||||
bench_tool "core" "image-to-base64" "small-r${run}" "$S" '{"outputFormat":"webp"}'
|
||||
bench_tool "core" "image-to-base64" "large-r${run}" "$L" '{"outputFormat":"webp"}'
|
||||
|
||||
bench_tool "core" "content-aware-resize" "small-r${run}" "$S" '{"width":100}'
|
||||
bench_tool "core" "content-aware-resize" "large-r${run}" "$L" '{"width":100}'
|
||||
|
||||
bench_tool "core" "image-enhancement" "small-r${run}" "$S" '{"mode":"auto","intensity":50}'
|
||||
bench_tool "core" "image-enhancement" "large-r${run}" "$L" '{"mode":"auto","intensity":50}'
|
||||
done
|
||||
|
||||
log "=== TIER 5: Format Decode Benchmarks ==="
|
||||
|
||||
for fmt_file in "${F}/formats/"*; do
|
||||
fname=$(basename "$fmt_file")
|
||||
for run in 1 2 3; do
|
||||
bench_tool "format" "resize" "fmt-${fname}-r${run}" "$fmt_file" '{"width":200}'
|
||||
done
|
||||
done
|
||||
|
||||
log "=== TIER 6: Concurrent Load Benchmarks ==="
|
||||
|
||||
for concurrency in 1 3 5 10 20; do
|
||||
log "Concurrency: ${concurrency}"
|
||||
local_results=$(mktemp)
|
||||
|
||||
for i in $(seq 1 "$concurrency"); do
|
||||
(
|
||||
result=$(curl -s -X POST "${BASE_URL}/api/v1/tools/resize" \
|
||||
-H "Authorization: Bearer ${TOKEN}" \
|
||||
-F "file=@${L}" \
|
||||
-F 'settings={"width":800}' \
|
||||
-o /dev/null -w "%{http_code} %{time_total}")
|
||||
echo "$result"
|
||||
) >> "$local_results" &
|
||||
done
|
||||
wait
|
||||
|
||||
cid=$(get_container_id)
|
||||
mem_after=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
|
||||
times=()
|
||||
errors=0
|
||||
while IFS= read -r line; do
|
||||
code=$(echo "$line" | awk '{print $1}')
|
||||
t=$(echo "$line" | awk '{print $2}')
|
||||
times+=("$t")
|
||||
if [ "$code" != "200" ]; then
|
||||
errors=$((errors + 1))
|
||||
fi
|
||||
done < "$local_results"
|
||||
|
||||
sorted=$(printf '%s\n' "${times[@]}" | sort -n)
|
||||
count=${#times[@]}
|
||||
avg=$(printf '%s\n' "${times[@]}" | awk '{s+=$1} END {printf "%.3f", s/NR}')
|
||||
p95_idx=$(( (count * 95 / 100) ))
|
||||
[ "$p95_idx" -ge "$count" ] && p95_idx=$((count - 1))
|
||||
p95=$(echo "$sorted" | sed -n "$((p95_idx + 1))p")
|
||||
max=$(echo "$sorted" | tail -1)
|
||||
min=$(echo "$sorted" | head -1)
|
||||
|
||||
printf '{"system":"%s","tier":"concurrent","tool":"resize","variant":"c%d","concurrency":%d,"avg_s":%s,"p95_s":%s,"max_s":%s,"min_s":%s,"errors":%d,"mem_mb":%s}\n' \
|
||||
"$SYSTEM" "$concurrency" "$concurrency" "$avg" "${p95:-0}" "${max:-0}" "${min:-0}" "$errors" "$mem_after" >> "$RESULTS_FILE"
|
||||
|
||||
log "concurrent/c${concurrency}: avg=${avg}s p95=${p95}s max=${max}s errors=${errors} mem=${mem_after}MB"
|
||||
rm -f "$local_results"
|
||||
done
|
||||
|
||||
log "=== TIER 7: Sustained Load (50 sequential resizes) ==="
|
||||
|
||||
cid=$(get_container_id)
|
||||
mem_start=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
|
||||
for i in $(seq 1 50); do
|
||||
bench_tool "sustained" "resize" "seq-${i}" "$L" '{"width":800}'
|
||||
done
|
||||
|
||||
mem_end=$(docker_mem_mb "$cid" 2>/dev/null || echo "0")
|
||||
mem_delta=$(echo "$mem_end - $mem_start" | bc 2>/dev/null || echo "0")
|
||||
printf '{"system":"%s","tier":"sustained-summary","test":"50-resizes","mem_start_mb":%s,"mem_end_mb":%s,"mem_delta_mb":%s}\n' \
|
||||
"$SYSTEM" "$mem_start" "$mem_end" "$mem_delta" >> "$RESULTS_FILE"
|
||||
log "Sustained: start=${mem_start}MB end=${mem_end}MB delta=${mem_delta}MB"
|
||||
|
||||
log "=== TIER 3: Batch Processing ==="
|
||||
|
||||
for batch_size in 3 5 10; do
|
||||
files=()
|
||||
for i in $(seq 1 "$batch_size"); do
|
||||
files+=("$S")
|
||||
done
|
||||
bench_tool_multifile "batch" "resize" "small-b${batch_size}" '{"width":100}' "${files[@]}"
|
||||
done
|
||||
|
||||
for batch_size in 3 5; do
|
||||
files=()
|
||||
for i in $(seq 1 "$batch_size"); do
|
||||
files+=("$L")
|
||||
done
|
||||
bench_tool_multifile "batch" "resize" "large-b${batch_size}" '{"width":800}' "${files[@]}"
|
||||
done
|
||||
|
||||
bench_tool_multifile "batch" "convert" "mixed-5" '{"format":"webp","quality":80}' \
|
||||
"$J" "$S" "${F}/test-50x50.webp" "${F}/test-200x150.heic" "${F}/formats/sample.avif"
|
||||
|
||||
log "=== TIER 4: Pipeline Benchmarks ==="
|
||||
|
||||
bench_tool "pipeline" "resize" "1step" "$L" '{"width":800}'
|
||||
|
||||
log "=== Container Cold Start ==="
|
||||
# Record current time as baseline
|
||||
log "Cold start measurement requires container restart - skipping in automated run"
|
||||
|
||||
log "=== ALL BENCHMARKS COMPLETE for ${SYSTEM} ==="
|
||||
log "Results in: ${RESULTS_FILE}"
|
||||
wc -l "$RESULTS_FILE" | awk '{print $1 " benchmark records written"}'
|
||||
Reference in New Issue
Block a user