Suya OCR API — vLLM-backed, OpenAI-compatible OCR service

FastAPI service wrapping the Surya-OCR-2 model (datalab-to) served through vLLM: legacy /v1/api/ai/* endpoints, an OpenAI-compatible /v1/chat/completions endpoint, a coalescing request batcher, a local OCR CLI, Docker packaging, multilingual example outputs, and quantization/concurrency benchmarks. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-17 10:20:02 +04:00
commit 1a585693be
147 changed files with 13827 additions and 0 deletions
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# vLLM serving-parameter tuning sweep for the OCR latency report.
+# Serves the BF16 model under several vLLM flag combinations (CUDA graph vs eager,
+# prefix caching on/off, chunked prefill on/off), captures a fixed page subset
+# through the OCR API for each, and records per-config latency.
+#
+# Run INSIDE the bench container:  bash scripts/quant/tune_vllm.sh
+# Requires: api.py already running on :5002 pointed at :8000; compat libs on path.
+set -uo pipefail
+
+MODEL="${MODEL:-datalab-to/surya-ocr-2}"
+OCR_URL="http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/"
+N_PAGES="${N_PAGES:-3}"
+OUT_ROOT="results/quant/tuning"
+LOG=/tmp/tune_vllm.log
+export LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH:-}"
+
+mapfile -t ALL < <(sed '/^#/d;/^$/d' eval_set/manifest.txt)
+IMAGES=("${ALL[@]:0:$N_PAGES}")
+
+# config_name | extra vLLM args
+CONFIGS=(
+  "default|--enable-prefix-caching"
+  "eager|--enable-prefix-caching --enforce-eager"
+  "no_prefix_cache|--no-enable-prefix-caching"
+  "no_chunked_prefill|--enable-prefix-caching --no-enable-chunked-prefill"
+)
+
+base_args() {
+  echo "--host 127.0.0.1 --port 8000 --model $MODEL --served-model-name $MODEL \
+--max-model-len 18000 --max-num-seqs 16 --gpu-memory-utilization 0.85 \
+--mm-processor-kwargs {\"min_pixels\":3136,\"max_pixels\":6291456}"
+}
+
+stop_server() {
+  local pid
+  pid=$(ss -ltnp 2>/dev/null | grep ":8000 " | grep -oP 'pid=\K[0-9]+' | head -1)
+  [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null
+  pkill -9 -f "vllm.entrypoints" 2>/dev/null
+  # The EngineCore worker holds the GPU and outlives the API server; kill every
+  # remaining CUDA compute process, then poll until the memory is actually freed
+  # (kill returns immediately, GPU release lags).
+  for p in $(nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null); do
+    kill -9 "$p" 2>/dev/null
+  done
+  for _ in $(seq 1 30); do
+    used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)
+    [ "${used:-99999}" -lt 2000 ] && break
+    sleep 2
+  done
+}
+
+for entry in "${CONFIGS[@]}"; do
+  name="${entry%%|*}"
+  extra="${entry#*|}"
+  echo "=== tuning config: $name ($extra) ===" | tee -a "$LOG"
+  stop_server
+  # shellcheck disable=SC2046
+  nohup python3 -m vllm.entrypoints.openai.api_server $(base_args) $extra \
+    > "/tmp/tune_${name}.log" 2>&1 &
+  # wait for health or crash (max 7 min)
+  ok=0
+  for _ in $(seq 1 84); do
+    if curl -fs http://127.0.0.1:8000/health >/dev/null 2>&1; then ok=1; break; fi
+    if grep -qiE "Engine core initialization failed|RuntimeError" "/tmp/tune_${name}.log" 2>/dev/null; then break; fi
+    sleep 5
+  done
+  if [ "$ok" -ne 1 ]; then
+    echo "$name: FAILED to start" | tee -a "$LOG"
+    continue
+  fi
+  python3 -m scripts.quant.capture --url "$OCR_URL" --out-dir "$OUT_ROOT/$name" "${IMAGES[@]}" \
+    >> "$LOG" 2>&1
+  echo "$name: captured $(ls "$OUT_ROOT/$name"/*.json 2>/dev/null | wc -l) pages" | tee -a "$LOG"
+done
+stop_server
+echo "TUNE_DONE" | tee -a "$LOG"