#!/usr/bin/env bash
# Serve and score compressor-built checkpoints (int8/gptq/awq) over a page subset.
# Each method: hardened server teardown -> serve checkpoint -> health/fail watch ->
# capture N_PAGES through the OCR API. Run INSIDE the bench container:
#   bash scripts/quant/score_methods.sh int8 gptq awq
# Requires api.py running on :5002 pointed at :8000.
set -uo pipefail

export LD_LIBRARY_PATH="/usr/local/cuda/compat:${LD_LIBRARY_PATH:-}"
N_PAGES="${N_PAGES:-3}"
OCR_URL="http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/"
LOG=/tmp/score_methods.log

mapfile -t ALL < <(sed '/^#/d;/^$/d' eval_set/manifest.txt)
IMAGES=("${ALL[@]:0:$N_PAGES}")

stop_server() {
  local pid
  pid=$(ss -ltnp 2>/dev/null | grep ":8000 " | grep -oP 'pid=\K[0-9]+' | head -1)
  [ -n "$pid" ] && kill -9 "$pid" 2>/dev/null
  pkill -9 -f "vllm.entrypoints" 2>/dev/null
  # EngineCore outlives the API server and holds the GPU; kill compute procs and
  # poll until memory actually frees (kill returns before GPU release).
  for p in $(nvidia-smi --query-compute-apps=pid --format=csv,noheader 2>/dev/null); do
    kill -9 "$p" 2>/dev/null
  done
  for _ in $(seq 1 30); do
    used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)
    [ "${used:-99999}" -lt 2000 ] && break
    sleep 2
  done
}

for m in "$@"; do
  echo "=== scoring $m ===" | tee -a "$LOG"
  stop_server
  # VLLM_EXTRA_ARGS lets a caller add e.g. --dtype float16 (Exllama W4A16 kernel
  # on Ampere only supports float16 activations).
  nohup python3 -m vllm.entrypoints.openai.api_server --host 127.0.0.1 --port 8000 \
    --model "results/quant/models/$m" --served-model-name datalab-to/surya-ocr-2 \
    --max-model-len 18000 --max-num-seqs 16 --gpu-memory-utilization 0.85 \
    --enable-prefix-caching --mm-processor-kwargs '{"min_pixels":3136,"max_pixels":6291456}' \
    ${VLLM_EXTRA_ARGS:-} \
    > "/tmp/serve_${m}.log" 2>&1 &
  ok=0
  for _ in $(seq 1 84); do
    if curl -fs http://127.0.0.1:8000/health >/dev/null 2>&1; then ok=1; break; fi
    if grep -qiE "Engine core initialization failed" "/tmp/serve_${m}.log" 2>/dev/null; then break; fi
    sleep 5
  done
  if [ "$ok" -ne 1 ]; then
    echo "$m: FAILED to start" | tee -a "$LOG"
    continue
  fi
  python3 -m scripts.quant.capture --url "$OCR_URL" --out-dir "results/quant/results/$m" "${IMAGES[@]}" \
    >> "$LOG" 2>&1
  echo "$m: captured $(ls "results/quant/results/$m"/*.json 2>/dev/null | wc -l) pages" | tee -a "$LOG"
done
stop_server
echo "SCORE_DONE" | tee -a "$LOG"