surya-ocr/benchmarks/concurrency_test.py

import argparse
import base64
import json
import statistics
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Dict, List

import requests


DEFAULT_OLD_ENDPOINT = "http://127.0.0.1:5002/v1/api/ai/suya_ocr/"
DEFAULT_NEW_ENDPOINT = "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/"


def _encode_image(path: Path) -> str:
    return base64.b64encode(path.read_bytes()).decode("utf-8")


def _preflight() -> Dict[str, Any]:
    checks: Dict[str, Any] = {}
    commands = {
        "docker_runtimes": ["docker", "info", "--format", "{{json .Runtimes}}"],
        "nvidia_smi": ["nvidia-smi", "-L"],
        "nvidia_container_runtime": ["which", "nvidia-container-runtime"],
        "nvidia_ctk": ["which", "nvidia-ctk"],
        "docker_gpus": [
            "docker",
            "run",
            "--rm",
            "--gpus",
            "all",
            "--entrypoint",
            "nvidia-smi",
            "vllm/vllm-openai:v0.20.1",
            "-L",
        ],
    }
    for name, cmd in commands.items():
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
            checks[name] = {
                "ok": result.returncode == 0,
                "stdout": result.stdout.strip(),
                "stderr": result.stderr.strip(),
            }
        except Exception as exc:
            checks[name] = {"ok": False, "stdout": "", "stderr": str(exc)}
    runtimes = checks.get("docker_runtimes", {}).get("stdout", "")
    checks["docker_has_nvidia_runtime"] = '"nvidia"' in runtimes
    if not checks["docker_has_nvidia_runtime"] and checks.get("docker_gpus", {}).get("ok"):
        checks["recommended_fix"] = (
            "Docker GPU passthrough works with --gpus. Use the local "
            "scripts/docker_nvidia_runtime_compat.sh wrapper to strip the legacy "
            "--runtime nvidia flag from Surya's vLLM spawn command."
        )
    else:
        checks["recommended_fix"] = (
            "Run: sudo nvidia-ctk runtime configure --runtime=docker "
            "--config=/etc/docker/daemon.json && sudo systemctl restart docker"
        )
    return checks


def _percentile(values: List[float], percentile: float) -> float:
    if not values:
        return 0.0
    ordered = sorted(values)
    index = min(len(ordered) - 1, int(round((percentile / 100) * (len(ordered) - 1))))
    return ordered[index]


def _post_once(url: str, payload: Dict[str, Any], timeout: float) -> Dict[str, Any]:
    start = time.perf_counter()
    try:
        response = requests.post(url, json=payload, timeout=timeout)
        elapsed = time.perf_counter() - start
        body = response.json() if response.content else {}
        return {
            "ok": response.status_code == 200 and body.get("code") == 200,
            "status_code": response.status_code,
            "api_code": body.get("code"),
            "elapsed_seconds": elapsed,
            "error": body.get("message") if body.get("code") != 200 else None,
        }
    except Exception as exc:
        return {
            "ok": False,
            "status_code": None,
            "api_code": None,
            "elapsed_seconds": time.perf_counter() - start,
            "error": str(exc),
        }


def run_endpoint(
    name: str,
    url: str,
    payload: Dict[str, Any],
    requests_count: int,
    concurrency: int,
    timeout: float,
) -> Dict[str, Any]:
    start = time.perf_counter()
    samples: List[Dict[str, Any]] = []
    with ThreadPoolExecutor(max_workers=concurrency) as pool:
        futures = [
            pool.submit(_post_once, url, payload, timeout)
            for _ in range(requests_count)
        ]
        for future in as_completed(futures):
            samples.append(future.result())
    wall_seconds = time.perf_counter() - start
    latencies = [sample["elapsed_seconds"] for sample in samples]
    success_count = sum(1 for sample in samples if sample["ok"])
    return {
        "name": name,
        "url": url,
        "requests": requests_count,
        "concurrency": concurrency,
        "success": success_count,
        "failed": requests_count - success_count,
        "wall_seconds": wall_seconds,
        "throughput_rps": requests_count / wall_seconds if wall_seconds else 0.0,
        "latency_seconds": {
            "min": min(latencies) if latencies else 0.0,
            "mean": statistics.mean(latencies) if latencies else 0.0,
            "p50": _percentile(latencies, 50),
            "p95": _percentile(latencies, 95),
            "max": max(latencies) if latencies else 0.0,
        },
        "errors": [sample for sample in samples if not sample["ok"]][:10],
    }


def main() -> None:
    parser = argparse.ArgumentParser(description="Compare legacy and vLLM OCR endpoint concurrency.")
    parser.add_argument("--image", required=True, help="Path to an input png/jpg image.")
    parser.add_argument("--old-url", default=DEFAULT_OLD_ENDPOINT)
    parser.add_argument("--new-url", default=DEFAULT_NEW_ENDPOINT)
    parser.add_argument("--requests", type=int, default=20)
    parser.add_argument("--concurrency", type=int, default=8)
    parser.add_argument("--timeout", type=float, default=900)
    parser.add_argument("--output", default="concurrency_test_results.json")
    parser.add_argument("--skip-old", action="store_true")
    parser.add_argument("--skip-new", action="store_true")
    args = parser.parse_args()

    image_path = Path(args.image)
    image_type = image_path.suffix.lstrip(".").lower() or "png"
    payload = {
        "file": _encode_image(image_path),
        "type": "jpg" if image_type == "jpeg" else image_type,
        "skip_text_detection": False,
        "skip_table_detection": False,
        "recognize_math": False,
        "ocr_with_boxes": True,
    }

    results = {
        "started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
        "image": str(image_path),
        "requests": args.requests,
        "concurrency": args.concurrency,
        "preflight": _preflight(),
        "endpoints": [],
    }
    if not args.skip_old:
        results["endpoints"].append(
            run_endpoint("legacy", args.old_url, payload, args.requests, args.concurrency, args.timeout)
        )
    if not args.skip_new:
        results["endpoints"].append(
            run_endpoint("vllm", args.new_url, payload, args.requests, args.concurrency, args.timeout)
        )

    output_path = Path(args.output)
    output_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
    print(json.dumps(results, indent=2))


if __name__ == "__main__":
    main()