Suya OCR API — vLLM-backed, OpenAI-compatible OCR service
FastAPI service wrapping the Surya-OCR-2 model (datalab-to) served through vLLM: legacy /v1/api/ai/* endpoints, an OpenAI-compatible /v1/chat/completions endpoint, a coalescing request batcher, a local OCR CLI, Docker packaging, multilingual example outputs, and quantization/concurrency benchmarks. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Fu Dai
@@ -0,0 +1,267 @@
|
||||
"""Concurrency benchmark + old-vs-new comparison for the OCR service.
|
||||
|
||||
Two subcommands:
|
||||
|
||||
sweep Drive one running service across a list of concurrency levels and
|
||||
record latency/throughput/success per level. Reuses the request
|
||||
harness in concurrency_test.py.
|
||||
|
||||
compare Load two sweep result files (old + new) and emit a markdown table,
|
||||
a CSV, and comparison plots (latency & throughput vs concurrency,
|
||||
plus a per-level speedup bar).
|
||||
|
||||
Typical flow (see scripts/run_concurrency_comparison.sh):
|
||||
|
||||
# against the new vLLM service (its optimized endpoint)
|
||||
python concurrency_compare.py sweep --label new \
|
||||
--url http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/ \
|
||||
--image temp_image_c3c56948-282e-453b-8fac-6c482243d1e5.jpg \
|
||||
--levels 1,2,4,8,16 --out results/compare/new.json
|
||||
|
||||
# ... swap containers, then against the old service (its OCR endpoint)
|
||||
python concurrency_compare.py sweep --label old \
|
||||
--url http://127.0.0.1:5002/v1/api/ai/suya_ocr/ \
|
||||
--image temp_image_c3c56948-282e-453b-8fac-6c482243d1e5.jpg \
|
||||
--levels 1,2,4,8,16 --out results/compare/old.json
|
||||
|
||||
python concurrency_compare.py compare \
|
||||
--old results/compare/old.json --new results/compare/new.json \
|
||||
--out-dir results/compare
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from concurrency_test import _encode_image, run_endpoint
|
||||
|
||||
|
||||
def _parse_levels(value: str) -> List[int]:
|
||||
return [int(x.strip()) for x in value.split(",") if x.strip()]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- sweep -----
|
||||
|
||||
def run_sweep(args: argparse.Namespace) -> None:
|
||||
image_path = Path(args.image)
|
||||
image_type = image_path.suffix.lstrip(".").lower() or "png"
|
||||
payload = {
|
||||
"file": _encode_image(image_path),
|
||||
"type": "jpg" if image_type == "jpeg" else image_type,
|
||||
"skip_text_detection": False,
|
||||
"skip_table_detection": False,
|
||||
"recognize_math": False,
|
||||
"ocr_with_boxes": True,
|
||||
}
|
||||
|
||||
levels = _parse_levels(args.levels)
|
||||
rows: List[Dict[str, Any]] = []
|
||||
for concurrency in levels:
|
||||
requests_count = max(concurrency, concurrency * args.reqs_per_level)
|
||||
print(f"[{args.label}] concurrency={concurrency} requests={requests_count}", flush=True)
|
||||
row = run_endpoint(
|
||||
args.label, args.url, payload,
|
||||
requests_count=requests_count, concurrency=concurrency, timeout=args.timeout,
|
||||
)
|
||||
rows.append(row)
|
||||
lat = row["latency_seconds"]
|
||||
print(f" -> success={row['success']}/{row['requests']} "
|
||||
f"rps={row['throughput_rps']:.3f} mean={lat['mean']:.2f}s p95={lat['p95']:.2f}s",
|
||||
flush=True)
|
||||
|
||||
out = {
|
||||
"label": args.label,
|
||||
"url": args.url,
|
||||
"image": str(image_path),
|
||||
"levels": levels,
|
||||
"reqs_per_level": args.reqs_per_level,
|
||||
"started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"results": rows,
|
||||
}
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
|
||||
print(f"wrote {out_path}")
|
||||
|
||||
|
||||
# -------------------------------------------------------------- compare -----
|
||||
|
||||
def _index_by_concurrency(sweep: Dict[str, Any]) -> Dict[int, Dict[str, Any]]:
|
||||
return {row["concurrency"]: row for row in sweep["results"]}
|
||||
|
||||
|
||||
def _comparison_rows(old: Dict[str, Any], new: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
old_by_c = _index_by_concurrency(old)
|
||||
new_by_c = _index_by_concurrency(new)
|
||||
concurrencies = sorted(set(old_by_c) | set(new_by_c))
|
||||
rows = []
|
||||
for c in concurrencies:
|
||||
o = old_by_c.get(c)
|
||||
n = new_by_c.get(c)
|
||||
row: Dict[str, Any] = {"concurrency": c}
|
||||
row["old_mean_s"] = round(o["latency_seconds"]["mean"], 2) if o else None
|
||||
row["new_mean_s"] = round(n["latency_seconds"]["mean"], 2) if n else None
|
||||
row["old_p95_s"] = round(o["latency_seconds"]["p95"], 2) if o else None
|
||||
row["new_p95_s"] = round(n["latency_seconds"]["p95"], 2) if n else None
|
||||
row["old_rps"] = round(o["throughput_rps"], 3) if o else None
|
||||
row["new_rps"] = round(n["throughput_rps"], 3) if n else None
|
||||
row["old_fail"] = o["failed"] if o else None
|
||||
row["new_fail"] = n["failed"] if n else None
|
||||
if o and n and o["latency_seconds"]["mean"] and n["latency_seconds"]["mean"]:
|
||||
row["latency_speedup"] = round(o["latency_seconds"]["mean"] / n["latency_seconds"]["mean"], 2)
|
||||
else:
|
||||
row["latency_speedup"] = None
|
||||
if o and n and o["throughput_rps"]:
|
||||
row["throughput_gain"] = round(n["throughput_rps"] / o["throughput_rps"], 2)
|
||||
else:
|
||||
row["throughput_gain"] = None
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _write_table(rows: List[Dict[str, Any]], old_label: str, new_label: str, path: Path) -> str:
|
||||
cols = [
|
||||
("concurrency", "conc"),
|
||||
("old_mean_s", f"{old_label} mean(s)"),
|
||||
("new_mean_s", f"{new_label} mean(s)"),
|
||||
("latency_speedup", "latency ×"),
|
||||
("old_p95_s", f"{old_label} p95(s)"),
|
||||
("new_p95_s", f"{new_label} p95(s)"),
|
||||
("old_rps", f"{old_label} rps"),
|
||||
("new_rps", f"{new_label} rps"),
|
||||
("throughput_gain", "rps ×"),
|
||||
("old_fail", f"{old_label} fail"),
|
||||
("new_fail", f"{new_label} fail"),
|
||||
]
|
||||
header = "| " + " | ".join(label for _, label in cols) + " |"
|
||||
sep = "| " + " | ".join("---" for _ in cols) + " |"
|
||||
lines = [header, sep]
|
||||
for r in rows:
|
||||
cells = []
|
||||
for key, _ in cols:
|
||||
v = r.get(key)
|
||||
cells.append("" if v is None else str(v))
|
||||
lines.append("| " + " | ".join(cells) + " |")
|
||||
md = "\n".join(lines) + "\n"
|
||||
path.write_text(md, encoding="utf-8")
|
||||
return md
|
||||
|
||||
|
||||
def _write_csv(rows: List[Dict[str, Any]], path: Path) -> None:
|
||||
if not rows:
|
||||
return
|
||||
with path.open("w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def _write_plots(rows, old_label, new_label, out_dir: Path) -> List[Path]:
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
x = [r["concurrency"] for r in rows]
|
||||
paths = []
|
||||
|
||||
# 1. latency (mean + p95) vs concurrency
|
||||
fig, ax = plt.subplots(figsize=(9, 6))
|
||||
ax.plot(x, [r["old_mean_s"] for r in rows], marker="o", color="#c0392b", label=f"{old_label} mean")
|
||||
ax.plot(x, [r["old_p95_s"] for r in rows], marker="^", color="#c0392b", linestyle="--", label=f"{old_label} p95")
|
||||
ax.plot(x, [r["new_mean_s"] for r in rows], marker="o", color="#27ae60", label=f"{new_label} mean")
|
||||
ax.plot(x, [r["new_p95_s"] for r in rows], marker="^", color="#27ae60", linestyle="--", label=f"{new_label} p95")
|
||||
ax.set_xlabel("Concurrency (simultaneous requests)")
|
||||
ax.set_ylabel("Latency per request (s)")
|
||||
ax.set_title("OCR latency vs concurrency — old vs new")
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
p = out_dir / "latency_vs_concurrency.png"
|
||||
fig.savefig(p, dpi=160)
|
||||
plt.close(fig)
|
||||
paths.append(p)
|
||||
|
||||
# 2. throughput vs concurrency
|
||||
fig, ax = plt.subplots(figsize=(9, 6))
|
||||
ax.plot(x, [r["old_rps"] for r in rows], marker="o", color="#c0392b", label=old_label)
|
||||
ax.plot(x, [r["new_rps"] for r in rows], marker="o", color="#27ae60", label=new_label)
|
||||
ax.set_xlabel("Concurrency (simultaneous requests)")
|
||||
ax.set_ylabel("Throughput (requests/s)")
|
||||
ax.set_title("OCR throughput vs concurrency — old vs new")
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
p = out_dir / "throughput_vs_concurrency.png"
|
||||
fig.savefig(p, dpi=160)
|
||||
plt.close(fig)
|
||||
paths.append(p)
|
||||
|
||||
# 3. per-level latency speedup bar
|
||||
fig, ax = plt.subplots(figsize=(9, 6))
|
||||
speedups = [r["latency_speedup"] or 0 for r in rows]
|
||||
ax.bar([str(c) for c in x], speedups, color="#2980b9")
|
||||
ax.axhline(1.0, color="gray", linestyle="--", linewidth=1)
|
||||
for i, v in enumerate(speedups):
|
||||
ax.text(i, v, f"{v:.2f}×", ha="center", va="bottom")
|
||||
ax.set_xlabel("Concurrency")
|
||||
ax.set_ylabel(f"Latency speedup ({old_label} mean / {new_label} mean)")
|
||||
ax.set_title("Per-level latency speedup (>1 = new is faster)")
|
||||
ax.grid(True, axis="y", alpha=0.3)
|
||||
fig.tight_layout()
|
||||
p = out_dir / "latency_speedup.png"
|
||||
fig.savefig(p, dpi=160)
|
||||
plt.close(fig)
|
||||
paths.append(p)
|
||||
|
||||
return paths
|
||||
|
||||
|
||||
def run_compare(args: argparse.Namespace) -> None:
|
||||
old = json.loads(Path(args.old).read_text(encoding="utf-8"))
|
||||
new = json.loads(Path(args.new).read_text(encoding="utf-8"))
|
||||
old_label = old.get("label", "old")
|
||||
new_label = new.get("label", "new")
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
rows = _comparison_rows(old, new)
|
||||
md = _write_table(rows, old_label, new_label, out_dir / "comparison_table.md")
|
||||
_write_csv(rows, out_dir / "comparison_table.csv")
|
||||
plots = _write_plots(rows, old_label, new_label, out_dir)
|
||||
|
||||
print(md)
|
||||
print("plots:")
|
||||
for p in plots:
|
||||
print(f" {p}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sp = sub.add_parser("sweep", help="benchmark one running service across concurrency levels")
|
||||
sp.add_argument("--label", required=True, help="short name for this service, e.g. old / new")
|
||||
sp.add_argument("--url", required=True, help="full endpoint URL to POST to")
|
||||
sp.add_argument("--image", required=True, help="path to a png/jpg test image")
|
||||
sp.add_argument("--levels", default="1,2,4,8,16", help="comma-separated concurrency levels")
|
||||
sp.add_argument("--reqs-per-level", type=int, default=2, help="requests = level * this (>=level)")
|
||||
sp.add_argument("--timeout", type=float, default=900)
|
||||
sp.add_argument("--out", required=True, help="output JSON path")
|
||||
sp.set_defaults(func=run_sweep)
|
||||
|
||||
cp = sub.add_parser("compare", help="compare two sweep result files")
|
||||
cp.add_argument("--old", required=True, help="old-service sweep JSON")
|
||||
cp.add_argument("--new", required=True, help="new-service sweep JSON")
|
||||
cp.add_argument("--out-dir", default="results/compare")
|
||||
cp.set_defaults(func=run_compare)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from concurrency_test import DEFAULT_NEW_ENDPOINT, _encode_image, run_endpoint
|
||||
|
||||
|
||||
def parse_levels(value: str) -> list[int]:
|
||||
return [int(item.strip()) for item in value.split(",") if item.strip()]
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||
columns = [
|
||||
"concurrency",
|
||||
"requests",
|
||||
"success",
|
||||
"failed",
|
||||
"wall_seconds",
|
||||
"throughput_rps",
|
||||
"latency_min",
|
||||
"latency_mean",
|
||||
"latency_p50",
|
||||
"latency_p95",
|
||||
"latency_max",
|
||||
]
|
||||
with path.open("w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=columns)
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
lat = row["latency_seconds"]
|
||||
writer.writerow(
|
||||
{
|
||||
"concurrency": row["concurrency"],
|
||||
"requests": row["requests"],
|
||||
"success": row["success"],
|
||||
"failed": row["failed"],
|
||||
"wall_seconds": row["wall_seconds"],
|
||||
"throughput_rps": row["throughput_rps"],
|
||||
"latency_min": lat["min"],
|
||||
"latency_mean": lat["mean"],
|
||||
"latency_p50": lat["p50"],
|
||||
"latency_p95": lat["p95"],
|
||||
"latency_max": lat["max"],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def write_plot(path: Path, rows: list[dict]) -> None:
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
x = [row["concurrency"] for row in rows]
|
||||
mean = [row["latency_seconds"]["mean"] for row in rows]
|
||||
p50 = [row["latency_seconds"]["p50"] for row in rows]
|
||||
p95 = [row["latency_seconds"]["p95"] for row in rows]
|
||||
|
||||
plt.figure(figsize=(10, 6))
|
||||
plt.plot(x, mean, marker="o", label="mean")
|
||||
plt.plot(x, p50, marker="o", label="p50")
|
||||
plt.plot(x, p95, marker="o", label="p95")
|
||||
plt.xlabel("Concurrency")
|
||||
plt.ylabel("Latency per request (seconds)")
|
||||
plt.title("vLLM OCR latency vs concurrency")
|
||||
plt.grid(True, alpha=0.3)
|
||||
plt.legend()
|
||||
plt.tight_layout()
|
||||
plt.savefig(path, dpi=160)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Sweep vLLM OCR endpoint concurrency levels.")
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--url", default=DEFAULT_NEW_ENDPOINT)
|
||||
parser.add_argument("--levels", default="1,2,4,6,8,10,12,20,30,40,50")
|
||||
parser.add_argument("--timeout", type=float, default=900)
|
||||
parser.add_argument("--output-json", default="concurrency_sweep_results.json")
|
||||
parser.add_argument("--output-csv", default="concurrency_sweep_results.csv")
|
||||
parser.add_argument("--output-plot", default="concurrency_sweep_latency.png")
|
||||
args = parser.parse_args()
|
||||
|
||||
image_path = Path(args.image)
|
||||
image_type = image_path.suffix.lstrip(".").lower() or "png"
|
||||
payload = {
|
||||
"file": _encode_image(image_path),
|
||||
"type": "jpg" if image_type == "jpeg" else image_type,
|
||||
"skip_text_detection": False,
|
||||
"skip_table_detection": False,
|
||||
"recognize_math": False,
|
||||
"ocr_with_boxes": True,
|
||||
}
|
||||
|
||||
levels = parse_levels(args.levels)
|
||||
rows = []
|
||||
for concurrency in levels:
|
||||
print(f"running concurrency={concurrency}", flush=True)
|
||||
rows.append(
|
||||
run_endpoint(
|
||||
"vllm",
|
||||
args.url,
|
||||
payload,
|
||||
requests_count=concurrency,
|
||||
concurrency=concurrency,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
|
||||
result = {
|
||||
"started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"image": str(image_path),
|
||||
"url": args.url,
|
||||
"levels": levels,
|
||||
"results": rows,
|
||||
}
|
||||
Path(args.output_json).write_text(json.dumps(result, indent=2), encoding="utf-8")
|
||||
write_csv(Path(args.output_csv), rows)
|
||||
write_plot(Path(args.output_plot), rows)
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 91 KiB |
@@ -0,0 +1,12 @@
|
||||
concurrency,requests,success,failed,wall_seconds,throughput_rps,latency_min,latency_mean,latency_p50,latency_p95,latency_max
|
||||
1,1,1,0,7.6577067924663424,0.13058739738949013,7.656610286794603,7.656610286794603,7.656610286794603,7.656610286794603,7.656610286794603
|
||||
2,2,2,0,11.936453193426132,0.16755395992349534,11.934747929684818,11.935045637656003,11.934747929684818,11.935343345627189,11.935343345627189
|
||||
4,4,4,0,20.956660461612046,0.19087010582278188,20.942796256393194,20.951773355016485,20.954541454091668,20.95546007528901,20.95546007528901
|
||||
6,6,6,0,29.489064288325608,0.20346525550406608,29.45817438978702,29.47708616297071,29.48345213010907,29.487088727764785,29.487088727764785
|
||||
8,8,8,0,41.46563811413944,0.192930830534405,11.84945331979543,34.050638656364754,41.457184289582074,41.46130123361945,41.46130123361945
|
||||
10,10,10,0,50.19571524951607,0.19922019141058875,12.104127056896687,42.553733562212436,50.15600565075874,50.19275312870741,50.19275312870741
|
||||
12,12,12,0,59.99224958010018,0.20002583807059768,25.362822842784226,45.54706535985073,59.93020099774003,59.95823861565441,59.98695467971265
|
||||
20,20,20,0,99.85960746835917,0.20028117981874766,16.85126264579594,66.68330737100914,55.060471390374005,92.9106959477067,99.76074412371963
|
||||
30,30,30,0,147.6483807535842,0.20318543181362822,20.873554840683937,90.49832232653473,97.37336550559849,147.4727698881179,147.48508568760008
|
||||
40,40,40,0,195.21150322351605,0.20490595758693703,25.34921144787222,114.89935769808945,102.41363409627229,194.96774306707084,194.99985321611166
|
||||
50,50,50,0,241.15029445569962,0.20733957680978582,88.28880738746375,149.28542947791516,126.26868482958525,240.86813501361758,240.87839913833886
|
||||
|
@@ -0,0 +1,218 @@
|
||||
{
|
||||
"started_at": "2026-06-10T15:58:07+0400",
|
||||
"image": "/path/to/suya-ocr-api/temp_image_c3c56948-282e-453b-8fac-6c482243d1e5.jpg",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"levels": [
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
6,
|
||||
8,
|
||||
10,
|
||||
12,
|
||||
20,
|
||||
30,
|
||||
40,
|
||||
50
|
||||
],
|
||||
"results": [
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 1,
|
||||
"concurrency": 1,
|
||||
"success": 1,
|
||||
"failed": 0,
|
||||
"wall_seconds": 7.6577067924663424,
|
||||
"throughput_rps": 0.13058739738949013,
|
||||
"latency_seconds": {
|
||||
"min": 7.656610286794603,
|
||||
"mean": 7.656610286794603,
|
||||
"p50": 7.656610286794603,
|
||||
"p95": 7.656610286794603,
|
||||
"max": 7.656610286794603
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 2,
|
||||
"concurrency": 2,
|
||||
"success": 2,
|
||||
"failed": 0,
|
||||
"wall_seconds": 11.936453193426132,
|
||||
"throughput_rps": 0.16755395992349534,
|
||||
"latency_seconds": {
|
||||
"min": 11.934747929684818,
|
||||
"mean": 11.935045637656003,
|
||||
"p50": 11.934747929684818,
|
||||
"p95": 11.935343345627189,
|
||||
"max": 11.935343345627189
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 4,
|
||||
"concurrency": 4,
|
||||
"success": 4,
|
||||
"failed": 0,
|
||||
"wall_seconds": 20.956660461612046,
|
||||
"throughput_rps": 0.19087010582278188,
|
||||
"latency_seconds": {
|
||||
"min": 20.942796256393194,
|
||||
"mean": 20.951773355016485,
|
||||
"p50": 20.954541454091668,
|
||||
"p95": 20.95546007528901,
|
||||
"max": 20.95546007528901
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 6,
|
||||
"concurrency": 6,
|
||||
"success": 6,
|
||||
"failed": 0,
|
||||
"wall_seconds": 29.489064288325608,
|
||||
"throughput_rps": 0.20346525550406608,
|
||||
"latency_seconds": {
|
||||
"min": 29.45817438978702,
|
||||
"mean": 29.47708616297071,
|
||||
"p50": 29.48345213010907,
|
||||
"p95": 29.487088727764785,
|
||||
"max": 29.487088727764785
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 8,
|
||||
"concurrency": 8,
|
||||
"success": 8,
|
||||
"failed": 0,
|
||||
"wall_seconds": 41.46563811413944,
|
||||
"throughput_rps": 0.192930830534405,
|
||||
"latency_seconds": {
|
||||
"min": 11.84945331979543,
|
||||
"mean": 34.050638656364754,
|
||||
"p50": 41.457184289582074,
|
||||
"p95": 41.46130123361945,
|
||||
"max": 41.46130123361945
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 10,
|
||||
"concurrency": 10,
|
||||
"success": 10,
|
||||
"failed": 0,
|
||||
"wall_seconds": 50.19571524951607,
|
||||
"throughput_rps": 0.19922019141058875,
|
||||
"latency_seconds": {
|
||||
"min": 12.104127056896687,
|
||||
"mean": 42.553733562212436,
|
||||
"p50": 50.15600565075874,
|
||||
"p95": 50.19275312870741,
|
||||
"max": 50.19275312870741
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 12,
|
||||
"concurrency": 12,
|
||||
"success": 12,
|
||||
"failed": 0,
|
||||
"wall_seconds": 59.99224958010018,
|
||||
"throughput_rps": 0.20002583807059768,
|
||||
"latency_seconds": {
|
||||
"min": 25.362822842784226,
|
||||
"mean": 45.54706535985073,
|
||||
"p50": 59.93020099774003,
|
||||
"p95": 59.95823861565441,
|
||||
"max": 59.98695467971265
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 20,
|
||||
"concurrency": 20,
|
||||
"success": 20,
|
||||
"failed": 0,
|
||||
"wall_seconds": 99.85960746835917,
|
||||
"throughput_rps": 0.20028117981874766,
|
||||
"latency_seconds": {
|
||||
"min": 16.85126264579594,
|
||||
"mean": 66.68330737100914,
|
||||
"p50": 55.060471390374005,
|
||||
"p95": 92.9106959477067,
|
||||
"max": 99.76074412371963
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 30,
|
||||
"concurrency": 30,
|
||||
"success": 30,
|
||||
"failed": 0,
|
||||
"wall_seconds": 147.6483807535842,
|
||||
"throughput_rps": 0.20318543181362822,
|
||||
"latency_seconds": {
|
||||
"min": 20.873554840683937,
|
||||
"mean": 90.49832232653473,
|
||||
"p50": 97.37336550559849,
|
||||
"p95": 147.4727698881179,
|
||||
"max": 147.48508568760008
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 40,
|
||||
"concurrency": 40,
|
||||
"success": 40,
|
||||
"failed": 0,
|
||||
"wall_seconds": 195.21150322351605,
|
||||
"throughput_rps": 0.20490595758693703,
|
||||
"latency_seconds": {
|
||||
"min": 25.34921144787222,
|
||||
"mean": 114.89935769808945,
|
||||
"p50": 102.41363409627229,
|
||||
"p95": 194.96774306707084,
|
||||
"max": 194.99985321611166
|
||||
},
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"name": "vllm",
|
||||
"url": "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/",
|
||||
"requests": 50,
|
||||
"concurrency": 50,
|
||||
"success": 50,
|
||||
"failed": 0,
|
||||
"wall_seconds": 241.15029445569962,
|
||||
"throughput_rps": 0.20733957680978582,
|
||||
"latency_seconds": {
|
||||
"min": 88.28880738746375,
|
||||
"mean": 149.28542947791516,
|
||||
"p50": 126.26868482958525,
|
||||
"p95": 240.86813501361758,
|
||||
"max": 240.87839913833886
|
||||
},
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_OLD_ENDPOINT = "http://127.0.0.1:5002/v1/api/ai/suya_ocr/"
|
||||
DEFAULT_NEW_ENDPOINT = "http://127.0.0.1:5002/v1/api/ai/suya_ocr_vllm/"
|
||||
|
||||
|
||||
def _encode_image(path: Path) -> str:
|
||||
return base64.b64encode(path.read_bytes()).decode("utf-8")
|
||||
|
||||
|
||||
def _preflight() -> Dict[str, Any]:
|
||||
checks: Dict[str, Any] = {}
|
||||
commands = {
|
||||
"docker_runtimes": ["docker", "info", "--format", "{{json .Runtimes}}"],
|
||||
"nvidia_smi": ["nvidia-smi", "-L"],
|
||||
"nvidia_container_runtime": ["which", "nvidia-container-runtime"],
|
||||
"nvidia_ctk": ["which", "nvidia-ctk"],
|
||||
"docker_gpus": [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--gpus",
|
||||
"all",
|
||||
"--entrypoint",
|
||||
"nvidia-smi",
|
||||
"vllm/vllm-openai:v0.20.1",
|
||||
"-L",
|
||||
],
|
||||
}
|
||||
for name, cmd in commands.items():
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||
checks[name] = {
|
||||
"ok": result.returncode == 0,
|
||||
"stdout": result.stdout.strip(),
|
||||
"stderr": result.stderr.strip(),
|
||||
}
|
||||
except Exception as exc:
|
||||
checks[name] = {"ok": False, "stdout": "", "stderr": str(exc)}
|
||||
runtimes = checks.get("docker_runtimes", {}).get("stdout", "")
|
||||
checks["docker_has_nvidia_runtime"] = '"nvidia"' in runtimes
|
||||
if not checks["docker_has_nvidia_runtime"] and checks.get("docker_gpus", {}).get("ok"):
|
||||
checks["recommended_fix"] = (
|
||||
"Docker GPU passthrough works with --gpus. Use the local "
|
||||
"scripts/docker_nvidia_runtime_compat.sh wrapper to strip the legacy "
|
||||
"--runtime nvidia flag from Surya's vLLM spawn command."
|
||||
)
|
||||
else:
|
||||
checks["recommended_fix"] = (
|
||||
"Run: sudo nvidia-ctk runtime configure --runtime=docker "
|
||||
"--config=/etc/docker/daemon.json && sudo systemctl restart docker"
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def _percentile(values: List[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, int(round((percentile / 100) * (len(ordered) - 1))))
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def _post_once(url: str, payload: Dict[str, Any], timeout: float) -> Dict[str, Any]:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
response = requests.post(url, json=payload, timeout=timeout)
|
||||
elapsed = time.perf_counter() - start
|
||||
body = response.json() if response.content else {}
|
||||
return {
|
||||
"ok": response.status_code == 200 and body.get("code") == 200,
|
||||
"status_code": response.status_code,
|
||||
"api_code": body.get("code"),
|
||||
"elapsed_seconds": elapsed,
|
||||
"error": body.get("message") if body.get("code") != 200 else None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"status_code": None,
|
||||
"api_code": None,
|
||||
"elapsed_seconds": time.perf_counter() - start,
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
|
||||
def run_endpoint(
|
||||
name: str,
|
||||
url: str,
|
||||
payload: Dict[str, Any],
|
||||
requests_count: int,
|
||||
concurrency: int,
|
||||
timeout: float,
|
||||
) -> Dict[str, Any]:
|
||||
start = time.perf_counter()
|
||||
samples: List[Dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = [
|
||||
pool.submit(_post_once, url, payload, timeout)
|
||||
for _ in range(requests_count)
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
samples.append(future.result())
|
||||
wall_seconds = time.perf_counter() - start
|
||||
latencies = [sample["elapsed_seconds"] for sample in samples]
|
||||
success_count = sum(1 for sample in samples if sample["ok"])
|
||||
return {
|
||||
"name": name,
|
||||
"url": url,
|
||||
"requests": requests_count,
|
||||
"concurrency": concurrency,
|
||||
"success": success_count,
|
||||
"failed": requests_count - success_count,
|
||||
"wall_seconds": wall_seconds,
|
||||
"throughput_rps": requests_count / wall_seconds if wall_seconds else 0.0,
|
||||
"latency_seconds": {
|
||||
"min": min(latencies) if latencies else 0.0,
|
||||
"mean": statistics.mean(latencies) if latencies else 0.0,
|
||||
"p50": _percentile(latencies, 50),
|
||||
"p95": _percentile(latencies, 95),
|
||||
"max": max(latencies) if latencies else 0.0,
|
||||
},
|
||||
"errors": [sample for sample in samples if not sample["ok"]][:10],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Compare legacy and vLLM OCR endpoint concurrency.")
|
||||
parser.add_argument("--image", required=True, help="Path to an input png/jpg image.")
|
||||
parser.add_argument("--old-url", default=DEFAULT_OLD_ENDPOINT)
|
||||
parser.add_argument("--new-url", default=DEFAULT_NEW_ENDPOINT)
|
||||
parser.add_argument("--requests", type=int, default=20)
|
||||
parser.add_argument("--concurrency", type=int, default=8)
|
||||
parser.add_argument("--timeout", type=float, default=900)
|
||||
parser.add_argument("--output", default="concurrency_test_results.json")
|
||||
parser.add_argument("--skip-old", action="store_true")
|
||||
parser.add_argument("--skip-new", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
image_path = Path(args.image)
|
||||
image_type = image_path.suffix.lstrip(".").lower() or "png"
|
||||
payload = {
|
||||
"file": _encode_image(image_path),
|
||||
"type": "jpg" if image_type == "jpeg" else image_type,
|
||||
"skip_text_detection": False,
|
||||
"skip_table_detection": False,
|
||||
"recognize_math": False,
|
||||
"ocr_with_boxes": True,
|
||||
}
|
||||
|
||||
results = {
|
||||
"started_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"image": str(image_path),
|
||||
"requests": args.requests,
|
||||
"concurrency": args.concurrency,
|
||||
"preflight": _preflight(),
|
||||
"endpoints": [],
|
||||
}
|
||||
if not args.skip_old:
|
||||
results["endpoints"].append(
|
||||
run_endpoint("legacy", args.old_url, payload, args.requests, args.concurrency, args.timeout)
|
||||
)
|
||||
if not args.skip_new:
|
||||
results["endpoints"].append(
|
||||
run_endpoint("vllm", args.new_url, payload, args.requests, args.concurrency, args.timeout)
|
||||
)
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.write_text(json.dumps(results, indent=2), encoding="utf-8")
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user