Fu DaiandClaude Opus 4.8
1a585693be
FastAPI service wrapping the Surya-OCR-2 model (datalab-to) served through vLLM: legacy /v1/api/ai/* endpoints, an OpenAI-compatible /v1/chat/completions endpoint, a coalescing request batcher, a local OCR CLI, Docker packaging, multilingual example outputs, and quantization/concurrency benchmarks. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
60 lines
1.9 KiB
Python
60 lines
1.9 KiB
Python
"""Character error rate (CER) between a reference baseline and a candidate.
|
|
|
|
CER = levenshtein(reference, hypothesis) / len(reference).
|
|
Used to gate work-reduction changes: keep a change only if CER <= 0.005.
|
|
|
|
Usage:
|
|
python scripts/cer_divergence.py baseline_outputs/ candidate_outputs/
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def _levenshtein(a: str, b: str) -> int:
|
|
if a == b:
|
|
return 0
|
|
if not a:
|
|
return len(b)
|
|
if not b:
|
|
return len(a)
|
|
prev = list(range(len(b) + 1))
|
|
for i, ca in enumerate(a, 1):
|
|
cur = [i]
|
|
for j, cb in enumerate(b, 1):
|
|
cost = 0 if ca == cb else 1
|
|
cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost))
|
|
prev = cur
|
|
return prev[-1]
|
|
|
|
|
|
def cer(reference: str, hypothesis: str) -> float:
|
|
if not reference:
|
|
return 0.0 if not hypothesis else 1.0
|
|
return _levenshtein(reference, hypothesis) / len(reference)
|
|
|
|
|
|
def cer_over_dirs(baseline_dir: Path, candidate_dir: Path) -> dict:
|
|
per_file = {}
|
|
for ref_path in sorted(baseline_dir.glob("*.txt")):
|
|
cand_path = candidate_dir / ref_path.name
|
|
ref = ref_path.read_text(encoding="utf-8")
|
|
hyp = cand_path.read_text(encoding="utf-8") if cand_path.exists() else ""
|
|
per_file[ref_path.name] = cer(ref, hyp)
|
|
mean = sum(per_file.values()) / len(per_file) if per_file else 0.0
|
|
return {"mean_cer": mean, "max_cer": max(per_file.values(), default=0.0), "per_file": per_file}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="CER divergence between two output dirs.")
|
|
parser.add_argument("baseline_dir", type=Path)
|
|
parser.add_argument("candidate_dir", type=Path)
|
|
args = parser.parse_args()
|
|
print(json.dumps(cer_over_dirs(args.baseline_dir, args.candidate_dir), indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|