""" Generate a self-contained HTML verification report from olmOCR-bench results. Requires results saved with --save-raw-outputs. Usage: # 1. Run benchmark with raw outputs saved python benchmark/ocr/bench_sglang.py --port 30000 --split arxiv_math \\ --max-samples 20 --save-raw-outputs # 2. Generate HTML report for a single split python benchmark/ocr/generate_report.py --split arxiv_math # 3. Generate HTML report for all splits in a results directory python benchmark/ocr/generate_report.py --results-dir ./ocr_bench_results # 4. Show only failing tests python benchmark/ocr/generate_report.py --split arxiv_math --failures-only Open the generated .html file in any browser — formulas are rendered via MathJax. """ import argparse import html import json import re import sys from pathlib import Path # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- _MATH_DELIM_RE = re.compile( r"(\\\[[\s\S]*?\\\]" # \[...\] r"|\\\([\s\S]*?\\\)" # \(...\) r"|\$\$[\s\S]*?\$\$" # $$...$$ r"|\$[^$\n]+?\$)", # $...$ re.DOTALL, ) def _ocr_to_html(text: str) -> str: """ Convert raw OCR output to readable HTML. Preserves LaTeX delimiters for MathJax, strips bounding-box annotations, and wraps block-level elements in

tags. """ lines = text.splitlines() out_lines = [] for line in lines: # Strip bounding box annotations like text[[x1, y1, x2, y2]] line = re.sub(r"^\s*\w[\w_]*\[\[\d+,\s*\d+,\s*\d+,\s*\d+\]\]\s*", "", line) # HTML-escape everything EXCEPT LaTeX delimiters parts = _MATH_DELIM_RE.split(line) escaped = "" for i, part in enumerate(parts): if i % 2 == 0: escaped += html.escape(part) else: escaped += part # LaTeX — pass through for MathJax out_lines.append(escaped) return "
".join(out_lines) def _latex_to_display(latex: str) -> str: """Wrap a raw LaTeX string in display-math delimiters for MathJax.""" stripped = latex.strip() # Already delimited → pass through if stripped.startswith(("\\[", "$$", "\\(")): return stripped return f"\\[ {stripped} \\]" # --------------------------------------------------------------------------- # HTML template # --------------------------------------------------------------------------- _HTML_HEAD = """\ {title}

{title}

""" _HTML_TAIL = """\ """ def _render_sample(sample: dict, failures_only: bool) -> str: """Render one sample (PDF page) as an HTML block.""" pdf = sample.get("pdf", sample.get("pdf_rel", "?")) page = sample.get("page", "?") passed = sample.get("passed", 0) total = sample.get("total", 0) error = sample.get("error") ocr_output = sample.get("ocr_output", "") test_results = sample.get("test_results", []) test_inputs = sample.get("test_inputs", []) if failures_only and passed == total and not error: return "" pct = f"{100 * passed // total}%" if total else "—" header_cls = "fail" if (error or passed < total) else "pass" parts = [f'
'] parts.append( f"
" f'' f"📄 {html.escape(pdf)}  ·  page {page}" f"" f'{passed}/{total}  {pct}' f"" f"" ) parts.append('
') if error: parts.append( f'

ERROR: {html.escape(str(error))}

' ) else: parts.append('
') for i, tr in enumerate(test_results): if failures_only and tr.get("passed"): continue ttype = tr.get("type", "?") ok = tr.get("passed", False) badge = ( 'PASS' if ok else 'FAIL' ) test_err = tr.get("error", "") # Get expected values from test_inputs (parallel list) ti = test_inputs[i] if i < len(test_inputs) else {} test_cls = "pass" if ok else "fail" parts.append(f'
') parts.append( f'
' f"{badge}   type={ttype!r}" + ( f'   {html.escape(test_err)}' if test_err else "" ) + f"
" ) # Expected pane parts.append('
') parts.append('
') parts.append("

Expected

") if ttype in ("math", "math_formula_accuracy"): latex = ti.get("math", "") parts.append(f"
{html.escape(latex)}
") if latex: parts.append( f'
{_latex_to_display(latex)}
' ) elif ttype in ("present", "absent", "text_presence", "text_absence"): parts.append(f"
{html.escape(ti.get('text', ''))}
") elif ttype in ("order", "natural_reading_order"): before = ti.get("before", "") after = ti.get("after", "") parts.append( f"
before: {html.escape(before)}\nafter:  {html.escape(after)}
" ) parts.append("
") # Right pane — OCR formulas extracted (if math type) parts.append('
') parts.append("

OCR extracted formulas

") if ttype in ("math", "math_formula_accuracy") and ocr_output: matches = _MATH_DELIM_RE.findall(ocr_output) if matches: for m in matches[:6]: # show at most 6 matches parts.append(f'
{m}
') if len(matches) > 6: parts.append( f'

… and {len(matches) - 6} more

' ) else: parts.append( '

No LaTeX delimiters found in OCR output.

' ) else: parts.append( '

(see full OCR output below)

' ) parts.append("
") parts.append("
") # test-body parts.append("
") # test parts.append("
") # tests # Full OCR output if ocr_output: parts.append('
') parts.append(f"

Full OCR output ({len(ocr_output)} chars)

") parts.append(f'
{_ocr_to_html(ocr_output)}
') parts.append("
") elif not error: parts.append( '

No raw OCR output stored. ' "Re-run with --save-raw-outputs to include it here.

" ) parts.append("
") # sample-body parts.append("
") parts.append("
") # sample return "\n".join(parts) # --------------------------------------------------------------------------- # Report generation # --------------------------------------------------------------------------- def generate_report( split_name: str, split_result: dict, out_path: Path, failures_only: bool = False, ) -> None: total_passed = split_result.get("total_passed", 0) total_tests = split_result.get("total_tests", 0) overall = split_result.get("overall_score", 0.0) error_samples = split_result.get("error_samples", 0) total_samples = split_result.get("total_samples", 0) title = f"OCR Bench — {split_name}" body = _HTML_HEAD.format(title=title) # Summary bar by_type = split_result.get("by_type", {}) type_badges = "   ".join( f'{t}: {v:.1f}%' for t, v in by_type.items() ) body += f"""
{total_passed}/{total_tests}tests passed
{overall:.1f}%overall score
{error_samples}/{total_samples}error samples
{type_badges}
""" if failures_only: body += '

Showing failures only.

' samples = split_result.get("samples", []) if not samples: body += '

No per-sample data found. The result JSON may not include sample-level details.

' else: for sample in samples: body += _render_sample(sample, failures_only) body += _HTML_TAIL out_path.write_text(body, encoding="utf-8") print(f"Report → {out_path}") # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- def main() -> None: parser = argparse.ArgumentParser( description="Generate HTML verification report from olmOCR-bench results", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( "--results-dir", default="./ocr_bench_results", help="Directory containing result JSON files", ) parser.add_argument( "--split", default=None, help="Single split to report (default: all JSONs in results-dir)", ) parser.add_argument( "--output-dir", default=None, help="Where to write HTML files (default: same as results-dir)", ) parser.add_argument( "--failures-only", action="store_true", help="Include only samples/tests that failed", ) args = parser.parse_args() results_dir = Path(args.results_dir) output_dir = Path(args.output_dir) if args.output_dir else results_dir output_dir.mkdir(parents=True, exist_ok=True) if args.split: json_files = [results_dir / f"{args.split}.json"] else: json_files = sorted(results_dir.glob("*.json")) json_files = [f for f in json_files if f.stem != "summary"] if not json_files: sys.exit(f"No result JSON files found in {results_dir}") for jf in json_files: if not jf.exists(): print(f" SKIP (not found): {jf}") continue split_name = jf.stem with open(jf, encoding="utf-8") as fh: data = json.load(fh) suffix = "_failures" if args.failures_only else "" out_path = output_dir / f"{split_name}{suffix}_report.html" generate_report(split_name, data, out_path, failures_only=args.failures_only) if __name__ == "__main__": main()