In this tutorial, we implement OmniParseBench, Datalab’s open benchmark for document parsers, which grades a parser’s output with 16,288 small pass-or-fail tests across 2,937 pages, from scanned forms and handwriting to arXiv math and dense tables. Rather than trust the leaderboard, we rebuild it from the verdicts Datalab released for 27 parsers, after feeding the scorer a hand-made page to see which mistakes flip a verdict. We then measure ranking certainty, slice it by document type, test whether the two models the card names as helping build tests behave differently on tests a model corrected, and run Tesseract and Indra OCR ourselves on a 50-page sample. Along the way, we find that the top two parsers are statistically tied, the leader changes on most slices, and one launch figure doesn’t match the released files.
import os
import re
import sys
import json
import time
import html
import shutil
import platform
import traceback
import subprocess
import collections
import urllib.request
from pathlib import Path
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, bool(held)))
return held
banner("1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision")
SYSTEM = platform.system()
OPB_VERSION = "1.0.0rc1" # a pre-release: a bare `pip install omni-parse-bench` resolves to 0.2.0, whose API differs
REPO = "datalab-to/omni_parse_bench"
REV = "961c0f89a2da701f5ce53afbd5e80aa469b0369a" # the dataset with the published verdicts of 27 parsers
INDRA = ["indra-ocr==0.3.2", "numpy==2.5.4", "onnxruntime==1.31.0", "opencv-python-headless==5.0.0.93",
"pypdfium2==5.14.0", "huggingface_hub==2.2.0"] # the pins of the package's own Indra server image
WORK = Path(os.environ.get("OPB_WORK") or ("/content/opb" if os.path.isdir("/content") else "opb_work")).resolve()
WORK.mkdir(parents=True, exist_ok=True)
subprocess.run([sys.executable, "-m", "pip", "install", "-q", f"omni-parse-bench[benchmark]=={OPB_VERSION}", "uv"], check=True)
if SYSTEM == "Linux" and shutil.which("tesseract") is None:
scripts = ("arab armn beng cans cher cyrl deva ethi frak geor grek gujr guru hang hang-vert hans hans-vert hant "
"hant-vert hebr jpan jpan-vert knda khmr laoo latn mlym mymr orya sinh syrc taml telu thaa thai tibt viet").split()
subprocess.run(["apt-get", "-qq", "update"], check=True)
subprocess.run(["apt-get", "-qq", "install", "-y", "tesseract-ocr", "tesseract-ocr-eng", "tesseract-ocr-osd"]
+ [f"tesseract-ocr-script-{s}" for s in scripts], check=True, stdout=subprocess.DEVNULL)
import numpy as np
import pandas as pd
import importlib.metadata as md
from huggingface_hub import snapshot_download
from omni_parse_bench import benchmark, data, harness, markdown, vocab
from omni_parse_bench.scorers import unit_tests
print(f"omni-parse-bench {md.version('omni-parse-bench')}, Python {platform.python_version()}, {SYSTEM} {platform.machine()}, "
f"{os.cpu_count()} CPU cores")
if SYSTEM == "Linux":
print("os:", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
tess = subprocess.run(["tesseract", "--version"], capture_output=True, text=True)
print("tesseract:", (tess.stdout or tess.stderr).splitlines()[0])
TESS_MAJOR = int((tess.stdout or tess.stderr).split()[1].split(".")[0])
print("test families:", {f: [t for t, g in vocab.FAMILIES.items() if g == f] for f in dict.fromkeys(vocab.FAMILIES.values())})
print(f"dataset {REPO} @ {REV[:7]}; work folder {WORK}")
RESULTS["1. Install omni-parse-bench 1.0.0rc1 and Tesseract, and pin the dataset revision"] = (
f"omni-parse-bench {md.version('omni-parse-bench')}, tesseract {(tess.stdout or tess.stderr).split()[1]}, dataset @ {REV[:7]}")
We install omni-parse-bench pinned to 1.0.0rc1, because a bare pip install still resolves to 0.2.0, an older release with a different API. The benchmark extra brings the parser adapters, the Hugging Face client, and pyarrow for the manifest, and on Colab we add Tesseract with its script models through apt. We also pin the dataset to one commit, since the repository changes as parsers are added. Printing the families shows the structure the benchmark rests on: present, order, and repeat tests form the text family; table_cell forms the tables family; and layout_kind forms the layout family.
@section("2. How the scorer decides: one hand-built page, five test types, one change at a time")
def scorer_anatomy():
T = lambda i, tt, out, tags, args, st=None: {"id": i, "test_type": tt, "output_type": out, "derivation": "authored",
"tags": tags, "args": args, "settings": st or {}}
exact = lambda s: {"text": s, "max_diffs": 0}
gold = [
T("title", "present", "html", ["heading"], {"content": exact("Quarterly Report")}),
T("total", "present", "html", ["eng"], {"content": exact("Total due")}),
T("growth", "present", "html", ["eng"], {"content": {"number": "-4.8%"}}),
T("amount", "present", "html", ["eng"], {"content": {"number": "1,240.00"}}),
T("eq_ratio", "present", "html", ["math"], {"content": {"latex": r"\alpha_{1} + \beta"}}),
T("eq_half", "present", "html", ["math"], {"content": {"latex": r"\frac{1}{2} \to x"}}),
T("order", "order", "html", ["eng"], {"before": exact("Introduction: this report covers the second quarter."),
"after": exact("Conclusion: the plan holds.")}),
T("repeat", "repeat", "html", ["eng"], {"content": exact("Revenue")}, {"expected_count": 2}),
T("cell_top", "table_cell", "html", ["table"], {"cell": exact("1,240"), "top_heading": exact("2024")}),
T("cell_left", "table_cell", "html", ["table"], {"cell": exact("1,240"), "left_heading": exact("Income")}),
T("cell_price", "table_cell", "html", ["table"], {"cell": exact("9.50"), "top_heading": exact("Price"),
"left": exact("Widget")}),
T("block_title", "layout_kind", "blocks", ["heading"],
{"content": dict(exact("Quarterly Report"), place=[[0.10, 0.04, 0.90, 0.09]])}, {"kind": "heading"}),
T("block_total", "layout_kind", "blocks", ["eng"],
{"content": dict(exact("Total due"), place=[[0.10, 0.12, 0.90, 0.16]])}, {"kind": "text"}),
]
gold_file = WORK / "fixture" / "gold" / "demo-p1.json" # a gold file is named after its page
gold_file.parent.mkdir(parents=True, exist_ok=True)
gold_file.write_text(json.dumps({"tests": gold}, indent=1))
page = data.Page(page_id="demo-p1", document="demo", suite="demo", doc_path=WORK / "fixture" / "demo-p1.pdf",
gt_path=gold_file, outputs=("blocks", "html"))
tests = unit_tests.gold(page)
table_a = ("<table><tr><th></th><th>2023</th><th>2024</th></tr><tr><th>Income</th><td>1,000</td><td>1,240</td></tr>"
"<tr><th>Costs</th><td>600</td><td>700</td></tr></table>")
table_b = "<table><tr><th>Product</th><th>Price</th></tr><tr><td>Widget</td><td>9.50</td></tr></table>"
md_table_b = "| Product | Price |\n|---|---|\n| Widget | 9.50 |"
parts = {"title": "<h1>Quarterly Report</h1>", "total": "<p>Total due: 1,240.00 USD.</p>",
"growth": "<p>Growth was -4.8% year on year.</p>", "ratio": r"<p>Ratio: 1+ holds.</p>",
"half": r"<p>Half: 12x in the model.</p>",
"intro": "<p>Introduction: this report covers the second quarter.</p>",
"revenue": "<p>Revenue is shown in thousands. Revenue grew in the second quarter.</p>",
"table_a": table_a, "conclusion": "<p>Conclusion: the plan holds.</p>", "table_b": table_b}
blocks = [{"label": "heading", "bbox": [0.10, 0.04, 0.90, 0.09], "text": "Quarterly Report"},
{"label": "text", "bbox": [0.10, 0.12, 0.90, 0.16], "text": "Total due: 1,240.00 USD."},
{"label": "table", "bbox": [0.10, 0.40, 0.90, 0.60], "text": "Income 1,000 1,240 Costs 600 700"}]
def pred(change=None, order=None, new_blocks=None):
p = dict(parts, **(change or {}))
return {"html": "\n".join(p[k] for k in (order or list(parts))), "blocks": blocks if new_blocks is None else new_blocks}
base = {v.id: v.result for v in unit_tests.score(pred(), tests)}
print(f"base prediction: {sum(r == 'pass' for r in base.values())} of {len(tests)} tests pass\n")
swapped = ["title", "total", "growth", "ratio", "half", "conclusion", "intro", "revenue", "table_a", "table_b"]
variants = [
("flip", "a typo: 'Total dve'", pred({"total": "<p>Total dve: 1,240.00 USD.</p>"}), ()),
("flip", "the minus sign dropped: 4.8%", pred({"growth": "<p>Growth was 4.8% year on year.</p>"}), ()),
("flip", "another symbol: alpha_2", pred({"ratio": r"<p>Ratio: 2+ holds.</p>"}), ()),
("flip", "conclusion before introduction", pred(order=swapped), ()),
("flip", "'Revenues' for the second 'Revenue'",
pred({"revenue": "<p>Revenue is shown in thousands. Revenues grew in the second quarter.</p>"}), ()),
("flip", "2023 and 2024 headings swapped", pred({"table_a": table_a.replace("<th>2023</th><th>2024</th>",
"<th>2024</th><th>2023</th>")}), ()),
("flip", "row label 'Margin' for 'Income'", pred({"table_a": table_a.replace("<th>Income</th>", "<th>Margin</th>")}), ()),
("flip", "a markdown table left in the html", pred({"table_b": md_table_b}), ()),
("flip", "the title's block labelled text", pred(new_blocks=[dict(blocks[0], label="text")] + blocks[1:]), ()),
("control", r"\dfrac12\rightarrow for \frac{1}{2}\to", pred({"half": r"<p>Half: $\dfrac12\rightarrow x$ in the model.</p>"}), ()),
("control", "'TOTAL DUE' and '1240.00'", pred({"total": "<p>TOTAL DUE: 1240.00 USD.</p>"}), ()),
("control", "markdown table through markdown.to_html", pred({"table_b": markdown.to_html(md_table_b)}), ()),
("unsupported", "no blocks, declared unsupported", {"html": pred()["html"]}, ("blocks",)),
]
as_expected = 0
for kind, name, p, unsupported in variants:
changed = [v for v in unit_tests.score(p, tests, unsupported) if v.result != base[v.id]]
if kind == "flip":
ok = len(changed) == 1 and changed[0].result == "fail"
elif kind == "control":
ok = not changed
else:
ok = bool(changed) and all(v.result == "unsupported" for v in changed)
as_expected += ok
shown = "; ".join(f"{v.id} -> {v.result}" + (f" ({v.why})" if v.result == "fail" else "") for v in changed) or "no verdict changes"
print(f"{kind:11s} {name:42s} {shown}")
s = unit_tests.summarize(unit_tests.lines(unit_tests.score(variants[0][2], tests), tests, page))
print(f"\nsummary of the typo variant: headline {s['headline']:.4f} = mean of "
+ ", ".join(f"{k} {v['score']:.3f} ({v['tests']} tests)" for k, v in s["families"].items()))
claim("every one-change variant flips exactly one verdict to fail, and no control flips any", as_expected == len(variants))
return f"{as_expected} of {len(variants)} hand-built variants behave as expected (9 flips, 3 controls, 1 unsupported)"
scorer_anatomy()
Before trusting a score, we want to see what a verdict means, so we write a gold file by hand with thirteen tests on an invented page and grade predictions with unit_tests.score, the function the benchmark itself uses. A base prediction passes everything, and each variant changes one thing. A one-letter typo, a lost minus sign, a different symbol in an equation, swapped headings and a mislabelled block each flip exactly one test to fail, with a reason attached. The controls still pass: capitals, a dropped thousands separator and dfrac in place of frac, because text is folded and equations are compared after KaTeX renders them.
@section("3. The dataset: 16,288 tests, what they check, and how each answer was established")
def dataset_step():
global DATA, PAGES, TESTS, CARD_TEXT
snapshot_download(REPO, repo_type="dataset", revision=REV, local_dir=str(WORK / "dataset"),
allow_patterns=["manifest.parquet", "gold/*", "results.parquet", "README.md"])
DATA, PAGES = data.read_manifest(str(WORK / "dataset"))
rows = []
for p in PAGES:
for t in unit_tests.gold(p):
rows.append(dict(test_id=t.id, page_id=p.page_id, suite=p.suite, document=p.document, test_type=t.test_type,
derivation=t.derivation, tags=list(t.tags), pdf=p.doc_path.suffix == ".pdf"))
TESTS = pd.DataFrame(rows)
tags = collections.Counter(g for ts in TESTS.tags for g in ts)
langs = {g for g in tags if re.fullmatch(r"[a-z]{3}", g)}
scripts = {g for g in tags if re.fullmatch(r"[A-Z][a-z]{3}", g)}
n_pdf = sum(p.doc_path.suffix == ".pdf" for p in PAGES)
print(f"{len(TESTS):,} tests on {TESTS.page_id.nunique():,} pages from {TESTS.document.nunique():,} documents in "
f"{TESTS.suite.nunique()} suites; {n_pdf:,} PDF pages and {len(PAGES) - n_pdf:,} images")
print(f"tags: {len(langs)} language codes ({len(langs - {'und', 'zxx'})} without und and zxx), {len(scripts)} scripts")
counts = lambda col: {k: int(v) for k, v in col.value_counts().items()}
print("\ntests by type: ", counts(TESTS.test_type))
print("tests by family: ", counts(TESTS.test_type.map(vocab.FAMILIES)))
print("structure and role tags:", {g: n for g, n in tags.most_common() if g not in langs | scripts})
print("\nhow each test's answer was established (derivation):")
for d, n in TESTS.derivation.value_counts().items():
print(f" {d:22s} {n:6,d} {', '.join(f'{k} {v}' for k, v in TESTS[TESTS.derivation == d].test_type.value_counts().items())}")
CARD_TEXT = (DATA / "README.md").read_text()
claim("16,288 tests on 2,937 pages from 2,343 documents in 22 suites, as the card says",
(len(TESTS), TESTS.page_id.nunique(), TESTS.document.nunique(), TESTS.suite.nunique()) == (16288, 2937, 2343, 22))
return f"{len(TESTS):,} tests, {TESTS.page_id.nunique():,} pages, {TESTS.document.nunique():,} documents, {len(langs - {'und', 'zxx'})} languages"
dataset_step()
With the scorer understood, we download the manifest, the gold files, and the published verdicts at the pinned revision, skip the 1.5 GB of page images, and read every page’s tests through unit_tests.gold. The counts match the dataset card, including 92 languages once the undetermined and no-language codes are removed. Two fields deserve attention. Tags such as handwriting, rotated, or table never decide a verdict; they only let us slice results. Derivation records how each answer was established, from publisher markup and text layers to model panels and human review, which lets us test later whether test construction favored any parser.
COLS = ["page_id", "suite", "document", "test_id", "test_type", "output_type", "derivation", "tags", "result", "why"]
STARRED = ["Gemini 3.8 Flash", "Claude Sonnet 5.5"] # the card: models "used in creating and verifying at least some tests"
@section("4. Recompute the published leaderboard from 439,776 verdicts")
def recompute_table():
global VERDICTS, TABLE
VERDICTS = pd.read_parquet(DATA / "results.parquet")
VERDICTS["output_type"] = VERDICTS.test_type.map(vocab.READS)
VERDICTS["tags"] = VERDICTS.tags.apply(list)
VERDICTS["family