{"site":"https://www.extractingbench.com","generatedAt":null,"runId":null,"versions":null,"note":"No results published yet. The lineup lists the models and providers planned for the first run.","metrics":[{"id":"index","label":"Extracting Index","unit":"index","better":"higher","description":"75% accuracy on fields that are in the document plus 25% null honesty on fields that aren't, on the private test set, text input."},{"id":"fieldAccuracy","label":"Accuracy","unit":"pct","better":"higher","description":"Share of fields printed in the document that the model returned correctly, after the normalization rules on the methodology page. Every document type counts equally."},{"id":"nullHonesty","label":"Null honesty","unit":"pct","better":"higher","description":"Share of fields missing from the document where the model returned null instead of inventing a value."},{"id":"falseNull","label":"False nulls","unit":"pct","better":"lower","description":"Share of fields in the document that the model returned as null."},{"id":"ece","label":"Calibration error","unit":"ratio","better":"lower","description":"Expected calibration error of the model's confidence: how far its stated confidence is from how often it's right."},{"id":"coverage80","label":"80% range coverage","unit":"pct","better":"higher","description":"Share of true values that land inside the model's 80% range. Close to 80% is good; far above or below is not."},{"id":"messyDrop","label":"Messy-input drop","unit":"points","better":"lower","description":"Accuracy on clean documents minus accuracy on blurred, rotated, dark, crumpled and scanned versions of the same documents."},{"id":"longDrop","label":"Long-document drop","unit":"points","better":"lower","description":"Accuracy at about 1,000 tokens minus accuracy at about 48,000 tokens, on the same documents padded with similar ones."},{"id":"manyFieldsDrop","label":"Many-fields drop","unit":"points","better":"lower","description":"Accuracy on the same 5 fields when asked alone versus inside a list of 50."},{"id":"localeAccuracy","label":"Accuracy across locales","unit":"pct","better":"higher","description":"Accuracy on the same records rendered with US, UK, German, French and Japanese date, number and currency formats. The chart shows the worst locale."},{"id":"languageAccuracy","label":"Non-English accuracy","unit":"pct","better":"higher","description":"Accuracy on documents in German, French, Spanish, Japanese and Chinese, pooled."},{"id":"repeatChange","label":"Answers that change on a rerun","unit":"pct","better":"lower","description":"Share of fields whose answer changes when the same request is sent again."},{"id":"textImageAgreement","label":"Text and image agree","unit":"pct","better":"higher","description":"Share of fields where the answer from the text version matches the answer from the image version."},{"id":"renameChange","label":"Answers that change when fields are renamed or reordered","unit":"pct","better":"lower","description":"Share of answers that change when the field list is shuffled or the fields are renamed to synonyms."},{"id":"computedOneStep","label":"Math, one step","unit":"pct","better":"higher","description":"Accuracy on values that need one operation, such as a sum or a date difference."},{"id":"computedMultiStep","label":"Math, multi-step","unit":"pct","better":"higher","description":"Accuracy on values that need two or more operations."},{"id":"pointing","label":"Pointing","unit":"pct","better":"higher","description":"Share of returned text values whose character position, read back from the input, gives the returned value."},{"id":"hijackRate","label":"Hijack rate","unit":"pct","better":"lower","description":"Share of targeted fields where the model returned the value an instruction planted in the document asked for."},{"id":"latencyP50","label":"Latency, median","unit":"seconds","better":"lower","description":"Median wall time per document, timed from our server with the network included."},{"id":"latencyP90","label":"Latency, p90","unit":"seconds","better":"lower","description":"90th percentile wall time per document."},{"id":"throughput","label":"Throughput","unit":"count","better":"higher","description":"Documents finished per minute at concurrency 4, or lower where the provider's rate limit requires it."},{"id":"costPer1k","label":"Cost per 1,000 documents","unit":"usd","better":"lower","description":"List-price cost per 1,000 documents, from the usage each API reports."}],"lineup":[{"slug":"clairic","name":"Clairic","maker":"Cloudflare","weights":"proprietary","inputs":["text","image"],"providers":[{"name":"Cloudflare Workers AI","kind":"serverless","price":"$0.10 per million input tokens, no output charge","status":"ready"}]},{"slug":"nuextract3","name":"NuExtract3","maker":"NuMind","weights":"open","inputs":["text","image"],"providers":[{"name":"NuMind API","kind":"serverless","price":"$1 per million input tokens, $5 per million output tokens","status":"needs-key"},{"name":"Hugging Face Inference Endpoints","kind":"dedicated","price":"per GPU hour","status":"needs-key"},{"name":"Our Mac Mini (run it yourself)","kind":"self-run","price":"no per-document price","status":"optional"}]},{"slug":"gliner2","name":"GLiNER2","maker":"Fastino","weights":"open","inputs":["text"],"providers":[{"name":"Fastino API","kind":"serverless","price":"$0.03 per million input tokens, no output charge","status":"needs-key"},{"name":"Hugging Face Inference Endpoints","kind":"dedicated","price":"per CPU hour","status":"needs-key"},{"name":"Our Mac Mini (run it yourself)","kind":"self-run","price":"no per-document price","status":"optional"}]}],"results":[]}