{ "schemaVersion": 1, "benchmarkId": "2026-07-19-paddlevl16-v1", "datasetVersion": "2026-07-19.quality-v1", "systemVersion": "8584fdd", "provider": [ "paddle_vl" ], "model": [ "PaddleOCR-VL-1.6" ], "startedAt": "2026-07-19T14:00:04Z", "completedAt": "2026-07-19T14:01:06Z", "sampleCount": 10, "pageCount": 11, "successfulRunRate": 1, "cellExactMatchRate": 0.982, "rowExactMatchRate": 0.903, "totals": { "expectedCells": 218, "matchedCells": 214, "expectedRows": 31, "matchedRows": 28, "completedRuns": 10, "failedRuns": 0, "byFieldType": { "header": { "expected": 55, "matched": 54 }, "text": { "expected": 88, "matched": 88 }, "number": { "expected": 59, "matched": 56 }, "date": { "expected": 16, "matched": 16 } } }, "byFieldType": { "header": { "expected": 55, "matched": 54, "exactMatchRate": 0.982 }, "text": { "expected": 88, "matched": 88, "exactMatchRate": 1 }, "number": { "expected": 59, "matched": 56, "exactMatchRate": 0.949 }, "date": { "expected": 16, "matched": 16, "exactMatchRate": 1 } }, "methodology": { "metric": "Expected-value exact-match recall plus contiguous full-row exact match.", "normalization": "Unicode NFKC, trimmed/collapsed whitespace, case-folded English text, canonical decimal numbers, ISO dates.", "emptyReferenceCells": "Excluded from cell scoring.", "structure": "Header sequence and complete row values must appear contiguously within one output row.", "independence": "Reference workbooks are not used as model outputs; only manifest-declared expected values are scored against saved actual workbooks." }, "limitations": [ "Ten synthetic cases and eleven pages are too small for a general product accuracy claim.", "The four scan-quality cases derive from one source layout; handwriting, camera perspective, language variation, and naturally noisy archives are not represented.", "Expected-value recall does not penalize unrelated extra cells and is not a semantic field-mapping metric.", "Each document type has one sample, so document-type rates are descriptive cases rather than stable estimates." ], "results": [ { "id": "scanned-purchase-register", "documentType": "scanned-document", "sourceMode": "image-only", "scanQuality": "clean-image-only", "pages": 1, "status": "completed", "durationMs": 10146, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 20, "matchedCells": 20, "expectedRows": 3, "matchedRows": 3, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 5, "matched": 5, "exactMatchRate": 1 }, "text": { "expected": 9, "matched": 9, "exactMatchRate": 1 }, "number": { "expected": 3, "matched": 3, "exactMatchRate": 1 }, "date": { "expected": 3, "matched": 3, "exactMatchRate": 1 } }, "failureCases": [] }, { "id": "multipage-shipment-register", "documentType": "multi-page-table", "sourceMode": "selectable-text", "scanQuality": "digital-selectable-text", "pages": 2, "status": "completed", "durationMs": 5831, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 35, "matchedCells": 35, "expectedRows": 6, "matchedRows": 6, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 5, "matched": 5, "exactMatchRate": 1 }, "text": { "expected": 18, "matched": 18, "exactMatchRate": 1 }, "number": { "expected": 12, "matched": 12, "exactMatchRate": 1 }, "date": { "expected": 0, "matched": 0, "exactMatchRate": null } }, "failureCases": [] }, { "id": "bank-statement", "documentType": "bank-statement", "sourceMode": "selectable-text", "scanQuality": "digital-selectable-text", "pages": 1, "status": "completed", "durationMs": 6994, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 25, "matchedCells": 25, "expectedRows": 4, "matchedRows": 4, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 6, "matched": 6, "exactMatchRate": 1 }, "text": { "expected": 8, "matched": 8, "exactMatchRate": 1 }, "number": { "expected": 7, "matched": 7, "exactMatchRate": 1 }, "date": { "expected": 4, "matched": 4, "exactMatchRate": 1 } }, "failureCases": [] }, { "id": "commercial-invoice", "documentType": "invoice", "sourceMode": "selectable-text", "scanQuality": "digital-selectable-text", "pages": 1, "status": "completed", "durationMs": 3027, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 24, "matchedCells": 24, "expectedRows": 3, "matchedRows": 3, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 6, "matched": 6, "exactMatchRate": 1 }, "text": { "expected": 6, "matched": 6, "exactMatchRate": 1 }, "number": { "expected": 12, "matched": 12, "exactMatchRate": 1 }, "date": { "expected": 0, "matched": 0, "exactMatchRate": null } }, "failureCases": [] }, { "id": "customs-declaration", "documentType": "customs-declaration", "sourceMode": "selectable-text", "scanQuality": "digital-selectable-text", "pages": 1, "status": "completed", "durationMs": 3024, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 18, "matchedCells": 18, "expectedRows": 2, "matchedRows": 2, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 6, "matched": 6, "exactMatchRate": 1 }, "text": { "expected": 8, "matched": 8, "exactMatchRate": 1 }, "number": { "expected": 4, "matched": 4, "exactMatchRate": 1 }, "date": { "expected": 0, "matched": 0, "exactMatchRate": null } }, "failureCases": [] }, { "id": "bill-of-lading", "documentType": "bill-of-lading", "sourceMode": "selectable-text", "scanQuality": "digital-selectable-text", "pages": 1, "status": "completed", "durationMs": 5882, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 18, "matchedCells": 18, "expectedRows": 2, "matchedRows": 2, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 6, "matched": 6, "exactMatchRate": 1 }, "text": { "expected": 8, "matched": 8, "exactMatchRate": 1 }, "number": { "expected": 4, "matched": 4, "exactMatchRate": 1 }, "date": { "expected": 0, "matched": 0, "exactMatchRate": null } }, "failureCases": [] }, { "id": "air-waybill", "documentType": "air-waybill", "sourceMode": "selectable-text", "scanQuality": "digital-selectable-text", "pages": 1, "status": "completed", "durationMs": 7875, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 18, "matchedCells": 18, "expectedRows": 2, "matchedRows": 2, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 6, "matched": 6, "exactMatchRate": 1 }, "text": { "expected": 4, "matched": 4, "exactMatchRate": 1 }, "number": { "expected": 8, "matched": 8, "exactMatchRate": 1 }, "date": { "expected": 0, "matched": 0, "exactMatchRate": null } }, "failureCases": [] }, { "id": "scanned-purchase-register-moderate", "documentType": "scanned-document", "sourceMode": "image-only", "scanQuality": "moderate-resampled-skew", "pages": 1, "status": "completed", "durationMs": 4523, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 20, "matchedCells": 20, "expectedRows": 3, "matchedRows": 3, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 5, "matched": 5, "exactMatchRate": 1 }, "text": { "expected": 9, "matched": 9, "exactMatchRate": 1 }, "number": { "expected": 3, "matched": 3, "exactMatchRate": 1 }, "date": { "expected": 3, "matched": 3, "exactMatchRate": 1 } }, "failureCases": [] }, { "id": "scanned-purchase-register-poor", "documentType": "scanned-document", "sourceMode": "image-only", "scanQuality": "poor-low-contrast-blur-skew", "pages": 1, "status": "completed", "durationMs": 9128, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 20, "matchedCells": 20, "expectedRows": 3, "matchedRows": 3, "headerSequenceMatched": true, "cellExactMatchRate": 1, "rowExactMatchRate": 1, "fieldTypes": { "header": { "expected": 5, "matched": 5, "exactMatchRate": 1 }, "text": { "expected": 9, "matched": 9, "exactMatchRate": 1 }, "number": { "expected": 3, "matched": 3, "exactMatchRate": 1 }, "date": { "expected": 3, "matched": 3, "exactMatchRate": 1 } }, "failureCases": [] }, { "id": "scanned-purchase-register-cropped", "documentType": "scanned-document", "sourceMode": "image-only", "scanQuality": "failure-right-edge-cropped", "pages": 1, "status": "completed", "durationMs": 4989, "provider": "paddle_vl", "model": "PaddleOCR-VL-1.6", "expectedCells": 20, "matchedCells": 16, "expectedRows": 3, "matchedRows": 0, "headerSequenceMatched": false, "cellExactMatchRate": 0.8, "rowExactMatchRate": 0, "fieldTypes": { "header": { "expected": 5, "matched": 4, "exactMatchRate": 0.8 }, "text": { "expected": 9, "matched": 9, "exactMatchRate": 1 }, "number": { "expected": 3, "matched": 0, "exactMatchRate": 0 }, "date": { "expected": 3, "matched": 3, "exactMatchRate": 1 } }, "failureCases": [ { "kind": "missing-header", "column": 5, "expected": "Amount USD" }, { "kind": "missing-cell", "row": 1, "column": "Amount USD", "fieldType": "number", "expected": "425.30" }, { "kind": "missing-cell", "row": 2, "column": "Amount USD", "fieldType": "number", "expected": "1180.00" }, { "kind": "missing-cell", "row": 3, "column": "Amount USD", "fieldType": "number", "expected": "690.75" }, { "kind": "missing-row", "row": 1, "expected": [ "PR-001", "Northwind Test Supply", "2026-07-03", "Packaging", "425.30" ] }, { "kind": "missing-row", "row": 2, "expected": [ "PR-002", "Cedar Example Works", "2026-07-08", "Parts", "1180.00" ] }, { "kind": "missing-row", "row": 3, "expected": [ "PR-003", "Beacon Synthetic Freight", "2026-07-14", "Transport", "690.75" ] } ] } ] }