mtmd: Add DeepSeekOCR 2 Support (#20975)
* mtmd: DeepSeek-OCR 2 support, with multi-tile dynamic resolution * introduced clip_image_f32::add_viewsep * address PR review - drop redundant ggml_cpy ops in both deepseekocr versions build - drop no-op ggml_cont in build_sam - assert num_image_tokens deepseekocr2 - view_seperator as (1, n_embd) at conversion (for both versions) - drop redundant ggml_reshape_2d * Update tools/mtmd/models/deepseekocr2.cpp Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com> --------- Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com>
This commit is contained in:
Saba Fallah
co-authored by
Xuan-Son Nguyen
Xuan-Son Nguyen
parent
6ed481eea4
commit
da3f990a47
@@ -3,7 +3,7 @@
|
||||
Evaluates llama.cpp's DeepSeek-OCR by comparing its output for a test
|
||||
image to the actual text in part of that image.
|
||||
|
||||
Runs the test image through mtmd-cli, calculates CER and chrF for
|
||||
Runs each test image through mtmd-cli, calculates CER and chrF for
|
||||
its output, and holds them against the HF model's scores.
|
||||
"""
|
||||
|
||||
@@ -12,24 +12,81 @@ import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("deepseek-ocr-test")
|
||||
|
||||
DEFAULT_IMAGE = "test-1.jpeg"
|
||||
DEFAULT_EXPECTED_TEXT = "test-1-ground-truth.txt"
|
||||
RUN_TIMEOUT = 300
|
||||
|
||||
# DeepSeek-OCR reference scores on the test image.
|
||||
# This is the baseline the implementation should keep up with.
|
||||
HF_REFERENCE_CER = 0.3030
|
||||
HF_REFERENCE_CHRF = 67.52
|
||||
|
||||
CER_TOLERANCE = 0.02
|
||||
CHRF_TOLERANCE = 2.0
|
||||
@dataclass
|
||||
class ModelSpec:
|
||||
key: str
|
||||
label: str
|
||||
model_arg: str
|
||||
mmproj_arg: str
|
||||
model_default: str
|
||||
mmproj_default: str
|
||||
|
||||
CER_MAX = HF_REFERENCE_CER + CER_TOLERANCE
|
||||
CHRF_MIN = HF_REFERENCE_CHRF - CHRF_TOLERANCE
|
||||
|
||||
@dataclass
|
||||
class TestCase:
|
||||
model_key: str
|
||||
label: str
|
||||
image: str
|
||||
ground_truth: str
|
||||
hf_cer: float
|
||||
hf_chrf: float
|
||||
cer_tol: float
|
||||
chrf_tol: float
|
||||
|
||||
@property
|
||||
def cer_max(self) -> float:
|
||||
return self.hf_cer + self.cer_tol
|
||||
|
||||
@property
|
||||
def chrf_min(self) -> float:
|
||||
return self.hf_chrf - self.chrf_tol
|
||||
|
||||
|
||||
MODELS = {
|
||||
"v1": ModelSpec(
|
||||
key="v1", label="DeepSeek-OCR",
|
||||
model_arg="--llama-model", mmproj_arg="--mmproj",
|
||||
model_default="gguf_models/deepseek-ai/deepseek-ocr-bf16.gguf",
|
||||
mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-bf16.gguf",
|
||||
),
|
||||
"v2": ModelSpec(
|
||||
key="v2", label="DeepSeek-OCR-2",
|
||||
model_arg="--llama-model-2", mmproj_arg="--mmproj-2",
|
||||
model_default="gguf_models/deepseek-ai/deepseek-ocr-2-bf16.gguf",
|
||||
mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-2-bf16.gguf",
|
||||
),
|
||||
}
|
||||
|
||||
CASES = [
|
||||
TestCase(
|
||||
model_key="v1", label="single-view scan",
|
||||
image="tools/mtmd/test-1.jpeg",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
hf_cer=0.3030, hf_chrf=67.52, cer_tol=0.02, chrf_tol=2.0,
|
||||
),
|
||||
TestCase(
|
||||
model_key="v2", label="single-view scan",
|
||||
image="tools/mtmd/test-1.jpeg",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
# 640x488 is below the 768 tiling threshold -- single 1024 global view.
|
||||
# hf_cer/hf_chrf are the deepseek-ai repo's own scores (ImageOps.pad);
|
||||
# the transformers HF processor is *not* the reference -- its pad_to_square
|
||||
# is one pixel off and lands at ~0.69 instead.
|
||||
hf_cer=0.7761, hf_chrf=28.70, cer_tol=0.12, chrf_tol=8.0,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def arg_dest(flag: str) -> str:
|
||||
return flag.lstrip("-").replace("-", "_")
|
||||
|
||||
|
||||
def verdict(ok: bool) -> str:
|
||||
@@ -84,6 +141,14 @@ def run_mtmd_cli(model_path, mmproj_path, image_path, bin_path) -> str:
|
||||
"--temp", "0",
|
||||
"--flash-attn", "off", # match the HF "eager" attention reference
|
||||
"--no-warmup",
|
||||
"-n", "512", # cap loops on hard images (KV would otherwise fill)
|
||||
# HF decodes with no_repeat_ngram_size; llama.cpp's analog is DRY.
|
||||
# Default DRY breakers include "\n", so they are cleared below.
|
||||
"--dry-multiplier", "0.8",
|
||||
"--dry-base", "1.75",
|
||||
"--dry-allowed-length", "2",
|
||||
"--dry-penalty-last-n", "-1",
|
||||
"--dry-sequence-breaker", "none",
|
||||
]
|
||||
logger.debug(f" command: {' '.join(cmd)}")
|
||||
|
||||
@@ -110,7 +175,7 @@ def read_expected_text(file_path: Path) -> str:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def evaluate(expected: str, ocr_out: str) -> bool:
|
||||
def evaluate(case: "TestCase", expected: str, ocr_out: str) -> bool:
|
||||
expected = normalize_text(expected)
|
||||
ocr_out = normalize_text(ocr_out)
|
||||
aligned = locally_align(expected, ocr_out)
|
||||
@@ -122,16 +187,16 @@ def evaluate(expected: str, ocr_out: str) -> bool:
|
||||
cer = compute_cer(expected, aligned)
|
||||
chrf = compute_chrf(expected, aligned)
|
||||
|
||||
cer_pass = cer <= CER_MAX
|
||||
chrf_pass = chrf >= CHRF_MIN
|
||||
cer_pass = cer <= case.cer_max
|
||||
chrf_pass = chrf >= case.chrf_min
|
||||
passed = cer_pass and chrf_pass
|
||||
|
||||
logger.info("")
|
||||
logger.info("=" * 60)
|
||||
logger.info("Free OCR evaluation:")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f" CER {cer:>7.4f} (<= {CER_MAX:>7.4f} -> {verdict(cer_pass)})")
|
||||
logger.info(f" chrF (0-100) {chrf:>7.2f} (>= {CHRF_MIN:>7.2f} -> {verdict(chrf_pass)})")
|
||||
logger.info(f" CER {cer:>7.4f} (HF {case.hf_cer:.4f}, <= {case.cer_max:>7.4f} -> {verdict(cer_pass)})")
|
||||
logger.info(f" chrF (0-100) {chrf:>7.2f} (HF {case.hf_chrf:.2f}, >= {case.chrf_min:>7.2f} -> {verdict(chrf_pass)})")
|
||||
logger.info(f" Expected chars {len(expected):>7}")
|
||||
logger.info(f" Aligned chars {len(aligned):>7} (of {len(ocr_out)} OCR chars)")
|
||||
logger.info("")
|
||||
@@ -142,12 +207,13 @@ def evaluate(expected: str, ocr_out: str) -> bool:
|
||||
|
||||
def argument_parser() -> argparse.ArgumentParser:
|
||||
ap = argparse.ArgumentParser(description="Compare llama.cpp DeepSeek-OCR output with a ground-truth transcript")
|
||||
ap.add_argument("--llama-model", default="gguf_models/deepseek-ai/deepseek-ocr-bf16.gguf",
|
||||
help="Path to llama.cpp GGUF model (relative to repo root or absolute)")
|
||||
ap.add_argument("--mmproj", default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-bf16.gguf",
|
||||
help="Path to mmproj GGUF file (relative to repo root or absolute)")
|
||||
ap.add_argument("--llama-bin", default="build/bin/llama-mtmd-cli",
|
||||
help="Path to llama-mtmd-cli binary (relative to repo root or absolute)")
|
||||
for spec in MODELS.values():
|
||||
ap.add_argument(spec.model_arg, default=spec.model_default,
|
||||
help=f"Path to the {spec.label} GGUF model (relative to repo root or absolute)")
|
||||
ap.add_argument(spec.mmproj_arg, default=spec.mmproj_default,
|
||||
help=f"Path to the {spec.label} mmproj GGUF file (relative to repo root or absolute)")
|
||||
ap.add_argument("--verbose", action="store_true",
|
||||
help="Also log the expected, OCR, and aligned text")
|
||||
return ap
|
||||
@@ -167,53 +233,60 @@ def main() -> int:
|
||||
args = argument_parser().parse_args()
|
||||
configure_logging(args.verbose)
|
||||
|
||||
tests_dir = Path(__file__).parent # tools/mtmd/tests
|
||||
mtmd_dir = tests_dir.parent # tools/mtmd
|
||||
repo_root = mtmd_dir.parent.parent # repo root
|
||||
repo_root = Path(__file__).resolve().parents[3] # tests -> mtmd -> tools -> repo root
|
||||
binary = resolve_path(args.llama_bin, repo_root)
|
||||
|
||||
inputs = [
|
||||
("image", resolve_path(DEFAULT_IMAGE, mtmd_dir)),
|
||||
("expected-text", resolve_path(DEFAULT_EXPECTED_TEXT, tests_dir)),
|
||||
("model", resolve_path(args.llama_model, repo_root)),
|
||||
("mmproj", resolve_path(args.mmproj, repo_root)),
|
||||
("binary", resolve_path(args.llama_bin, repo_root)),
|
||||
]
|
||||
for label, path in inputs:
|
||||
if not path.exists():
|
||||
logger.error(f"Error: {label} not found: {path}")
|
||||
return 1
|
||||
paths = dict(inputs)
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("DeepSeek-OCR: llama.cpp vs ground-truth comparison")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"HF baselines: CER {HF_REFERENCE_CER:.4f}, chrF {HF_REFERENCE_CHRF:.2f}")
|
||||
logger.info(f"Test thresholds: CER <= {CER_MAX:.4f}, chrF >= {CHRF_MIN:.2f}")
|
||||
|
||||
logger.debug("")
|
||||
logger.debug("Resolved test inputs:")
|
||||
for label, path in inputs:
|
||||
logger.debug(f" {label:<14} {path}")
|
||||
|
||||
logger.info("")
|
||||
logger.info("[1/3] Running llama.cpp 'Free OCR'")
|
||||
try:
|
||||
ocr_out = run_mtmd_cli(paths["model"], paths["mmproj"],
|
||||
paths["image"], paths["binary"])
|
||||
except RuntimeError as e:
|
||||
logger.error(f"Error: {e}")
|
||||
if not binary.exists():
|
||||
logger.error(f"Error: binary not found: {binary}")
|
||||
return 1
|
||||
|
||||
logger.info("")
|
||||
logger.info("[2/3] Reading expected output")
|
||||
expected = read_expected_text(paths["expected-text"])
|
||||
logger.info(f" expected: {len(expected)} chars")
|
||||
logger.info("=" * 60)
|
||||
logger.info("DeepSeek-OCR: llama.cpp vs HF parity check")
|
||||
logger.info("=" * 60)
|
||||
|
||||
results = {}
|
||||
for case in CASES:
|
||||
model_spec = MODELS[case.model_key]
|
||||
title = f"{model_spec.label} -- {case.label}"
|
||||
|
||||
logger.info("")
|
||||
logger.info(f"=== {title} ===")
|
||||
|
||||
model = resolve_path(getattr(args, arg_dest(model_spec.model_arg)), repo_root)
|
||||
mmproj = resolve_path(getattr(args, arg_dest(model_spec.mmproj_arg)), repo_root)
|
||||
image = resolve_path(case.image, repo_root)
|
||||
ground_truth = resolve_path(case.ground_truth, repo_root)
|
||||
|
||||
missing = [(lbl, p) for lbl, p in [("model", model), ("mmproj", mmproj),
|
||||
("image", image), ("ground-truth", ground_truth)]
|
||||
if not p.exists()]
|
||||
if missing:
|
||||
for lbl, p in missing:
|
||||
logger.error(f" Error: {lbl} not found: {p}")
|
||||
results[title] = False
|
||||
continue
|
||||
|
||||
expected = read_expected_text(ground_truth)
|
||||
logger.info(f" Image: {case.image}")
|
||||
logger.info(f" Expected text: {len(expected)} chars")
|
||||
logger.info(" Running llama.cpp 'Free OCR'")
|
||||
try:
|
||||
ocr_out = run_mtmd_cli(model, mmproj, image, binary)
|
||||
except RuntimeError as e:
|
||||
logger.error(f" Error: {e}")
|
||||
results[title] = False
|
||||
continue
|
||||
|
||||
results[title] = evaluate(case, expected, ocr_out)
|
||||
|
||||
logger.info("")
|
||||
logger.info("[3/3] Computing OCR metrics")
|
||||
ok = evaluate(expected, ocr_out)
|
||||
logger.info("=== Summary ===")
|
||||
for title, ok in results.items():
|
||||
logger.info(f" {title:<48} {verdict(ok)}")
|
||||
all_passed = all(results.values())
|
||||
logger.info(f"Overall: {verdict(all_passed)}")
|
||||
|
||||
return 0 if ok else 1
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user