mtmd: Add DeepSeekOCR 2 Support (#20975)
* mtmd: DeepSeek-OCR 2 support, with multi-tile dynamic resolution * introduced clip_image_f32::add_viewsep * address PR review - drop redundant ggml_cpy ops in both deepseekocr versions build - drop no-op ggml_cont in build_sam - assert num_image_tokens deepseekocr2 - view_seperator as (1, n_embd) at conversion (for both versions) - drop redundant ggml_reshape_2d * Update tools/mtmd/models/deepseekocr2.cpp Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com> --------- Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com>
This commit is contained in:
Saba Fallah
co-authored by
Xuan-Son Nguyen
Xuan-Son Nguyen
parent
6ed481eea4
commit
da3f990a47
@@ -237,6 +237,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"AudioFlamingo3ForConditionalGeneration": "ultravox",
|
||||
"CogVLMForCausalLM": "cogvlm",
|
||||
"DeepseekOCR2ForCausalLM": "deepseek",
|
||||
"DeepseekOCRForCausalLM": "deepseek",
|
||||
"DotsOCRForCausalLM": "dotsocr",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
|
||||
+1
-1
@@ -1140,7 +1140,7 @@ class TextModel(ModelBase):
|
||||
# Skip multimodal tensors
|
||||
if name.startswith(("mlp", "vit.", "vpm.", "siglip2.", "conformer.", "merger.", "resampler.", "sound_encoder.", "sound_projection.", "speech_embeddings.")) \
|
||||
or "visual." in name or "vision." in name or "audio." in name or "talker." in name \
|
||||
or "vision_" in name or "audio_" in name or "sam_model" in name \
|
||||
or "vision_" in name or "audio_" in name \
|
||||
or "token2wav." in name or "code2wav." in name \
|
||||
or "projector." in name or "pre_mm_projector_norm" in name \
|
||||
or "image_newline" in name or "view_seperator" in name \
|
||||
|
||||
+56
-12
@@ -16,10 +16,14 @@ from .qwen import QwenModel
|
||||
|
||||
@ModelBase.register("DeepseekOCRForCausalLM")
|
||||
class DeepseekOCRVisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
hparams = self.hparams
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEKOCR)
|
||||
self.gguf_writer.add_clip_projector_type(self.clip_projector_type)
|
||||
# default values below are taken from HF tranformers code
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("layer_norm_eps", 1e-6))
|
||||
self.gguf_writer.add_vision_use_gelu(True)
|
||||
@@ -49,22 +53,27 @@ class DeepseekOCRVisionModel(MmprojModel):
|
||||
raise ValueError("DeepseekOCR model requires 'vision_config' in the model configuration, but it was not found")
|
||||
|
||||
vision_config['sam'] = vision_config['width']['sam_vit_b']
|
||||
vision_config.update(vision_config['width']['clip-l-14-224'])
|
||||
vision_config['hidden_size'] = vision_config['width']
|
||||
vision_config['num_heads'] = vision_config['heads']
|
||||
vision_config['intermediate_size'] = vision_config['heads'] * 4
|
||||
if vision_config['width'].get('clip-l-14-224') is not None:
|
||||
vision_config.update(vision_config['width']['clip-l-14-224'])
|
||||
if isinstance(vision_config['width'], int):
|
||||
vision_config['hidden_size'] = vision_config['width']
|
||||
if vision_config.get('heads') is not None:
|
||||
vision_config['num_heads'] = vision_config['heads']
|
||||
vision_config['intermediate_size'] = vision_config['heads'] * 4
|
||||
|
||||
return vision_config
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".embeddings." in name or 'pos_embed' in name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
if ".rel_pos_h" in name or '.rel_pos_w' in name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
if ".neck." in name or ".net_" in name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
for nq_name in ('.embeddings.', 'pos_embed', '.rel_pos_h', '.rel_pos_w', '.neck.', '.net_'):
|
||||
if nq_name in name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.endswith("view_seperator"):
|
||||
data_torch = data_torch.unsqueeze(0)
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
@@ -81,6 +90,33 @@ class DeepseekOCRVisionModel(MmprojModel):
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekOCR2ForCausalLM")
|
||||
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
# the vision tower's qwen2 encoder is built from fixed defaults,
|
||||
# see build_qwen2_decoder_as_encoder() in deepencoderv2.py
|
||||
if self.hparams.get("patch_size") is None:
|
||||
self.hparams["patch_size"] = 16
|
||||
if self.hparams.get("intermediate_size") is None:
|
||||
self.hparams["intermediate_size"] = 4864
|
||||
if self.hparams.get("num_attention_heads") is None:
|
||||
self.hparams["num_attention_heads"] = 14
|
||||
super().set_gguf_parameters()
|
||||
# qwen2 encoder is GQA: 14 Q heads, 2 KV heads
|
||||
self.gguf_writer.add_vision_head_count_kv(2)
|
||||
|
||||
def get_vision_config(self) -> dict[str, Any]:
|
||||
vision_config = super().get_vision_config()
|
||||
vision_config['hidden_size'] = vision_config['width']['qwen2-0-5b']['dim']
|
||||
if vision_config.get('layers') is None:
|
||||
vision_config['layers'] = 24
|
||||
return vision_config
|
||||
|
||||
|
||||
@ModelBase.register("DeepseekForCausalLM")
|
||||
class DeepseekModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.DEEPSEEK
|
||||
@@ -188,13 +224,21 @@ class DeepseekV2Model(TextModel):
|
||||
self.origin_hf_arch = hparams.get('architectures', [None])[0]
|
||||
|
||||
# special handling for Deepseek OCR
|
||||
if self.origin_hf_arch == "DeepseekOCRForCausalLM":
|
||||
if self.origin_hf_arch in ("DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM"):
|
||||
self.model_arch = gguf.MODEL_ARCH.DEEPSEEK2OCR
|
||||
self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch]
|
||||
self.gguf_writer.add_architecture()
|
||||
# default jinja template
|
||||
self.gguf_writer.add_chat_template("{% for m in messages %}{{m['content']}}{% endfor %}")
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# DeepSeek-OCR vision encoder (SAM + DeepSeek-OCR-2 qwen2 tower)
|
||||
if "sam_model" in name or "qwen2_model" in name:
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def set_vocab(self):
|
||||
try:
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
Reference in New Issue
Block a user