nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 16 Packages Projects Releases Wiki Activity

mtmd : DeepSeek-OCR image processing fixes, img_tool::resize padding refactor (#23345)

Browse Source 
* mtmd : deepseek-ocr fixes, improvements and refactoring

- image processing changes to achieve full parity with Pillow (reference impl)
- SAM mask casting only when flash-attn is on
- SAM refactor (build_sam() extracted so deepseek-ocr-2 can reuse it)
- llama-chat changes to fix server/WebUI issue (new media_markers_first())
- adapted test-chat-template and added test cases for deepseek-ocr
- changed regression test for deepseek-ocr to use CER+chrF scores for ground-truth comparison; removed embedding-model
- ty.toml ignore unresolved-import for tools/mtmd/tests/**

* image-text reordering fix removed

* refactor bool add_padding + pad_rounding enum into a single pad_style enum
This commit is contained in:
Saba Fallah
2026-05-20 17:37:10 +02:00
committed by GitHub
parent acd604fb27
commit a8681a0ed2
11 changed files with 443 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+8 -10
View File
@@ -1233,12 +1233,12 @@ struct clip_model_loader {
hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM;
hparams.image_pad_color = {122, 116, 104};
if (!hparams.image_res_candidates.empty()) {
hparams.image_resize_pad = true;
hparams.image_resize_pad = PAD_CEIL;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
} else {
// llava-1.6 default params
hparams.image_pad_ov = false;
hparams.image_pad_rf = true;
hparams.image_pad_ov = PAD_NONE;
hparams.image_pad_rf = PAD_CEIL;
hparams.image_pad_color_rf = {122, 116, 104};
hparams.image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
hparams.image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
@@ -1246,7 +1246,7 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_GLM_EDGE:
{
hparams.image_resize_pad = true;
hparams.image_resize_pad = PAD_CEIL;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
} break;
case PROJECTOR_TYPE_MINICPMV:
@@ -1441,7 +1441,7 @@ struct clip_model_loader {
{
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
hparams.image_resize_pad = false;
hparams.image_resize_pad = PAD_NONE;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size, true);
std::vector<int> wa_layer_indexes_vec;
@@ -1461,7 +1461,7 @@ struct clip_model_loader {
// reka model performs better when using resize_bicubic, which stretches
// the image to fit fixed square size
hparams.image_resize_pad = false;
hparams.image_resize_pad = PAD_NONE;
} break;
case PROJECTOR_TYPE_GLM4V:
{
@@ -1516,9 +1516,7 @@ struct clip_model_loader {
hparams.image_size = 1024;
hparams.warmup_image_size = 1024;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
hparams.image_pad_color[0] = hparams.image_mean[0];
hparams.image_pad_color[1] = hparams.image_mean[1];
hparams.image_pad_color[2] = hparams.image_mean[2];
hparams.image_pad_color = {127, 127, 127};
get_u32(KEY_SAM_N_BLOCK, hparams.sam_n_layer, true);
get_u32(KEY_SAM_N_HEAD, hparams.sam_n_head, true);
@@ -1537,7 +1535,7 @@ struct clip_model_loader {
{
hparams.n_merge = 2;
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
hparams.image_resize_pad = false;
hparams.image_resize_pad = PAD_NONE;
hparams.ffn_op = FFN_GELU;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
hparams.set_limit_image_tokens(256, 16384);