mtmd : DeepSeek-OCR image processing fixes, img_tool::resize padding refactor (#23345)

* mtmd : deepseek-ocr fixes, improvements and refactoring

- image processing changes to achieve full parity with Pillow (reference impl)
- SAM mask casting only when flash-attn is on
- SAM refactor (build_sam() extracted so deepseek-ocr-2 can reuse it)
- llama-chat changes to fix server/WebUI issue (new media_markers_first())
- adapted test-chat-template and added test cases for deepseek-ocr
- changed regression test for deepseek-ocr to use CER+chrF scores for ground-truth comparison; removed embedding-model
- ty.toml ignore unresolved-import for tools/mtmd/tests/**

* image-text reordering fix removed

* refactor bool add_padding + pad_rounding enum into a single pad_style enum
This commit is contained in:
Saba Fallah
2026-05-20 17:37:10 +02:00
committed by GitHub
parent acd604fb27
commit a8681a0ed2
11 changed files with 443 additions and 482 deletions
+13 -3
View File
@@ -35,6 +35,16 @@ enum resize_algo {
// RESIZE_ALGO_LANCZOS, // TODO
};
// Padding style for img_tool::resize
// PAD_NONE - no padding; direct resize to target dimensions
// PAD_CEIL - aspect-preserving pad (default)
// PAD_NEAREST - aspect-preserving pad with nearest-integer rounding (Pillow byte-parity)
enum pad_style {
PAD_NONE,
PAD_CEIL,
PAD_NEAREST,
};
struct clip_hparams {
int32_t image_size = 0;
int32_t patch_size = 0;
@@ -52,7 +62,7 @@ struct clip_hparams {
int32_t image_min_pixels = -1;
int32_t image_max_pixels = -1;
resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
bool image_resize_pad = true; // if false, center-crop will be applied when resizing
pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
// (preprocessor) for llava-uhd style models
@@ -61,8 +71,8 @@ struct clip_hparams {
int32_t preproc_max_tiles = 0;
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
resize_algo image_resize_algo_ov = RESIZE_ALGO_BILINEAR;
bool image_pad_rf = true; // if true, refined image will be padded (e.g. llava-1.6)
bool image_pad_ov = false; // if true, overview image will be padded (e.g. llava-1.6)
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
pad_style image_pad_ov = PAD_NONE; // padding style for the overview image (e.g. llava-1.6)
std::array<uint8_t, 3> image_pad_color_rf = {0, 0, 0}; // padding color for refined image
std::array<uint8_t, 3> image_pad_color_ov = {0, 0, 0}; // padding color for overview image