nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 16 Packages Projects Releases Wiki Activity

mtmd : DeepSeek-OCR image processing fixes, img_tool::resize padding refactor (#23345)

Browse Source 
* mtmd : deepseek-ocr fixes, improvements and refactoring

- image processing changes to achieve full parity with Pillow (reference impl)
- SAM mask casting only when flash-attn is on
- SAM refactor (build_sam() extracted so deepseek-ocr-2 can reuse it)
- llama-chat changes to fix server/WebUI issue (new media_markers_first())
- adapted test-chat-template and added test cases for deepseek-ocr
- changed regression test for deepseek-ocr to use CER+chrF scores for ground-truth comparison; removed embedding-model
- ty.toml ignore unresolved-import for tools/mtmd/tests/**

* image-text reordering fix removed

* refactor bool add_padding + pad_rounding enum into a single pad_style enum
This commit is contained in:
Saba Fallah
2026-05-20 17:37:10 +02:00
committed by GitHub
parent acd604fb27
commit a8681a0ed2
11 changed files with 443 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-image.cpp
+56 -38
View File
@@ -38,7 +38,7 @@ struct img_tool {
clip_image_u8 & dst,
const clip_image_size & target_resolution,
resize_algo algo,
bool add_padding = true, // TODO: define the behavior for add_padding = false
pad_style padding = PAD_CEIL,
std::array<uint8_t, 3> pad_color = {0, 0, 0}) {
dst.nx = target_resolution.width;
dst.ny = target_resolution.height;
@@ -50,7 +50,7 @@ struct img_tool {
return;
}
if (!add_padding) {
if (padding == PAD_NONE) {
// direct resize
switch (algo) {
case RESIZE_ALGO_BILINEAR:
@@ -71,8 +71,15 @@ struct img_tool {
float scale_w = static_cast<float>(target_resolution.width) / src.nx;
float scale_h = static_cast<float>(target_resolution.height) / src.ny;
float scale = std::min(scale_w, scale_h);
int new_width = std::min(static_cast<int>(std::ceil(src.nx * scale)), target_resolution.width);
int new_height = std::min(static_cast<int>(std::ceil(src.ny * scale)), target_resolution.height);
int new_width, new_height;
if (padding == PAD_NEAREST) {
new_width = std::min(static_cast<int>(std::round(src.nx * scale)), target_resolution.width);
new_height = std::min(static_cast<int>(std::round(src.ny * scale)), target_resolution.height);
} else {
new_width = std::min(static_cast<int>(std::ceil(src.nx * scale)), target_resolution.width);
new_height = std::min(static_cast<int>(std::ceil(src.ny * scale)), target_resolution.height);
}
switch (algo) {
case RESIZE_ALGO_BILINEAR:
@@ -91,9 +98,14 @@ struct img_tool {
// fill dst with pad_color
fill(dst, pad_color);
int offset_x = (target_resolution.width - new_width) / 2;
int offset_y = (target_resolution.height - new_height) / 2;
int offset_x, offset_y;
if (padding == PAD_NEAREST) {
offset_x = static_cast<int>(std::round((target_resolution.width - new_width) / 2.0f));
offset_y = static_cast<int>(std::round((target_resolution.height - new_height) / 2.0f));
} else {
offset_x = (target_resolution.width - new_width) / 2;
offset_y = (target_resolution.height - new_height) / 2;
}
composite(dst, resized_image, offset_x, offset_y);
}
}
@@ -356,10 +368,10 @@ private:
GGML_ASSERT(inSize > 0 && outSize > 0);
double support, scale, filterscale;
double center, ww, ss;
int xx, x, ksize, xmin, xmax, xcnt;
int xx, x, ksize, xmin, xmax;
// Calculate scaling factor: ratio of input range to output size
filterscale = scale = (double)inSize / outSize;
filterscale = scale = static_cast<double>(inSize) / outSize;
// For upsampling (scale < 1), keep filterscale = 1 to maintain filter sharpness
// For downsampling (scale > 1), widen filter to prevent aliasing
if (filterscale < 1.0) {
@@ -373,6 +385,7 @@ private:
std::vector<double> pre_weights(outSize * ksize); // Temporary weights
bounds.resize(outSize * 2);
// For each output pixel, compute its filter coefficients
for (xx = 0; xx < outSize; xx++) {
// Calculate the center position in input space (pixel-center convention: +0.5)
@@ -391,10 +404,10 @@ private:
xmax = inSize;
}
xcnt = xmax - xmin;
xmax -= xmin;
// Compute filter weights for each contributing input pixel
for (x = 0; x < xcnt; x++) {
for (x = 0; x < xmax; x++) {
// Distance from input pixel center to output pixel center in input space
double w = bicubic_filter((x + xmin - center + 0.5) * ss);
pre_weights[xx * ksize + x] = w;
@@ -402,7 +415,7 @@ private:
}
// Normalize weights to sum to 1.0 (preserves brightness)
for (x = 0; x < xcnt; x++) {
for (x = 0; x < xmax; x++) {
if (ww != 0.0) {
pre_weights[xx * ksize + x] /= ww;
}
@@ -415,18 +428,27 @@ private:
// Store input pixel range for this output pixel
bounds[xx * 2 + 0] = xmin;
bounds[xx * 2 + 1] = xcnt;
bounds[xx * 2 + 1] = xmax;
}
// Convert floating-point coefficients to fixed-point integers
// Formula: int32 = round(float * 2^PRECISION_BITS)
weights.resize(outSize * ksize);
const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS
for (int i = 0; i < outSize * ksize; i++) {
double tmp_val = pre_weights[i] * fxp_scale;
if (pre_weights[i] < 0) {
weights[i] = static_cast<int32_t>(-0.5 + pre_weights[i] * (1 << PRECISION_BITS));
tmp_val -= 0.5;
} else {
weights[i] = static_cast<int32_t>(0.5 + pre_weights[i] * (1 << PRECISION_BITS));
tmp_val += 0.5;
}
tmp_val = std::round(tmp_val);
tmp_val = std::clamp(tmp_val,
static_cast<double>(std::numeric_limits<int32_t>::min()),
static_cast<double>(std::numeric_limits<int32_t>::max()));
weights[i] = static_cast<int32_t>(tmp_val);
}
return ksize;
@@ -1083,35 +1105,31 @@ bool mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img, cli
//
bool mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {
const std::vector native_resolutions = {
/*512 tiny , 640 small, */ 1024 /* base */, 1280 /* large */
};
// original image size
const clip_image_size original_size{img.nx, img.ny};
const int orig_w = original_size.width;
const int orig_h = original_size.height;
const int orig_area = orig_h * orig_w;
static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ };
// TODO: support 512 (tiny) and 640 (small) once we have eval data for them
size_t mode_i = 0;
int min_diff = orig_area;
const int64_t orig_area = static_cast<int64_t>(img.nx) * img.ny;
for (size_t i = 0; i < native_resolutions.size(); i++) {
int r = native_resolutions[i];
if (std::abs(orig_area - r * r) < min_diff) {
mode_i = i;
min_diff = std::abs(orig_area - r * r);
size_t mode_i = 0;
int64_t min_diff = std::numeric_limits<int64_t>::max();
for (size_t i = 0; i < std::size(native_resolutions); i++) {
const int64_t r = native_resolutions[i];
const int64_t diff = std::abs(orig_area - r * r);
if (diff < min_diff) {
mode_i = i;
min_diff = diff;
}
}
/* Native Resolution (Base/Large) */
const int image_size = native_resolutions[mode_i];
// scaled and padded image
clip_image_u8_ptr scaled_img(clip_image_u8_init());
img_tool::resize(img, *scaled_img, clip_image_size{image_size, image_size}, hparams.image_resize_algo);
// Aspect-preserving fit-and-pad. Pillow bicubic + PAD_NEAREST for
// byte-parity with the upstream deepseek-ai/DeepSeek-OCR HF preprocessor.
clip_image_u8 padded;
img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW,
PAD_NEAREST, hparams.image_pad_color);
clip_image_f32_ptr res(clip_image_f32_init());
img_u8_to_f32(*scaled_img, *res, hparams.image_mean, hparams.image_std);
img_u8_to_f32(padded, *res, hparams.image_mean, hparams.image_std);
output.entries.push_back(std::move(res));
output.grid_x = 1;
@@ -1246,7 +1264,7 @@ clip_image_u8 mtmd_image_preprocessor_step3vl::prepare_image(const clip_image_u8
std::max(1, static_cast<int>(std::floor(resized.ny * scale))),
};
clip_image_u8 scaled;
img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, false);
img_tool::resize(resized, scaled, new_size, RESIZE_ALGO_BILINEAR, PAD_NONE);
resized = std::move(scaled);
}
@@ -1347,7 +1365,7 @@ bool mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img, clip
clip_image_u8 img_for_crop = prepared;
if (instructions.refined_size.width != prepared.nx || instructions.refined_size.height != prepared.ny) {
clip_image_u8 refined;
img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, false);
img_tool::resize(prepared, refined, instructions.refined_size, RESIZE_ALGO_BILINEAR, PAD_NONE);
img_for_crop = std::move(refined);
}