nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

mtmd, model: allow skip build_vit() (#24077)

Browse Source 
* add model

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-06-03 17:10:35 +02:00
committed by GitHub
parent ee4cf705bb
commit a731805ced
20 changed files with 363 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/CMakeLists.txt
+2
View File
@@ -21,6 +21,8 @@ add_library(mtmd
models/exaone4_5.cpp
models/gemma4a.cpp
models/gemma4v.cpp
models/gemma4ua.cpp
models/gemma4uv.cpp
models/glm4v.cpp
models/granite-speech.cpp
models/hunyuanvl.cpp
tools/mtmd/clip-impl.h
+5
View File
@@ -83,6 +83,7 @@
#define TN_PATCH_EMBD_1 "v.patch_embd.weight.1"
#define TN_PATCH_BIAS "v.patch_embd.bias"
#define TN_NORM_EMBD "v.norm_embd.%s"
#define TN_PATCH_NORM "v.patch_norm.%d.%s"
#define TN_ATTN_QKV "%s.blk.%d.attn_qkv.%s"
#define TN_ATTN_K "%s.blk.%d.attn_k.%s"
#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"
@@ -317,6 +318,8 @@ enum projector_type {
PROJECTOR_TYPE_GEMMA3NA,
PROJECTOR_TYPE_GEMMA4V,
PROJECTOR_TYPE_GEMMA4A,
PROJECTOR_TYPE_GEMMA4UV,
PROJECTOR_TYPE_GEMMA4UA,
PROJECTOR_TYPE_PHI4,
PROJECTOR_TYPE_IDEFICS3,
PROJECTOR_TYPE_PIXTRAL,
@@ -369,6 +372,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_GEMMA3NA, "gemma3na"},
{ PROJECTOR_TYPE_GEMMA4V, "gemma4v"},
{ PROJECTOR_TYPE_GEMMA4A, "gemma4a"},
{ PROJECTOR_TYPE_GEMMA4UV, "gemma4uv"},
{ PROJECTOR_TYPE_GEMMA4UA, "gemma4ua"},
{ PROJECTOR_TYPE_PHI4, "phi4"},
{ PROJECTOR_TYPE_IDEFICS3, "idefics3"},
{ PROJECTOR_TYPE_PIXTRAL, "pixtral"},
tools/mtmd/clip-model.h
+8
View File
@@ -339,6 +339,14 @@ struct clip_model {
ggml_tensor * norm_embd_w = nullptr;
ggml_tensor * norm_embd_b = nullptr;
// "indexed" patch embedding norms
ggml_tensor * patch_norm_1_w = nullptr;
ggml_tensor * patch_norm_1_b = nullptr;
ggml_tensor * patch_norm_2_w = nullptr;
ggml_tensor * patch_norm_2_b = nullptr;
ggml_tensor * patch_norm_3_w = nullptr;
ggml_tensor * patch_norm_3_b = nullptr;
ggml_tensor * pre_ln_w = nullptr;
ggml_tensor * pre_ln_b = nullptr;
tools/mtmd/clip.cpp
+46 -2
View File
@@ -866,6 +866,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_gemma4v>(ctx, img);
} break;
case PROJECTOR_TYPE_GEMMA4UV:
{
builder = std::make_unique<clip_graph_gemma4uv>(ctx, img);
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_LIGHTONOCR:
{
@@ -969,6 +973,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_gemma4a>(ctx, img);
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
builder = std::make_unique<clip_graph_gemma4ua>(ctx, img);
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
builder = std::make_unique<clip_graph_granite_speech>(ctx, img);
@@ -1386,13 +1394,19 @@ struct clip_model_loader {
} break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
{
hparams.rope_theta = 100.0f;
hparams.n_merge = 3; // pooling_kernel_size
hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false);
if (model.proj_type == PROJECTOR_TYPE_GEMMA4UV) {
// for "unified" variant, we directly use a bigger patch size, because the "token merging" is done directly on conv layer
hparams.patch_size = hparams.patch_size * hparams.n_merge;
hparams.n_merge = 1;
}
// @ngxson : the model performs quite poor with small images, we need to bump minimum image tokens to 40 to avoid that
hparams.set_limit_image_tokens(252, 280);
hparams.set_limit_image_tokens(40, 280);
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
@@ -1586,6 +1600,14 @@ struct clip_model_loader {
// since all gemma4a models use 1e-6, we just hardcode it here to avoid re-conversion
hparams.eps = 1e-6f;
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
// Encoder-free: raw 16 kHz waveform chunked into 640-sample frames.
hparams.audio_chunk_len = 0;
hparams.audio_sample_rate = 16000;
hparams.eps = 1e-6f;
hparams.n_mel_bins = 640;
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
hparams.audio_chunk_len = 0;
@@ -2097,6 +2119,16 @@ struct clip_model_loader {
}
}
} break;
case PROJECTOR_TYPE_GEMMA4UV:
{
model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ);
model.patch_norm_1_w = get_tensor(string_format(TN_PATCH_NORM, 1, "weight"));
model.patch_norm_1_b = get_tensor(string_format(TN_PATCH_NORM, 1, "bias"));
model.patch_norm_2_w = get_tensor(string_format(TN_PATCH_NORM, 2, "weight"));
model.patch_norm_2_b = get_tensor(string_format(TN_PATCH_NORM, 2, "bias"));
model.patch_norm_3_w = get_tensor(string_format(TN_PATCH_NORM, 3, "weight")); // pos_norm
model.patch_norm_3_b = get_tensor(string_format(TN_PATCH_NORM, 3, "bias")); // pos_norm
} break;
case PROJECTOR_TYPE_GEMMA3NV:
{
model.mobilenet_stem_conv_w = get_tensor(TN_MNV5_STEM_CONV, false);
@@ -2510,6 +2542,10 @@ struct clip_model_loader {
}
}
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
model.mm_input_proj_w = get_tensor(string_format(TN_A_MM_INP_PROJ, "weight"));
} break;
case PROJECTOR_TYPE_LFM2A:
{
for (int i : {0, 2, 3, 5, 6}) {
@@ -3218,6 +3254,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
} break;
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_INTERNVL:
case PROJECTOR_TYPE_NEMOTRON_V2_VL:
@@ -3350,6 +3387,10 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
}
n_patches = n;
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
n_patches = img->nx; // no downsampling: one token per raw waveform frame
} break;
case PROJECTOR_TYPE_GRANITE_SPEECH:
{
const int ws = ctx->model.hparams.audio_proj_window_size;
@@ -3917,6 +3958,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
set_input_i32("patches", patches);
} break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
{
// set (col, row) patch positions for learned positional embedding
const int n_cols = image_size_width / patch_size;
@@ -3998,6 +4040,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
case PROJECTOR_TYPE_PHI4:
case PROJECTOR_TYPE_COGVLM:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_GEMMA4UA:
{
// do nothing
} break;
@@ -4303,6 +4346,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_GEMMA3NV:
return ctx->model.mm_input_proj_w->ne[0];
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
return ctx->model.mm_input_proj_w->ne[1];
case PROJECTOR_TYPE_IDEFICS3:
return ctx->model.mm_fc_w->ne[1];
@@ -4337,7 +4381,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_fc_w->ne[1];
case PROJECTOR_TYPE_LFM2A:
return ctx->model.position_embeddings->ne[0];
case PROJECTOR_TYPE_GEMMA4A:
case PROJECTOR_TYPE_GEMMA4UA:
return ctx->model.hparams.projection_dim;
case PROJECTOR_TYPE_GRANITE_SPEECH:
return ctx->model.qf_proj_linear_w->ne[1];
tools/mtmd/models/gemma4ua.cpp
+19
View File
@@ -0,0 +1,19 @@
#include "models.h"
#include <cmath>
ggml_cgraph * clip_graph_gemma4ua::build() {
ggml_tensor * inp = build_inp_raw(1);
auto cur = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
// Gemma4UnifiedMultimodalEmbedder
{
// embedding_pre_projection_norm
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
cur = build_mm(model.mm_input_proj_w, cur);
cb(cur, "projected", -1);
}
ggml_build_forward_expand(gf, cur);
return gf;
}
tools/mtmd/models/gemma4uv.cpp
+71
View File
@@ -0,0 +1,71 @@
#include "models.h"
#include <cmath>
ggml_cgraph * clip_graph_gemma4uv::build() {
ggml_tensor * inp_raw = build_inp_raw();
// Gemma4UnifiedVisionEmbedder uses default pytorch LayerNorm, not RMSNorm
float eps = 1e-5f; // default eps for pytorch LayerNorm
ggml_tensor * inp = nullptr;
{
// note: we cannot use ggml_conv_2d here because we need to apply norm after im2col
auto c = inp_raw->ne[2];
ggml_tensor * kernel = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, patch_size, patch_size, c);
inp = ggml_im2col(ctx0, kernel, inp_raw, patch_size, patch_size, 0, 0, 1, 1, true, inp_raw->type);
// inp shape: [patch_size * patch_size * c, n_patches_w, n_patches_h]
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1] * inp->ne[2] * inp->ne[3]);
inp = build_norm(inp, model.patch_norm_1_w, model.patch_norm_1_b, NORM_TYPE_NORMAL, eps, -1);
// inp shape: [patch_size * patch_size * c, n_patches]
inp = ggml_mul_mat(ctx0, model.patch_embeddings_0, inp);
inp = ggml_add(ctx0, inp, model.patch_bias);
// inp shape: [n_embd, n_patches]
inp = build_norm(inp, model.patch_norm_2_w, model.patch_norm_2_b, NORM_TYPE_NORMAL, eps, -1);
}
ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_x, "pos_x");
ggml_set_input(pos_x);
ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
ggml_set_name(pos_y, "pos_y");
ggml_set_input(pos_y);
{
const int64_t pos_size = model.position_embeddings->ne[1];
const size_t nb1 = ggml_row_size(model.position_embeddings->type, n_embd);
// positional embeddings are stored as lookup tables (one for x, one for y)
ggml_tensor * tbl_x = ggml_view_2d(ctx0, model.position_embeddings,
n_embd, pos_size, nb1, 0);
ggml_tensor * tbl_y = ggml_view_2d(ctx0, model.position_embeddings,
n_embd, pos_size, nb1, pos_size * nb1);
// ggml_get_rows: [n_embd, n_patches]
ggml_tensor * emb_x = ggml_get_rows(ctx0, tbl_x, pos_x);
ggml_tensor * emb_y = ggml_get_rows(ctx0, tbl_y, pos_y);
inp = ggml_add(ctx0, inp, emb_x);
inp = ggml_add(ctx0, inp, emb_y);
cb(inp, "pos_embd", -1);
// pos_norm
inp = build_norm(inp, model.patch_norm_3_w, model.patch_norm_3_b, NORM_TYPE_NORMAL, eps, -1);
}
auto cur = inp;
// Gemma4UnifiedMultimodalEmbedder
{
// embedding_pre_projection_norm
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
cur = build_mm(model.mm_input_proj_w, cur);
cb(cur, "projected", -1);
}
ggml_build_forward_expand(gf, cur);
return gf;
}
tools/mtmd/models/models.h
+10
View File
@@ -18,6 +18,11 @@ struct clip_graph_gemma4v : clip_graph {
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
};
struct clip_graph_gemma4uv : clip_graph {
clip_graph_gemma4uv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_pixtral : clip_graph {
clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
@@ -142,6 +147,11 @@ struct clip_graph_gemma4a : clip_graph {
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
};
struct clip_graph_gemma4ua : clip_graph {
clip_graph_gemma4ua(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_glm4v : clip_graph {
clip_graph_glm4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
tools/mtmd/mtmd-audio.cpp
+38
View File
@@ -942,6 +942,44 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s
return true;
}
//
// mtmd_audio_preprocessor_gemma4ua
//
void mtmd_audio_preprocessor_gemma4ua::initialize() {
// no-op: no FFT or filterbank needed
}
bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
if (n_samples == 0) {
return false;
}
const int frame_size = hparams.n_mel_bins; // 640 samples per token @ 16 kHz = 40 ms
const int n_tokens = ((int)n_samples + frame_size - 1) / frame_size;
mtmd_audio_mel mel;
mel.n_len = n_tokens;
mel.n_len_org = n_tokens;
mel.n_mel = frame_size;
mel.data.assign((size_t)frame_size * n_tokens, 0.0f);
// Store mel-major (data[f * n_tokens + t]) so the ggml tensor loads as
// [n_tokens, frame_size] with ne[0]=n_tokens, ne[1]=frame_size.
// The graph builder transposes before RMSNorm so normalization is over frame_size.
for (int t = 0; t < n_tokens; t++) {
for (int f = 0; f < frame_size; f++) {
size_t src = (size_t)t * frame_size + f;
mel.data[(size_t)f * n_tokens + t] = (src < n_samples) ? samples[src] : 0.0f;
}
}
output.push_back(std::move(mel));
return true;
}
//
// mtmd_audio_streaming_istft implementation
//
tools/mtmd/mtmd-audio.h
+6
View File
@@ -96,6 +96,12 @@ struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
};
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
tools/mtmd/mtmd.cpp
+7
View File
@@ -482,6 +482,7 @@ struct mtmd_context {
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);
} break;
case PROJECTOR_TYPE_GEMMA4V:
case PROJECTOR_TYPE_GEMMA4UV:
{
// <|image> ... (image embeddings) ... <image|>
img_beg = "<|image>";
@@ -576,6 +577,12 @@ struct mtmd_context {
aud_end = "<audio|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4a>(ctx_a);
} break;
case PROJECTOR_TYPE_GEMMA4UA:
{
aud_beg = "<|audio>";
aud_end = "<audio|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4ua>(ctx_a);
} break;
default:
throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));
}