nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 16 Packages Projects Releases Wiki Activity

mtmd: fit_params now take into account mmproj (#21489)

Browse Source 
* mtmd: fit_params now take into account mmproj

* rename alloc_compute_meta to reserve_compute_meta

* rm unused functions

* add ggml_backend_dev_t support

* add debug log
This commit is contained in:
Xuan-Son Nguyen
2026-05-20 11:27:44 +02:00
committed by GitHub
parent 7e50ef7d79
commit e2b129e1bf
5 changed files with 183 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd.cpp
+101 -45
View File
@@ -21,6 +21,7 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <climits>
#include <vector>
// represents raw image data, layout is RGBRGBRGB...
@@ -139,13 +140,13 @@ mtmd_context_params mtmd_context_params_default() {
struct mtmd_context {
struct clip_ctx * ctx_v; // vision
struct clip_ctx * ctx_a; // audio
const struct llama_model * text_model;
std::vector<float> image_embd_v; // image embedding vector
bool print_timings;
int n_threads;
std::string media_marker;
const int n_embd_text;
const int n_embd_text = -1; // -1 means llm context not provided, skip checking this
const llama_vocab * vocab = nullptr; // can be nullptr if text_model is not provided
mtmd_pos_type pos_type;
// these are not token, but strings used to mark the beginning and end of image/audio embeddings
@@ -178,12 +179,13 @@ struct mtmd_context {
mtmd_context(const char * mmproj_fname,
const llama_model * text_model,
const mtmd_context_params & ctx_params) :
text_model (text_model),
const mtmd_context_params & ctx_params,
bool no_alloc = false) :
print_timings(ctx_params.print_timings),
n_threads (ctx_params.n_threads),
media_marker (ctx_params.media_marker),
n_embd_text (llama_model_n_embd_inp(text_model))
n_embd_text (text_model ? llama_model_n_embd_inp(text_model) : -1),
vocab (text_model ? llama_model_get_vocab(text_model) : nullptr)
{
if (ctx_params.image_marker != nullptr) {
throw std::runtime_error("custom image_marker is not supported anymore, use media_marker instead");
@@ -193,21 +195,23 @@ struct mtmd_context {
throw std::runtime_error("media_marker must not be empty");
}
auto decoder_rope_type = llama_model_rope_type(text_model);
switch (decoder_rope_type) {
case LLAMA_ROPE_TYPE_NONE:
case LLAMA_ROPE_TYPE_NORM:
case LLAMA_ROPE_TYPE_NEOX:
{
pos_type = MTMD_POS_TYPE_NORMAL;
} break;
case LLAMA_ROPE_TYPE_MROPE:
case LLAMA_ROPE_TYPE_IMROPE:
{
pos_type = MTMD_POS_TYPE_MROPE;
} break;
default:
throw std::runtime_error(string_format("unsupported decoder rope type: %d\n", decoder_rope_type));
if (text_model) {
auto decoder_rope_type = llama_model_rope_type(text_model);
switch (decoder_rope_type) {
case LLAMA_ROPE_TYPE_NONE:
case LLAMA_ROPE_TYPE_NORM:
case LLAMA_ROPE_TYPE_NEOX:
{
pos_type = MTMD_POS_TYPE_NORMAL;
} break;
case LLAMA_ROPE_TYPE_MROPE:
case LLAMA_ROPE_TYPE_IMROPE:
{
pos_type = MTMD_POS_TYPE_MROPE;
} break;
default:
throw std::runtime_error(string_format("unsupported decoder rope type: %d\n", decoder_rope_type));
}
}
clip_context_params ctx_clip_params {
@@ -218,6 +222,7 @@ struct mtmd_context {
/* warmup */ ctx_params.warmup,
/* cb_eval */ ctx_params.cb_eval,
/* cb_eval_user_data */ ctx_params.cb_eval_user_data,
/* no_alloc */ no_alloc,
};
auto res = clip_init(mmproj_fname, ctx_clip_params);
@@ -241,7 +246,7 @@ struct mtmd_context {
// since we already validate n_embd of vision and audio mmproj,
// we can safely assume that they are the same
int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a);
if (n_embd_text != n_embd_clip) {
if (n_embd_text > 0 && n_embd_text != n_embd_clip) {
throw std::runtime_error(string_format(
"mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n"
"hint: you may be using wrong mmproj\n",
@@ -279,7 +284,7 @@ struct mtmd_context {
} break;
case PROJECTOR_TYPE_MINICPMV:
{
int minicpmv_version = clip_is_minicpmv(ctx_v);
int minicpmv_version = clip_get_hparams(ctx_v)->minicpmv_version;
if (minicpmv_version == 2) {
// minicpmv 2.5 format:
// <image> (overview) </image><slice><image> (slice) </image><image> (slice) </image>\n ... </slice>
@@ -594,7 +599,11 @@ struct mtmd_context {
private:
llama_token lookup_token(const std::string & token_text) {
const llama_vocab * vocab = llama_model_get_vocab(text_model);
if (vocab == nullptr) {
// TODO @ngxson : this case is currently hit by mtmd_get_memory_usage
// but we should reconsider this if this case is needed in other places in the future
return LLAMA_TOKEN_NULL;
}
const int n_vocab = llama_vocab_n_tokens(vocab);
for (int i = 0; i < n_vocab; i++) {
if (token_to_piece(vocab, i, true) == token_text) {
@@ -605,6 +614,9 @@ private:
}
std::string token_to_piece(const llama_vocab * vocab, llama_token token, bool special) {
if (vocab == nullptr) {
throw std::runtime_error("llama_vocab is not provided");
}
std::string piece;
piece.resize(piece.capacity()); // using string internal cache, 15 bytes + '\n'
const int n_chars = llama_token_to_piece(vocab, token, &piece[0], piece.size(), 0, special);
@@ -653,7 +665,7 @@ struct mtmd_tokenizer {
add_special = text->add_special;
parse_special = text->parse_special;
input_text = text->text;
vocab = llama_model_get_vocab(ctx->text_model);
vocab = ctx->vocab;
}
int32_t tokenize(mtmd_input_chunks * output) {
@@ -679,27 +691,29 @@ struct mtmd_tokenizer {
}
}
if (add_special && llama_vocab_get_add_bos(vocab)) {
// if first chunk is text, we add BOS token to first text chunk
// otherwise, create a new text chunk with BOS token
if (!cur.entries.empty() && cur.entries[0].type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
// add BOS token to the beginning of first text chunk
cur.entries[0].tokens_text.insert(cur.entries[0].tokens_text.begin(), llama_vocab_bos(vocab));
} else {
// create a new text chunk with BOS token at the beginning
mtmd_input_chunk bos_chunk{
MTMD_INPUT_CHUNK_TYPE_TEXT,
{llama_vocab_bos(vocab)},
nullptr, // image tokens
nullptr, // audio tokens
};
cur.entries.insert(cur.entries.begin(), std::move(bos_chunk));
if (vocab != nullptr) {
if (add_special && llama_vocab_get_add_bos(vocab)) {
// if first chunk is text, we add BOS token to first text chunk
// otherwise, create a new text chunk with BOS token
if (!cur.entries.empty() && cur.entries[0].type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
// add BOS token to the beginning of first text chunk
cur.entries[0].tokens_text.insert(cur.entries[0].tokens_text.begin(), llama_vocab_bos(vocab));
} else {
// create a new text chunk with BOS token at the beginning
mtmd_input_chunk bos_chunk{
MTMD_INPUT_CHUNK_TYPE_TEXT,
{llama_vocab_bos(vocab)},
nullptr, // image tokens
nullptr, // audio tokens
};
cur.entries.insert(cur.entries.begin(), std::move(bos_chunk));
}
}
}
if (add_special && llama_vocab_get_add_eos(vocab)) {
// if last chunk is text, we add EOS token to it
add_text({llama_vocab_eos(vocab)});
if (add_special && llama_vocab_get_add_eos(vocab)) {
// if last chunk is text, we add EOS token to it
add_text({llama_vocab_eos(vocab)});
}
}
if (i_bm != bitmaps.size()) {
@@ -714,6 +728,9 @@ struct mtmd_tokenizer {
}
void add_text(const std::string & txt, bool parse_special) {
if (vocab == nullptr) {
throw std::runtime_error("llama_vocab is not provided");
}
LOG_DBG("%s: %s\n", __func__, txt.c_str());
auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special);
add_text(tokens);
@@ -1002,10 +1019,16 @@ struct mtmd_tokenizer {
const std::string & text,
bool add_special,
bool parse_special) {
if (vocab == nullptr) {
throw std::runtime_error("llama_vocab is not provided");
}
// upper limit for the number of tokens
int n_tokens = text.length() + 2 * add_special;
std::vector<llama_token> result(n_tokens);
n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special);
if (n_tokens == std::numeric_limits<int32_t>::min()) {
throw std::runtime_error("Tokenization failed: input text too large, tokenization result exceeds int32_t limit");
}
if (n_tokens < 0) {
result.resize(-n_tokens);
int check = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special);
@@ -1067,8 +1090,8 @@ int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens)
bool ok = false;
if (clip_is_llava(ctx_clip)
|| clip_is_minicpmv(ctx_clip)
|| clip_is_glm(ctx_clip)
|| proj_type == PROJECTOR_TYPE_MINICPMV
|| proj_type == PROJECTOR_TYPE_GLM_EDGE
|| proj_type == PROJECTOR_TYPE_INTERNVL) {
// TODO @ngxson : llava does not support batched encoding ; this should be fixed inside clip_image_batch_encode()
const auto & entries = image_tokens->batch_f32.entries;
@@ -1542,3 +1565,36 @@ void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector<float> &
}
}
}
static void stub_log_callback(enum ggml_log_level, const char *, void *) {
// do nothing
}
std::map<ggml_backend_dev_t, size_t> mtmd_get_memory_usage(const char * mmproj_fname,
struct mtmd_context_params ctx_params) {
mtmd::context_ptr ctx;
auto saved_log_callback = g_logger_state.log_callback;
auto saved_log_user_data = g_logger_state.log_callback_user_data;
try {
mtmd_log_set(stub_log_callback, nullptr); // suppress logging
ctx.reset(new mtmd_context(mmproj_fname, nullptr, ctx_params));
mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback
std::map<ggml_backend_dev_t, size_t> total_mem;
auto merge = [&](const struct clip_ctx * c) {
for (auto & [dev, size] : clip_get_mem_usage(c)) {
total_mem[dev] += size;
}
};
if (ctx->ctx_v) {
merge(ctx->ctx_v);
}
if (ctx->ctx_a) {
merge(ctx->ctx_a);
}
return total_mem;
} catch (const std::exception & e) {
mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback
LOG_ERR("%s: error: %s\n", __func__, e.what());
return {};
}
}