nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 18 Packages Projects Releases Wiki Activity

mtmd: fix mtmd_get_memory_usage (#24867)

Browse Source
This commit is contained in:
Xuan-Son Nguyen
2026-06-21 14:12:15 +02:00
committed by GitHub
parent bf533823cd
commit 0d135df48c
3 changed files with 37 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+33 -29
View File
@@ -2796,7 +2796,7 @@ struct clip_model_loader {
} }
// load data // load data
if (!ctx_clip.no_alloc) { {
std::vector<uint8_t> read_buf; std::vector<uint8_t> read_buf;
// start loading event // start loading event
@@ -2814,38 +2814,42 @@ struct clip_model_loader {
ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(ctx_clip.backend); ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(ctx_clip.backend);
ctx_clip.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_clip.ctx_data.get(), buft)); ctx_clip.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_clip.ctx_data.get(), buft));
ggml_backend_buffer_set_usage(ctx_clip.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); ggml_backend_buffer_set_usage(ctx_clip.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
size_t data_loaded = 0; // read the weight from file
for (auto & t : tensors_to_load) { if (!ctx_clip.no_alloc) {
ggml_tensor * cur = ggml_get_tensor(ctx_clip.ctx_data.get(), t->name); size_t data_loaded = 0;
GGML_ASSERT(cur && "tensor not found in ctx_data"); for (auto & t : tensors_to_load) {
auto it_off = tensor_offset.find(t->name); ggml_tensor * cur = ggml_get_tensor(ctx_clip.ctx_data.get(), t->name);
GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); GGML_ASSERT(cur && "tensor not found in ctx_data");
const size_t offset = it_off->second; auto it_off = tensor_offset.find(t->name);
fin.seekg(offset, std::ios::beg); GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor");
if (!fin) { const size_t offset = it_off->second;
throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name)); fin.seekg(offset, std::ios::beg);
} if (!fin) {
size_t num_bytes = ggml_nbytes(cur); throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name));
if (ggml_backend_buft_is_host(buft)) { }
// for the CPU and Metal backend, we can read directly into the tensor size_t num_bytes = ggml_nbytes(cur);
fin.read(reinterpret_cast<char *>(cur->data), num_bytes); if (ggml_backend_buft_is_host(buft)) {
} else { // for the CPU and Metal backend, we can read directly into the tensor
// read into a temporary buffer first, then copy to device memory fin.read(reinterpret_cast<char *>(cur->data), num_bytes);
read_buf.resize(num_bytes); } else {
fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes); // read into a temporary buffer first, then copy to device memory
ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes); read_buf.resize(num_bytes);
} fin.read(reinterpret_cast<char *>(read_buf.data()), num_bytes);
data_loaded += num_bytes; ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);
if (progress_callback && total_data_size > 0) { }
const float progress = (float)data_loaded / (float)total_data_size; data_loaded += num_bytes;
if (!progress_callback(progress, progress_callback_user_data)) { if (progress_callback && total_data_size > 0) {
throw std::runtime_error(string_format("%s: model loading cancelled by progress_callback\n", __func__)); const float progress = (float)data_loaded / (float)total_data_size;
if (!progress_callback(progress, progress_callback_user_data)) {
throw std::runtime_error(string_format("%s: model loading cancelled by progress_callback\n", __func__));
}
} }
} }
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
} else {
LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
} }
fin.close(); fin.close();
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
} }
} }
tools/mtmd/mtmd.cpp
+1 -2
View File
@@ -2142,8 +2142,7 @@ std::map<ggml_backend_dev_t, size_t> mtmd_get_memory_usage(const char * mmproj_f
try { try {
mtmd_log_set(stub_log_callback, nullptr); // suppress logging mtmd_log_set(stub_log_callback, nullptr); // suppress logging
// TODO @ngxson : fix no_alloc here ctx.reset(new mtmd_context(mmproj_fname, nullptr, ctx_params, true));
ctx.reset(new mtmd_context(mmproj_fname, nullptr, ctx_params));
mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback mtmd_log_set(saved_log_callback, saved_log_user_data); // restore log callback
std::map<ggml_backend_dev_t, size_t> total_mem; std::map<ggml_backend_dev_t, size_t> total_mem;
auto merge = [&](const struct clip_ctx * c) { auto merge = [&](const struct clip_ctx * c) {
tools/server/server-context.cpp
+3 -1
View File
@@ -926,13 +926,15 @@ private:
// optionally get the memory usage of mmproj // optionally get the memory usage of mmproj
if (has_mmproj && params_base.fit_params) { if (has_mmproj && params_base.fit_params) {
int64_t t_start = ggml_time_us();
auto mmproj_mem = mtmd_get_memory_usage(mmproj_path.c_str(), mparams); auto mmproj_mem = mtmd_get_memory_usage(mmproj_path.c_str(), mparams);
int64_t t_elapsed = ggml_time_us() - t_start;
if (!mmproj_mem.empty()) { if (!mmproj_mem.empty()) {
size_t total = 0; size_t total = 0;
for (auto & [dev, size] : mmproj_mem) { for (auto & [dev, size] : mmproj_mem) {
total += size; total += size;
} }
SRV_INF("[mtmd] estimated worst-case memory usage of mmproj is %.2f MiB\n", total / (1024.0 * 1024.0)); SRV_INF("[mtmd] estimated worst-case memory usage of mmproj is %.2f MiB (took %.2f ms)\n", total / (1024.0 * 1024.0), t_elapsed / 1000.0);
GGML_ASSERT(!params_base.fit_params_target.empty()); GGML_ASSERT(!params_base.fit_params_target.empty());
for (auto & [dev, size] : mmproj_mem) { for (auto & [dev, size] : mmproj_mem) {
for (size_t i = 0; i < ggml_backend_dev_count(); i++) { for (size_t i = 0; i < ggml_backend_dev_count(); i++) {