nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

server : enable multi-modal prompt caching (#19877)

Browse Source
This commit is contained in:
Georgi Gerganov
2026-02-25 15:15:42 +02:00
committed by GitHub
parent d7d826b3c1
commit f20469d919
2 changed files with 1 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
-3
View File
@@ -995,9 +995,6 @@ private:
// don't update the cache if the slot's context is empty // don't update the cache if the slot's context is empty
update_cache = update_cache && tokens.size() > 0; update_cache = update_cache && tokens.size() > 0;
// TODO: mtmd does not support prompt cache
update_cache = update_cache && (ret->mctx == nullptr);
if (update_cache) { if (update_cache) {
SRV_WRN("%s", "updating prompt cache\n"); SRV_WRN("%s", "updating prompt cache\n");
tools/server/server-task.cpp
+1 -2
View File
@@ -1900,10 +1900,9 @@ server_prompt * server_prompt_cache::alloc(const server_prompt & prompt, size_t
return nullptr; return nullptr;
} }
// TODO: for some reason we can't copy server_tokens, so we have to do this workaround
auto & cur = states.emplace_back(); auto & cur = states.emplace_back();
cur = { cur = {
/*.tokens =*/ server_tokens(prompt.tokens.get_text_tokens(), false), /*.tokens =*/ prompt.tokens.clone(),
/*.data =*/ std::move(state_data), /*.data =*/ std::move(state_data),
/*.checkpoints =*/ prompt.checkpoints, /*.checkpoints =*/ prompt.checkpoints,
}; };