nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

mtmd, server: add "placeholder bitmap" for counting tokens , add */input_tokens API (#23913)

Browse Source 
* mtmd: add "placeholder bitmap" for counting tokens w/o preprocessing

* fast path skip preproc for placeholder

* fix build

* correct the api

* add server endpoint + tests

* add object name

* update docs

* add proxy handling

* fix build

* fix audio input path

* use is_placeholder in process_mtmd_prompt()

* nits

* nits (2)

* docs: clarify chat/completions/input_tokens is not official

* fix merge problem
This commit is contained in:
Xuan-Son Nguyen
2026-06-06 11:06:51 +02:00
committed by GitHub
parent 5a69c97439
commit f5c6ae1827
26 changed files with 732 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/README.md
+30
View File
@@ -1447,6 +1447,36 @@ See [OpenAI Embeddings API documentation](https://platform.openai.com/docs/api-r
}'
```
### POST `/v1/responses/input_tokens`: Token Counting
Similar to [Response input token counts API](https://developers.openai.com/api/reference/python/resources/responses/subresources/input_tokens/methods/count).
Example response:
```json
{
"object": "response.input_tokens",
"input_tokens": 11
}
```
### POST `/v1/chat/completions/input_tokens`: Token Counting
Similar to [Response input token counts API](https://developers.openai.com/api/reference/python/resources/responses/subresources/input_tokens/methods/count), but accepts a chat completion body as input.
Note: This is not an official OAI endpoint, but is added for completeness and convenience.
Example response:
```json
{
"object": "response.input_tokens",
"input_tokens": 11
}
```
## Anthropic-compatible API Endpoints
### POST `/v1/messages`: Anthropic-compatible Messages API
Given a list of `messages`, returns the assistant's response. Streaming is supported via Server-Sent Events. While no strong claims of compatibility with the Anthropic API spec are made, in our experience it suffices to support many apps.
tools/server/server-common.cpp
+2 -2
View File
@@ -713,10 +713,10 @@ static std::string fnv_hash(const uint8_t * data, size_t len) {
return std::to_string(hash);
}
server_tokens process_mtmd_prompt(mtmd_context * mctx, std::string prompt, std::vector<raw_buffer> files) {
server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) {
mtmd::bitmaps bitmaps;
for (auto & file : files) {
mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size()));
mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder));
if (!bmp.ptr) {
throw std::runtime_error("Failed to load image or audio file");
}
tools/server/server-common.h
+2 -1
View File
@@ -258,7 +258,8 @@ llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_prompt,
size_t validate_utf8(const std::string& text);
// process mtmd prompt, return the server_tokens containing both text tokens and media chunks
server_tokens process_mtmd_prompt(mtmd_context * mctx, std::string prompt, std::vector<raw_buffer> files);
// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder = false);
/**
* break the input "prompt" object into multiple prompt if needed, then tokenize them
tools/server/server-context.cpp
+60 -14
View File
@@ -4333,6 +4333,10 @@ void server_routes::init_routes() {
TASK_RESPONSE_TYPE_OAI_CHAT);
};
this->post_chat_completions_tok = [this](const server_http_req & req) {
return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_CHAT);
};
this->post_control = [this](const server_http_req & req) {
auto res = create_response();
const json body = json::parse(req.body);
@@ -4388,6 +4392,10 @@ void server_routes::init_routes() {
TASK_RESPONSE_TYPE_OAI_RESP);
};
this->post_responses_tok_oai = [this](const server_http_req & req) {
return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_RESP);
};
this->post_transcriptions_oai = [this](const server_http_req & req) {
auto res = create_response();
@@ -4435,20 +4443,7 @@ void server_routes::init_routes() {
};
this->post_anthropic_count_tokens = [this](const server_http_req & req) {
auto res = create_response();
std::vector<raw_buffer> files;
json body = server_chat_convert_anthropic_to_oai(json::parse(req.body));
SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions");
SRV_DBG("converted request: %s\n", body.dump().c_str());
json body_parsed = oaicompat_chat_params_parse(
body,
meta->chat_params,
files);
json prompt = body_parsed.at("prompt");
llama_tokens tokens = tokenize_mixed(ctx_server.vocab, prompt, true, true);
res->ok({{"input_tokens", static_cast<int>(tokens.size())}});
return res;
return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_ANTHROPIC);
};
// same with handle_chat_completions, but without inference part
@@ -4928,3 +4923,54 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
res->ok(root);
return res;
}
std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type) {
auto res = create_response();
std::vector<raw_buffer> files;
json body = json::parse(req.body);
bool is_oai = false;
switch (res_type) {
case TASK_RESPONSE_TYPE_OAI_CHAT:
{
is_oai = true;
} break;
case TASK_RESPONSE_TYPE_OAI_RESP:
{
is_oai = true;
body = server_chat_convert_responses_to_chatcmpl(body);
} break;
case TASK_RESPONSE_TYPE_ANTHROPIC:
{
body = server_chat_convert_anthropic_to_oai(body);
} break;
default:
res->error(format_error_response("invalid res_type", ERROR_TYPE_INVALID_REQUEST));
return res;
}
json body_parsed = oaicompat_chat_params_parse(
body,
meta->chat_params,
files);
json prompt = body_parsed.at("prompt");
// SRV_DBG("prompt = %s\n", prompt.dump().c_str());
// TODO @ngxson : refactor this code block, move this to server-common and reuse it in other places
size_t n_tokens;
if (mctx != nullptr) {
if (!prompt.is_string()) {
throw std::runtime_error("for mtmd, input prompt must be a string.");
}
n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, true).size();
} else {
n_tokens = tokenize_mixed(vocab, prompt, true, true).size();
}
json response = {{"input_tokens", static_cast<int>(n_tokens)}};
if (is_oai) {
response["object"] = "response.input_tokens";
}
res->ok(response);
return res;
}
tools/server/server-context.h
+3
View File
@@ -110,8 +110,10 @@ struct server_routes {
server_http_context::handler_t post_completions;
server_http_context::handler_t post_completions_oai;
server_http_context::handler_t post_chat_completions;
server_http_context::handler_t post_chat_completions_tok;
server_http_context::handler_t post_control;
server_http_context::handler_t post_responses_oai;
server_http_context::handler_t post_responses_tok_oai;
server_http_context::handler_t post_transcriptions_oai;
server_http_context::handler_t post_anthropic_messages;
server_http_context::handler_t post_anthropic_count_tokens;
@@ -139,6 +141,7 @@ private:
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type);
// using unique_ptr to allow late initialization of const
std::unique_ptr<const server_context_meta> meta;
tools/server/server.cpp
+8 -1
View File
@@ -161,6 +161,8 @@ int llama_server(int argc, char ** argv) {
routes.post_tokenize = models_routes->proxy_post;
routes.post_detokenize = models_routes->proxy_post;
routes.post_apply_template = models_routes->proxy_post;
routes.post_chat_completions_tok = models_routes->proxy_post;
routes.post_responses_tok_oai = models_routes->proxy_post;
routes.get_lora_adapters = models_routes->proxy_get;
routes.post_lora_adapters = models_routes->proxy_post;
routes.get_slots = models_routes->proxy_get;
@@ -192,7 +194,6 @@ int llama_server(int argc, char ** argv) {
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
ctx_http.post("/infill", ex_wrapper(routes.post_infill));
ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy
ctx_http.post("/embeddings", ex_wrapper(routes.post_embeddings));
@@ -204,6 +205,12 @@ int llama_server(int argc, char ** argv) {
ctx_http.post("/tokenize", ex_wrapper(routes.post_tokenize));
ctx_http.post("/detokenize", ex_wrapper(routes.post_detokenize));
ctx_http.post("/apply-template", ex_wrapper(routes.post_apply_template));
// token counting
ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));
ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));
ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
// LoRA adapters hotswap
ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters));
ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters));
tools/server/tests/unit/test_chat_completion.py
+16
View File
@@ -573,3 +573,19 @@ def test_chat_completions_multiple_choices():
for choice in res.body["choices"]:
assert "assistant" == choice["message"]["role"]
assert choice["finish_reason"] == "length"
def test_chat_completions_token_count():
global server
server.start()
# make sure cache can be reused across multiple choices and multiple requests
# ref: https://github.com/ggml-org/llama.cpp/pull/18663
for _ in range(2):
res = server.make_request("POST", "/chat/completions/input_tokens", data={
"messages": [
{"role": "system", "content": "Book"},
{"role": "user", "content": "What is the best book"},
],
})
assert res.status_code == 200
assert res.body["input_tokens"] > 5
tools/server/tests/unit/test_vision_api.py
+19
View File
@@ -98,6 +98,25 @@ def test_vision_chat_completion(prompt, image_url, success, re_content):
assert res.status_code != 200
def test_vision_chat_completion_token_count():
global server
server.start()
res = server.make_request("POST", "/chat/completions/input_tokens", data={
"temperature": 0.0,
"top_k": 1,
"messages": [
{"role": "user", "content": [
{"type": "text", "text": "What is this:"},
{"type": "image_url", "image_url": {
"url": get_img_url("IMG_URL_0"),
}},
]},
],
})
assert res.status_code == 200
assert res.body["input_tokens"] > 10
@pytest.mark.parametrize(
"prompt, image_data, success, re_content",
[