nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

chat: add dedicated Cohere2MoE (North Code) parser (#24615)

Browse Source 
* chat: add dedicated Cohere2MoE (North Code) parser

* Some renames to make @CISC happy :>
This commit is contained in:
Piotr Wilkin (ilintar)
2026-06-14 20:17:40 +02:00
committed by GitHub
parent 8edaca9034
commit aedb2a5e9c
3 changed files with 506 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
common/chat.cpp
View File

@@ -1979,6 +1979,146 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
return data;
}
// Cohere2 MoE (a.k.a. "North Code") parser.
//
// The assistant turn is fully marker-wrapped:
// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
// <|START_THINKING|>{reasoning}<|END_THINKING|>
// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|>
// OR tool calls: <|START_ACTION|>[
// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...
// ]<|END_ACTION|>
// <|END_OF_TURN_TOKEN|>
//
// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is
// the template default), so the model's output continues from *inside* the thinking block. The
// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix
// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,
// regardless of whether they came from the generation prompt or the generated text.
static common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
const std::string TURN_START = "<|START_OF_TURN_TOKEN|>";
const std::string TURN_END = "<|END_OF_TURN_TOKEN|>";
const std::string CHATBOT = "<|CHATBOT_TOKEN|>";
const std::string USER = "<|USER_TOKEN|>";
const std::string SYSTEM = "<|SYSTEM_TOKEN|>";
const std::string THINK_START = "<|START_THINKING|>";
const std::string THINK_END = "<|END_THINKING|>";
const std::string TEXT_START = "<|START_TEXT|>";
const std::string TEXT_END = "<|END_TEXT|>";
const std::string ACTION_START = "<|START_ACTION|>";
const std::string ACTION_END = "<|END_ACTION|>";
const std::string RESULT_START = "<|START_TOOL_RESULT|>";
const std::string RESULT_END = "<|END_TOOL_RESULT|>";
// Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.
const std::string GEN_PREFIX = TURN_START + CHATBOT;
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = THINK_START;
data.thinking_end_tag = THINK_END;
data.preserved_tokens = {
TURN_START, TURN_END, CHATBOT, USER, SYSTEM,
THINK_START, THINK_END,
TEXT_START, TEXT_END,
ACTION_START, ACTION_END,
RESULT_START, RESULT_END,
};
// Split the rendered prompt into per-role message spans. Tool results are rendered with the
// system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before
// the plain "system" one (it is a strict superset, and the role split tries delimiters in order).
data.message_spans = common_chat_split_by_role(data.prompt, {
{ "assistant", GEN_PREFIX },
{ "user", TURN_START + USER },
{ "tool", TURN_START + SYSTEM + RESULT_START },
{ "system", TURN_START + SYSTEM },
});
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
if (inputs.has_continuation()) {
const auto & msg = inputs.continue_msg;
data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
data.generation_prompt += THINK_END + TEXT_START + msg.render_content();
}
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(GEN_PREFIX);
auto end = p.end();
// The thinking block is always present (the generation prompt forces <|START_THINKING|>).
// When extracting reasoning, capture its body; otherwise keep the whole block (markers
// included) inline as content, matching reasoning_format=NONE conventions.
common_peg_parser reasoning = p.eps();
if (extract_reasoning) {
reasoning = p.optional(p.literal(THINK_START) +
p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +
p.optional(p.literal(THINK_END)));
} else {
reasoning = p.optional(p.content(p.literal(THINK_START) +
p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +
p.optional(p.literal(THINK_END))));
}
auto text_content = p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;
}
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
// <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>
auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,
/* force_tool_calls = */ true,
/* name_key = */ "tool_name",
/* args_key = */ "parameters",
/* array_wrapped = */ true,
/* function_is_key = */ false,
/* call_id_key = */ "",
/* gen_call_id_key = */ "tool_call_id",
/* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });
// Content and tool calls are mutually exclusive in this format.
common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });
return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
});
data.parser = parser.save();
if (include_grammar) {
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
foreach_function(inputs.tools, [&](const json & tool) {
const auto & function = tool.at("function");
auto schema = function.at("parameters");
builder.resolve_refs(schema);
});
parser.build_grammar(builder, data.grammar_lazy);
});
data.grammar_triggers = {
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }
};
}
return data;
}
namespace workaround {
static void map_developer_role_to_system(json & messages) {
@@ -2227,6 +2367,15 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_kimi_k2(tmpl, params);
}
// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and
// <|START_ACTION|> JSON tool calls. <|START_TEXT|> is unique to this template (the older
// Command-R templates use <|START_RESPONSE|>).
if (src.find("<|START_TEXT|>") != std::string::npos &&
src.find("<|START_ACTION|>") != std::string::npos) {
LOG_DBG("Using specialized template: Cohere2 MoE\n");
return common_chat_params_init_cohere2moe(tmpl, params);
}
if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);