nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

speculative : fix n_outputs_max and remove draft-simple auto-enable (#23988)

Browse Source 
* speculative : add common_speculative_n_max helper function

Extract the speculative max-draft-size logic from server_n_outputs_max
into a reusable common_speculative_n_max() function in common/speculative.

Assisted-by: llama.cpp:local pi

* cont : draft context always has n_parallel outputs

* llama : log n_outputs_max

* speculative : remove draft-simple auto-enable

* ci : enable server tests on PRs
This commit is contained in:
Georgi Gerganov
2026-06-01 22:26:58 +03:00
committed by GitHub
parent 5aa3a64596
commit 5dcb711666
6 changed files with 40 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+2 -32
View File
@@ -45,35 +45,7 @@ static uint32_t server_n_outputs_max(const common_params & params) {
return n_batch;
}
uint32_t n_outputs_per_seq = 1;
for (const auto type : params.speculative.types) {
switch (type) {
case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE:
case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + std::max(0, params.speculative.draft.n_max));
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_simple.size_m);
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_map_k.size_m);
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:
n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_map_k4v.size_m);
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_MOD:
n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + std::max(0, params.speculative.ngram_mod.n_max));
break;
case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + 8);
break;
case COMMON_SPECULATIVE_TYPE_NONE:
case COMMON_SPECULATIVE_TYPE_COUNT:
break;
}
}
const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(&params.speculative);
const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq;
@@ -862,9 +834,7 @@ private:
measure_model_bytes = false;
}
if (!has_draft) {
params_dft.n_outputs_max = params_base.n_parallel;
}
params_dft.n_outputs_max = params_base.n_parallel;
auto mparams_dft = common_model_params_to_llama(params_dft);
auto cparams_dft = common_context_params_to_llama(params_dft);