From 5dcb71166686799f0d873eab7386234302d05ecf Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 1 Jun 2026 22:26:58 +0300 Subject: [PATCH] speculative : fix n_outputs_max and remove draft-simple auto-enable (#23988) * speculative : add common_speculative_n_max helper function Extract the speculative max-draft-size logic from server_n_outputs_max into a reusable common_speculative_n_max() function in common/speculative. Assisted-by: llama.cpp:local pi * cont : draft context always has n_parallel outputs * llama : log n_outputs_max * speculative : remove draft-simple auto-enable * ci : enable server tests on PRs --- .github/workflows/server.yml | 3 --- common/arg.cpp | 4 --- common/speculative.cpp | 46 ++++++++++++++++++++++++--------- common/speculative.h | 3 +++ src/llama-context.cpp | 1 + tools/server/server-context.cpp | 34 ++---------------------- 6 files changed, 40 insertions(+), 51 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index f42b30d5c..5a02cc15a 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -102,7 +102,6 @@ jobs: - name: Tests id: server_integration_tests - if: ${{ !github.event.pull_request }} run: | cd tools/server/tests pytest -v -x -m "not slow" @@ -116,7 +115,6 @@ jobs: - name: Tests (Backend sampling) id: server_integration_tests_backend_sampling - if: ${{ !github.event.pull_request }} run: | cd tools/server/tests export LLAMA_ARG_BACKEND_SAMPLING=1 @@ -169,7 +167,6 @@ jobs: - name: Tests id: server_integration_tests - if: ${{ !github.event.pull_request }} run: | cd tools/server/tests $env:PYTHONIOENCODING = ":replace" diff --git a/common/arg.cpp b/common/arg.cpp index e0f6c6066..2df446d9d 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1041,11 +1041,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex // we define here to make sure it's included in llama-gen-docs if (ex == LLAMA_EXAMPLE_COMPLETION) { params.use_jinja = false; // disable jinja by default - } else if (ex == LLAMA_EXAMPLE_MTMD) { params.use_jinja = false; // disable jinja by default params.sampling.temp = 0.2; // lower temp by default for better quality - } else if (ex == LLAMA_EXAMPLE_SERVER) { params.n_parallel = -1; // auto by default } @@ -1066,7 +1064,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex sampler_type_names.pop_back(); // remove last semicolon } - /** * filter options by example * rules: @@ -1080,7 +1077,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } }; - add_opt(common_arg( {"-h", "--help", "--usage"}, "print usage and exit", diff --git a/common/speculative.cpp b/common/speculative.cpp index 253a5ecec..73830fda6 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1317,6 +1317,40 @@ static uint32_t common_get_enabled_speculative_configs(const std::vectortypes) { + switch (type) { + case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: + case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: + n_max = std::max(n_max, std::max(0, spec->draft.n_max)); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: + n_max = std::max(n_max, (int32_t) spec->ngram_simple.size_m); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: + n_max = std::max(n_max, (int32_t) spec->ngram_map_k.size_m); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: + n_max = std::max(n_max, (int32_t) spec->ngram_map_k4v.size_m); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: + n_max = std::max(n_max, std::max(0, spec->ngram_mod.n_max)); + break; + case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: + n_max = std::max(n_max, (int32_t) 8); + break; + case COMMON_SPECULATIVE_TYPE_NONE: + case COMMON_SPECULATIVE_TYPE_COUNT: + break; + } + } + + return n_max; +} + // initialization of the speculative decoding system // common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) { @@ -1325,8 +1359,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, { uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types); - bool has_draft_model_path = !params.draft.mparams.path.empty(); - bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3 bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; @@ -1359,16 +1391,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, if (has_ngram_cache) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params)); } - if (has_draft_simple) { - if (!has_draft_model_path) { - LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__); - has_draft_simple = false; - } - } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) { - LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__); - has_draft_simple = true; - } - if (has_draft_simple) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params)); } diff --git a/common/speculative.h b/common/speculative.h index f24bac79e..deba7dac7 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -20,6 +20,9 @@ enum common_speculative_type common_speculative_type_from_name(const std::string // convert type to string std::string common_speculative_type_to_str(enum common_speculative_type type); +// return the max number of draft tokens based on the speculative parameters +int32_t common_speculative_n_max(const common_params_speculative * spec); + common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq); void common_speculative_free(common_speculative * spec); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index a7dfb2492..946c2d0ef 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -229,6 +229,7 @@ llama_context::llama_context( LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base); LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale); LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq); + LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max); if (cparams.n_ctx_seq < hparams.n_ctx_train) { LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n", diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a5a688254..44fca83c6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -45,35 +45,7 @@ static uint32_t server_n_outputs_max(const common_params & params) { return n_batch; } - uint32_t n_outputs_per_seq = 1; - - for (const auto type : params.speculative.types) { - switch (type) { - case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: - case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: - case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: - n_outputs_per_seq = std::max(n_outputs_per_seq, 1 + std::max(0, params.speculative.draft.n_max)); - break; - case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: - n_outputs_per_seq = std::max(n_outputs_per_seq, 1 + params.speculative.ngram_simple.size_m); - break; - case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: - n_outputs_per_seq = std::max(n_outputs_per_seq, 1 + params.speculative.ngram_map_k.size_m); - break; - case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: - n_outputs_per_seq = std::max(n_outputs_per_seq, 1 + params.speculative.ngram_map_k4v.size_m); - break; - case COMMON_SPECULATIVE_TYPE_NGRAM_MOD: - n_outputs_per_seq = std::max(n_outputs_per_seq, 1 + std::max(0, params.speculative.ngram_mod.n_max)); - break; - case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: - n_outputs_per_seq = std::max(n_outputs_per_seq, 1 + 8); - break; - case COMMON_SPECULATIVE_TYPE_NONE: - case COMMON_SPECULATIVE_TYPE_COUNT: - break; - } - } + const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(¶ms.speculative); const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq; @@ -862,9 +834,7 @@ private: measure_model_bytes = false; } - if (!has_draft) { - params_dft.n_outputs_max = params_base.n_parallel; - } + params_dft.n_outputs_max = params_base.n_parallel; auto mparams_dft = common_model_params_to_llama(params_dft); auto cparams_dft = common_context_params_to_llama(params_dft);