server: in SSE mode, send HTTP headers when slot starts (#23884)
* server: in SSE mode, send HTTP headers when slot starts * ref to pr * stream should be false by default
This commit is contained in:
@@ -47,7 +47,7 @@ enum stop_type {
|
||||
};
|
||||
|
||||
struct task_params {
|
||||
bool stream = true;
|
||||
bool stream = false;
|
||||
bool include_usage = false;
|
||||
bool cache_prompt = true; // remember the prompt to avoid reprocessing all prompt
|
||||
bool return_tokens = false;
|
||||
@@ -418,6 +418,8 @@ struct server_task_result_cmpl_partial : server_task_result {
|
||||
|
||||
bool post_sampling_probs;
|
||||
bool is_progress = false;
|
||||
bool is_begin = false; // whether to send 200 status to HTTP client (begin of SSE stream)
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/23884
|
||||
completion_token_output prob_output;
|
||||
result_timings timings;
|
||||
result_prompt_progress progress;
|
||||
|
||||
Reference in New Issue
Block a user