nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

server: refactor child --> router communication (#24821)

Browse Source 
* server: refactor child --> router communication

* fix wakeup case

* add docs

* improve update_status()

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-06-20 01:02:26 +02:00
committed by GitHub
parent 4b48a53b6c
commit 2b686a9120
8 changed files with 173 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-models.h
+23 -7
View File
@@ -171,8 +171,12 @@ public:
void download(common_params_model && model, common_download_opts && opts);
// update the status of a model instance (thread-safe)
void update_status(const std::string & name, server_model_status status, int exit_code);
void update_loaded_info(const std::string & name, std::string & raw_info);
struct update_status_args {
server_model_status status;
int exit_code = 0; // only valid if status == UNLOADED
json loaded_info = nullptr;
};
void update_status(const std::string & name, const update_status_args & args);
void update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok = true);
// remove a cache model from disk and update the list (thread-safe)
@@ -193,15 +197,27 @@ public:
// proxy an HTTP request to the model instance
server_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used);
// handle message sent from server_child::notify_to_router()
// raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string
// this function is not thread-safe, must be called from instance's monitoring thread
// payload per state:
// state = loading -> payload = {} (TODO: add progress info)
// state = ready -> payload = model_info (json), or {} if wakeup from sleeping
// state = sleeping -> payload = {}
void handle_child_state(const std::string & name, const std::string & raw_input);
};
struct server_child {
// return true if the current process is a child server instance
static bool is_child_server();
bool is_child();
// notify the router server that a model instance is ready
// register the shutdown_handler to be called by the router
// return the monitoring thread (to be joined by the caller)
static std::thread setup_child_server(const std::function<void(int)> & shutdown_handler, const json & model_info);
std::thread setup(const std::function<void(int)> & shutdown_handler);
// notify the router server that the sleeping state has changed
static void notify_router_sleeping_state(bool sleeping);
// notify router server for status changes (e.g. loading, downloading, sleeping, etc.)
// message will be handled by server_models::handle_child_state() on the router side
void notify_to_router(const std::string & state_name, const json & payload);
};
struct server_models_routes {