server: fix deadlock in load_models() when erasing a finished download (#25358)

* server: fix deadlock in load_models() when erasing a finished download

The download monitoring thread acquires the models mutex on its way out,
but load_models() joined it from the erase loop while holding that mutex.
Join it outside the lock via threads_to_join like the other monitoring
threads.

* server: add default timeout to test requests

A hung server now fails the test after 10 minutes instead of stalling
the CI job for hours. Explicit timeouts are unchanged.

tools/server/tests/unit/test_router.py

@@ -314,7 +314,6 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i
     return False
 
 
-@pytest.mark.skip(reason="sse_thread sometimes hangs on GH actions, to be investigated")
 def test_router_download_model():
     """Case 1: download a model, verify SSE events and GET /models."""
     global server
@@ -358,7 +357,6 @@ def test_router_download_model():
     assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download"
 
 
-@pytest.mark.skip(reason="sse_thread sometimes hangs on GH actions, to be investigated")
 def test_router_delete_model():
     """Case 2: delete the downloaded model, verify it disappears from GET /models."""
     global server

tools/server/tests/utils.py

@@ -31,6 +31,9 @@ import wget
 
 DEFAULT_HTTP_TIMEOUT = 60
 
+# per-request timeout, a hung server fails the test instead of stalling the CI for hours
+DEFAULT_REQUEST_TIMEOUT = 600
+
 
 class ServerResponse:
     headers: dict
@@ -330,7 +333,7 @@ class ServerProcess:
         path: str,
         data: dict | Any | None = None,
         headers: dict | None = None,
-        timeout: float | None = None,
+        timeout: float | None = DEFAULT_REQUEST_TIMEOUT,
     ) -> ServerResponse:
         url = f"http://{self.server_host}:{self.server_port}{path}"
         parse_body = False
@@ -389,7 +392,7 @@ class ServerProcess:
         path: str,
         data: dict | None = None,
         headers: dict | None = None,
-        timeout: float | None = None,
+        timeout: float | None = DEFAULT_REQUEST_TIMEOUT,
     ) -> dict:
         stream = data.get('stream', False)
         if stream: