[CI] feat: add early exit to wait_for_server when process dies (#18602)

docs/advanced_features/lora.ipynb

@@ -102,7 +102,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -162,7 +162,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -240,7 +240,7 @@
     ")\n",
     "\n",
     "url = f\"http://127.0.0.1:{port}\"\n",
-    "wait_for_server(url)"
+    "wait_for_server(url, process=server_process)"
    ]
   },
   {
@@ -453,7 +453,7 @@
     "\n",
     "\n",
     "url = f\"http://127.0.0.1:{port}\"\n",
-    "wait_for_server(url)"
+    "wait_for_server(url, process=server_process)"
    ]
   },
   {
@@ -610,7 +610,7 @@
     ")\n",
     "\n",
     "url = f\"http://127.0.0.1:{port}\"\n",
-    "wait_for_server(url)"
+    "wait_for_server(url, process=server_process)"
    ]
   },
   {

docs/advanced_features/separate_reasoning.ipynb

@@ -70,7 +70,7 @@
     "    \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {

docs/advanced_features/speculative_decoding.ipynb

@@ -111,7 +111,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -167,7 +167,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -227,7 +227,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -283,7 +283,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -338,7 +338,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -402,7 +402,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -466,7 +466,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -538,7 +538,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {

docs/advanced_features/structured_outputs.ipynb

@@ -54,7 +54,7 @@
     "    \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")"
    ]
   },

docs/advanced_features/structured_outputs_for_reasoning_models.ipynb

@@ -50,7 +50,7 @@
     "    \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")"
    ]
   },

docs/advanced_features/tool_parser.ipynb

@@ -60,7 +60,7 @@
     "server_process, port = launch_server_cmd(\n",
     "    \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"  # qwen25\n",
     ")\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -550,7 +550,9 @@
     "server_process_tool_choice, port_tool_choice = launch_server_cmd(\n",
     "    \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0  --log-level warning\"\n",
     ")\n",
-    "wait_for_server(f\"http://localhost:{port_tool_choice}\")\n",
+    "wait_for_server(\n",
+    "    f\"http://localhost:{port_tool_choice}\", process=server_process_tool_choice\n",
+    ")\n",
     "\n",
     "# Initialize client for tool choice examples\n",
     "client_tool_choice = OpenAI(\n",
@@ -695,7 +697,7 @@
     "server_process, port = launch_server_cmd(\n",
     "    \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1  --log-level warning\"  # llama-3.2-1b-instruct\n",
     ")\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "\n",
     "tools = [\n",
     "    {\n",

docs/basic_usage/native_api.ipynb

@@ -49,7 +49,7 @@
     "    \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {
@@ -282,7 +282,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)"
    ]
   },
   {
@@ -331,7 +331,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=reranker_process)"
    ]
   },
   {
@@ -399,7 +399,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=score_process)"
    ]
   },
   {
@@ -462,7 +462,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=reward_process)"
    ]
   },
   {
@@ -526,7 +526,7 @@
     "    \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=expert_record_server_process)"
    ]
   },
   {
@@ -581,7 +581,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)"
    ]
   },
   {

docs/basic_usage/openai_api_completions.ipynb

@@ -39,7 +39,7 @@
     "    \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "print(f\"Server started on http://localhost:{port}\")"
    ]
   },

docs/basic_usage/openai_api_embeddings.ipynb

@@ -37,7 +37,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=embedding_process)"
    ]
   },
   {

docs/basic_usage/openai_api_vision.ipynb

@@ -39,7 +39,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=vision_process)"
    ]
   },
   {

docs/basic_usage/send_request.ipynb

@@ -38,7 +38,7 @@
     "\"\"\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")"
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)"
    ]
   },
   {

docs/references/frontend/frontend_tutorial.ipynb

@@ -42,7 +42,7 @@
     "    \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "print(f\"Server started on http://localhost:{port}\")"
    ]
   },
@@ -398,7 +398,7 @@
     "    \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "print(f\"Server started on http://localhost:{port}\")"
    ]
   },

examples/runtime/hidden_states/hidden_states_server.py

@@ -25,7 +25,7 @@ def main():
     server_process, port = launch_server_cmd(
         "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct --enable-return-hidden-states --host 0.0.0.0"
     )
-    wait_for_server(f"http://localhost:{port}")
+    wait_for_server(f"http://localhost:{port}", process=server_process)
 
     prompts = [
         "Hello, my name is",

examples/runtime/token_in_token_out/token_in_token_out_llm_server.py

@@ -25,7 +25,7 @@ def main():
     server_process, port = launch_server_cmd(
         f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0"
     )
-    wait_for_server(f"http://localhost:{port}")
+    wait_for_server(f"http://localhost:{port}", process=server_process)
 
     # Sample prompts.
     prompts = [

examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py

@@ -47,7 +47,7 @@ def main():
     server_process, port = launch_server_cmd(
         f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0"
     )
-    wait_for_server(f"http://localhost:{port}")
+    wait_for_server(f"http://localhost:{port}", process=server_process)
 
     input_ids, image_data = get_input_ids()
 

python/sglang/test/server_fixtures/disaggregation_fixture.py

@@ -5,8 +5,6 @@ import time
 import warnings
 from urllib.parse import urlparse
 
-import requests
-
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.test_utils import (
@@ -16,6 +14,7 @@ from sglang.test.test_utils import (
     is_in_ci,
     popen_with_error_check,
 )
+from sglang.utils import wait_for_http_ready
 
 logger = logging.getLogger(__name__)
 
@@ -72,23 +71,14 @@ class PDDisaggregationServerBase(CustomTestCase):
         ]
         print("Starting load balancer:", shlex.join(lb_command))
         cls.process_lb = popen_with_error_check(lb_command)
-        cls.wait_server_ready(cls.lb_url + "/health")
+        cls.wait_server_ready(cls.lb_url + "/health", process=cls.process_lb)
 
     @classmethod
-    def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH):
+    def wait_server_ready(
-        start_time = time.perf_counter()
+        cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, process=None
-        while True:
+    ):
-            try:
+        wait_for_http_ready(url=url, timeout=timeout, process=process)
-                response = requests.get(url)
+        print(f"Server {url} is ready")
-                if response.status_code == 200:
-                    print(f"Server {url} is ready")
-                    return
-            except Exception:
-                pass
-
-            if time.perf_counter() - start_time > timeout:
-                raise RuntimeError(f"Server {url} failed to start in {timeout}s")
-            time.sleep(1)
 
     @classmethod
     def tearDownClass(cls):

python/sglang/utils.py

@@ -464,37 +464,82 @@ def terminate_process(process):
         release_port(lock_socket)
 
 
-def wait_for_server(base_url: str, timeout: int = None) -> None:
+def _raise_if_process_exited(process: Optional[Any]) -> None:
+    if process is None:
+        return
+
+    if hasattr(process, "poll"):
+        return_code = process.poll()
+        if return_code is not None:
+            raise RuntimeError(f"Server process exited with code {return_code}")
+        return
+
+    if hasattr(process, "is_alive") and not process.is_alive():
+        return_code = getattr(process, "exitcode", None)
+        if return_code is None:
+            raise RuntimeError("Server process exited")
+        raise RuntimeError(f"Server process exited with code {return_code}")
+
+
+def _is_wait_timeout(start_time: float, timeout: Optional[int]) -> bool:
+    if timeout is None:
+        return False
+    return time.perf_counter() - start_time > timeout
+
+
+def wait_for_http_ready(
+    url: str,
+    timeout: Optional[int] = None,
+    process: Optional[Any] = None,
+    headers: Optional[dict] = None,
+    request_timeout: int = 5,
+) -> None:
+    """Wait for an HTTP endpoint to return status 200."""
+    start_time = time.perf_counter()
+    while True:
+        _raise_if_process_exited(process)
+        try:
+            response = requests.get(url, headers=headers, timeout=request_timeout)
+            if response.status_code == 200:
+                return
+        except requests.exceptions.RequestException:
+            _raise_if_process_exited(process)
+
+        if _is_wait_timeout(start_time, timeout):
+            raise TimeoutError(
+                f"Endpoint {url} did not become ready within timeout period"
+            )
+        time.sleep(1)
+
+
+def wait_for_server(
+    base_url: str,
+    timeout: int = None,
+    process: Optional[subprocess.Popen] = None,
+) -> None:
     """Wait for the server to be ready by polling the /v1/models endpoint.
 
     Args:
-        base_url: The base URL of the server
+        base_url: The base URL of the server.
         timeout: Maximum time to wait in seconds. None means wait forever.
+        process: Optional server process used for early-exit checks.
     """
-    start_time = time.perf_counter()
+    wait_for_http_ready(
-    while True:
+        url=f"{base_url}/v1/models",
-        try:
+        timeout=timeout,
-            response = requests.get(
+        process=process,
-                f"{base_url}/v1/models",
+        headers={"Authorization": "Bearer None"},
-                headers={"Authorization": "Bearer None"},
+    )
-            )
+    time.sleep(5)
-            if response.status_code == 200:
+    print_highlight(
-                time.sleep(5)
+        """\n
-                print_highlight(
+        NOTE: Typically, the server runs in a separate terminal.
-                    """\n
+        In this notebook, we run the server and notebook code together, so their outputs are combined.
-                    NOTE: Typically, the server runs in a separate terminal.
+        To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue.
-                    In this notebook, we run the server and notebook code together, so their outputs are combined.
+        To reduce the log length, we set the log level to warning for the server, the default log level is info.
-                    To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue.
+        We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance.
-                    To reduce the log length, we set the log level to warning for the server, the default log level is info.
+        """
-                    We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance.
+    )
-                    """
-                )
-                break
-
-            if timeout and time.perf_counter() - start_time > timeout:
-                raise TimeoutError("Server did not become ready within timeout period")
-        except requests.exceptions.RequestException:
-            time.sleep(1)
 
 
 class TypeBasedDispatcher:

scripts/playground/frontend_reasoning.ipynb

@@ -34,7 +34,7 @@
     "    \"python3 -m sglang.launch_server --model-path Qwen/Qwen3-4B --reasoning-parser qwen3 --host 0.0.0.0\"\n",
     ")\n",
     "\n",
-    "wait_for_server(f\"http://localhost:{port}\")\n",
+    "wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
     "print(f\"Server started on http://localhost:{port}\")"
    ]
   },

test/manual/hicache/test_disaggregation_hicache.py

@@ -33,8 +33,8 @@ class DisaggregationHiCacheBase(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py

@@ -25,8 +25,8 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Wait for both to be ready
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/manual/test_mori_transfer_engine_e2e.py

@@ -52,10 +52,14 @@ class TestMoriTransferEngineE2E(PDDisaggregationServerBase):
         cls.start_decode()
 
         cls.wait_server_ready(
-            cls.prefill_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+            cls.prefill_url + "/health",
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            process=cls.process_prefill,
         )
         cls.wait_server_ready(
-            cls.decode_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+            cls.decode_url + "/health",
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            process=cls.process_decode,
         )
 
         cls.launch_lb()
@@ -85,7 +89,9 @@ class TestMoriTransferEngineE2E(PDDisaggregationServerBase):
         print("Starting load balancer:", " ".join(lb_command))
         cls.process_lb = subprocess.Popen(lb_command, stdout=None, stderr=None)
         cls.wait_server_ready(
-            cls.lb_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+            cls.lb_url + "/health",
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            process=cls.process_lb,
         )
 
     @classmethod
@@ -190,10 +196,14 @@ class TestMoriTransferEngineTPMismatchE2E(PDDisaggregationServerBase):
         cls.start_decode()
 
         cls.wait_server_ready(
-            cls.prefill_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+            cls.prefill_url + "/health",
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            process=cls.process_prefill,
         )
         cls.wait_server_ready(
-            cls.decode_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+            cls.decode_url + "/health",
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            process=cls.process_decode,
         )
         cls.launch_lb()
 
@@ -222,7 +232,9 @@ class TestMoriTransferEngineTPMismatchE2E(PDDisaggregationServerBase):
         print("Starting load balancer:", " ".join(lb_command))
         cls.process_lb = subprocess.Popen(lb_command, stdout=None, stderr=None)
         cls.wait_server_ready(
-            cls.lb_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+            cls.lb_url + "/health",
+            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+            process=cls.process_lb,
         )
 
     @classmethod

test/registered/amd/disaggregation/test_disaggregation_basic.py

@@ -44,8 +44,8 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -238,8 +238,8 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -346,8 +346,8 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/amd/disaggregation/test_disaggregation_pp.py

@@ -40,8 +40,8 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -129,8 +129,8 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -219,8 +219,8 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/disaggregation/test_disaggregation_basic.py

@@ -34,8 +34,8 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -204,8 +204,8 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -306,8 +306,8 @@ class TestDisaggregationMooncakeSpec(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -375,8 +375,8 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/distributed/test_disaggregation_aarch64.py

@@ -29,8 +29,8 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/distributed/test_disaggregation_different_tp.py

@@ -32,8 +32,8 @@ class TestDisaggregationMooncakePrefillLargerTP(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -103,8 +103,8 @@ class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -174,8 +174,8 @@ class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -245,8 +245,8 @@ class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/distributed/test_disaggregation_dp_attention.py

@@ -34,8 +34,8 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/distributed/test_disaggregation_hybrid_attention.py

@@ -29,8 +29,8 @@ class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -97,8 +97,8 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -175,8 +175,8 @@ class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase)
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/distributed/test_disaggregation_pp.py

@@ -28,8 +28,8 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -101,8 +101,8 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -175,8 +175,8 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase):
         cls.start_decode()
 
         # Block until both
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/distributed/test_epd_disaggregation.py

@@ -44,9 +44,9 @@ class TestEPDDisaggregationOneEncoder(PDDisaggregationServerBase):
         decode_thread.join()
 
         # Wait for all servers to be ready
-        cls.wait_server_ready(cls.encode_url + "/health")
+        cls.wait_server_ready(cls.encode_url + "/health", process=cls.process_encode)
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 
@@ -249,10 +249,10 @@ class TestEPDDisaggregationMultiEncoders(PDDisaggregationServerBase):
         prefill_thread.join()
         decode_thread.join()
 
-        cls.wait_server_ready(cls.encode_url1 + "/health")
+        cls.wait_server_ready(cls.encode_url1 + "/health", process=cls.process_encode1)
-        cls.wait_server_ready(cls.encode_url2 + "/health")
+        cls.wait_server_ready(cls.encode_url2 + "/health", process=cls.process_encode2)
-        cls.wait_server_ready(cls.prefill_url + "/health")
+        cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
-        cls.wait_server_ready(cls.decode_url + "/health")
+        cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
 
         cls.launch_lb()
 

test/registered/hicache/test_hicache_storage_file_backend.py

@@ -29,6 +29,7 @@ from sglang.test.test_utils import (
     is_in_ci,
     popen_launch_server,
 )
+from sglang.utils import wait_for_http_ready
 
 register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu")
 register_amd_ci(est_time=526, suite="stage-b-test-large-2-gpu-amd")
@@ -53,7 +54,7 @@ class HiCacheStorageBaseMixin:
 
         # Launch server with HiCache enabled and cache report
         cls.process = cls._launch_server_with_hicache()
-        cls._wait_for_server_ready()
+        cls._wait_for_server_ready(process=cls.process)
 
         print(f"Test server launched successfully at {cls.base_url}")
         print(f"Cache directory: {cls.temp_dir}")
@@ -128,18 +129,14 @@ class HiCacheStorageBaseMixin:
         )
 
     @classmethod
-    def _wait_for_server_ready(cls, timeout: int = 60) -> bool:
+    def _wait_for_server_ready(cls, timeout: int = 60, process=None) -> bool:
         """Wait for server to be ready"""
-        start_time = time.time()
+        wait_for_http_ready(
-        while time.time() - start_time < timeout:
+            url=f"{cls.base_url}/health",
-            try:
+            timeout=timeout,
-                response = requests.get(f"{cls.base_url}/health", timeout=5)
+            process=process,
-                if response.status_code == 200:
+        )
-                    return True
+        return True
-            except requests.RequestException:
-                pass
-            time.sleep(2)
-        raise TimeoutError("Server failed to start within timeout")
 
     def send_request(
         self, prompt: str, max_tokens: int = 100, temperature: float = 0.0

test/registered/hicache/test_hicache_storage_runtime_attach_detach.py

@@ -26,6 +26,7 @@ from sglang.test.test_utils import (
     find_available_port,
     popen_launch_server,
 )
+from sglang.utils import wait_for_http_ready
 
 register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu")
 
@@ -68,17 +69,15 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
         shutil.rmtree(cls.temp_dir, ignore_errors=True)
 
     @classmethod
-    def _wait_for_server_ready(cls, base_url: str, timeout: int = 60) -> bool:
+    def _wait_for_server_ready(
-        start_time = time.time()
+        cls, base_url: str, timeout: int = 60, process=None
-        while time.time() - start_time < timeout:
+    ) -> bool:
-            try:
+        wait_for_http_ready(
-                code, _body = cls._http_get(f"{base_url}/health", timeout=5)
+            url=f"{base_url}/health",
-                if code == 200:
+            timeout=timeout,
-                    return True
+            process=process,
-            except Exception:
+        )
-                pass
+        return True
-            time.sleep(2)
-        raise TimeoutError("Server failed to start within timeout")
 
     @staticmethod
     def _http_get(url: str, timeout: int = 10, headers: dict | None = None):
@@ -204,7 +203,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
             env=self.env,
         )
         try:
-            self._wait_for_server_ready(self.base_url)
+            self._wait_for_server_ready(self.base_url, process=process1)
 
             code_info, _body_info = self._http_get(
                 f"{self.base_url}/hicache/storage-backend", timeout=10
@@ -234,7 +233,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
             env=self.env,
         )
         try:
-            self._wait_for_server_ready(base_url2)
+            self._wait_for_server_ready(base_url2, process=process2)
 
             # 1) Initially disabled (but unauthorized without admin key)
             code_info2_unauth, _ = self._http_get(

test/srt/experiment_runner.py

@@ -11,9 +11,10 @@ from datetime import datetime
 from typing import List, Optional, Tuple
 
 import psutil
-import requests
 import yaml
 
+from sglang.utils import wait_for_http_ready
+
 
 @dataclass
 class ServerConfig:
@@ -182,18 +183,20 @@ class ExperimentRunner:
         self.process_manager = ProcessManager()
         self.logger = logging.getLogger(__name__)
 
-    def wait_for_server(self, port: int, timeout: int = 300) -> bool:
+    def wait_for_server(
-        start_time = time.perf_counter()
+        self, port: int, timeout: int = 300, process: Optional[subprocess.Popen] = None
-
+    ) -> bool:
-        while time.perf_counter() - start_time < timeout:
+        try:
-            try:
+            wait_for_http_ready(
-                response = requests.get(f"http://localhost:{port}/health")
+                url=f"http://localhost:{port}/health",
-                if response.status_code == 200:
+                timeout=timeout,
-                    self.logger.debug(f"Server ready on port {port}")
+                process=process,
-                    return True
+            )
-            except requests.RequestException:
+            self.logger.debug(f"Server ready on port {port}")
-                time.sleep(2)
+            return True
-        return False
+        except (RuntimeError, TimeoutError) as e:
+            self.logger.error("Server failed to become ready: %s", e)
+            return False
 
     def run_task(self, config: TaskConfig) -> TaskResult:
         start_time = time.perf_counter()
@@ -216,7 +219,9 @@ class ExperimentRunner:
                 self.process_manager.start_process(config.server_cmd, "SERVER")
             )
 
-            if not self.wait_for_server(port):
+            if not self.wait_for_server(
+                port, process=self.process_manager.server_process
+            ):
                 raise TimeoutError("Server startup timeout")
 
             time.sleep(10)