minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] feat: add early exit to wait_for_server when process dies (#18602)

Browse Source
This commit is contained in:
shuwenn
2026-02-13 16:46:09 -08:00
committed by GitHub
parent eccf875d49
commit 3299c4f9c1
33 changed files with 229 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/advanced_features/lora.ipynb
+5 -5
View File
@@ -102,7 +102,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -162,7 +162,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -240,7 +240,7 @@
")\n",
"\n",
"url = f\"http://127.0.0.1:{port}\"\n",
"wait_for_server(url)"
"wait_for_server(url, process=server_process)"
]
},
{
@@ -453,7 +453,7 @@
"\n",
"\n",
"url = f\"http://127.0.0.1:{port}\"\n",
"wait_for_server(url)"
"wait_for_server(url, process=server_process)"
]
},
{
@@ -610,7 +610,7 @@
")\n",
"\n",
"url = f\"http://127.0.0.1:{port}\"\n",
"wait_for_server(url)"
"wait_for_server(url, process=server_process)"
]
},
{
docs/advanced_features/separate_reasoning.ipynb
+1 -1
View File
@@ -70,7 +70,7 @@
" \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
docs/advanced_features/speculative_decoding.ipynb
+8 -8
View File
@@ -111,7 +111,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -167,7 +167,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -227,7 +227,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -283,7 +283,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -338,7 +338,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -402,7 +402,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -466,7 +466,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -538,7 +538,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
docs/advanced_features/structured_outputs.ipynb
+1 -1
View File
@@ -54,7 +54,7 @@
" \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")"
]
},
docs/advanced_features/structured_outputs_for_reasoning_models.ipynb
+1 -1
View File
@@ -50,7 +50,7 @@
" \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")"
]
},
docs/advanced_features/tool_parser.ipynb
+5 -3
View File
@@ -60,7 +60,7 @@
"server_process, port = launch_server_cmd(\n",
" \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\" # qwen25\n",
")\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -550,7 +550,9 @@
"server_process_tool_choice, port_tool_choice = launch_server_cmd(\n",
" \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"\n",
")\n",
"wait_for_server(f\"http://localhost:{port_tool_choice}\")\n",
"wait_for_server(\n",
" f\"http://localhost:{port_tool_choice}\", process=server_process_tool_choice\n",
")\n",
"\n",
"# Initialize client for tool choice examples\n",
"client_tool_choice = OpenAI(\n",
@@ -695,7 +697,7 @@
"server_process, port = launch_server_cmd(\n",
" \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1 --log-level warning\" # llama-3.2-1b-instruct\n",
")\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"\n",
"tools = [\n",
" {\n",
docs/basic_usage/native_api.ipynb
+7 -7
View File
@@ -49,7 +49,7 @@
" \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
@@ -282,7 +282,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=embedding_process)"
]
},
{
@@ -331,7 +331,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=reranker_process)"
]
},
{
@@ -399,7 +399,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=score_process)"
]
},
{
@@ -462,7 +462,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=reward_process)"
]
},
{
@@ -526,7 +526,7 @@
" \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=expert_record_server_process)"
]
},
{
@@ -581,7 +581,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=tokenizer_free_server_process)"
]
},
{
docs/basic_usage/openai_api_completions.ipynb
+1 -1
View File
@@ -39,7 +39,7 @@
" \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"print(f\"Server started on http://localhost:{port}\")"
]
},
docs/basic_usage/openai_api_embeddings.ipynb
+1 -1
View File
@@ -37,7 +37,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=embedding_process)"
]
},
{
docs/basic_usage/openai_api_vision.ipynb
+1 -1
View File
@@ -39,7 +39,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=vision_process)"
]
},
{
docs/basic_usage/send_request.ipynb
+1 -1
View File
@@ -38,7 +38,7 @@
"\"\"\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")"
"wait_for_server(f\"http://localhost:{port}\", process=server_process)"
]
},
{
docs/references/frontend/frontend_tutorial.ipynb
+2 -2
View File
@@ -42,7 +42,7 @@
" \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"print(f\"Server started on http://localhost:{port}\")"
]
},
@@ -398,7 +398,7 @@
" \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"print(f\"Server started on http://localhost:{port}\")"
]
},
examples/runtime/hidden_states/hidden_states_server.py
+1 -1
View File
@@ -25,7 +25,7 @@ def main():
server_process, port = launch_server_cmd(
"python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct --enable-return-hidden-states --host 0.0.0.0"
)
wait_for_server(f"http://localhost:{port}")
wait_for_server(f"http://localhost:{port}", process=server_process)
prompts = [
"Hello, my name is",
examples/runtime/token_in_token_out/token_in_token_out_llm_server.py
+1 -1
View File
@@ -25,7 +25,7 @@ def main():
server_process, port = launch_server_cmd(
f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0"
)
wait_for_server(f"http://localhost:{port}")
wait_for_server(f"http://localhost:{port}", process=server_process)
# Sample prompts.
prompts = [
examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py
+1 -1
View File
@@ -47,7 +47,7 @@ def main():
server_process, port = launch_server_cmd(
f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0"
)
wait_for_server(f"http://localhost:{port}")
wait_for_server(f"http://localhost:{port}", process=server_process)
input_ids, image_data = get_input_ids()
python/sglang/test/server_fixtures/disaggregation_fixture.py
+7 -17
View File
@@ -5,8 +5,6 @@ import time
import warnings
from urllib.parse import urlparse
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
@@ -16,6 +14,7 @@ from sglang.test.test_utils import (
is_in_ci,
popen_with_error_check,
)
from sglang.utils import wait_for_http_ready
logger = logging.getLogger(__name__)
@@ -72,23 +71,14 @@ class PDDisaggregationServerBase(CustomTestCase):
]
print("Starting load balancer:", shlex.join(lb_command))
cls.process_lb = popen_with_error_check(lb_command)
cls.wait_server_ready(cls.lb_url + "/health")
cls.wait_server_ready(cls.lb_url + "/health", process=cls.process_lb)
@classmethod
def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH):
start_time = time.perf_counter()
while True:
try:
response = requests.get(url)
if response.status_code == 200:
print(f"Server {url} is ready")
return
except Exception:
pass
if time.perf_counter() - start_time > timeout:
raise RuntimeError(f"Server {url} failed to start in {timeout}s")
time.sleep(1)
def wait_server_ready(
cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, process=None
):
wait_for_http_ready(url=url, timeout=timeout, process=process)
print(f"Server {url} is ready")
@classmethod
def tearDownClass(cls):
python/sglang/utils.py
+71 -26
View File
@@ -464,37 +464,82 @@ def terminate_process(process):
release_port(lock_socket)
def wait_for_server(base_url: str, timeout: int = None) -> None:
def _raise_if_process_exited(process: Optional[Any]) -> None:
if process is None:
return
if hasattr(process, "poll"):
return_code = process.poll()
if return_code is not None:
raise RuntimeError(f"Server process exited with code {return_code}")
return
if hasattr(process, "is_alive") and not process.is_alive():
return_code = getattr(process, "exitcode", None)
if return_code is None:
raise RuntimeError("Server process exited")
raise RuntimeError(f"Server process exited with code {return_code}")
def _is_wait_timeout(start_time: float, timeout: Optional[int]) -> bool:
if timeout is None:
return False
return time.perf_counter() - start_time > timeout
def wait_for_http_ready(
url: str,
timeout: Optional[int] = None,
process: Optional[Any] = None,
headers: Optional[dict] = None,
request_timeout: int = 5,
) -> None:
"""Wait for an HTTP endpoint to return status 200."""
start_time = time.perf_counter()
while True:
_raise_if_process_exited(process)
try:
response = requests.get(url, headers=headers, timeout=request_timeout)
if response.status_code == 200:
return
except requests.exceptions.RequestException:
_raise_if_process_exited(process)
if _is_wait_timeout(start_time, timeout):
raise TimeoutError(
f"Endpoint {url} did not become ready within timeout period"
)
time.sleep(1)
def wait_for_server(
base_url: str,
timeout: int = None,
process: Optional[subprocess.Popen] = None,
) -> None:
"""Wait for the server to be ready by polling the /v1/models endpoint.
Args:
base_url: The base URL of the server
base_url: The base URL of the server.
timeout: Maximum time to wait in seconds. None means wait forever.
process: Optional server process used for early-exit checks.
"""
start_time = time.perf_counter()
while True:
try:
response = requests.get(
f"{base_url}/v1/models",
headers={"Authorization": "Bearer None"},
)
if response.status_code == 200:
time.sleep(5)
print_highlight(
"""\n
NOTE: Typically, the server runs in a separate terminal.
In this notebook, we run the server and notebook code together, so their outputs are combined.
To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue.
To reduce the log length, we set the log level to warning for the server, the default log level is info.
We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance.
"""
)
break
if timeout and time.perf_counter() - start_time > timeout:
raise TimeoutError("Server did not become ready within timeout period")
except requests.exceptions.RequestException:
time.sleep(1)
wait_for_http_ready(
url=f"{base_url}/v1/models",
timeout=timeout,
process=process,
headers={"Authorization": "Bearer None"},
)
time.sleep(5)
print_highlight(
"""\n
NOTE: Typically, the server runs in a separate terminal.
In this notebook, we run the server and notebook code together, so their outputs are combined.
To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue.
To reduce the log length, we set the log level to warning for the server, the default log level is info.
We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance.
"""
)
class TypeBasedDispatcher:
scripts/playground/frontend_reasoning.ipynb
+1 -1
View File
@@ -34,7 +34,7 @@
" \"python3 -m sglang.launch_server --model-path Qwen/Qwen3-4B --reasoning-parser qwen3 --host 0.0.0.0\"\n",
")\n",
"\n",
"wait_for_server(f\"http://localhost:{port}\")\n",
"wait_for_server(f\"http://localhost:{port}\", process=server_process)\n",
"print(f\"Server started on http://localhost:{port}\")"
]
},
test/manual/hicache/test_disaggregation_hicache.py
+2 -2
View File
@@ -33,8 +33,8 @@ class DisaggregationHiCacheBase(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py
+2 -2
View File
@@ -25,8 +25,8 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase):
cls.start_decode()
# Wait for both to be ready
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/manual/test_mori_transfer_engine_e2e.py
+18 -6
View File
@@ -52,10 +52,14 @@ class TestMoriTransferEngineE2E(PDDisaggregationServerBase):
cls.start_decode()
cls.wait_server_ready(
cls.prefill_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
cls.prefill_url + "/health",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
process=cls.process_prefill,
)
cls.wait_server_ready(
cls.decode_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
cls.decode_url + "/health",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
process=cls.process_decode,
)
cls.launch_lb()
@@ -85,7 +89,9 @@ class TestMoriTransferEngineE2E(PDDisaggregationServerBase):
print("Starting load balancer:", " ".join(lb_command))
cls.process_lb = subprocess.Popen(lb_command, stdout=None, stderr=None)
cls.wait_server_ready(
cls.lb_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
cls.lb_url + "/health",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
process=cls.process_lb,
)
@classmethod
@@ -190,10 +196,14 @@ class TestMoriTransferEngineTPMismatchE2E(PDDisaggregationServerBase):
cls.start_decode()
cls.wait_server_ready(
cls.prefill_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
cls.prefill_url + "/health",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
process=cls.process_prefill,
)
cls.wait_server_ready(
cls.decode_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
cls.decode_url + "/health",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
process=cls.process_decode,
)
cls.launch_lb()
@@ -222,7 +232,9 @@ class TestMoriTransferEngineTPMismatchE2E(PDDisaggregationServerBase):
print("Starting load balancer:", " ".join(lb_command))
cls.process_lb = subprocess.Popen(lb_command, stdout=None, stderr=None)
cls.wait_server_ready(
cls.lb_url + "/health", timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
cls.lb_url + "/health",
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
process=cls.process_lb,
)
@classmethod
test/registered/amd/disaggregation/test_disaggregation_basic.py
+6 -6
View File
@@ -44,8 +44,8 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -238,8 +238,8 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -346,8 +346,8 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/amd/disaggregation/test_disaggregation_pp.py
+6 -6
View File
@@ -40,8 +40,8 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -129,8 +129,8 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -219,8 +219,8 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/disaggregation/test_disaggregation_basic.py
+8 -8
View File
@@ -34,8 +34,8 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -204,8 +204,8 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -306,8 +306,8 @@ class TestDisaggregationMooncakeSpec(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -375,8 +375,8 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/distributed/test_disaggregation_aarch64.py
+2 -2
View File
@@ -29,8 +29,8 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/distributed/test_disaggregation_different_tp.py
+8 -8
View File
@@ -32,8 +32,8 @@ class TestDisaggregationMooncakePrefillLargerTP(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -103,8 +103,8 @@ class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -174,8 +174,8 @@ class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -245,8 +245,8 @@ class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/distributed/test_disaggregation_dp_attention.py
+2 -2
View File
@@ -34,8 +34,8 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/distributed/test_disaggregation_hybrid_attention.py
+6 -6
View File
@@ -29,8 +29,8 @@ class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -97,8 +97,8 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -175,8 +175,8 @@ class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase)
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/distributed/test_disaggregation_pp.py
+6 -6
View File
@@ -28,8 +28,8 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -101,8 +101,8 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -175,8 +175,8 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase):
cls.start_decode()
# Block until both
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/distributed/test_epd_disaggregation.py
+7 -7
View File
@@ -44,9 +44,9 @@ class TestEPDDisaggregationOneEncoder(PDDisaggregationServerBase):
decode_thread.join()
# Wait for all servers to be ready
cls.wait_server_ready(cls.encode_url + "/health")
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.encode_url + "/health", process=cls.process_encode)
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
@@ -249,10 +249,10 @@ class TestEPDDisaggregationMultiEncoders(PDDisaggregationServerBase):
prefill_thread.join()
decode_thread.join()
cls.wait_server_ready(cls.encode_url1 + "/health")
cls.wait_server_ready(cls.encode_url2 + "/health")
cls.wait_server_ready(cls.prefill_url + "/health")
cls.wait_server_ready(cls.decode_url + "/health")
cls.wait_server_ready(cls.encode_url1 + "/health", process=cls.process_encode1)
cls.wait_server_ready(cls.encode_url2 + "/health", process=cls.process_encode2)
cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill)
cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode)
cls.launch_lb()
test/registered/hicache/test_hicache_storage_file_backend.py
+9 -12
View File
@@ -29,6 +29,7 @@ from sglang.test.test_utils import (
is_in_ci,
popen_launch_server,
)
from sglang.utils import wait_for_http_ready
register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu")
register_amd_ci(est_time=526, suite="stage-b-test-large-2-gpu-amd")
@@ -53,7 +54,7 @@ class HiCacheStorageBaseMixin:
# Launch server with HiCache enabled and cache report
cls.process = cls._launch_server_with_hicache()
cls._wait_for_server_ready()
cls._wait_for_server_ready(process=cls.process)
print(f"Test server launched successfully at {cls.base_url}")
print(f"Cache directory: {cls.temp_dir}")
@@ -128,18 +129,14 @@ class HiCacheStorageBaseMixin:
)
@classmethod
def _wait_for_server_ready(cls, timeout: int = 60) -> bool:
def _wait_for_server_ready(cls, timeout: int = 60, process=None) -> bool:
"""Wait for server to be ready"""
start_time = time.time()
while time.time() - start_time < timeout:
try:
response = requests.get(f"{cls.base_url}/health", timeout=5)
if response.status_code == 200:
return True
except requests.RequestException:
pass
time.sleep(2)
raise TimeoutError("Server failed to start within timeout")
wait_for_http_ready(
url=f"{cls.base_url}/health",
timeout=timeout,
process=process,
)
return True
def send_request(
self, prompt: str, max_tokens: int = 100, temperature: float = 0.0
test/registered/hicache/test_hicache_storage_runtime_attach_detach.py
+12 -13
View File
@@ -26,6 +26,7 @@ from sglang.test.test_utils import (
find_available_port,
popen_launch_server,
)
from sglang.utils import wait_for_http_ready
register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu")
@@ -68,17 +69,15 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
shutil.rmtree(cls.temp_dir, ignore_errors=True)
@classmethod
def _wait_for_server_ready(cls, base_url: str, timeout: int = 60) -> bool:
start_time = time.time()
while time.time() - start_time < timeout:
try:
code, _body = cls._http_get(f"{base_url}/health", timeout=5)
if code == 200:
return True
except Exception:
pass
time.sleep(2)
raise TimeoutError("Server failed to start within timeout")
def _wait_for_server_ready(
cls, base_url: str, timeout: int = 60, process=None
) -> bool:
wait_for_http_ready(
url=f"{base_url}/health",
timeout=timeout,
process=process,
)
return True
@staticmethod
def _http_get(url: str, timeout: int = 10, headers: dict | None = None):
@@ -204,7 +203,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
env=self.env,
)
try:
self._wait_for_server_ready(self.base_url)
self._wait_for_server_ready(self.base_url, process=process1)
code_info, _body_info = self._http_get(
f"{self.base_url}/hicache/storage-backend", timeout=10
@@ -234,7 +233,7 @@ class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
env=self.env,
)
try:
self._wait_for_server_ready(base_url2)
self._wait_for_server_ready(base_url2, process=process2)
# 1) Initially disabled (but unauthorized without admin key)
code_info2_unauth, _ = self._http_get(
test/srt/experiment_runner.py
+19 -14
View File
@@ -11,9 +11,10 @@ from datetime import datetime
from typing import List, Optional, Tuple
import psutil
import requests
import yaml
from sglang.utils import wait_for_http_ready
@dataclass
class ServerConfig:
@@ -182,18 +183,20 @@ class ExperimentRunner:
self.process_manager = ProcessManager()
self.logger = logging.getLogger(__name__)
def wait_for_server(self, port: int, timeout: int = 300) -> bool:
start_time = time.perf_counter()
while time.perf_counter() - start_time < timeout:
try:
response = requests.get(f"http://localhost:{port}/health")
if response.status_code == 200:
self.logger.debug(f"Server ready on port {port}")
return True
except requests.RequestException:
time.sleep(2)
return False
def wait_for_server(
self, port: int, timeout: int = 300, process: Optional[subprocess.Popen] = None
) -> bool:
try:
wait_for_http_ready(
url=f"http://localhost:{port}/health",
timeout=timeout,
process=process,
)
self.logger.debug(f"Server ready on port {port}")
return True
except (RuntimeError, TimeoutError) as e:
self.logger.error("Server failed to become ready: %s", e)
return False
def run_task(self, config: TaskConfig) -> TaskResult:
start_time = time.perf_counter()
@@ -216,7 +219,9 @@ class ExperimentRunner:
self.process_manager.start_process(config.server_cmd, "SERVER")
)
if not self.wait_for_server(port):
if not self.wait_for_server(
port, process=self.process_manager.server_process
):
raise TimeoutError("Server startup timeout")
time.sleep(10)