[rust-server] warm up each rust listener in-process (#43004)

Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
Sage
2026-10-08 14:09:46 -07:00
committed by GitHub
parent e902207a4e
commit 5a5ea81036
4 changed files with 132 additions and 102 deletions
+7 -32
View File
@@ -2268,10 +2268,6 @@ def _execute_server_warmup(server_args: ServerArgs):
url = server_args.url()
if get_serving().api_key:
headers["Authorization"] = f"Bearer {get_serving().api_key}"
if envs.SGLANG_RUST_SERVER.get():
# The Rust listener binds before this request so /model_info is
# available, but health stays 503 until this marked request succeeds.
headers["x-sglang-startup-warmup"] = "1"
ssl_verify = ssl_verify_of(server_args)
@@ -2333,16 +2329,7 @@ def _execute_server_warmup(server_args: ServerArgs):
and get_disagg().disaggregation_mode == "null"
and model_info["is_generation"]
):
served_model_name = ""
if not envs.SGLANG_RUST_SERVER.get():
served_model_name = _global_state.tokenizer_manager.served_model_name
else:
# _global_state.tokenizer_manager is not initialized in the rust server,
# so we need to get the model name from the model_info
served_model_name = model_info.get(
"model_path", get_serving().served_model_name
)
served_model_name = served_model_name or get_model().model_path
served_model_name = _global_state.tokenizer_manager.served_model_name
# TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now
# Only use chat completions format for generation models, not embedding models
json_data = {
@@ -2397,9 +2384,7 @@ def _execute_server_warmup(server_args: ServerArgs):
verify=ssl_verify,
)
assert res.status_code == 200, f"{res.text}"
# Skip server_status update for Rust server
if not envs.SGLANG_RUST_SERVER.get():
_global_state.tokenizer_manager.server_status = ServerStatus.Up
_global_state.tokenizer_manager.server_status = ServerStatus.Up
else:
logger.info(f"Start of pd disaggregation warmup ...")
@@ -2424,14 +2409,9 @@ def _execute_server_warmup(server_args: ServerArgs):
get_disagg().disaggregation_mode,
failed_status_codes,
)
# In rust-server mode there is no TokenizerManager (readiness is
# the Rust server's own /health), so skip the status update.
if not envs.SGLANG_RUST_SERVER.get():
_global_state.tokenizer_manager.server_status = (
ServerStatus.Up
if not failed_status_codes
else ServerStatus.UnHealthy
)
_global_state.tokenizer_manager.server_status = (
ServerStatus.Up if not failed_status_codes else ServerStatus.UnHealthy
)
except Exception:
last_traceback = get_exception_traceback()
@@ -2926,13 +2906,8 @@ def launch_server(
if envs.SGLANG_RUST_SERVER.get():
# The Rust server serves api-server, tokenizer, and detokenizer, so the
# main process has no Python HTTP server / tokenizer manager to run.
# Run a warmup /generate before advertising readiness: the Rust /health
# and /get_model_info endpoints are static (200 as soon as the server
# binds, before any forward pass), so without this the first real request
# pays the cold-start cost (observed as a >60s first generation).
if not get_serving().skip_server_warmup:
_execute_server_warmup(server_args)
logger.info("The server is fired up and ready to roll!")
# Each Rust listener runs its own startup warmup and keeps /health at
# 503 until it succeeds, so the main process does not warm up here.
if launch_callback is not None:
launch_callback()
scheduler_init_result.block_until_scheduler_exits()