[Rust] Gate health on startup warmup completion (#37994)
This commit is contained in:
@@ -2205,6 +2205,10 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
url = server_args.url()
|
||||
if get_serving().api_key:
|
||||
headers["Authorization"] = f"Bearer {get_serving().api_key}"
|
||||
if envs.SGLANG_RUST_SERVER.get():
|
||||
# The Rust listener binds before this request so /model_info is
|
||||
# available, but health stays 503 until this marked request succeeds.
|
||||
headers["x-sglang-startup-warmup"] = "1"
|
||||
|
||||
ssl_verify = ssl_verify_of(server_args)
|
||||
|
||||
|
||||
@@ -62,6 +62,7 @@ def _build_server_args(scheduler: Scheduler) -> ServerArgs:
|
||||
tokenizer_worker_num=get_serving().tokenizer_worker_num,
|
||||
detokenizer_worker_num=get_serving().detokenizer_worker_num,
|
||||
skip_tokenizer_init=get_serving().skip_tokenizer_init,
|
||||
skip_server_warmup=get_serving().skip_server_warmup,
|
||||
incremental_streaming_output=get_serving().incremental_streaming_output,
|
||||
disaggregation_mode=disaggregation_mode,
|
||||
model_config=ext.ModelConfig(
|
||||
|
||||
Reference in New Issue
Block a user