[rust-server] warm up each rust listener in-process (#43004)

Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
Sage
2026-10-08 14:09:46 -07:00
committed by GitHub
parent e902207a4e
commit 5a5ea81036
4 changed files with 132 additions and 102 deletions
+7 -32
View File
@@ -2268,10 +2268,6 @@ def _execute_server_warmup(server_args: ServerArgs):
url = server_args.url()
if get_serving().api_key:
headers["Authorization"] = f"Bearer {get_serving().api_key}"
if envs.SGLANG_RUST_SERVER.get():
# The Rust listener binds before this request so /model_info is
# available, but health stays 503 until this marked request succeeds.
headers["x-sglang-startup-warmup"] = "1"
ssl_verify = ssl_verify_of(server_args)
@@ -2333,16 +2329,7 @@ def _execute_server_warmup(server_args: ServerArgs):
and get_disagg().disaggregation_mode == "null"
and model_info["is_generation"]
):
served_model_name = ""
if not envs.SGLANG_RUST_SERVER.get():
served_model_name = _global_state.tokenizer_manager.served_model_name
else:
# _global_state.tokenizer_manager is not initialized in the rust server,
# so we need to get the model name from the model_info
served_model_name = model_info.get(
"model_path", get_serving().served_model_name
)
served_model_name = served_model_name or get_model().model_path
served_model_name = _global_state.tokenizer_manager.served_model_name
# TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now
# Only use chat completions format for generation models, not embedding models
json_data = {
@@ -2397,9 +2384,7 @@ def _execute_server_warmup(server_args: ServerArgs):
verify=ssl_verify,
)
assert res.status_code == 200, f"{res.text}"
# Skip server_status update for Rust server
if not envs.SGLANG_RUST_SERVER.get():
_global_state.tokenizer_manager.server_status = ServerStatus.Up
_global_state.tokenizer_manager.server_status = ServerStatus.Up
else:
logger.info(f"Start of pd disaggregation warmup ...")
@@ -2424,14 +2409,9 @@ def _execute_server_warmup(server_args: ServerArgs):
get_disagg().disaggregation_mode,
failed_status_codes,
)
# In rust-server mode there is no TokenizerManager (readiness is
# the Rust server's own /health), so skip the status update.
if not envs.SGLANG_RUST_SERVER.get():
_global_state.tokenizer_manager.server_status = (
ServerStatus.Up
if not failed_status_codes
else ServerStatus.UnHealthy
)
_global_state.tokenizer_manager.server_status = (
ServerStatus.Up if not failed_status_codes else ServerStatus.UnHealthy
)
except Exception:
last_traceback = get_exception_traceback()
@@ -2926,13 +2906,8 @@ def launch_server(
if envs.SGLANG_RUST_SERVER.get():
# The Rust server serves api-server, tokenizer, and detokenizer, so the
# main process has no Python HTTP server / tokenizer manager to run.
# Run a warmup /generate before advertising readiness: the Rust /health
# and /get_model_info endpoints are static (200 as soon as the server
# binds, before any forward pass), so without this the first real request
# pays the cold-start cost (observed as a >60s first generation).
if not get_serving().skip_server_warmup:
_execute_server_warmup(server_args)
logger.info("The server is fired up and ready to roll!")
# Each Rust listener runs its own startup warmup and keeps /health at
# 503 until it succeeds, so the main process does not warm up here.
if launch_callback is not None:
launch_callback()
scheduler_init_result.block_until_scheduler_exits()
@@ -121,9 +121,8 @@ impl CoreHandle {
self.inner.startup_ready.load(Ordering::Acquire)
}
/// Record successful completion of the main process's startup warmup.
/// The adapter decides which response qualifies as that warmup; the state
/// itself is shared by every transport.
/// Record successful completion of the startup warmup. The state is
/// shared by every transport.
pub(crate) fn mark_ready(&self) {
self.inner.startup_ready.store(true, Ordering::Release);
}
@@ -294,6 +293,43 @@ impl CoreHandle {
}
}
/// One generation mirroring the Python `_execute_server_warmup` text request;
/// VLMs warm only the text path. The caller decides readiness.
pub(crate) async fn warm_up(&self, skip_tokenizer_init: bool) -> Result<(), CoreError> {
let (text, input_ids) = if skip_tokenizer_init {
(None, Some(vec![10, 11, 12]))
} else {
(Some("The capital city of France is".to_string()), None)
};
let mut call = self
.generate(GenerateRequest {
rid: Rid::new(),
text,
input_ids,
sampling_params: SamplingParams {
max_new_tokens: Some(8),
temperature: 0.0,
..Default::default()
},
stream: false,
bootstrap_host: self
.inner
.is_disaggregation
.then(|| FAKE_BOOTSTRAP_HOST.into()),
bootstrap_room: self.inner.is_disaggregation.then_some(0),
..Default::default()
})
.await?;
loop {
match call.recv().await {
Some(CoreEvent::Finished(_)) => return Ok(()),
Some(CoreEvent::Failed(e)) => return Err(e),
Some(CoreEvent::Delta(_)) => {}
None => return Err(CoreError::ResponseTruncated),
}
}
}
/// Submit one already-parsed in-process request.
///
/// The async send preserves TokenizerManager inbox backpressure. Once the
@@ -961,6 +997,60 @@ mod tests {
));
}
#[tokio::test]
async fn warm_up_succeeds_on_finished_generation() {
let harness = Harness::unbounded(8);
let handle = harness.handle.clone();
let warm_up = tokio::spawn(async move { handle.warm_up(false).await });
let request = accept_intake(harness.intake_rx.recv_async().await.unwrap());
let RequestKind::Generate(warmup) = &request.kind else {
panic!("warmup must submit a generation request");
};
assert_eq!(
warmup.text.as_deref(),
Some("The capital city of France is")
);
assert_eq!(warmup.input_ids, None);
assert_eq!(warmup.sampling_params.max_new_tokens, Some(8));
assert_eq!(warmup.bootstrap_host, None);
request
.sink
.try_send(ResponseItem::Done(ChunkEvent::default()))
.unwrap();
assert!(warm_up.await.unwrap().is_ok());
assert!(!harness.handle.is_ready());
}
#[tokio::test]
async fn pd_warm_up_without_tokenizer_fails_on_truncated_stream() {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, _abort_rx) = flume::unbounded();
let handle = configured_handle(
intake_tx,
abort_tx,
8,
Default::default(),
false,
true,
BTreeMap::new(),
);
let warm_up = tokio::spawn(async move { handle.warm_up(true).await });
let request = accept_intake(intake_rx.recv_async().await.unwrap());
let RequestKind::Generate(warmup) = &request.kind else {
panic!("warmup must submit a generation request");
};
assert_eq!(warmup.text, None);
assert_eq!(warmup.input_ids, Some(vec![10, 11, 12]));
assert_eq!(warmup.bootstrap_host.as_deref(), Some(FAKE_BOOTSTRAP_HOST));
assert_eq!(warmup.bootstrap_room, Some(0));
drop(request);
assert!(warm_up.await.unwrap().is_err());
}
#[tokio::test]
async fn control_operation_returns_transport_neutral_payload() {
let harness = Harness::unbounded(8);
+29 -67
View File
@@ -3,20 +3,16 @@
//! pre-bound listener until shutdown.
use std::sync::Arc;
use std::time::Duration;
use axum::{
Router,
extract::{Request, State},
http::StatusCode,
middleware::Next,
response::Response,
};
use axum::Router;
use super::{info, native_api, openai};
use crate::api_server::core::CoreHandle;
use crate::api_server::disaggregation::bootstrap as pd_bootstrap;
use crate::api_server::log;
use crate::message::config::ServerArgs;
use crate::utils::environ;
/// HTTP adapter state: the shared core capability, immutable server
/// configuration needed for HTTP request preparation, and the chat formatter.
@@ -31,28 +27,29 @@ pub(super) struct AppState {
pub(super) chat_formatter: Option<openai::ChatFormatter>,
}
/// Private marker attached by the main process to its startup warmup request.
/// The middleware flips readiness only after that request returns successfully.
const STARTUP_WARMUP_HEADER: &str = "x-sglang-startup-warmup";
async fn mark_startup_ready(
State(state): State<Arc<AppState>>,
req: Request,
next: Next,
) -> Response {
let is_startup_warmup = req.headers().contains_key(STARTUP_WARMUP_HEADER)
&& matches!(
req.uri().path(),
"/generate" | "/encode" | "/v1/chat/completions"
);
let response = next.run(req).await;
record_startup_warmup_status(&state.core, is_startup_warmup, response.status());
response
}
fn record_startup_warmup_status(core: &CoreHandle, is_startup_warmup: bool, status: StatusCode) {
if is_startup_warmup && status.is_success() {
core.mark_ready();
/// Every listener warms its own rank; health stays 503 until it succeeds,
/// and a failed or timed-out warmup leaves the listener unready.
async fn startup_warmup(core: CoreHandle, server_args: Arc<ServerArgs>) {
if core.is_ready() {
return;
}
let timeout = match environ::env_i64("SGLANG_WARMUP_TIMEOUT", -1) {
t if t > 0 => t as u64,
_ if server_args.is_disaggregation() => 1800,
_ => 600,
};
match tokio::time::timeout(
Duration::from_secs(timeout),
core.warm_up(server_args.skip_tokenizer_init),
)
.await
{
Ok(Ok(())) => {
core.mark_ready();
tracing::info!("The server is fired up and ready to roll!");
}
Ok(Err(e)) => tracing::error!(error = %e, "startup warmup failed"),
Err(_) => tracing::error!(timeout_secs = timeout, "startup warmup timed out"),
}
}
@@ -72,6 +69,9 @@ pub async fn serve(
server_args: server_args.clone(),
chat_formatter,
});
// Cancelled with the api runtime on shutdown.
tokio::spawn(startup_warmup(state.core.clone(), server_args.clone()));
// Each endpoint module registers its own routes and merges here.
let router = Router::new()
.merge(info::routes())
@@ -85,10 +85,6 @@ pub async fn serve(
// No body limit, matching the Python server.
let mut app = router
.layer(axum::extract::DefaultBodyLimit::disable())
.layer(axum::middleware::from_fn_with_state(
state.clone(),
mark_startup_ready,
))
.with_state(state);
// Prefill-only KV bootstrap registry. Merged AFTER `with_state` — its
@@ -130,37 +126,3 @@ pub async fn serve(
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn core_handle() -> CoreHandle {
CoreHandle::new(
flume::unbounded().0,
flume::unbounded().0,
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::api_server::core::CoreMetadata::default(),
},
)
}
#[test]
fn only_a_successful_recognized_warmup_marks_frontend_ready() {
let core = core_handle();
record_startup_warmup_status(&core, false, StatusCode::OK);
assert!(!core.is_ready());
record_startup_warmup_status(&core, true, StatusCode::INTERNAL_SERVER_ERROR);
assert!(!core.is_ready());
record_startup_warmup_status(&core, true, StatusCode::OK);
assert!(core.is_ready());
}
}
+3
View File
@@ -464,6 +464,9 @@ mod tests {
fn test_server_args() -> ServerArgs {
ServerArgs {
skip_tokenizer_init: true,
// No scheduler answers here, so a startup warmup would only
// occupy the scheduler ring that the tests inspect.
skip_server_warmup: true,
..Default::default()
}
}