[rust-server] warm up each rust listener in-process (#43004)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
@@ -2268,10 +2268,6 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
url = server_args.url()
|
||||
if get_serving().api_key:
|
||||
headers["Authorization"] = f"Bearer {get_serving().api_key}"
|
||||
if envs.SGLANG_RUST_SERVER.get():
|
||||
# The Rust listener binds before this request so /model_info is
|
||||
# available, but health stays 503 until this marked request succeeds.
|
||||
headers["x-sglang-startup-warmup"] = "1"
|
||||
|
||||
ssl_verify = ssl_verify_of(server_args)
|
||||
|
||||
@@ -2333,16 +2329,7 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
and get_disagg().disaggregation_mode == "null"
|
||||
and model_info["is_generation"]
|
||||
):
|
||||
served_model_name = ""
|
||||
if not envs.SGLANG_RUST_SERVER.get():
|
||||
served_model_name = _global_state.tokenizer_manager.served_model_name
|
||||
else:
|
||||
# _global_state.tokenizer_manager is not initialized in the rust server,
|
||||
# so we need to get the model name from the model_info
|
||||
served_model_name = model_info.get(
|
||||
"model_path", get_serving().served_model_name
|
||||
)
|
||||
served_model_name = served_model_name or get_model().model_path
|
||||
served_model_name = _global_state.tokenizer_manager.served_model_name
|
||||
# TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now
|
||||
# Only use chat completions format for generation models, not embedding models
|
||||
json_data = {
|
||||
@@ -2397,9 +2384,7 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
verify=ssl_verify,
|
||||
)
|
||||
assert res.status_code == 200, f"{res.text}"
|
||||
# Skip server_status update for Rust server
|
||||
if not envs.SGLANG_RUST_SERVER.get():
|
||||
_global_state.tokenizer_manager.server_status = ServerStatus.Up
|
||||
_global_state.tokenizer_manager.server_status = ServerStatus.Up
|
||||
|
||||
else:
|
||||
logger.info(f"Start of pd disaggregation warmup ...")
|
||||
@@ -2424,14 +2409,9 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
get_disagg().disaggregation_mode,
|
||||
failed_status_codes,
|
||||
)
|
||||
# In rust-server mode there is no TokenizerManager (readiness is
|
||||
# the Rust server's own /health), so skip the status update.
|
||||
if not envs.SGLANG_RUST_SERVER.get():
|
||||
_global_state.tokenizer_manager.server_status = (
|
||||
ServerStatus.Up
|
||||
if not failed_status_codes
|
||||
else ServerStatus.UnHealthy
|
||||
)
|
||||
_global_state.tokenizer_manager.server_status = (
|
||||
ServerStatus.Up if not failed_status_codes else ServerStatus.UnHealthy
|
||||
)
|
||||
|
||||
except Exception:
|
||||
last_traceback = get_exception_traceback()
|
||||
@@ -2926,13 +2906,8 @@ def launch_server(
|
||||
if envs.SGLANG_RUST_SERVER.get():
|
||||
# The Rust server serves api-server, tokenizer, and detokenizer, so the
|
||||
# main process has no Python HTTP server / tokenizer manager to run.
|
||||
# Run a warmup /generate before advertising readiness: the Rust /health
|
||||
# and /get_model_info endpoints are static (200 as soon as the server
|
||||
# binds, before any forward pass), so without this the first real request
|
||||
# pays the cold-start cost (observed as a >60s first generation).
|
||||
if not get_serving().skip_server_warmup:
|
||||
_execute_server_warmup(server_args)
|
||||
logger.info("The server is fired up and ready to roll!")
|
||||
# Each Rust listener runs its own startup warmup and keeps /health at
|
||||
# 503 until it succeeds, so the main process does not warm up here.
|
||||
if launch_callback is not None:
|
||||
launch_callback()
|
||||
scheduler_init_result.block_until_scheduler_exits()
|
||||
|
||||
@@ -121,9 +121,8 @@ impl CoreHandle {
|
||||
self.inner.startup_ready.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Record successful completion of the main process's startup warmup.
|
||||
/// The adapter decides which response qualifies as that warmup; the state
|
||||
/// itself is shared by every transport.
|
||||
/// Record successful completion of the startup warmup. The state is
|
||||
/// shared by every transport.
|
||||
pub(crate) fn mark_ready(&self) {
|
||||
self.inner.startup_ready.store(true, Ordering::Release);
|
||||
}
|
||||
@@ -294,6 +293,43 @@ impl CoreHandle {
|
||||
}
|
||||
}
|
||||
|
||||
/// One generation mirroring the Python `_execute_server_warmup` text request;
|
||||
/// VLMs warm only the text path. The caller decides readiness.
|
||||
pub(crate) async fn warm_up(&self, skip_tokenizer_init: bool) -> Result<(), CoreError> {
|
||||
let (text, input_ids) = if skip_tokenizer_init {
|
||||
(None, Some(vec![10, 11, 12]))
|
||||
} else {
|
||||
(Some("The capital city of France is".to_string()), None)
|
||||
};
|
||||
let mut call = self
|
||||
.generate(GenerateRequest {
|
||||
rid: Rid::new(),
|
||||
text,
|
||||
input_ids,
|
||||
sampling_params: SamplingParams {
|
||||
max_new_tokens: Some(8),
|
||||
temperature: 0.0,
|
||||
..Default::default()
|
||||
},
|
||||
stream: false,
|
||||
bootstrap_host: self
|
||||
.inner
|
||||
.is_disaggregation
|
||||
.then(|| FAKE_BOOTSTRAP_HOST.into()),
|
||||
bootstrap_room: self.inner.is_disaggregation.then_some(0),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
loop {
|
||||
match call.recv().await {
|
||||
Some(CoreEvent::Finished(_)) => return Ok(()),
|
||||
Some(CoreEvent::Failed(e)) => return Err(e),
|
||||
Some(CoreEvent::Delta(_)) => {}
|
||||
None => return Err(CoreError::ResponseTruncated),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit one already-parsed in-process request.
|
||||
///
|
||||
/// The async send preserves TokenizerManager inbox backpressure. Once the
|
||||
@@ -961,6 +997,60 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warm_up_succeeds_on_finished_generation() {
|
||||
let harness = Harness::unbounded(8);
|
||||
let handle = harness.handle.clone();
|
||||
let warm_up = tokio::spawn(async move { handle.warm_up(false).await });
|
||||
|
||||
let request = accept_intake(harness.intake_rx.recv_async().await.unwrap());
|
||||
let RequestKind::Generate(warmup) = &request.kind else {
|
||||
panic!("warmup must submit a generation request");
|
||||
};
|
||||
assert_eq!(
|
||||
warmup.text.as_deref(),
|
||||
Some("The capital city of France is")
|
||||
);
|
||||
assert_eq!(warmup.input_ids, None);
|
||||
assert_eq!(warmup.sampling_params.max_new_tokens, Some(8));
|
||||
assert_eq!(warmup.bootstrap_host, None);
|
||||
request
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(ChunkEvent::default()))
|
||||
.unwrap();
|
||||
|
||||
assert!(warm_up.await.unwrap().is_ok());
|
||||
assert!(!harness.handle.is_ready());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pd_warm_up_without_tokenizer_fails_on_truncated_stream() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, _abort_rx) = flume::unbounded();
|
||||
let handle = configured_handle(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
8,
|
||||
Default::default(),
|
||||
false,
|
||||
true,
|
||||
BTreeMap::new(),
|
||||
);
|
||||
let warm_up = tokio::spawn(async move { handle.warm_up(true).await });
|
||||
|
||||
let request = accept_intake(intake_rx.recv_async().await.unwrap());
|
||||
let RequestKind::Generate(warmup) = &request.kind else {
|
||||
panic!("warmup must submit a generation request");
|
||||
};
|
||||
assert_eq!(warmup.text, None);
|
||||
assert_eq!(warmup.input_ids, Some(vec![10, 11, 12]));
|
||||
assert_eq!(warmup.bootstrap_host.as_deref(), Some(FAKE_BOOTSTRAP_HOST));
|
||||
assert_eq!(warmup.bootstrap_room, Some(0));
|
||||
drop(request);
|
||||
|
||||
assert!(warm_up.await.unwrap().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_operation_returns_transport_neutral_payload() {
|
||||
let harness = Harness::unbounded(8);
|
||||
|
||||
@@ -3,20 +3,16 @@
|
||||
//! pre-bound listener until shutdown.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
extract::{Request, State},
|
||||
http::StatusCode,
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use axum::Router;
|
||||
|
||||
use super::{info, native_api, openai};
|
||||
use crate::api_server::core::CoreHandle;
|
||||
use crate::api_server::disaggregation::bootstrap as pd_bootstrap;
|
||||
use crate::api_server::log;
|
||||
use crate::message::config::ServerArgs;
|
||||
use crate::utils::environ;
|
||||
|
||||
/// HTTP adapter state: the shared core capability, immutable server
|
||||
/// configuration needed for HTTP request preparation, and the chat formatter.
|
||||
@@ -31,28 +27,29 @@ pub(super) struct AppState {
|
||||
pub(super) chat_formatter: Option<openai::ChatFormatter>,
|
||||
}
|
||||
|
||||
/// Private marker attached by the main process to its startup warmup request.
|
||||
/// The middleware flips readiness only after that request returns successfully.
|
||||
const STARTUP_WARMUP_HEADER: &str = "x-sglang-startup-warmup";
|
||||
|
||||
async fn mark_startup_ready(
|
||||
State(state): State<Arc<AppState>>,
|
||||
req: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
let is_startup_warmup = req.headers().contains_key(STARTUP_WARMUP_HEADER)
|
||||
&& matches!(
|
||||
req.uri().path(),
|
||||
"/generate" | "/encode" | "/v1/chat/completions"
|
||||
);
|
||||
let response = next.run(req).await;
|
||||
record_startup_warmup_status(&state.core, is_startup_warmup, response.status());
|
||||
response
|
||||
}
|
||||
|
||||
fn record_startup_warmup_status(core: &CoreHandle, is_startup_warmup: bool, status: StatusCode) {
|
||||
if is_startup_warmup && status.is_success() {
|
||||
core.mark_ready();
|
||||
/// Every listener warms its own rank; health stays 503 until it succeeds,
|
||||
/// and a failed or timed-out warmup leaves the listener unready.
|
||||
async fn startup_warmup(core: CoreHandle, server_args: Arc<ServerArgs>) {
|
||||
if core.is_ready() {
|
||||
return;
|
||||
}
|
||||
let timeout = match environ::env_i64("SGLANG_WARMUP_TIMEOUT", -1) {
|
||||
t if t > 0 => t as u64,
|
||||
_ if server_args.is_disaggregation() => 1800,
|
||||
_ => 600,
|
||||
};
|
||||
match tokio::time::timeout(
|
||||
Duration::from_secs(timeout),
|
||||
core.warm_up(server_args.skip_tokenizer_init),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => {
|
||||
core.mark_ready();
|
||||
tracing::info!("The server is fired up and ready to roll!");
|
||||
}
|
||||
Ok(Err(e)) => tracing::error!(error = %e, "startup warmup failed"),
|
||||
Err(_) => tracing::error!(timeout_secs = timeout, "startup warmup timed out"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +69,9 @@ pub async fn serve(
|
||||
server_args: server_args.clone(),
|
||||
chat_formatter,
|
||||
});
|
||||
// Cancelled with the api runtime on shutdown.
|
||||
tokio::spawn(startup_warmup(state.core.clone(), server_args.clone()));
|
||||
|
||||
// Each endpoint module registers its own routes and merges here.
|
||||
let router = Router::new()
|
||||
.merge(info::routes())
|
||||
@@ -85,10 +85,6 @@ pub async fn serve(
|
||||
// No body limit, matching the Python server.
|
||||
let mut app = router
|
||||
.layer(axum::extract::DefaultBodyLimit::disable())
|
||||
.layer(axum::middleware::from_fn_with_state(
|
||||
state.clone(),
|
||||
mark_startup_ready,
|
||||
))
|
||||
.with_state(state);
|
||||
|
||||
// Prefill-only KV bootstrap registry. Merged AFTER `with_state` — its
|
||||
@@ -130,37 +126,3 @@ pub async fn serve(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn core_handle() -> CoreHandle {
|
||||
CoreHandle::new(
|
||||
flume::unbounded().0,
|
||||
flume::unbounded().0,
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::api_server::core::CoreMetadata::default(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_successful_recognized_warmup_marks_frontend_ready() {
|
||||
let core = core_handle();
|
||||
|
||||
record_startup_warmup_status(&core, false, StatusCode::OK);
|
||||
assert!(!core.is_ready());
|
||||
|
||||
record_startup_warmup_status(&core, true, StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert!(!core.is_ready());
|
||||
|
||||
record_startup_warmup_status(&core, true, StatusCode::OK);
|
||||
assert!(core.is_ready());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -464,6 +464,9 @@ mod tests {
|
||||
fn test_server_args() -> ServerArgs {
|
||||
ServerArgs {
|
||||
skip_tokenizer_init: true,
|
||||
// No scheduler answers here, so a startup warmup would only
|
||||
// occupy the scheduler ring that the tests inspect.
|
||||
skip_server_warmup: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user