From 5a5ea81036c13c8f812df3817c679fe3f0b9afb2 Mon Sep 17 00:00:00 2001 From: Sage <80211083+sagearc@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:09:46 +0300 Subject: [PATCH] [rust-server] warm up each rust listener in-process (#43004) Signed-off-by: Sage Ahrac --- python/sglang/srt/entrypoints/http_server.py | 39 ++------ .../src/api_server/core/handle.rs | 96 ++++++++++++++++++- rust/sglang-server/src/api_server/http/app.rs | 96 ++++++------------- rust/sglang-server/src/utils/runtime.rs | 3 + 4 files changed, 132 insertions(+), 102 deletions(-) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index adc489b4f3..b659c9add8 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -2268,10 +2268,6 @@ def _execute_server_warmup(server_args: ServerArgs): url = server_args.url() if get_serving().api_key: headers["Authorization"] = f"Bearer {get_serving().api_key}" - if envs.SGLANG_RUST_SERVER.get(): - # The Rust listener binds before this request so /model_info is - # available, but health stays 503 until this marked request succeeds. - headers["x-sglang-startup-warmup"] = "1" ssl_verify = ssl_verify_of(server_args) @@ -2333,16 +2329,7 @@ def _execute_server_warmup(server_args: ServerArgs): and get_disagg().disaggregation_mode == "null" and model_info["is_generation"] ): - served_model_name = "" - if not envs.SGLANG_RUST_SERVER.get(): - served_model_name = _global_state.tokenizer_manager.served_model_name - else: - # _global_state.tokenizer_manager is not initialized in the rust server, - # so we need to get the model name from the model_info - served_model_name = model_info.get( - "model_path", get_serving().served_model_name - ) - served_model_name = served_model_name or get_model().model_path + served_model_name = _global_state.tokenizer_manager.served_model_name # TODO: ChatCompletionRequest does not have bootstrap info required by disaggregation mode, disable image-warmup for now # Only use chat completions format for generation models, not embedding models json_data = { @@ -2397,9 +2384,7 @@ def _execute_server_warmup(server_args: ServerArgs): verify=ssl_verify, ) assert res.status_code == 200, f"{res.text}" - # Skip server_status update for Rust server - if not envs.SGLANG_RUST_SERVER.get(): - _global_state.tokenizer_manager.server_status = ServerStatus.Up + _global_state.tokenizer_manager.server_status = ServerStatus.Up else: logger.info(f"Start of pd disaggregation warmup ...") @@ -2424,14 +2409,9 @@ def _execute_server_warmup(server_args: ServerArgs): get_disagg().disaggregation_mode, failed_status_codes, ) - # In rust-server mode there is no TokenizerManager (readiness is - # the Rust server's own /health), so skip the status update. - if not envs.SGLANG_RUST_SERVER.get(): - _global_state.tokenizer_manager.server_status = ( - ServerStatus.Up - if not failed_status_codes - else ServerStatus.UnHealthy - ) + _global_state.tokenizer_manager.server_status = ( + ServerStatus.Up if not failed_status_codes else ServerStatus.UnHealthy + ) except Exception: last_traceback = get_exception_traceback() @@ -2926,13 +2906,8 @@ def launch_server( if envs.SGLANG_RUST_SERVER.get(): # The Rust server serves api-server, tokenizer, and detokenizer, so the # main process has no Python HTTP server / tokenizer manager to run. - # Run a warmup /generate before advertising readiness: the Rust /health - # and /get_model_info endpoints are static (200 as soon as the server - # binds, before any forward pass), so without this the first real request - # pays the cold-start cost (observed as a >60s first generation). - if not get_serving().skip_server_warmup: - _execute_server_warmup(server_args) - logger.info("The server is fired up and ready to roll!") + # Each Rust listener runs its own startup warmup and keeps /health at + # 503 until it succeeds, so the main process does not warm up here. if launch_callback is not None: launch_callback() scheduler_init_result.block_until_scheduler_exits() diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index 4cecaa397b..06ea61b597 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -121,9 +121,8 @@ impl CoreHandle { self.inner.startup_ready.load(Ordering::Acquire) } - /// Record successful completion of the main process's startup warmup. - /// The adapter decides which response qualifies as that warmup; the state - /// itself is shared by every transport. + /// Record successful completion of the startup warmup. The state is + /// shared by every transport. pub(crate) fn mark_ready(&self) { self.inner.startup_ready.store(true, Ordering::Release); } @@ -294,6 +293,43 @@ impl CoreHandle { } } + /// One generation mirroring the Python `_execute_server_warmup` text request; + /// VLMs warm only the text path. The caller decides readiness. + pub(crate) async fn warm_up(&self, skip_tokenizer_init: bool) -> Result<(), CoreError> { + let (text, input_ids) = if skip_tokenizer_init { + (None, Some(vec![10, 11, 12])) + } else { + (Some("The capital city of France is".to_string()), None) + }; + let mut call = self + .generate(GenerateRequest { + rid: Rid::new(), + text, + input_ids, + sampling_params: SamplingParams { + max_new_tokens: Some(8), + temperature: 0.0, + ..Default::default() + }, + stream: false, + bootstrap_host: self + .inner + .is_disaggregation + .then(|| FAKE_BOOTSTRAP_HOST.into()), + bootstrap_room: self.inner.is_disaggregation.then_some(0), + ..Default::default() + }) + .await?; + loop { + match call.recv().await { + Some(CoreEvent::Finished(_)) => return Ok(()), + Some(CoreEvent::Failed(e)) => return Err(e), + Some(CoreEvent::Delta(_)) => {} + None => return Err(CoreError::ResponseTruncated), + } + } + } + /// Submit one already-parsed in-process request. /// /// The async send preserves TokenizerManager inbox backpressure. Once the @@ -961,6 +997,60 @@ mod tests { )); } + #[tokio::test] + async fn warm_up_succeeds_on_finished_generation() { + let harness = Harness::unbounded(8); + let handle = harness.handle.clone(); + let warm_up = tokio::spawn(async move { handle.warm_up(false).await }); + + let request = accept_intake(harness.intake_rx.recv_async().await.unwrap()); + let RequestKind::Generate(warmup) = &request.kind else { + panic!("warmup must submit a generation request"); + }; + assert_eq!( + warmup.text.as_deref(), + Some("The capital city of France is") + ); + assert_eq!(warmup.input_ids, None); + assert_eq!(warmup.sampling_params.max_new_tokens, Some(8)); + assert_eq!(warmup.bootstrap_host, None); + request + .sink + .try_send(ResponseItem::Done(ChunkEvent::default())) + .unwrap(); + + assert!(warm_up.await.unwrap().is_ok()); + assert!(!harness.handle.is_ready()); + } + + #[tokio::test] + async fn pd_warm_up_without_tokenizer_fails_on_truncated_stream() { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, _abort_rx) = flume::unbounded(); + let handle = configured_handle( + intake_tx, + abort_tx, + 8, + Default::default(), + false, + true, + BTreeMap::new(), + ); + let warm_up = tokio::spawn(async move { handle.warm_up(true).await }); + + let request = accept_intake(intake_rx.recv_async().await.unwrap()); + let RequestKind::Generate(warmup) = &request.kind else { + panic!("warmup must submit a generation request"); + }; + assert_eq!(warmup.text, None); + assert_eq!(warmup.input_ids, Some(vec![10, 11, 12])); + assert_eq!(warmup.bootstrap_host.as_deref(), Some(FAKE_BOOTSTRAP_HOST)); + assert_eq!(warmup.bootstrap_room, Some(0)); + drop(request); + + assert!(warm_up.await.unwrap().is_err()); + } + #[tokio::test] async fn control_operation_returns_transport_neutral_payload() { let harness = Harness::unbounded(8); diff --git a/rust/sglang-server/src/api_server/http/app.rs b/rust/sglang-server/src/api_server/http/app.rs index 9d16ad9205..a4a0a05910 100644 --- a/rust/sglang-server/src/api_server/http/app.rs +++ b/rust/sglang-server/src/api_server/http/app.rs @@ -3,20 +3,16 @@ //! pre-bound listener until shutdown. use std::sync::Arc; +use std::time::Duration; -use axum::{ - Router, - extract::{Request, State}, - http::StatusCode, - middleware::Next, - response::Response, -}; +use axum::Router; use super::{info, native_api, openai}; use crate::api_server::core::CoreHandle; use crate::api_server::disaggregation::bootstrap as pd_bootstrap; use crate::api_server::log; use crate::message::config::ServerArgs; +use crate::utils::environ; /// HTTP adapter state: the shared core capability, immutable server /// configuration needed for HTTP request preparation, and the chat formatter. @@ -31,28 +27,29 @@ pub(super) struct AppState { pub(super) chat_formatter: Option, } -/// Private marker attached by the main process to its startup warmup request. -/// The middleware flips readiness only after that request returns successfully. -const STARTUP_WARMUP_HEADER: &str = "x-sglang-startup-warmup"; - -async fn mark_startup_ready( - State(state): State>, - req: Request, - next: Next, -) -> Response { - let is_startup_warmup = req.headers().contains_key(STARTUP_WARMUP_HEADER) - && matches!( - req.uri().path(), - "/generate" | "/encode" | "/v1/chat/completions" - ); - let response = next.run(req).await; - record_startup_warmup_status(&state.core, is_startup_warmup, response.status()); - response -} - -fn record_startup_warmup_status(core: &CoreHandle, is_startup_warmup: bool, status: StatusCode) { - if is_startup_warmup && status.is_success() { - core.mark_ready(); +/// Every listener warms its own rank; health stays 503 until it succeeds, +/// and a failed or timed-out warmup leaves the listener unready. +async fn startup_warmup(core: CoreHandle, server_args: Arc) { + if core.is_ready() { + return; + } + let timeout = match environ::env_i64("SGLANG_WARMUP_TIMEOUT", -1) { + t if t > 0 => t as u64, + _ if server_args.is_disaggregation() => 1800, + _ => 600, + }; + match tokio::time::timeout( + Duration::from_secs(timeout), + core.warm_up(server_args.skip_tokenizer_init), + ) + .await + { + Ok(Ok(())) => { + core.mark_ready(); + tracing::info!("The server is fired up and ready to roll!"); + } + Ok(Err(e)) => tracing::error!(error = %e, "startup warmup failed"), + Err(_) => tracing::error!(timeout_secs = timeout, "startup warmup timed out"), } } @@ -72,6 +69,9 @@ pub async fn serve( server_args: server_args.clone(), chat_formatter, }); + // Cancelled with the api runtime on shutdown. + tokio::spawn(startup_warmup(state.core.clone(), server_args.clone())); + // Each endpoint module registers its own routes and merges here. let router = Router::new() .merge(info::routes()) @@ -85,10 +85,6 @@ pub async fn serve( // No body limit, matching the Python server. let mut app = router .layer(axum::extract::DefaultBodyLimit::disable()) - .layer(axum::middleware::from_fn_with_state( - state.clone(), - mark_startup_ready, - )) .with_state(state); // Prefill-only KV bootstrap registry. Merged AFTER `with_state` — its @@ -130,37 +126,3 @@ pub async fn serve( } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn core_handle() -> CoreHandle { - CoreHandle::new( - flume::unbounded().0, - flume::unbounded().0, - crate::api_server::core::CoreConfig { - response_capacity: 8, - response_activity: Default::default(), - startup_ready: false, - is_disaggregation: false, - mm_limits: Default::default(), - metadata: crate::api_server::core::CoreMetadata::default(), - }, - ) - } - - #[test] - fn only_a_successful_recognized_warmup_marks_frontend_ready() { - let core = core_handle(); - - record_startup_warmup_status(&core, false, StatusCode::OK); - assert!(!core.is_ready()); - - record_startup_warmup_status(&core, true, StatusCode::INTERNAL_SERVER_ERROR); - assert!(!core.is_ready()); - - record_startup_warmup_status(&core, true, StatusCode::OK); - assert!(core.is_ready()); - } -} diff --git a/rust/sglang-server/src/utils/runtime.rs b/rust/sglang-server/src/utils/runtime.rs index 811362abfe..a55bbbcf3d 100644 --- a/rust/sglang-server/src/utils/runtime.rs +++ b/rust/sglang-server/src/utils/runtime.rs @@ -464,6 +464,9 @@ mod tests { fn test_server_args() -> ServerArgs { ServerArgs { skip_tokenizer_init: true, + // No scheduler answers here, so a startup warmup would only + // occupy the scheduler ring that the tests inspect. + skip_server_warmup: true, ..Default::default() } }