diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index a95acd5c18..e910eef213 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -6,7 +6,7 @@ import json import logging import os from types import ModuleType -from typing import TYPE_CHECKING, Callable, List, Optional, Tuple +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.managers.utils import compute_num_reserved_tokens @@ -27,7 +27,10 @@ logger = logging.getLogger(__name__) def _build_server_args( - scheduler: Scheduler, *, extension: Optional[ModuleType] = None + scheduler: Scheduler, + *, + extension: Optional[ModuleType] = None, + health_generate_headers: Optional[Dict[str, str]] = None, ) -> ServerArgs: """The typed launch configuration for the scheduler's embedded Rust server: the ``server_args`` fields it reads, the already-resolved @@ -74,6 +77,7 @@ def _build_server_args( detokenizer_worker_num=get_serving().detokenizer_worker_num, skip_tokenizer_init=get_serving().skip_tokenizer_init, skip_server_warmup=get_serving().skip_server_warmup, + health_generate_headers=dict(health_generate_headers or {}), incremental_streaming_output=get_serving().incremental_streaming_output, disaggregation_mode=disaggregation_mode, model_config=ext.ModelConfig( diff --git a/python/sglang/srt/rust_server/server.py b/python/sglang/srt/rust_server/server.py index 48c2f58369..6d72b0c78b 100644 --- a/python/sglang/srt/rust_server/server.py +++ b/python/sglang/srt/rust_server/server.py @@ -68,6 +68,12 @@ class RustServer: self._multimodal_enabled = mm_spec is not None self._max_per_poll = max_per_poll + @classmethod + def _health_generate_headers(cls, scheduler: Scheduler) -> Dict[str, str]: + """Headers a healthy ``/health_generate`` carries; a deployment's + router may read its metadata off the deep health probe.""" + return {} + @classmethod def _load_extension(cls) -> ModuleType: from sglang.srt.rust_extensions import load_rust_extension @@ -167,7 +173,11 @@ class RustServer: extension = cls._load_extension() server = extension.Server( - _build_server_args(scheduler, extension=extension), + _build_server_args( + scheduler, + extension=extension, + health_generate_headers=cls._health_generate_headers(scheduler), + ), # None runs unpinned; otherwise the list carries the pinning decision. cores=server_cores, port_offset=local_dp_rank, diff --git a/rust/sglang-server/src/api_server/http/native_api.rs b/rust/sglang-server/src/api_server/http/native_api.rs index 1b63b66185..d48d7b2e13 100644 --- a/rust/sglang-server/src/api_server/http/native_api.rs +++ b/rust/sglang-server/src/api_server/http/native_api.rs @@ -26,7 +26,7 @@ use axum::{ Json, Router, extract::State, extract::rejection::JsonRejection, - http::StatusCode, + http::{HeaderName, HeaderValue, StatusCode}, response::{ IntoResponse, Response, sse::{Event, Sse}, @@ -102,12 +102,12 @@ fn health_routes() -> Router> { let timeout = std::time::Duration::from_secs( environ::env_i64("SGLANG_HEALTH_CHECK_TIMEOUT", 20).max(0) as u64, ); - let probe = get(move |state: State>| health_generate(state, timeout)); let health = if environ::env_bool("SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION", true) { - probe.clone() + get(move |state: State>| health_generate(state, timeout, false)) } else { get(health_without_generation) }; + let probe = get(move |state: State>| health_generate(state, timeout, true)); Router::new() .route("/health", health) .route("/health_generate", probe) @@ -125,7 +125,8 @@ async fn health_without_generation(State(state): State>) -> Respon /// producing output. 200 if the response heartbeat advances within `timeout` /// (from `SGLANG_HEALTH_CHECK_TIMEOUT`, frozen at router build), else 503. /// It also returns 503 until startup warmup completes. (`/health` uses the same -/// handler when its env gate is on.) +/// handler when its env gate is on, without `metadata_headers`.) A healthy +/// `/health_generate` carries the configured `health_generate_headers`. /// /// Fires a pre-tokenized 1-token probe (`input_ids = [0]`, skips the tokenizer) so /// an idle pipeline produces a frame, then watches the *global* @@ -135,9 +136,24 @@ async fn health_without_generation(State(state): State>) -> Respon async fn health_generate( State(state): State>, timeout: std::time::Duration, + metadata_headers: bool, ) -> Response { match state.core.probe_health(timeout).await { - Ok(HealthStatus::Healthy) => StatusCode::OK.into_response(), + Ok(HealthStatus::Healthy) => { + let mut response = StatusCode::OK.into_response(); + if metadata_headers { + // Validated at startup (`ServerArgs::validate`). + for (name, value) in &state.server_args.health_generate_headers { + if let (Ok(name), Ok(value)) = ( + HeaderName::from_bytes(name.as_bytes()), + HeaderValue::from_str(value), + ) { + response.headers_mut().insert(name, value); + } + } + } + response + } Ok(HealthStatus::NotReady | HealthStatus::Stalled) => { StatusCode::SERVICE_UNAVAILABLE.into_response() } @@ -551,12 +567,61 @@ mod tests { chat_formatter: None, }); - let response = health_generate(State(state), Duration::ZERO).await; + let response = health_generate(State(state), Duration::ZERO, true).await; assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(intake_rx.try_recv().is_err()); assert!(abort_rx.try_recv().is_err()); } + /// A healthy `/health_generate` carries the configured metadata headers; + /// `/health` sharing the probe does not. + #[tokio::test] + async fn healthy_probe_carries_configured_headers() { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, _abort_rx) = flume::unbounded(); + let activity: crate::tokenizer_manager::from_scheduler::ActivityCounter = + Default::default(); + let server_args = crate::message::config::ServerArgs { + health_generate_headers: [ + ("x-custom-header".to_string(), "v".to_string()), + ("x-rust-server".to_string(), "health".to_string()), + ] + .into(), + ..Default::default() + }; + let state = Arc::new(AppState { + core: crate::api_server::core::CoreHandle::new( + intake_tx, + abort_tx, + crate::api_server::core::CoreConfig { + response_capacity: 8, + response_activity: activity.clone(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: crate::api_server::core::CoreMetadata::default(), + }, + ), + server_args: Arc::new(server_args), + chat_formatter: None, + }); + // Each probe that reaches intake counts as scheduler output. + tokio::spawn(async move { + while intake_rx.recv_async().await.is_ok() { + activity.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + }); + + let response = health_generate(State(state.clone()), Duration::from_secs(5), true).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["x-custom-header"], "v"); + assert_eq!(response.headers()["x-rust-server"], "health"); + + let response = health_generate(State(state), Duration::from_secs(5), false).await; + assert_eq!(response.status(), StatusCode::OK); + assert!(response.headers().get("x-custom-header").is_none()); + } + #[tokio::test] async fn unary_http_generate_round_trips_through_frontend_handle() { let (intake_tx, intake_rx) = flume::unbounded(); diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 8bedcfc8a7..27fca165e6 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -135,6 +135,9 @@ pub struct ServerArgs { /// Start accepting health checks immediately instead of waiting for the /// main process's startup warmup request to finish. pub skip_server_warmup: bool, + /// Extra headers on a successful `/health_generate` response, for a router + /// that reads deployment metadata off the deep health probe. + pub health_generate_headers: BTreeMap, /// Streamed `/generate` frames carry per-step deltas instead of cumulative /// text. Matches the Python `TokenizerManager`. pub incremental_streaming_output: bool, @@ -191,6 +194,7 @@ impl ServerArgs { detokenizer_worker_num, skip_tokenizer_init, skip_server_warmup, + health_generate_headers, incremental_streaming_output, disaggregation_mode, model_config, @@ -225,6 +229,7 @@ impl ServerArgs { detokenizer_worker_num: usize, skip_tokenizer_init: bool, skip_server_warmup: bool, + health_generate_headers: BTreeMap, incremental_streaming_output: bool, disaggregation_mode: DisaggregationMode, model_config: ModelConfig, @@ -257,6 +262,7 @@ impl ServerArgs { detokenizer_worker_num, skip_tokenizer_init, skip_server_warmup, + health_generate_headers, incremental_streaming_output, disaggregation_mode, model_config, @@ -297,6 +303,7 @@ impl Default for ServerArgs { detokenizer_worker_num: 1, skip_tokenizer_init: false, skip_server_warmup: false, + health_generate_headers: BTreeMap::new(), incremental_streaming_output: false, disaggregation_mode: DisaggregationMode::Null, model_config: ModelConfig::default(), @@ -591,6 +598,15 @@ impl ServerArgs { if self.served_model_name.is_empty() { return Err("empty 'served_model_name' in server_args".into()); } + for (name, value) in &self.health_generate_headers { + if http::HeaderName::from_bytes(name.as_bytes()).is_err() + || http::HeaderValue::from_str(value).is_err() + { + return Err(format!( + "invalid 'health_generate_headers' entry {name:?}: {value:?}" + )); + } + } if let Some(grpc_port) = self.grpc_port { if grpc_port == 0 { return Err("'grpc_port' must be between 1 and 65535".into()); @@ -704,13 +720,22 @@ mod tests { } #[test] - fn validate_requires_served_model_name() { + fn validate_requires_served_model_name_and_valid_health_headers() { assert!(ServerArgs::default().validate().is_err()); let sa = ServerArgs { served_model_name: "m".into(), ..Default::default() }; assert!(sa.validate().is_ok()); + let bad = ServerArgs { + health_generate_headers: [("bad header".into(), "v".into())].into(), + ..sa + }; + assert!( + bad.validate() + .unwrap_err() + .contains("health_generate_headers") + ); } #[test] diff --git a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py index 0d2a29e363..9ca78de010 100644 --- a/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py +++ b/test/registered/unit/multimodal/rust/shared/test_rust_server_extension.py @@ -81,7 +81,9 @@ class TestRustServerExtension(CustomTestCase): with launch_context(ModelServer) as build_args: instance = ModelServer.launch(scheduler) - build_args.assert_called_once_with(scheduler, extension=extension) + build_args.assert_called_once_with( + scheduler, extension=extension, health_generate_headers={} + ) extension.Server.assert_called_once_with( sentinel.args, cores=None, port_offset=1 )