[Feat] rust-server: Attach configured metadata headers to a healthy /health_generate (#44315)

Co-authored-by: fei-xx <fei-xx@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-15 12:04:23 -07:00
committed by GitHub
co-authored by fei-xx shodoco
parent 173e0f2e9b
commit 02a43eeef6
5 changed files with 117 additions and 11 deletions
+6 -2
View File
@@ -6,7 +6,7 @@ import json
import logging
import os
from types import ModuleType
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.managers.utils import compute_num_reserved_tokens
@@ -27,7 +27,10 @@ logger = logging.getLogger(__name__)
def _build_server_args(
scheduler: Scheduler, *, extension: Optional[ModuleType] = None
scheduler: Scheduler,
*,
extension: Optional[ModuleType] = None,
health_generate_headers: Optional[Dict[str, str]] = None,
) -> ServerArgs:
"""The typed launch configuration for the scheduler's embedded Rust server:
the ``server_args`` fields it reads, the already-resolved
@@ -74,6 +77,7 @@ def _build_server_args(
detokenizer_worker_num=get_serving().detokenizer_worker_num,
skip_tokenizer_init=get_serving().skip_tokenizer_init,
skip_server_warmup=get_serving().skip_server_warmup,
health_generate_headers=dict(health_generate_headers or {}),
incremental_streaming_output=get_serving().incremental_streaming_output,
disaggregation_mode=disaggregation_mode,
model_config=ext.ModelConfig(
+11 -1
View File
@@ -68,6 +68,12 @@ class RustServer:
self._multimodal_enabled = mm_spec is not None
self._max_per_poll = max_per_poll
@classmethod
def _health_generate_headers(cls, scheduler: Scheduler) -> Dict[str, str]:
"""Headers a healthy ``/health_generate`` carries; a deployment's
router may read its metadata off the deep health probe."""
return {}
@classmethod
def _load_extension(cls) -> ModuleType:
from sglang.srt.rust_extensions import load_rust_extension
@@ -167,7 +173,11 @@ class RustServer:
extension = cls._load_extension()
server = extension.Server(
_build_server_args(scheduler, extension=extension),
_build_server_args(
scheduler,
extension=extension,
health_generate_headers=cls._health_generate_headers(scheduler),
),
# None runs unpinned; otherwise the list carries the pinning decision.
cores=server_cores,
port_offset=local_dp_rank,
@@ -26,7 +26,7 @@ use axum::{
Json, Router,
extract::State,
extract::rejection::JsonRejection,
http::StatusCode,
http::{HeaderName, HeaderValue, StatusCode},
response::{
IntoResponse, Response,
sse::{Event, Sse},
@@ -102,12 +102,12 @@ fn health_routes() -> Router<Arc<AppState>> {
let timeout = std::time::Duration::from_secs(
environ::env_i64("SGLANG_HEALTH_CHECK_TIMEOUT", 20).max(0) as u64,
);
let probe = get(move |state: State<Arc<AppState>>| health_generate(state, timeout));
let health = if environ::env_bool("SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION", true) {
probe.clone()
get(move |state: State<Arc<AppState>>| health_generate(state, timeout, false))
} else {
get(health_without_generation)
};
let probe = get(move |state: State<Arc<AppState>>| health_generate(state, timeout, true));
Router::new()
.route("/health", health)
.route("/health_generate", probe)
@@ -125,7 +125,8 @@ async fn health_without_generation(State(state): State<Arc<AppState>>) -> Respon
/// producing output. 200 if the response heartbeat advances within `timeout`
/// (from `SGLANG_HEALTH_CHECK_TIMEOUT`, frozen at router build), else 503.
/// It also returns 503 until startup warmup completes. (`/health` uses the same
/// handler when its env gate is on.)
/// handler when its env gate is on, without `metadata_headers`.) A healthy
/// `/health_generate` carries the configured `health_generate_headers`.
///
/// Fires a pre-tokenized 1-token probe (`input_ids = [0]`, skips the tokenizer) so
/// an idle pipeline produces a frame, then watches the *global*
@@ -135,9 +136,24 @@ async fn health_without_generation(State(state): State<Arc<AppState>>) -> Respon
async fn health_generate(
State(state): State<Arc<AppState>>,
timeout: std::time::Duration,
metadata_headers: bool,
) -> Response {
match state.core.probe_health(timeout).await {
Ok(HealthStatus::Healthy) => StatusCode::OK.into_response(),
Ok(HealthStatus::Healthy) => {
let mut response = StatusCode::OK.into_response();
if metadata_headers {
// Validated at startup (`ServerArgs::validate`).
for (name, value) in &state.server_args.health_generate_headers {
if let (Ok(name), Ok(value)) = (
HeaderName::from_bytes(name.as_bytes()),
HeaderValue::from_str(value),
) {
response.headers_mut().insert(name, value);
}
}
}
response
}
Ok(HealthStatus::NotReady | HealthStatus::Stalled) => {
StatusCode::SERVICE_UNAVAILABLE.into_response()
}
@@ -551,12 +567,61 @@ mod tests {
chat_formatter: None,
});
let response = health_generate(State(state), Duration::ZERO).await;
let response = health_generate(State(state), Duration::ZERO, true).await;
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(intake_rx.try_recv().is_err());
assert!(abort_rx.try_recv().is_err());
}
/// A healthy `/health_generate` carries the configured metadata headers;
/// `/health` sharing the probe does not.
#[tokio::test]
async fn healthy_probe_carries_configured_headers() {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, _abort_rx) = flume::unbounded();
let activity: crate::tokenizer_manager::from_scheduler::ActivityCounter =
Default::default();
let server_args = crate::message::config::ServerArgs {
health_generate_headers: [
("x-custom-header".to_string(), "v".to_string()),
("x-rust-server".to_string(), "health".to_string()),
]
.into(),
..Default::default()
};
let state = Arc::new(AppState {
core: crate::api_server::core::CoreHandle::new(
intake_tx,
abort_tx,
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: activity.clone(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::api_server::core::CoreMetadata::default(),
},
),
server_args: Arc::new(server_args),
chat_formatter: None,
});
// Each probe that reaches intake counts as scheduler output.
tokio::spawn(async move {
while intake_rx.recv_async().await.is_ok() {
activity.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
});
let response = health_generate(State(state.clone()), Duration::from_secs(5), true).await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()["x-custom-header"], "v");
assert_eq!(response.headers()["x-rust-server"], "health");
let response = health_generate(State(state), Duration::from_secs(5), false).await;
assert_eq!(response.status(), StatusCode::OK);
assert!(response.headers().get("x-custom-header").is_none());
}
#[tokio::test]
async fn unary_http_generate_round_trips_through_frontend_handle() {
let (intake_tx, intake_rx) = flume::unbounded();
+26 -1
View File
@@ -135,6 +135,9 @@ pub struct ServerArgs {
/// Start accepting health checks immediately instead of waiting for the
/// main process's startup warmup request to finish.
pub skip_server_warmup: bool,
/// Extra headers on a successful `/health_generate` response, for a router
/// that reads deployment metadata off the deep health probe.
pub health_generate_headers: BTreeMap<String, String>,
/// Streamed `/generate` frames carry per-step deltas instead of cumulative
/// text. Matches the Python `TokenizerManager`.
pub incremental_streaming_output: bool,
@@ -191,6 +194,7 @@ impl ServerArgs {
detokenizer_worker_num,
skip_tokenizer_init,
skip_server_warmup,
health_generate_headers,
incremental_streaming_output,
disaggregation_mode,
model_config,
@@ -225,6 +229,7 @@ impl ServerArgs {
detokenizer_worker_num: usize,
skip_tokenizer_init: bool,
skip_server_warmup: bool,
health_generate_headers: BTreeMap<String, String>,
incremental_streaming_output: bool,
disaggregation_mode: DisaggregationMode,
model_config: ModelConfig,
@@ -257,6 +262,7 @@ impl ServerArgs {
detokenizer_worker_num,
skip_tokenizer_init,
skip_server_warmup,
health_generate_headers,
incremental_streaming_output,
disaggregation_mode,
model_config,
@@ -297,6 +303,7 @@ impl Default for ServerArgs {
detokenizer_worker_num: 1,
skip_tokenizer_init: false,
skip_server_warmup: false,
health_generate_headers: BTreeMap::new(),
incremental_streaming_output: false,
disaggregation_mode: DisaggregationMode::Null,
model_config: ModelConfig::default(),
@@ -591,6 +598,15 @@ impl ServerArgs {
if self.served_model_name.is_empty() {
return Err("empty 'served_model_name' in server_args".into());
}
for (name, value) in &self.health_generate_headers {
if http::HeaderName::from_bytes(name.as_bytes()).is_err()
|| http::HeaderValue::from_str(value).is_err()
{
return Err(format!(
"invalid 'health_generate_headers' entry {name:?}: {value:?}"
));
}
}
if let Some(grpc_port) = self.grpc_port {
if grpc_port == 0 {
return Err("'grpc_port' must be between 1 and 65535".into());
@@ -704,13 +720,22 @@ mod tests {
}
#[test]
fn validate_requires_served_model_name() {
fn validate_requires_served_model_name_and_valid_health_headers() {
assert!(ServerArgs::default().validate().is_err());
let sa = ServerArgs {
served_model_name: "m".into(),
..Default::default()
};
assert!(sa.validate().is_ok());
let bad = ServerArgs {
health_generate_headers: [("bad header".into(), "v".into())].into(),
..sa
};
assert!(
bad.validate()
.unwrap_err()
.contains("health_generate_headers")
);
}
#[test]
@@ -81,7 +81,9 @@ class TestRustServerExtension(CustomTestCase):
with launch_context(ModelServer) as build_args:
instance = ModelServer.launch(scheduler)
build_args.assert_called_once_with(scheduler, extension=extension)
build_args.assert_called_once_with(
scheduler, extension=extension, health_generate_headers={}
)
extension.Server.assert_called_once_with(
sentinel.args, cores=None, port_offset=1
)