[Feat] rust-server: Attach configured metadata headers to a healthy /health_generate (#44315)
Co-authored-by: fei-xx <fei-xx@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
fei-xx
shodoco
parent
173e0f2e9b
commit
02a43eeef6
@@ -6,7 +6,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.managers.utils import compute_num_reserved_tokens
|
||||
@@ -27,7 +27,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _build_server_args(
|
||||
scheduler: Scheduler, *, extension: Optional[ModuleType] = None
|
||||
scheduler: Scheduler,
|
||||
*,
|
||||
extension: Optional[ModuleType] = None,
|
||||
health_generate_headers: Optional[Dict[str, str]] = None,
|
||||
) -> ServerArgs:
|
||||
"""The typed launch configuration for the scheduler's embedded Rust server:
|
||||
the ``server_args`` fields it reads, the already-resolved
|
||||
@@ -74,6 +77,7 @@ def _build_server_args(
|
||||
detokenizer_worker_num=get_serving().detokenizer_worker_num,
|
||||
skip_tokenizer_init=get_serving().skip_tokenizer_init,
|
||||
skip_server_warmup=get_serving().skip_server_warmup,
|
||||
health_generate_headers=dict(health_generate_headers or {}),
|
||||
incremental_streaming_output=get_serving().incremental_streaming_output,
|
||||
disaggregation_mode=disaggregation_mode,
|
||||
model_config=ext.ModelConfig(
|
||||
|
||||
@@ -68,6 +68,12 @@ class RustServer:
|
||||
self._multimodal_enabled = mm_spec is not None
|
||||
self._max_per_poll = max_per_poll
|
||||
|
||||
@classmethod
|
||||
def _health_generate_headers(cls, scheduler: Scheduler) -> Dict[str, str]:
|
||||
"""Headers a healthy ``/health_generate`` carries; a deployment's
|
||||
router may read its metadata off the deep health probe."""
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _load_extension(cls) -> ModuleType:
|
||||
from sglang.srt.rust_extensions import load_rust_extension
|
||||
@@ -167,7 +173,11 @@ class RustServer:
|
||||
|
||||
extension = cls._load_extension()
|
||||
server = extension.Server(
|
||||
_build_server_args(scheduler, extension=extension),
|
||||
_build_server_args(
|
||||
scheduler,
|
||||
extension=extension,
|
||||
health_generate_headers=cls._health_generate_headers(scheduler),
|
||||
),
|
||||
# None runs unpinned; otherwise the list carries the pinning decision.
|
||||
cores=server_cores,
|
||||
port_offset=local_dp_rank,
|
||||
|
||||
@@ -26,7 +26,7 @@ use axum::{
|
||||
Json, Router,
|
||||
extract::State,
|
||||
extract::rejection::JsonRejection,
|
||||
http::StatusCode,
|
||||
http::{HeaderName, HeaderValue, StatusCode},
|
||||
response::{
|
||||
IntoResponse, Response,
|
||||
sse::{Event, Sse},
|
||||
@@ -102,12 +102,12 @@ fn health_routes() -> Router<Arc<AppState>> {
|
||||
let timeout = std::time::Duration::from_secs(
|
||||
environ::env_i64("SGLANG_HEALTH_CHECK_TIMEOUT", 20).max(0) as u64,
|
||||
);
|
||||
let probe = get(move |state: State<Arc<AppState>>| health_generate(state, timeout));
|
||||
let health = if environ::env_bool("SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION", true) {
|
||||
probe.clone()
|
||||
get(move |state: State<Arc<AppState>>| health_generate(state, timeout, false))
|
||||
} else {
|
||||
get(health_without_generation)
|
||||
};
|
||||
let probe = get(move |state: State<Arc<AppState>>| health_generate(state, timeout, true));
|
||||
Router::new()
|
||||
.route("/health", health)
|
||||
.route("/health_generate", probe)
|
||||
@@ -125,7 +125,8 @@ async fn health_without_generation(State(state): State<Arc<AppState>>) -> Respon
|
||||
/// producing output. 200 if the response heartbeat advances within `timeout`
|
||||
/// (from `SGLANG_HEALTH_CHECK_TIMEOUT`, frozen at router build), else 503.
|
||||
/// It also returns 503 until startup warmup completes. (`/health` uses the same
|
||||
/// handler when its env gate is on.)
|
||||
/// handler when its env gate is on, without `metadata_headers`.) A healthy
|
||||
/// `/health_generate` carries the configured `health_generate_headers`.
|
||||
///
|
||||
/// Fires a pre-tokenized 1-token probe (`input_ids = [0]`, skips the tokenizer) so
|
||||
/// an idle pipeline produces a frame, then watches the *global*
|
||||
@@ -135,9 +136,24 @@ async fn health_without_generation(State(state): State<Arc<AppState>>) -> Respon
|
||||
async fn health_generate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
timeout: std::time::Duration,
|
||||
metadata_headers: bool,
|
||||
) -> Response {
|
||||
match state.core.probe_health(timeout).await {
|
||||
Ok(HealthStatus::Healthy) => StatusCode::OK.into_response(),
|
||||
Ok(HealthStatus::Healthy) => {
|
||||
let mut response = StatusCode::OK.into_response();
|
||||
if metadata_headers {
|
||||
// Validated at startup (`ServerArgs::validate`).
|
||||
for (name, value) in &state.server_args.health_generate_headers {
|
||||
if let (Ok(name), Ok(value)) = (
|
||||
HeaderName::from_bytes(name.as_bytes()),
|
||||
HeaderValue::from_str(value),
|
||||
) {
|
||||
response.headers_mut().insert(name, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
response
|
||||
}
|
||||
Ok(HealthStatus::NotReady | HealthStatus::Stalled) => {
|
||||
StatusCode::SERVICE_UNAVAILABLE.into_response()
|
||||
}
|
||||
@@ -551,12 +567,61 @@ mod tests {
|
||||
chat_formatter: None,
|
||||
});
|
||||
|
||||
let response = health_generate(State(state), Duration::ZERO).await;
|
||||
let response = health_generate(State(state), Duration::ZERO, true).await;
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert!(intake_rx.try_recv().is_err());
|
||||
assert!(abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
/// A healthy `/health_generate` carries the configured metadata headers;
|
||||
/// `/health` sharing the probe does not.
|
||||
#[tokio::test]
|
||||
async fn healthy_probe_carries_configured_headers() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, _abort_rx) = flume::unbounded();
|
||||
let activity: crate::tokenizer_manager::from_scheduler::ActivityCounter =
|
||||
Default::default();
|
||||
let server_args = crate::message::config::ServerArgs {
|
||||
health_generate_headers: [
|
||||
("x-custom-header".to_string(), "v".to_string()),
|
||||
("x-rust-server".to_string(), "health".to_string()),
|
||||
]
|
||||
.into(),
|
||||
..Default::default()
|
||||
};
|
||||
let state = Arc::new(AppState {
|
||||
core: crate::api_server::core::CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: activity.clone(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::api_server::core::CoreMetadata::default(),
|
||||
},
|
||||
),
|
||||
server_args: Arc::new(server_args),
|
||||
chat_formatter: None,
|
||||
});
|
||||
// Each probe that reaches intake counts as scheduler output.
|
||||
tokio::spawn(async move {
|
||||
while intake_rx.recv_async().await.is_ok() {
|
||||
activity.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
});
|
||||
|
||||
let response = health_generate(State(state.clone()), Duration::from_secs(5), true).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()["x-custom-header"], "v");
|
||||
assert_eq!(response.headers()["x-rust-server"], "health");
|
||||
|
||||
let response = health_generate(State(state), Duration::from_secs(5), false).await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(response.headers().get("x-custom-header").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unary_http_generate_round_trips_through_frontend_handle() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
|
||||
@@ -135,6 +135,9 @@ pub struct ServerArgs {
|
||||
/// Start accepting health checks immediately instead of waiting for the
|
||||
/// main process's startup warmup request to finish.
|
||||
pub skip_server_warmup: bool,
|
||||
/// Extra headers on a successful `/health_generate` response, for a router
|
||||
/// that reads deployment metadata off the deep health probe.
|
||||
pub health_generate_headers: BTreeMap<String, String>,
|
||||
/// Streamed `/generate` frames carry per-step deltas instead of cumulative
|
||||
/// text. Matches the Python `TokenizerManager`.
|
||||
pub incremental_streaming_output: bool,
|
||||
@@ -191,6 +194,7 @@ impl ServerArgs {
|
||||
detokenizer_worker_num,
|
||||
skip_tokenizer_init,
|
||||
skip_server_warmup,
|
||||
health_generate_headers,
|
||||
incremental_streaming_output,
|
||||
disaggregation_mode,
|
||||
model_config,
|
||||
@@ -225,6 +229,7 @@ impl ServerArgs {
|
||||
detokenizer_worker_num: usize,
|
||||
skip_tokenizer_init: bool,
|
||||
skip_server_warmup: bool,
|
||||
health_generate_headers: BTreeMap<String, String>,
|
||||
incremental_streaming_output: bool,
|
||||
disaggregation_mode: DisaggregationMode,
|
||||
model_config: ModelConfig,
|
||||
@@ -257,6 +262,7 @@ impl ServerArgs {
|
||||
detokenizer_worker_num,
|
||||
skip_tokenizer_init,
|
||||
skip_server_warmup,
|
||||
health_generate_headers,
|
||||
incremental_streaming_output,
|
||||
disaggregation_mode,
|
||||
model_config,
|
||||
@@ -297,6 +303,7 @@ impl Default for ServerArgs {
|
||||
detokenizer_worker_num: 1,
|
||||
skip_tokenizer_init: false,
|
||||
skip_server_warmup: false,
|
||||
health_generate_headers: BTreeMap::new(),
|
||||
incremental_streaming_output: false,
|
||||
disaggregation_mode: DisaggregationMode::Null,
|
||||
model_config: ModelConfig::default(),
|
||||
@@ -591,6 +598,15 @@ impl ServerArgs {
|
||||
if self.served_model_name.is_empty() {
|
||||
return Err("empty 'served_model_name' in server_args".into());
|
||||
}
|
||||
for (name, value) in &self.health_generate_headers {
|
||||
if http::HeaderName::from_bytes(name.as_bytes()).is_err()
|
||||
|| http::HeaderValue::from_str(value).is_err()
|
||||
{
|
||||
return Err(format!(
|
||||
"invalid 'health_generate_headers' entry {name:?}: {value:?}"
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(grpc_port) = self.grpc_port {
|
||||
if grpc_port == 0 {
|
||||
return Err("'grpc_port' must be between 1 and 65535".into());
|
||||
@@ -704,13 +720,22 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_requires_served_model_name() {
|
||||
fn validate_requires_served_model_name_and_valid_health_headers() {
|
||||
assert!(ServerArgs::default().validate().is_err());
|
||||
let sa = ServerArgs {
|
||||
served_model_name: "m".into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(sa.validate().is_ok());
|
||||
let bad = ServerArgs {
|
||||
health_generate_headers: [("bad header".into(), "v".into())].into(),
|
||||
..sa
|
||||
};
|
||||
assert!(
|
||||
bad.validate()
|
||||
.unwrap_err()
|
||||
.contains("health_generate_headers")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -81,7 +81,9 @@ class TestRustServerExtension(CustomTestCase):
|
||||
with launch_context(ModelServer) as build_args:
|
||||
instance = ModelServer.launch(scheduler)
|
||||
|
||||
build_args.assert_called_once_with(scheduler, extension=extension)
|
||||
build_args.assert_called_once_with(
|
||||
scheduler, extension=extension, health_generate_headers={}
|
||||
)
|
||||
extension.Server.assert_called_once_with(
|
||||
sentinel.args, cores=None, port_offset=1
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user