Files
sglang/rust/sglang-server/src/api_server/common.rs
T
2026-09-29 13:40:04 -07:00

357 lines
13 KiB
Rust

//! Common control-plane endpoints — `/server_info`, `/get_model_info`
//! (+ `/model_info` alias), backed by typed operations on
//! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`,
//! which round-trips a generate probe) live in the sibling `native_api` and
//! `openai` modules; the shared `AppState` lives in the parent
//! `api_server` module.
use axum::{
Router,
extract::State,
http::StatusCode,
response::{IntoResponse, Response},
routing::get,
};
use std::sync::Arc;
use super::app::AppState;
use super::frontend_error_status;
use super::native_api::native_error;
use crate::frontend::{FrontendError, ServerInfo};
/// The routes this module owns, mounted by `api_server::serve`.
pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new()
// Control-plane: the frontend performs the typed operation and this
// adapter renders its result as one non-streamed JSON response.
.route("/server_info", get(server_info))
// Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info`
// alias).
.route("/get_model_info", get(model_info))
.route("/model_info", get(model_info))
}
/// Await the frontend's complete semantic server-info result. HTTP status and
/// serialization stay here; scheduler control bytes stay behind the contract.
async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
match state.frontend.server_info().await {
Ok(server_info) => Ok(server_info),
Err(FrontendError::Unavailable) => Err(native_error(
StatusCode::SERVICE_UNAVAILABLE,
"service unavailable",
false,
)),
// Preserve the established HTTP behavior for a scheduler-side control
// stream that disappears, while non-HTTP adapters see Internal via
// FrontendError::kind().
Err(FrontendError::ResponseTruncated) => {
Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response())
}
Err(error @ FrontendError::InvalidResponse(_)) => {
tracing::error!(%error, "server_info: invalid runtime response");
Err((
StatusCode::INTERNAL_SERVER_ERROR,
"bad server_info response",
)
.into_response())
}
Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()),
}
}
/// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static
/// model metadata (no scheduler round-trip).
async fn model_info(State(state): State<Arc<AppState>>) -> Response {
let info = state.frontend.model_info();
(
StatusCode::OK,
[("content-type", "application/json")],
serde_json::to_vec(&info).unwrap_or_default(),
)
.into_response()
}
/// `GET /server_info` — serialize typed, allowlisted runtime metrics plus
/// curated launch metadata. The raw scheduler state embeds
/// `api_key`/`admin_api_key` and is never visible to this HTTP adapter.
///
/// TODO(server_info): Python also includes `kv_events`; add once plumbed.
async fn server_info(State(state): State<Arc<AppState>>) -> Response {
let info = match await_server_info(&state).await {
Ok(info) => info,
Err(resp) => return resp,
};
match serde_json::to_vec(&info) {
Ok(json) => (StatusCode::OK, [("content-type", "application/json")], json).into_response(),
Err(e) => {
tracing::error!(error = %e, "server_info: shaping failed");
(
StatusCode::INTERNAL_SERVER_ERROR,
"bad server_info response",
)
.into_response()
}
}
}
#[cfg(test)]
mod tests {
use axum::{
body::{Body, to_bytes},
http::{Request as HttpRequest, header::CONTENT_TYPE},
};
use bytes::Bytes;
use tower::ServiceExt;
use super::*;
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs};
use crate::message::request::RequestKind;
use crate::message::response::ResponseItem;
use crate::tokenizer_manager::wiring::TmEvent;
fn test_state(
server_args: ServerArgs,
) -> (
Arc<AppState>,
flume::Receiver<TmEvent>,
flume::Receiver<crate::tokenizer_manager::wiring::AbortSource>,
) {
let server_args = Arc::new(server_args);
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::from(server_args.as_ref()),
},
);
(
Arc::new(AppState {
frontend,
server_args,
chat_formatter: None,
}),
intake_rx,
abort_rx,
)
}
async fn response_json(response: Response) -> serde_json::Value {
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers().get(CONTENT_TYPE).unwrap(),
"application/json"
);
let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
serde_json::from_slice(&body).unwrap()
}
#[tokio::test]
async fn model_info_routes_render_the_typed_frontend_contract() {
let (state, _, _) = test_state(ServerArgs {
model_path: "/model".into(),
served_model_name: "served".into(),
tokenizer_path: "/tokenizer".into(),
weight_version: Some("weights-v1".into()),
load_format: Some("safetensors".into()),
reasoning_parser: Some("reasoner".into()),
tool_call_parser: Some("tools".into()),
disaggregation_mode: DisaggregationMode::Prefill,
..Default::default()
});
let app = routes().with_state(state);
for path in ["/get_model_info", "/model_info"] {
let response = app
.clone()
.oneshot(
HttpRequest::builder()
.uri(path)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json(response).await;
assert_eq!(body["model_path"], "/model");
assert_eq!(body["served_model_name"], "served");
assert_eq!(body["tokenizer_path"], "/tokenizer");
assert_eq!(body["is_generation"], true);
assert_eq!(body["weight_version"], "weights-v1");
assert_eq!(body["load_format"], "safetensors");
assert_eq!(body["reasoning_parser"], "reasoner");
assert_eq!(body["tool_call_parser"], "tools");
assert_eq!(body["disaggregation_mode"], "prefill");
assert!(body["preferred_sampling_params"].is_null());
}
}
#[tokio::test]
async fn server_info_route_serializes_only_the_typed_public_subset() {
// Native floats stay JSON numbers; NumPy scalars encoded by Python's
// msgspec enc_hook=str must remain JSON strings, as before the refactor.
assert_server_info_response(rmpv::Value::from(23.616), serde_json::json!(23.616)).await;
assert_server_info_response(rmpv::Value::from("23.616"), serde_json::json!("23.616")).await;
}
async fn assert_server_info_response(
kvcache: rmpv::Value,
expected_kvcache: serde_json::Value,
) {
let (state, intake_rx, abort_rx) = test_state(ServerArgs {
model_path: "/model".into(),
served_model_name: "served".into(),
tokenizer_path: "/tokenizer".into(),
model_config: ModelConfig {
context_len: 4096,
..Default::default()
},
max_total_num_tokens: 8192,
version: "1.2.3".into(),
..Default::default()
});
let request = tokio::spawn(
routes().with_state(state).oneshot(
HttpRequest::builder()
.uri("/server_info")
.body(Body::empty())
.unwrap(),
),
);
let TmEvent::Intake {
request: runtime_request,
admission,
} = intake_rx.recv_async().await.unwrap()
else {
panic!("server_info must submit one control request");
};
assert!(admission.try_accept());
assert!(matches!(runtime_request.kind, RequestKind::Control(_)));
let internal_state = rmpv::Value::Map(vec![
(
rmpv::Value::from("rust_mm_transport"),
rmpv::Value::Map(vec![(
rmpv::Value::from("inline_features"),
rmpv::Value::from(3),
)]),
),
(
rmpv::Value::from("api_key"),
rmpv::Value::from("must-not-leak"),
),
(
rmpv::Value::from("last_gen_throughput"),
rmpv::Value::from(1.5),
),
(
rmpv::Value::from("memory_usage"),
rmpv::Value::Map(vec![
(rmpv::Value::from("kvcache"), kvcache),
(rmpv::Value::from("token_capacity"), rmpv::Value::from(8192)),
(rmpv::Value::from("token_capacity_swa"), rmpv::Value::Nil),
(
rmpv::Value::from("future_uncontracted_metric"),
rmpv::Value::from(99),
),
]),
),
(
rmpv::Value::from("step_time_dict"),
rmpv::Value::Map(vec![(
rmpv::Value::from(4),
rmpv::Value::Array(vec![rmpv::Value::from(0.01)]),
)]),
),
]);
let envelope =
rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal_state)]);
let mut payload = Vec::new();
rmpv::encode::write_value(&mut payload, &envelope).unwrap();
runtime_request
.sink
.try_send(ResponseItem::Control(Bytes::from(payload)))
.unwrap();
let response = request.await.unwrap().unwrap();
let body = response_json(response).await;
assert_eq!(body["model_path"], "/model");
assert_eq!(body["served_model_name"], "served");
assert_eq!(body["tokenizer_path"], "/tokenizer");
assert_eq!(body["max_context_length"], 4096);
assert_eq!(body["max_total_num_tokens"], 8192);
assert_eq!(body["version"], "1.2.3");
assert_eq!(body["frontend"], "rust");
assert_eq!(
body["internal_states"][0]["rust_mm_transport"]["inline_features"],
3
);
assert_eq!(body["internal_states"][0]["last_gen_throughput"], 1.5);
assert_eq!(
body["internal_states"][0]["memory_usage"]["kvcache"],
expected_kvcache
);
assert_eq!(
body["internal_states"][0]["memory_usage"]["token_capacity"],
8192
);
assert_eq!(
body["internal_states"][0]["memory_usage"]["token_capacity_swa"],
serde_json::Value::Null
);
assert_eq!(
body["internal_states"][0]["step_time_dict"]["4"],
serde_json::json!([0.01])
);
assert!(!body.to_string().contains("must-not-leak"));
assert!(
body["internal_states"][0]["memory_usage"]
.get("future_uncontracted_metric")
.is_none(),
"uncontracted scheduler fields must not silently become public API"
);
assert!(abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn server_info_preserves_legacy_499_for_a_truncated_control_response() {
let (state, intake_rx, abort_rx) = test_state(ServerArgs::default());
let response = tokio::spawn(
routes().with_state(state).oneshot(
HttpRequest::builder()
.uri("/server_info")
.body(Body::empty())
.unwrap(),
),
);
let TmEvent::Intake {
request: runtime_request,
admission,
} = intake_rx.recv_async().await.unwrap()
else {
panic!("server_info must submit one control request");
};
assert!(admission.try_accept());
let rid = runtime_request.rid.clone();
drop(runtime_request);
let response = response.await.unwrap().unwrap();
assert_eq!(response.status().as_u16(), 499);
let body = to_bytes(response.into_body(), 1024).await.unwrap();
assert_eq!(body.as_ref(), b"request aborted");
assert!(matches!(
abort_rx.recv_async().await.unwrap(),
crate::tokenizer_manager::wiring::AbortSource::Guard(aborted) if aborted == rid
));
}
}