diff --git a/rust/sglang-server/src/api_server.rs b/rust/sglang-server/src/api_server.rs index db40d84fad..c23ced6f3a 100644 --- a/rust/sglang-server/src/api_server.rs +++ b/rust/sglang-server/src/api_server.rs @@ -1,9 +1,8 @@ //! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. Only //! this module knows HTTP. Its handlers use the transport-neutral //! [`crate::frontend::FrontendHandle`] to enter the shared runtime pipeline. -//! `/generate` submits a request then awaits one `Done` (unary) or relays SSE -//! frames (`data: {json}` … `[DONE]`), byte-compatible with Python -//! `http_server.generate_request`; `/server_info` reuses it for one control result. +//! Generation handlers render semantic frontend events as unary JSON or SSE; +//! control handlers serialize typed frontend results such as server metadata. pub mod app; mod common; mod disaggregation; @@ -11,3 +10,74 @@ mod frame; mod log; mod native_api; mod openai; + +use axum::http::StatusCode; + +use crate::frontend::{FrontendError, FrontendErrorKind}; + +/// Map transport-neutral failures into HTTP status codes. Response bodies and +/// stream framing remain the responsibility of each HTTP API surface. +pub(crate) fn frontend_error_status(error: &FrontendError) -> StatusCode { + if let FrontendError::RuntimeRejected { + legacy_http_status, .. + } = error + { + return StatusCode::from_u16(*legacy_http_status) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + } + + match error.kind() { + FrontendErrorKind::InvalidArgument => StatusCode::BAD_REQUEST, + FrontendErrorKind::NotFound => StatusCode::NOT_FOUND, + FrontendErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED, + // Existing intake backpressure has historically surfaced as 503. + FrontendErrorKind::ResourceExhausted | FrontendErrorKind::Unavailable => { + StatusCode::SERVICE_UNAVAILABLE + } + FrontendErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"), + FrontendErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT, + FrontendErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frontend_error_categories_map_at_the_http_boundary() { + assert_eq!( + frontend_error_status(&FrontendError::InvalidArgument("bad".into())), + StatusCode::BAD_REQUEST + ); + assert_eq!( + frontend_error_status(&FrontendError::Unavailable), + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!( + frontend_error_status(&FrontendError::Overloaded("full".into())), + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!( + frontend_error_status(&FrontendError::Cancelled("gone".into())).as_u16(), + 499 + ); + assert_eq!( + frontend_error_status(&FrontendError::RuntimeRejected { + kind: FrontendErrorKind::InvalidArgument, + message: "runtime rejected request".into(), + legacy_http_status: 432, + }) + .as_u16(), + 432 + ); + assert_eq!( + frontend_error_status(&FrontendError::InvalidResponse("bad reply".into())), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + frontend_error_status(&FrontendError::Internal("bug".into())), + StatusCode::INTERNAL_SERVER_ERROR + ); + } +} diff --git a/rust/sglang-server/src/api_server/app.rs b/rust/sglang-server/src/api_server/app.rs index 19b84c7ca4..2fa0deb2a9 100644 --- a/rust/sglang-server/src/api_server/app.rs +++ b/rust/sglang-server/src/api_server/app.rs @@ -148,6 +148,7 @@ mod tests { startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), + metadata: crate::frontend::FrontendMetadata::default(), }, ) } diff --git a/rust/sglang-server/src/api_server/common.rs b/rust/sglang-server/src/api_server/common.rs index 38024e4cc7..20f38780d9 100644 --- a/rust/sglang-server/src/api_server/common.rs +++ b/rust/sglang-server/src/api_server/common.rs @@ -1,5 +1,5 @@ //! Common control-plane endpoints — `/server_info`, `/get_model_info` -//! (+ `/model_info` alias), plus the control-request path through +//! (+ `/model_info` alias), backed by typed operations on //! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`, //! which round-trips a generate probe) live in the sibling `native_api` and //! `openai` modules; the shared `AppState` lives in the parent @@ -15,17 +15,15 @@ use axum::{ use std::sync::Arc; use super::app::AppState; +use super::frontend_error_status; use super::native_api::native_error; -use crate::frontend::FrontendError; -use crate::message::config::ServerArgs; -use crate::message::ids::Rid; -use crate::message::io_struct::{ControlRequest, GetInternalStateReq}; +use crate::frontend::{FrontendError, ServerInfo}; /// The routes this module owns, mounted by `api_server::serve`. pub(super) fn routes() -> Router> { Router::new() - // Control-plane: reuses the request FSM (no tokenization), returns one - // non-streamed JSON result. Adding one = a route line + its struct tag. + // Control-plane: the frontend performs the typed operation and this + // adapter renders its result as one non-streamed JSON response. .route("/server_info", get(server_info)) // Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info` // alias). @@ -33,85 +31,57 @@ pub(super) fn routes() -> Router> { .route("/model_info", get(model_info)) } -/// Submit a control request through the request FSM (no tokenization) and await the -/// scheduler's single msgpack result (a `structs.asdict` named map). Returns the -/// raw bytes, or an error `Response` to return as-is. -async fn await_control_result( - state: &AppState, - control: ControlRequest, -) -> Result { - match state.frontend.control(control).await { - Ok(bytes) => Ok(bytes), +/// Await the frontend's complete semantic server-info result. HTTP status and +/// serialization stay here; scheduler control bytes stay behind the contract. +async fn await_server_info(state: &AppState) -> Result { + match state.frontend.server_info().await { + Ok(server_info) => Ok(server_info), Err(FrontendError::Unavailable) => Err(native_error( StatusCode::SERVICE_UNAVAILABLE, "service unavailable", false, )), - Err(FrontendError::Pipeline(error)) => { - let code = StatusCode::from_u16(error.http_status()) - .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - Err((code, error.to_string()).into_response()) - } - Err(FrontendError::ResponseClosed) => { + // Preserve the established HTTP behavior for a scheduler-side control + // stream that disappears, while non-HTTP adapters see Internal via + // FrontendError::kind(). + Err(FrontendError::ResponseTruncated) => { Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response()) } - Err(FrontendError::UnexpectedResponse(message)) => { - Err((StatusCode::INTERNAL_SERVER_ERROR, message).into_response()) - } - Err(FrontendError::InvalidArgument(error)) => { - Err((StatusCode::INTERNAL_SERVER_ERROR, error).into_response()) + Err(error @ FrontendError::InvalidResponse(_)) => { + tracing::error!(%error, "server_info: invalid runtime response"); + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + "bad server_info response", + ) + .into_response()) } + Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()), } } -/// `GET /get_model_info` (+ `/model_info` alias) — static model metadata from -/// `server_args` (no scheduler round-trip); `is_generation` always true. +/// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static +/// model metadata (no scheduler round-trip). async fn model_info(State(state): State>) -> Response { - let sa = &state.server_args; - let body = serde_json::json!({ - "model_path": sa.model_path, - "served_model_name": sa.served_model_name, - "tokenizer_path": sa.tokenizer_path, - "is_generation": true, - // Python's `TokenizerManager` merges this into every request - // (`{**preferred, **client}`); this server has no equivalent yet, so - // `RustServer.launch` REFUSES to start when it is set. It can therefore - // only be null here — echoing it keeps the field's shape. - "preferred_sampling_params": sa.preferred_sampling_params, - // Python answers this through `config_value`, so a control-plane write - // moves it there; here it is the launch value. - "weight_version": sa.weight_version, - "load_format": sa.load_format, - // `auto` never reaches the blob: `resolve_auto_parsers` writes the - // selected parser into `server_args` before the scheduler forks. - "reasoning_parser": sa.reasoning_parser, - "tool_call_parser": sa.tool_call_parser, - "disaggregation_mode": sa.disaggregation_mode, - }); + let info = state.frontend.model_info(); ( StatusCode::OK, [("content-type", "application/json")], - serde_json::to_vec(&body).unwrap_or_default(), + serde_json::to_vec(&info).unwrap_or_default(), ) .into_response() } -/// `GET /server_info` — surface only an allowlist ([`INTERNAL_STATE_ALLOWLIST`] + -/// curated [`ServerArgs`] accessors), never the raw server-args dump (embeds -/// `api_key`/`admin_api_key`; see [`shape_server_info`]). +/// `GET /server_info` — serialize typed, allowlisted runtime metrics plus +/// curated launch metadata. The raw scheduler state embeds +/// `api_key`/`admin_api_key` and is never visible to this HTTP adapter. /// /// TODO(server_info): Python also includes `kv_events`; add once plumbed. async fn server_info(State(state): State>) -> Response { - let bytes = match await_control_result( - &state, - ControlRequest::GetInternalStateReq(GetInternalStateReq::new(Rid::new().to_string())), - ) - .await - { - Ok(b) => b, + let info = match await_server_info(&state).await { + Ok(info) => info, Err(resp) => return resp, }; - match shape_server_info(&bytes, &state.server_args) { + match serde_json::to_vec(&info) { Ok(json) => (StatusCode::OK, [("content-type", "application/json")], json).into_response(), Err(e) => { tracing::error!(error = %e, "server_info: shaping failed"); @@ -124,116 +94,263 @@ async fn server_info(State(state): State>) -> Response { } } -/// Runtime-metric keys `get_internal_state` adds atop the server-args dump. We copy -/// ONLY these out of `internal_state` (an allowlist), so the co-mingled -/// `api_key`/`admin_api_key` can never reach the response. -const INTERNAL_STATE_ALLOWLIST: &[&str] = &[ - "last_gen_throughput", - "memory_usage", - "effective_max_running_requests_per_dp", - "avg_spec_accept_length", - "step_time_dict", - "rust_mm_transport", -]; - -fn shape_server_info(msgpack: &[u8], server_args: &ServerArgs) -> Result, String> { - // GetInternalStateReqOutput asdict → `{ "internal_state": { server-args dump + - // metrics }, ... }`. Pull that inner map out (it is NOT safe to expose whole). - let mut obj: serde_json::Map = - rmp_serde::from_slice(msgpack).map_err(|e| e.to_string())?; - let internal = match obj.remove("internal_state") { - Some(serde_json::Value::Object(m)) => m, - _ => serde_json::Map::new(), +#[cfg(test)] +mod tests { + use axum::{ + body::{Body, to_bytes}, + http::{Request as HttpRequest, header::CONTENT_TYPE}, }; + use bytes::Bytes; + use tower::ServiceExt; - // Copy only the allowlisted runtime metrics — never the raw server-args dump. - let mut state_out = serde_json::Map::new(); - for &k in INTERNAL_STATE_ALLOWLIST { - match internal.get(k) { - Some(v) if !v.is_null() => { - state_out.insert(k.to_string(), v.clone()); - } - _ => {} + use super::*; + use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata}; + use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs}; + use crate::message::request::RequestKind; + use crate::message::response::ResponseItem; + use crate::tokenizer_manager::wiring::TmEvent; + + fn test_state( + server_args: ServerArgs, + ) -> ( + Arc, + flume::Receiver, + flume::Receiver, + ) { + let server_args = Arc::new(server_args); + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, abort_rx) = flume::unbounded(); + let frontend = FrontendHandle::new( + intake_tx, + abort_tx, + FrontendConfig { + response_capacity: 8, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: FrontendMetadata::from(server_args.as_ref()), + }, + ); + ( + Arc::new(AppState { + frontend, + server_args, + chat_formatter: None, + }), + intake_rx, + abort_rx, + ) + } + + async fn response_json(response: Response) -> serde_json::Value { + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get(CONTENT_TYPE).unwrap(), + "application/json" + ); + let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap(); + serde_json::from_slice(&body).unwrap() + } + + #[tokio::test] + async fn model_info_routes_render_the_typed_frontend_contract() { + let (state, _, _) = test_state(ServerArgs { + model_path: "/model".into(), + served_model_name: "served".into(), + tokenizer_path: "/tokenizer".into(), + weight_version: Some("weights-v1".into()), + load_format: Some("safetensors".into()), + reasoning_parser: Some("reasoner".into()), + tool_call_parser: Some("tools".into()), + disaggregation_mode: DisaggregationMode::Prefill, + ..Default::default() + }); + let app = routes().with_state(state); + + for path in ["/get_model_info", "/model_info"] { + let response = app + .clone() + .oneshot( + HttpRequest::builder() + .uri(path) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response_json(response).await; + assert_eq!(body["model_path"], "/model"); + assert_eq!(body["served_model_name"], "served"); + assert_eq!(body["tokenizer_path"], "/tokenizer"); + assert_eq!(body["is_generation"], true); + assert_eq!(body["weight_version"], "weights-v1"); + assert_eq!(body["load_format"], "safetensors"); + assert_eq!(body["reasoning_parser"], "reasoner"); + assert_eq!(body["tool_call_parser"], "tools"); + assert_eq!(body["disaggregation_mode"], "prefill"); + assert!(body["preferred_sampling_params"].is_null()); } } - // Top-level non-secret config from typed accessors (structurally can't surface - // a key field, unlike the raw dump). - let response = serde_json::json!({ - "model_path": server_args.model_path, - "served_model_name": server_args.served_model_name, - "tokenizer_path": server_args.tokenizer_path, - "max_context_length": server_args.model_config.context_len, - "max_total_num_tokens": server_args.max_total_num_tokens, - "version": server_args.version, - "frontend": "rust", - "internal_states": [serde_json::Value::Object(state_out)], - }); - serde_json::to_vec(&response).map_err(|e| e.to_string()) -} + #[tokio::test] + async fn server_info_route_serializes_only_the_typed_public_subset() { + // Native floats stay JSON numbers; NumPy scalars encoded by Python's + // msgspec enc_hook=str must remain JSON strings, as before the refactor. + assert_server_info_response(rmpv::Value::from(23.616), serde_json::json!(23.616)).await; + assert_server_info_response(rmpv::Value::from("23.616"), serde_json::json!("23.616")).await; + } -#[cfg(test)] -mod tests { - use super::*; + async fn assert_server_info_response( + kvcache: rmpv::Value, + expected_kvcache: serde_json::Value, + ) { + let (state, intake_rx, abort_rx) = test_state(ServerArgs { + model_path: "/model".into(), + served_model_name: "served".into(), + tokenizer_path: "/tokenizer".into(), + model_config: ModelConfig { + context_len: 4096, + ..Default::default() + }, + max_total_num_tokens: 8192, + version: "1.2.3".into(), + ..Default::default() + }); + let request = tokio::spawn( + routes().with_state(state).oneshot( + HttpRequest::builder() + .uri("/server_info") + .body(Body::empty()) + .unwrap(), + ), + ); - /// The scheduler's `internal_state` embeds the full server-args dump (incl. - /// `api_key`/`admin_api_key`). `/server_info` must surface only the allowlisted - /// runtime metrics + curated config — never the secrets — and must not re-nest - /// the dump under `internal_states[].internal_state`. - #[test] - fn shape_server_info_excludes_secrets_and_dump() { - // GetInternalStateReqOutput.asdict → { "internal_state": { …dump+metrics… } }. - let internal = rmpv::Value::Map(vec![ + let TmEvent::Intake { + request: runtime_request, + admission, + } = intake_rx.recv_async().await.unwrap() + else { + panic!("server_info must submit one control request"); + }; + assert!(admission.try_accept()); + assert!(matches!(runtime_request.kind, RequestKind::Control(_))); + + let internal_state = rmpv::Value::Map(vec![ + ( + rmpv::Value::from("rust_mm_transport"), + rmpv::Value::Map(vec![( + rmpv::Value::from("inline_features"), + rmpv::Value::from(3), + )]), + ), ( rmpv::Value::from("api_key"), - rmpv::Value::from("secret-token"), + rmpv::Value::from("must-not-leak"), ), - ( - rmpv::Value::from("admin_api_key"), - rmpv::Value::from("admin-token"), - ), - (rmpv::Value::from("model_path"), rmpv::Value::from("/m")), ( rmpv::Value::from("last_gen_throughput"), rmpv::Value::from(1.5), ), ( - rmpv::Value::from("effective_max_running_requests_per_dp"), - rmpv::Value::from(32), + rmpv::Value::from("memory_usage"), + rmpv::Value::Map(vec![ + (rmpv::Value::from("kvcache"), kvcache), + (rmpv::Value::from("token_capacity"), rmpv::Value::from(8192)), + (rmpv::Value::from("token_capacity_swa"), rmpv::Value::Nil), + ( + rmpv::Value::from("future_uncontracted_metric"), + rmpv::Value::from(99), + ), + ]), + ), + ( + rmpv::Value::from("step_time_dict"), + rmpv::Value::Map(vec![( + rmpv::Value::from(4), + rmpv::Value::Array(vec![rmpv::Value::from(0.01)]), + )]), ), ]); - let outer = rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal)]); - let mut msgpack = Vec::new(); - rmpv::encode::write_value(&mut msgpack, &outer).unwrap(); + let envelope = + rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal_state)]); + let mut payload = Vec::new(); + rmpv::encode::write_value(&mut payload, &envelope).unwrap(); + runtime_request + .sink + .try_send(ResponseItem::Control(Bytes::from(payload))) + .unwrap(); - // `api_key` is deliberately NOT a `ServerArgs` field — the typed schema - // cannot carry it — so the only place it could leak from is the raw - // scheduler dump shaped above. - let sa = ServerArgs { - model_path: "/m".into(), - ..Default::default() + let response = request.await.unwrap().unwrap(); + let body = response_json(response).await; + assert_eq!(body["model_path"], "/model"); + assert_eq!(body["served_model_name"], "served"); + assert_eq!(body["tokenizer_path"], "/tokenizer"); + assert_eq!(body["max_context_length"], 4096); + assert_eq!(body["max_total_num_tokens"], 8192); + assert_eq!(body["version"], "1.2.3"); + assert_eq!(body["frontend"], "rust"); + assert_eq!( + body["internal_states"][0]["rust_mm_transport"]["inline_features"], + 3 + ); + assert_eq!(body["internal_states"][0]["last_gen_throughput"], 1.5); + assert_eq!( + body["internal_states"][0]["memory_usage"]["kvcache"], + expected_kvcache + ); + assert_eq!( + body["internal_states"][0]["memory_usage"]["token_capacity"], + 8192 + ); + assert_eq!( + body["internal_states"][0]["memory_usage"]["token_capacity_swa"], + serde_json::Value::Null + ); + assert_eq!( + body["internal_states"][0]["step_time_dict"]["4"], + serde_json::json!([0.01]) + ); + assert!(!body.to_string().contains("must-not-leak")); + assert!( + body["internal_states"][0]["memory_usage"] + .get("future_uncontracted_metric") + .is_none(), + "uncontracted scheduler fields must not silently become public API" + ); + assert!(abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn server_info_preserves_legacy_499_for_a_truncated_control_response() { + let (state, intake_rx, abort_rx) = test_state(ServerArgs::default()); + let response = tokio::spawn( + routes().with_state(state).oneshot( + HttpRequest::builder() + .uri("/server_info") + .body(Body::empty()) + .unwrap(), + ), + ); + + let TmEvent::Intake { + request: runtime_request, + admission, + } = intake_rx.recv_async().await.unwrap() + else { + panic!("server_info must submit one control request"); }; - let out = shape_server_info(&msgpack, &sa).unwrap(); - let text = String::from_utf8(out.clone()).unwrap(); - // No secret leaks anywhere in the serialized response. - assert!(!text.contains("secret-token"), "api_key leaked: {text}"); - assert!( - !text.contains("admin-token"), - "admin_api_key leaked: {text}" - ); + assert!(admission.try_accept()); + let rid = runtime_request.rid.clone(); + drop(runtime_request); - let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); - // Allowlisted metric surfaced; the whole dump did not. - let state0 = &v["internal_states"][0]; - assert_eq!(state0["last_gen_throughput"], 1.5); - assert_eq!(state0["effective_max_running_requests_per_dp"], 32); - assert!( - state0.get("internal_state").is_none(), - "must not re-nest the dump under internal_state" - ); - assert!(state0.get("api_key").is_none()); - // Curated top-level config comes from typed accessors, not the dump. - assert_eq!(v["model_path"], "/m"); - assert_eq!(v["frontend"], "rust"); + let response = response.await.unwrap().unwrap(); + assert_eq!(response.status().as_u16(), 499); + let body = to_bytes(response.into_body(), 1024).await.unwrap(); + assert_eq!(body.as_ref(), b"request aborted"); + assert!(matches!( + abort_rx.recv_async().await.unwrap(), + crate::tokenizer_manager::wiring::AbortSource::Guard(aborted) if aborted == rid + )); } } diff --git a/rust/sglang-server/src/api_server/frame.rs b/rust/sglang-server/src/api_server/frame.rs index 64e5f7a422..b669ceecaa 100644 --- a/rust/sglang-server/src/api_server/frame.rs +++ b/rust/sglang-server/src/api_server/frame.rs @@ -1,10 +1,11 @@ //! Frame shaping for the native `/generate` protocol: the cumulative -//! [`OutputAccumulator`] plus the functions that render [`ChunkEvent`]s / +//! [`OutputAccumulator`] plus the functions that render [`FrontendOutput`]s / //! accumulated state into wire JSON (`meta_info`, logprob tuples, error and //! abort frames). No HTTP here — the sibling `native_api` module owns the handlers //! and streams; it calls these per frame. -use crate::message::response::{ChunkEvent, ChunkExtras}; +use crate::frontend::FrontendOutput; +use crate::message::response::ChunkExtras; /// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when /// `return_text_in_logprobs` supplied a text buffer, else `null`. @@ -159,10 +160,10 @@ fn hidden_states_rows(vals: &[f32], lens: &[u32]) -> serde_json::Value { serde_json::Value::Array(rows) } -/// Format a decoded [`ChunkEvent`] as one SGLang `/generate` frame's JSON. `rid` +/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid` /// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is /// just the shard routing key. -pub(super) fn frame_value(out: &ChunkEvent, rid: &str) -> serde_json::Value { +pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value { let mut v = serde_json::json!({ "text": out.text, "meta_info": { @@ -318,7 +319,7 @@ pub(super) fn tag_value(mut v: serde_json::Value, index: Option) -> Strin /// One streaming frame's JSON: cumulative ignores `delta`, incremental ships it. pub(super) fn stream_frame_string( - delta: ChunkEvent, + delta: FrontendOutput, acc: &OutputAccumulator, incremental: bool, rid_str: &str, @@ -344,7 +345,7 @@ pub(super) fn cumulative_frame_string( /// Format one streaming frame: the accumulator's cumulative view (default), or this /// step's delta with the cumulative token count in `meta_info` (matching Python). pub(super) fn stream_frame_value( - delta: ChunkEvent, + delta: FrontendOutput, acc: &OutputAccumulator, incremental: bool, rid_str: &str, @@ -358,15 +359,15 @@ pub(super) fn stream_frame_value( } } -/// Folds per-chunk [`ChunkEvent`] deltas into a cumulative view — used by the drain +/// Folds per-chunk [`FrontendOutput`] deltas into a cumulative view — used by the drain /// loops needing cumulative output (every unary response + the cumulative SGLang /// stream; OpenAI streaming forwards deltas and skips this). Holds a single -/// [`ChunkEvent`] so `snapshot` hands back a **borrow** per frame — no per-frame +/// [`FrontendOutput`] so `snapshot` hands back a **borrow** per frame — no per-frame /// clone of the growing buffers (that added O(T²) atop the wire's inherent O(T²)). /// Shared with the [`openai`] submodule. #[derive(Default)] pub(super) struct OutputAccumulator { - out: ChunkEvent, + out: FrontendOutput, /// Serialized cumulative `output_ids` body (`"1,2,3"`, no brackets), appended per /// delta so a frame memcpy's it instead of rebuilding the array — O(T), not O(T²). ids_json: String, @@ -414,7 +415,7 @@ impl OutputAccumulator { /// Fold one delta frame in. Output families concatenate; input families and /// hidden states are set-once / last-writer-wins (they ride the prefill/final /// chunk), matching the Python `meta_info` assignment. - pub(super) fn fold(&mut self, d: &ChunkEvent) { + pub(super) fn fold(&mut self, d: &FrontendOutput) { use std::fmt::Write; // Grow the memoized serializations alongside the raw cumulative buffers. @@ -427,7 +428,6 @@ impl OutputAccumulator { } let o = &mut self.out; - o.rid.clone_from(&d.rid); // constant across the request; keeps the accumulated view coherent o.text.push_str(&d.text); o.token_ids.extend_from_slice(&d.token_ids); // token_ids doubles as output_ids o.completion_tokens += d.completion_tokens; @@ -532,12 +532,12 @@ impl OutputAccumulator { } /// Borrow the cumulative output for an intermediate streaming frame. - pub(super) fn snapshot(&self) -> &ChunkEvent { + pub(super) fn snapshot(&self) -> &FrontendOutput { &self.out } /// Consume into the final cumulative output. - pub(super) fn into_output(self) -> ChunkEvent { + pub(super) fn into_output(self) -> FrontendOutput { self.out } } @@ -606,12 +606,12 @@ mod tests { ); } - /// End-to-end: a `ChunkEvent` carrying a prompt-logprob request (first input + /// End-to-end: a `FrontendOutput` carrying a prompt-logprob request (first input /// logprob is the `NaN` sentinel) formats without panicking and emits /// `input_token_logprobs` with a leading `[null, token_id, text]`. #[test] fn prompt_logprob_frame_emits_null_first() { - let out = ChunkEvent { + let out = FrontendOutput { extras: Some(Box::new(ChunkExtras { in_lp_val: vec![f32::NAN, -0.5], in_lp_idx: vec![10, 20], @@ -632,7 +632,7 @@ mod tests { #[test] fn accumulator_snapshot_is_cumulative() { let mut acc = OutputAccumulator::default(); - acc.fold(&ChunkEvent { + acc.fold(&FrontendOutput { text: "he".into(), token_ids: vec![1, 2], completion_tokens: 2, @@ -643,7 +643,7 @@ mod tests { assert_eq!(s.text, "he"); assert_eq!(s.token_ids, vec![1, 2]); } - acc.fold(&ChunkEvent { + acc.fold(&FrontendOutput { text: "llo".into(), token_ids: vec![3], completion_tokens: 1, @@ -681,32 +681,28 @@ mod tests { #[test] fn cumulative_frame_json_matches_serde() { let deltas = [ - ChunkEvent { - rid: "7".into(), + FrontendOutput { text: String::new(), token_ids: vec![], completion_tokens: 0, prompt_tokens: 128, ..Default::default() }, - ChunkEvent { - rid: "7".into(), + FrontendOutput { text: "He\"llo\n\t".into(), token_ids: vec![1000], completion_tokens: 1, prompt_tokens: 128, ..Default::default() }, - ChunkEvent { - rid: "7".into(), + FrontendOutput { text: " 世界 🌍 \\".into(), token_ids: vec![-2, 3], completion_tokens: 2, prompt_tokens: 128, ..Default::default() }, - ChunkEvent { - rid: "7".into(), + FrontendOutput { text: "!".into(), token_ids: vec![9], completion_tokens: 1, @@ -752,8 +748,7 @@ mod tests { }; let deltas = [ // Prefill: the set-once input families and a null top-k position. - ChunkEvent { - rid: "9".into(), + FrontendOutput { prompt_tokens: 4, extras: Some(Box::new(ChunkExtras { in_lp_val: vec![f32::NAN, -1.5], @@ -771,8 +766,7 @@ mod tests { })), ..Default::default() }, - ChunkEvent { - rid: "9".into(), + FrontendOutput { text: "He\"llo".into(), token_ids: vec![100], completion_tokens: 1, @@ -793,8 +787,7 @@ mod tests { })), ..Default::default() }, - ChunkEvent { - rid: "9".into(), + FrontendOutput { text: " 世界".into(), token_ids: vec![-2, 3], completion_tokens: 2, @@ -847,8 +840,7 @@ mod tests { #[test] fn mismatched_logprob_texts_fall_back_to_the_value_path() { let mut acc = OutputAccumulator::default(); - acc.fold(&ChunkEvent { - rid: "1".into(), + acc.fold(&FrontendOutput { extras: Some(Box::new(ChunkExtras { out_lp_val: vec![-0.5], out_lp_idx: vec![5], @@ -857,8 +849,7 @@ mod tests { ..Default::default() }); assert!(cumulative_frame_json(&acc, "1", None).is_some()); - acc.fold(&ChunkEvent { - rid: "1".into(), + acc.fold(&FrontendOutput { extras: Some(Box::new(ChunkExtras { out_lp_val: vec![-0.25], out_lp_idx: vec![6], diff --git a/rust/sglang-server/src/api_server/native_api.rs b/rust/sglang-server/src/api_server/native_api.rs index 0dc9d6c7b6..ca06f7bd80 100644 --- a/rust/sglang-server/src/api_server/native_api.rs +++ b/rust/sglang-server/src/api_server/native_api.rs @@ -14,10 +14,13 @@ use super::app::AppState; use super::frame::{ OutputAccumulator, cumulative_frame_string, frame_value, stream_frame_string, tag_value, }; -use crate::frontend::{FrontendCall, FrontendError, HealthStatus}; +use super::frontend_error_status; +use crate::frontend::{ + FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus, +}; +#[cfg(test)] use crate::message::ids::Rid; -use crate::message::request::{GenerateBody, GenerateRequest}; -use crate::message::response::{ChunkEvent, ResponseItem}; +use crate::message::request::GenerateBody; use crate::utils::{ environ, response::{error_response, error_value}, @@ -87,19 +90,7 @@ pub(super) fn native_error(code: StatusCode, message: &str, stream: bool) -> Res } fn native_frontend_error(error: FrontendError, stream: bool) -> Response { - match error { - FrontendError::Unavailable => { - native_error(StatusCode::SERVICE_UNAVAILABLE, &error.to_string(), stream) - } - FrontendError::InvalidArgument(_) => { - native_error(StatusCode::BAD_REQUEST, &error.to_string(), stream) - } - _ => native_error( - StatusCode::INTERNAL_SERVER_ERROR, - &error.to_string(), - stream, - ), - } + native_error(frontend_error_status(&error), &error.to_string(), stream) } /// `/health` + `/health_generate`. Both env knobs are resolved ONCE here, at @@ -196,10 +187,9 @@ async fn generate( // payloads. `is_batch` = list form → the response is a JSON array. let (payloads, is_batch) = match body.into_requests() { Ok(v) => v, - // The error carries its own status (a bad batch is `Validation` → 400). + // Request normalization only returns validation failures. Err(e) => { - let code = StatusCode::from_u16(e.http_status()).unwrap_or(StatusCode::BAD_REQUEST); - return native_error(code, &e.to_string(), stream); + return native_error(StatusCode::BAD_REQUEST, &e.to_string(), stream); } }; // Python starts APIServerReqTimeStats after request normalization and before @@ -225,7 +215,7 @@ async fn generate( /// SSE frames or fold to one unary response. async fn generate_single( state: &AppState, - req: GenerateRequest, + req: FrontendRequest, stream: bool, timing: RequestTiming, ) -> Response { @@ -235,7 +225,7 @@ async fn generate_single( Ok(call) => call, Err(error) => return native_frontend_error(error, stream), }; - let rid = call.rid().clone(); + let rid = call.public_id().to_owned(); // Cumulative frames (SGLang default) vs per-step deltas. let incremental = state.server_args.incremental_streaming_output; @@ -249,7 +239,7 @@ async fn generate_single( } else { // Unary: fold to the terminal and respond once. `FrontendCall` disarms // itself on a terminal item; truncation leaves it armed for drop cleanup. - let (status, value) = drain_unary(&mut call, rid.client_facing(), timing).await; + let (status, value) = drain_unary(&mut call, &rid, timing).await; (status, Json(value)).into_response() } } @@ -263,41 +253,28 @@ async fn drain_unary( let mut acc = OutputAccumulator::default(); while let Some(item) = call.recv().await { match item { - ResponseItem::Frame(out) => { + FrontendEvent::Delta(out) => { timing.observe_first_output(); acc.fold(&out); } - ResponseItem::Done(out) => { + FrontendEvent::Finished(out) => { timing.observe_first_output(); timing.finish(); acc.fold(&out); let final_out = acc.into_output(); - // A validation abort carries its own HTTP status + diagnostic. - if let Some((code, message)) = final_out - .finish_reason - .as_ref() - .and_then(|f| f.abort_status()) - { - let status = - StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - return (status, error_value(code, message)); - } let mut value = frame_value(&final_out, rid_str); add_e2e_latency(&mut value, &timing); return (StatusCode::OK, value); } - ResponseItem::Error(e) => { + FrontendEvent::Failed(error) => { timing.finish(); - let code = e.http_status(); - let status = - StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - return (status, error_value(code, &e.to_string())); + let status = frontend_error_status(&error); + return (status, error_value(status.as_u16(), &error.to_string())); } - ResponseItem::Control(_) | ResponseItem::Data(_) => continue, // never on `/generate` } } - // Sender dropped without a terminal item: the shard dropped this request (a - // truncation — a client disconnect would have dropped the handler future). + // Defensive fallback: FrontendCall normally turns premature runtime + // termination into a terminal Failed event before exposing stream end. ( StatusCode::INTERNAL_SERVER_ERROR, error_value(500, "response truncated before completion"), @@ -312,7 +289,7 @@ async fn drain_unary( /// own `{ "error": … }` entry; the batch response is 200. async fn generate_batch( state: &AppState, - requests: Vec, + requests: Vec, stream: bool, timing: RequestTiming, ) -> Response { @@ -341,7 +318,7 @@ async fn generate_batch( // its own terminal output promptly (important for per-item e2e_latency). let drained = futures::future::join_all(calls.into_iter().map( |(mut call, request_timing)| async move { - let client_rid = call.rid().client_facing().to_owned(); + let client_rid = call.public_id().to_owned(); let (_status, value) = drain_unary(&mut call, &client_rid, request_timing).await; value }, @@ -351,21 +328,20 @@ async fn generate_batch( } } -/// Await the next item from `call`, then drain whatever queued behind it (so the caller -/// can coalesce a backlog, as Python's `state.out_list` does), handing the call -/// back for `FuturesUnordered` to re-poll. Empty result = channel closed. +/// Await the next event from `call`, then drain whatever queued behind it (so the +/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the +/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic +/// stream was already exhausted. async fn recv_indexed( index: usize, mut call: FrontendCall, -) -> (usize, FrontendCall, Vec) { +) -> (usize, FrontendCall, Vec) { let mut items = Vec::new(); match call.recv().await { Some(item) => items.push(item), - None => return (index, call, items), // closed - } - while let Ok(item) = call.try_recv() { - items.push(item); + None => return (index, call, items), // already exhausted } + call.drain_ready(&mut items); (index, call, items) } @@ -381,9 +357,9 @@ fn generation_event_stream( use futures::StreamExt; let n = calls.len(); - let rid_strs: Vec = calls + let rid_strs: Vec = calls .iter() - .map(|(call, _)| call.rid().clone()) + .map(|(call, _)| call.public_id().to_owned()) .collect(); let mut timings: Vec = calls .iter() @@ -404,8 +380,9 @@ fn generation_event_stream( while let Some((i, call, items)) = futs.next().await { if items.is_empty() { - // Channel closed with no terminal → truncation for this item; - // leave its rid armed so the scheduler work is aborted. + // Defensive fallback: a premature runtime close is normally a + // Failed event produced by FrontendCall. Keep this item's call + // armed if its semantic stream somehow ends unexpectedly. yield tag_value(error_value(500, "response truncated before completion"), idx(i)); continue; } @@ -418,48 +395,45 @@ fn generation_event_stream( for item in items { match item { - ResponseItem::Frame(out) => { + FrontendEvent::Delta(out) => { timings[i].observe_first_output(); accs[i].fold(&out); if incremental { - yield stream_frame_string(out, &accs[i], true, rid_strs[i].client_facing(), idx(i)); + yield stream_frame_string(out, &accs[i], true, &rid_strs[i], idx(i)); } else { coalesced = true; } } - ResponseItem::Done(out) => { + FrontendEvent::Finished(out) => { timings[i].observe_first_output(); timings[i].finish(); accs[i].fold(&out); terminal = Some(out); } - ResponseItem::Error(e) => { + FrontendEvent::Failed(error) => { timings[i].finish(); - failed = Some(e); + failed = Some(error); } - ResponseItem::Control(_) | ResponseItem::Data(_) => {} // never on /generate } } - if let Some(e) = failed { - yield tag_value(error_value(e.http_status(), &e.to_string()), idx(i)); + if let Some(error) = failed { + let status = frontend_error_status(&error); + yield tag_value(error_value(status.as_u16(), &error.to_string()), idx(i)); } else if let Some(out) = terminal { - // A validation abort → an error object, not a frame. The final frame - // carries the full cumulative state, so any coalesced ones are moot. - yield match out.finish_reason.as_ref().and_then(|f| f.abort_status()) { - Some((code, message)) => tag_value(error_value(code, message), idx(i)), - None => terminal_stream_frame_string( - out, - &accs[i], - incremental, - rid_strs[i].client_facing(), - idx(i), - &timings[i], - ), - }; + // The final frame carries the full cumulative state, so any + // coalesced non-terminal frames are moot. + yield terminal_stream_frame_string( + out, + &accs[i], + incremental, + &rid_strs[i], + idx(i), + &timings[i], + ); } else { if coalesced { - yield cumulative_frame_string(&accs[i], rid_strs[i].client_facing(), idx(i)); + yield cumulative_frame_string(&accs[i], &rid_strs[i], idx(i)); } futs.push(recv_indexed(i, call)); // keep this item flowing } @@ -471,7 +445,7 @@ fn generation_event_stream( /// Python's `e2e_latency` is `finished_time - created_time`, in seconds, and is /// attached only when the request finishes. The Rust native API owns the same /// lifecycle boundary, so it adds the value while handling the terminal egress -/// item rather than putting API-only timing onto every scheduler `ChunkEvent`. +/// item rather than putting API-only timing onto every shared frontend output. fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) { let (time_to_first_token, e2e_latency) = timing .terminal_latencies() @@ -484,7 +458,7 @@ fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) { /// memoized fast path; the one terminal frame uses the Value path so it can carry /// the request-local `e2e_latency`, exactly as Python does. fn terminal_stream_frame_string( - out: ChunkEvent, + out: FrontendOutput, acc: &OutputAccumulator, incremental: bool, rid_str: &str, @@ -500,7 +474,7 @@ fn terminal_stream_frame_string( mod tests { use super::*; use crate::message::request::RequestKind; - use crate::message::response::ChunkEvent; + use crate::message::response::{ChunkEvent, ResponseItem}; use crate::tokenizer_manager::wiring::TmEvent; use crate::utils::error::Error; use axum::body::Body; @@ -568,6 +542,7 @@ mod tests { startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), + metadata: crate::frontend::FrontendMetadata::default(), }, ), server_args: Arc::new(crate::message::config::ServerArgs::default()), @@ -593,6 +568,7 @@ mod tests { startup_ready: true, is_disaggregation: false, mm_limits: Default::default(), + metadata: crate::frontend::FrontendMetadata::default(), }, ); let state = Arc::new(AppState { @@ -825,6 +801,32 @@ mod tests { assert_eq!(stream.next().await.unwrap(), "[DONE]"); } + #[tokio::test] + async fn scheduler_rejection_preserves_legacy_http_status() { + let (tx, rx) = mpsc::channel(8); + let calls = vec![timed_call(10, rx)]; + let stream = generation_event_stream(calls, false, false); + futures::pin_mut!(stream); + + tx.send(ResponseItem::Done(ChunkEvent { + finish_reason: serde_json::from_value(serde_json::json!({ + "type": "abort", + "message": "media decode failed", + "status_code": 432, + "err_type": "BadRequestError" + })) + .unwrap(), + ..Default::default() + })) + .await + .unwrap(); + + let failure = parse(&stream.next().await.unwrap()); + assert_eq!(failure["error"]["code"], 432); + assert_eq!(failure["error"]["message"], "media decode failed"); + assert_eq!(stream.next().await.unwrap(), "[DONE]"); + } + /// `incremental=true`: each frame carries this step's **delta** text/output_ids, /// but `meta_info.completion_tokens` stays cumulative (matching Python). #[tokio::test] @@ -905,6 +907,32 @@ mod tests { assert_eq!(stream.next().await.unwrap(), "[DONE]"); } + /// A runtime close must not overtake output that was already queued. The + /// cumulative adapter first exposes the produced text, then reports the + /// truncation on the next event, preserving the Step 2 HTTP behavior. + #[tokio::test] + async fn cumulative_output_precedes_truncation_error() { + let (tx, rx) = mpsc::channel(8); + let calls = vec![timed_call(10, rx)]; + let stream = generation_event_stream(calls, false, false); + futures::pin_mut!(stream); + + tx.send(frame(10, "already-produced")).await.unwrap(); + drop(tx); + + let output = parse(&stream.next().await.unwrap()); + assert_eq!(output["text"], "already-produced"); + + let failure = parse(&stream.next().await.unwrap()); + assert_eq!(failure["error"]["code"], 500); + assert_eq!( + failure["error"]["message"], + "response truncated before completion" + ); + assert_eq!(stream.next().await.unwrap(), "[DONE]"); + assert!(stream.next().await.is_none()); + } + /// Incremental frames are *deltas*, so a backlog must emit every one — dropping /// any would silently lose tokens. Only the cumulative protocol may coalesce. #[tokio::test] diff --git a/rust/sglang-server/src/api_server/openai.rs b/rust/sglang-server/src/api_server/openai.rs index 76fff2b320..589968065c 100644 --- a/rust/sglang-server/src/api_server/openai.rs +++ b/rust/sglang-server/src/api_server/openai.rs @@ -1,8 +1,8 @@ //! OpenAI-compatible generation endpoints. //! //! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI -//! request and response primitives. Scheduler [`ChunkEvent`] values remain the one -//! backend output type for both unary and streaming responses. +//! request and response primitives. [`FrontendOutput`] remains the one backend +//! output type for both unary and streaming responses. use axum::{Router, http::StatusCode, response::Response}; use futures::StreamExt; @@ -22,10 +22,11 @@ pub(super) use template::{ChatFormatter, ChatTemplateKwargs}; use super::app::AppState; use super::frame::OutputAccumulator; -use crate::frontend::{FrontendCall, FrontendError}; +use super::frontend_error_status; +use crate::frontend::{ + FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, +}; use crate::message::config::ServerArgs; -use crate::message::request::GenerateRequest; -use crate::message::response::{ChunkEvent, ResponseItem}; use crate::tokenizer_manager::tokenizer; use crate::utils::response::error_response; @@ -115,23 +116,21 @@ pub(super) fn openai_error(code: StatusCode, message: impl Into, stream: } /// Drain one submitted request to its terminal output, fold frames, and map -/// errors / validation aborts / truncation to `(status, message)` for the -/// OpenAI error shape. The call owns cancellation and disarms itself. -async fn collect_output(mut call: FrontendCall) -> Result { +/// semantic failures / truncation to `(status, message)` for the OpenAI error +/// shape. The call owns cancellation and disarms itself. +async fn collect_output(mut call: FrontendCall) -> Result { let mut accumulator = OutputAccumulator::default(); let output = loop { match call.recv().await { - Some(ResponseItem::Frame(output)) => accumulator.fold(&output), - Some(ResponseItem::Done(output)) => { + Some(FrontendEvent::Delta(output)) => accumulator.fold(&output), + Some(FrontendEvent::Finished(output)) => { accumulator.fold(&output); break accumulator.into_output(); } - Some(ResponseItem::Error(error)) => { - let status = StatusCode::from_u16(error.http_status()) - .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Some(FrontendEvent::Failed(error)) => { + let status = frontend_error_status(&error); return Err((status, error.to_string())); } - Some(ResponseItem::Control(_)) | Some(ResponseItem::Data(_)) => {} None => { return Err(( StatusCode::INTERNAL_SERVER_ERROR, @@ -140,22 +139,12 @@ async fn collect_output(mut call: FrontendCall) -> Result Result { match state.frontend.generate(request).await { @@ -179,18 +168,14 @@ async fn submit_generation( fn indexed_decode_stream( index: usize, call: FrontendCall, -) -> futures::stream::BoxStream<'static, (usize, Option)> { +) -> futures::stream::BoxStream<'static, (usize, FrontendEvent)> { futures::stream::unfold((call, false), move |(mut call, finished)| async move { if finished { return None; } - match call.recv().await { - Some(item) => { - let finished = matches!(item, ResponseItem::Done(_) | ResponseItem::Error(_)); - Some(((index, Some(item)), (call, finished))) - } - None => Some(((index, None), (call, true))), - } + let event = call.recv().await?; + let finished = event.is_terminal(); + Some(((index, event), (call, finished))) }) .boxed() } diff --git a/rust/sglang-server/src/api_server/openai/chat.rs b/rust/sglang-server/src/api_server/openai/chat.rs index defa1beb0d..76aabbe03e 100644 --- a/rust/sglang-server/src/api_server/openai/chat.rs +++ b/rust/sglang-server/src/api_server/openai/chat.rs @@ -36,11 +36,11 @@ use super::{ AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload, indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32, }; -use crate::frontend::FrontendCall; +use crate::api_server::frontend_error_status; +use crate::frontend::{FrontendCall, FrontendEvent, FrontendRequest}; use crate::message::config::{DefaultSamplingParams, ServerArgs}; use crate::message::ids::Rid; -use crate::message::request::GenerateRequest; -use crate::message::response::{ChunkExtras, ResponseItem}; +use crate::message::response::ChunkExtras; use crate::message::sampling::SamplingParams; use crate::message::types::OneOrMany; @@ -204,7 +204,7 @@ async fn chat_completions( .expect("chat prompt exists until the last choice") .clone() }; - let native = GenerateRequest { + let native = FrontendRequest { rid: rid.clone(), text: Some(choice_prompt), // Rendered templates own their special tokens — the pool must not @@ -580,47 +580,20 @@ pub(super) fn chat_event_stream( } let mut events = futures::stream::select_all(streams); - while let Some((index, item)) = events.next().await { - let Some(item) = item else { - yield Annotated { - data: None, - id: None, - event: None, - comment: None, - error: Some(error_payload(StatusCode::INTERNAL_SERVER_ERROR, "response truncated before completion").to_string()), - }; - continue; - }; - let output = match item { - ResponseItem::Frame(output) => output, - ResponseItem::Done(output) => output, - ResponseItem::Error(error) => { + while let Some((index, event)) = events.next().await { + let output = match event { + FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output, + FrontendEvent::Failed(error) => { yield Annotated { data: None, id: None, event: None, comment: None, - error: Some(error_payload(StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), error.to_string()).to_string()), + error: Some(error_payload(frontend_error_status(&error), error.to_string()).to_string()), }; continue; } - ResponseItem::Control(_) | ResponseItem::Data(_) => continue, }; - if let Some((code, message)) = output - .finish_reason - .as_ref() - .and_then(|reason| reason.abort_status()) - { - yield Annotated { - data: None, - id: None, - event: None, - comment: None, - error: Some(error_payload(StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), message).to_string()), - }; - continue; - } - if prompt_tokens == 0 { prompt_tokens = output.prompt_tokens; } diff --git a/rust/sglang-server/src/api_server/openai/completions.rs b/rust/sglang-server/src/api_server/openai/completions.rs index bd4691f2f2..dc2a4b50b9 100644 --- a/rust/sglang-server/src/api_server/openai/completions.rs +++ b/rust/sglang-server/src/api_server/openai/completions.rs @@ -24,14 +24,15 @@ use super::{ AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32, }; -use crate::frontend::{FrontendCall, FrontendError}; +use crate::api_server::frontend_error_status; +use crate::frontend::{ + FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, +}; use crate::message::finish_reason::Matched; use crate::message::ids::Rid; -use crate::message::request::GenerateRequest; -use crate::message::response::{ChunkEvent, ChunkExtras, ResponseItem}; +use crate::message::response::ChunkExtras; use crate::message::sampling::SamplingParams; use crate::message::types::{OneOrMany, TokenIds}; -use crate::utils::error::Error; pub(super) fn routes() -> Router> { Router::new().route("/v1/completions", post(completions)) @@ -163,7 +164,7 @@ async fn completions( Err(response) => return response, }; } - let native = GenerateRequest { + let native = FrontendRequest { rid: rid.clone(), text: text.clone(), input_ids: input_ids.clone(), @@ -234,13 +235,7 @@ async fn completions( /// or an `Error` (e.g. out-of-range ids → `Validation` → 400). async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result { match state.frontend.detokenize(token_ids).await { - Ok(payload) => String::from_utf8(payload.to_vec()).map_err(|_| { - openai_error( - StatusCode::INTERNAL_SERVER_ERROR, - "detokenized prompt is not valid UTF-8", - false, - ) - }), + Ok(text) => Ok(text), // Same rule as `submit_generation`: build the refusal in the OpenAI // error shape rather than forwarding the native-shaped response. Err(FrontendError::Unavailable) => Err(openai_error( @@ -248,30 +243,22 @@ async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result { + Err(FrontendError::InvalidArgument(message)) => { Err(openai_error(StatusCode::BAD_REQUEST, &message, false)) } - Err(FrontendError::Pipeline(error)) => { - let status = StatusCode::from_u16(error.http_status()) - .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Err(FrontendError::InvalidResponse(message)) => Err(openai_error( + StatusCode::INTERNAL_SERVER_ERROR, + message, + false, + )), + Err(error) => { + let status = frontend_error_status(&error); Err(openai_error( status, format!("failed to decode prompt for echo: {error}"), false, )) } - Err(FrontendError::ResponseClosed | FrontendError::UnexpectedResponse(_)) => { - Err(openai_error( - StatusCode::INTERNAL_SERVER_ERROR, - "failed to decode prompt for echo: reply channel closed", - false, - )) - } - Err(error @ FrontendError::InvalidArgument(_)) => Err(openai_error( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to decode prompt for echo: {error}"), - false, - )), } } @@ -419,7 +406,7 @@ pub(super) async fn unary_completion( fn completion_choice( index: usize, text: String, - output: &ChunkEvent, + output: &FrontendOutput, want_logprobs: bool, include_input_logprobs: bool, ) -> (Choice, ChoiceExtensions) { @@ -532,30 +519,15 @@ pub(super) fn completion_event_stream( } let mut events = futures::stream::select_all(streams); - while let Some((index, item)) = events.next().await { - let Some(item) = item else { - yield error_payload(StatusCode::INTERNAL_SERVER_ERROR, "response truncated before completion").to_string(); - continue; - }; - let output = match item { - ResponseItem::Frame(output) => output, - ResponseItem::Done(output) => output, - ResponseItem::Error(error) => { - yield error_payload(StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), error.to_string()).to_string(); + while let Some((index, event)) = events.next().await { + let output = match event { + FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output, + FrontendEvent::Failed(error) => { + yield error_payload(frontend_error_status(&error), error.to_string()).to_string(); continue; } - ResponseItem::Control(_) | ResponseItem::Data(_) => continue, }; - if let Some((code, message)) = output - .finish_reason - .as_ref() - .and_then(|reason| reason.abort_status()) - { - yield error_payload(StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), message).to_string(); - continue; - } - prompt_tokens_by_prompt .entry(prompt_indexes[index]) .or_insert(output.prompt_tokens); diff --git a/rust/sglang-server/src/api_server/openai/test_utils.rs b/rust/sglang-server/src/api_server/openai/test_utils.rs index 6d1eee0527..7b3d5bcab8 100644 --- a/rust/sglang-server/src/api_server/openai/test_utils.rs +++ b/rust/sglang-server/src/api_server/openai/test_utils.rs @@ -20,7 +20,7 @@ use serde_json::json; use tower::util::ServiceExt; use super::{indexed_decode_stream, openai_error, routes}; -use crate::frontend::{FrontendCall, FrontendHandle}; +use crate::frontend::{FrontendCall, FrontendEvent, FrontendHandle}; use crate::message::config::ServerArgs; use crate::message::response::{ChunkEvent, ResponseItem}; pub(super) fn frontend() -> FrontendHandle { @@ -33,6 +33,7 @@ pub(super) fn frontend() -> FrontendHandle { startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), + metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()), }, ) } @@ -130,6 +131,7 @@ pub(super) fn frontend_closed() -> FrontendHandle { startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), + metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()), }, ) } @@ -170,7 +172,7 @@ async fn dropping_indexed_stream_aborts_its_live_call() { response_tx.send(chunk("live", "x", false)).await.unwrap(); assert!(matches!( stream.next().await, - Some((0, Some(ResponseItem::Frame(_)))) + Some((0, FrontendEvent::Delta(_))) )); drop(stream); diff --git a/rust/sglang-server/src/api_server/openai/tools.rs b/rust/sglang-server/src/api_server/openai/tools.rs index c6d61963f7..ae2353f2eb 100644 --- a/rust/sglang-server/src/api_server/openai/tools.rs +++ b/rust/sglang-server/src/api_server/openai/tools.rs @@ -36,7 +36,7 @@ use dynamo_protocols::types::{ Role, }; -use crate::message::response::ChunkEvent; +use crate::frontend::FrontendOutput; use crate::message::sampling::SamplingParams; /// Canonicalize a tool-call parser name onto the dynamo-parsers registry keys. @@ -246,7 +246,7 @@ pub(super) async fn parse_chat_tool_calls( /// Map the scheduler's finish kind onto the OpenAI wire values. Length and /// content-filter keep their names; everything else (including a bare abort) /// reports as `stop`, matching Python's fallback. -pub(super) fn chat_finish_reason(output: &ChunkEvent) -> Option { +pub(super) fn chat_finish_reason(output: &FrontendOutput) -> Option { let kind = output .finish_reason .as_ref() @@ -264,7 +264,7 @@ mod tests { apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name, dynamo_tool_choice, parse_chat_tool_calls, }; - use crate::message::response::ChunkEvent; + use crate::frontend::FrontendOutput; use crate::message::sampling::SamplingParams; use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail}; use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition}; @@ -766,8 +766,7 @@ mod tests { #[test] fn chat_finish_reason_maps_scheduler_kinds() { - let output = |finish: serde_json::Value| ChunkEvent { - rid: "r".into(), + let output = |finish: serde_json::Value| FrontendOutput { text: "x".into(), token_ids: vec![1], prompt_tokens: 1, @@ -795,7 +794,7 @@ mod tests { Some(OpenAIFinishReason::Stop) ); assert_eq!( - chat_finish_reason(&ChunkEvent { + chat_finish_reason(&FrontendOutput { finish_reason: None, ..Default::default() }), diff --git a/rust/sglang-server/src/frontend.rs b/rust/sglang-server/src/frontend.rs index 56373af1f3..c7f1ea220c 100644 --- a/rust/sglang-server/src/frontend.rs +++ b/rust/sglang-server/src/frontend.rs @@ -1,10 +1,10 @@ //! Transport-neutral entry point into the Rust frontend pipeline. //! -//! Wire adapters normalize their protocol into the existing in-process request -//! types, submit them through [`FrontendHandle`], and render the returned -//! [`ResponseItem`] values for their own transport. This module owns shared -//! preprocessing, runtime capabilities, and request lifetime; it deliberately -//! knows nothing about Axum, HTTP response shapes, Tonic, or protobuf. +//! Wire adapters normalize their protocol into [`FrontendRequest`], submit it +//! through [`FrontendHandle`], and render semantic [`FrontendEvent`]s for their +//! own transport. This module owns shared preprocessing, runtime translation, +//! capabilities, and request lifetime; it deliberately knows nothing about +//! Axum, HTTP response shapes, Tonic, or protobuf. use std::collections::BTreeMap; use std::sync::{ @@ -13,12 +13,12 @@ use std::sync::{ }; use std::time::Duration; -use bytes::Bytes; use tokio::sync::mpsc; +use crate::message::config::{DisaggregationMode, ServerArgs}; use crate::message::ids::Rid; -use crate::message::io_struct::ControlRequest; -use crate::message::request::{GenerateRequest, Request, RequestKind}; +use crate::message::io_struct::{ControlRequest, GetInternalStateReq}; +use crate::message::request::{Request, RequestKind}; use crate::message::response::{ResponseItem, ResponseSink}; use crate::message::sampling::SamplingParams; use crate::message::types::TokenIds; @@ -27,50 +27,63 @@ use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent}; use crate::utils::error::Error; use crate::utils::fsm::RequestState; +mod contract; mod prefetch; +pub(crate) use contract::{ + FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus, + InternalState, ModelInfo, ServerInfo, +}; + /// Sentinel host that makes the KV connector no-op. Parity with /// `sglang.srt.disaggregation.utils.FAKE_BOOTSTRAP_HOST`. const FAKE_BOOTSTRAP_HOST: &str = "2.2.2.2"; -/// A transport-neutral failure while executing a frontend operation. -#[derive(Clone, Debug, thiserror::Error)] -pub(crate) enum FrontendError { - /// The TokenizerManager intake loop has shut down, so callers may retry on - /// another healthy server. - #[error("service unavailable")] - Unavailable, - - #[error("{0}")] - InvalidArgument(String), - - #[error(transparent)] - Pipeline(#[from] Error), - - #[error("response channel closed before terminal output")] - ResponseClosed, - - #[error("{0}")] - UnexpectedResponse(&'static str), -} - -/// Semantic result of a deep frontend health check. -/// -/// Expected lifecycle states are values rather than transport errors so each -/// adapter can render them according to its own protocol. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum HealthStatus { - Healthy, - NotReady, - Stalled, -} - pub(crate) struct FrontendConfig { pub(crate) response_capacity: usize, pub(crate) response_activity: ActivityCounter, pub(crate) startup_ready: bool, pub(crate) is_disaggregation: bool, pub(crate) mm_limits: BTreeMap, + pub(crate) metadata: FrontendMetadata, +} + +/// Immutable, transport-independent metadata retained by the frontend. +/// Keeping this snapshot narrow avoids making adapters or the shared handle +/// depend on the full launch-configuration object. +#[derive(Clone, Debug, Default)] +pub(crate) struct FrontendMetadata { + model_path: String, + served_model_name: String, + tokenizer_path: String, + preferred_sampling_params: Option, + weight_version: Option, + load_format: Option, + reasoning_parser: Option, + tool_call_parser: Option, + disaggregation_mode: DisaggregationMode, + max_context_length: u64, + max_total_num_tokens: u64, + version: String, +} + +impl From<&ServerArgs> for FrontendMetadata { + fn from(args: &ServerArgs) -> Self { + Self { + model_path: args.model_path.clone(), + served_model_name: args.served_model_name.clone(), + tokenizer_path: args.tokenizer_path.clone(), + preferred_sampling_params: args.preferred_sampling_params.clone(), + weight_version: args.weight_version.clone(), + load_format: args.load_format.clone(), + reasoning_parser: args.reasoning_parser.clone(), + tool_call_parser: args.tool_call_parser.clone(), + disaggregation_mode: args.disaggregation_mode, + max_context_length: args.model_config.context_len, + max_total_num_tokens: args.max_total_num_tokens, + version: args.version.clone(), + } + } } /// Cloneable capability for submitting work to the existing frontend runtime. @@ -90,6 +103,7 @@ struct FrontendInner { startup_ready: AtomicBool, is_disaggregation: bool, mm_limits: BTreeMap, + metadata: FrontendMetadata, } impl FrontendHandle { @@ -107,6 +121,7 @@ impl FrontendHandle { startup_ready: AtomicBool::new(config.startup_ready), is_disaggregation: config.is_disaggregation, mm_limits: config.mm_limits, + metadata: config.metadata, }), } } @@ -122,17 +137,19 @@ impl FrontendHandle { self.inner.startup_ready.store(true, Ordering::Release); } - pub(crate) fn response_activity(&self) -> u64 { + fn response_activity(&self) -> u64 { self.inner.response_activity.load(Ordering::Relaxed) } pub(crate) async fn generate( &self, - mut request: GenerateRequest, + mut request: FrontendRequest, ) -> Result { self.prepare_generations(std::slice::from_mut(&mut request)) .await?; - self.submit(RequestKind::Generate(Box::new(request))).await + self.submit(RequestKind::Generate(Box::new(request))) + .await + .map(FrontendCall::new) } /// Preprocess and submit a group atomically from the adapter's perspective: @@ -141,7 +158,7 @@ impl FrontendHandle { /// the already-created calls aborts those accepted requests. pub(crate) async fn generate_batch( &self, - mut requests: Vec, + mut requests: Vec, ) -> Result, FrontendError> { self.prepare_generations(&mut requests).await?; @@ -149,7 +166,8 @@ impl FrontendHandle { for request in requests { calls.push( self.submit(RequestKind::Generate(Box::new(request))) - .await?, + .await + .map(FrontendCall::new)?, ); } Ok(calls) @@ -157,37 +175,96 @@ impl FrontendHandle { async fn prepare_generations( &self, - requests: &mut [GenerateRequest], + requests: &mut [FrontendRequest], ) -> Result<(), FrontendError> { prefetch::prefetch_all(requests, &self.inner.mm_limits) .await .map_err(FrontendError::InvalidArgument) } - pub(crate) async fn control(&self, request: ControlRequest) -> Result { - let mut call = self.submit(RequestKind::Control(Box::new(request))).await?; - match call.recv().await { - Some(ResponseItem::Control(bytes)) => Ok(bytes), - Some(ResponseItem::Error(error)) => Err(FrontendError::Pipeline(error)), - Some(_) => Err(FrontendError::UnexpectedResponse( - "unexpected generation output for control request", - )), - None => Err(FrontendError::ResponseClosed), + /// Return static model metadata without exposing the full launch config. + pub(crate) fn model_info(&self) -> ModelInfo { + let metadata = &self.inner.metadata; + ModelInfo { + model_path: metadata.model_path.clone(), + served_model_name: metadata.served_model_name.clone(), + tokenizer_path: metadata.tokenizer_path.clone(), + is_generation: true, + preferred_sampling_params: metadata.preferred_sampling_params.clone(), + weight_version: metadata.weight_version.clone(), + load_format: metadata.load_format.clone(), + reasoning_parser: metadata.reasoning_parser.clone(), + tool_call_parser: metadata.tool_call_parser.clone(), + disaggregation_mode: metadata.disaggregation_mode, } } - pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result { - let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?; + /// Return public server metadata with the current scheduler metrics. + /// Raw control bytes and the scheduler's full launch-argument dump never + /// cross the frontend boundary. + pub(crate) async fn server_info(&self) -> Result { + let internal_state = self.internal_state().await?; + let metadata = &self.inner.metadata; + Ok(ServerInfo { + model_path: metadata.model_path.clone(), + served_model_name: metadata.served_model_name.clone(), + tokenizer_path: metadata.tokenizer_path.clone(), + max_context_length: metadata.max_context_length, + max_total_num_tokens: metadata.max_total_num_tokens, + version: metadata.version.clone(), + frontend: "rust", + internal_states: vec![internal_state], + }) + } + + async fn internal_state(&self) -> Result { + let bytes = self + .control(ControlRequest::GetInternalStateReq( + GetInternalStateReq::new(Rid::new().to_string()), + )) + .await?; + rmp_serde::from_slice::(&bytes) + .map(|response| response.internal_state) + .map_err(|error| { + FrontendError::InvalidResponse(format!("invalid internal-state response: {error}")) + }) + } + + async fn control(&self, request: ControlRequest) -> Result { + let mut call = self.submit(RequestKind::Control(Box::new(request))).await?; match call.recv().await { - Some(ResponseItem::Data(bytes)) => Ok(bytes), - Some(ResponseItem::Error(error)) => Err(FrontendError::Pipeline(error)), - Some(_) => Err(FrontendError::UnexpectedResponse( - "unexpected output for detokenize request", + Some(ResponseItem::Control(bytes)) => Ok(bytes), + Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)), + Some(_) => Err(FrontendError::InvalidResponse( + "unexpected generation output for control request".into(), )), - None => Err(FrontendError::ResponseClosed), + None => Err(FrontendError::ResponseTruncated), } } + /// Decode token IDs into text without exposing the runtime's byte payload. + pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result { + let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?; + let bytes = match call.recv().await { + Some(ResponseItem::Data(bytes)) => bytes, + Some(ResponseItem::Error(Error::Validation(message))) => { + return Err(FrontendError::InvalidArgument(message)); + } + Some(ResponseItem::Error(error)) => return Err(translate_runtime_error(error)), + Some(_) => { + return Err(FrontendError::InvalidResponse( + "unexpected output for detokenize request".into(), + )); + } + None => { + return Err(FrontendError::Internal("reply channel closed".into())); + } + }; + String::from_utf8(bytes.to_vec()).map_err(|_| { + FrontendError::InvalidResponse("detokenized prompt is not valid UTF-8".into()) + }) + } + /// Confirm that scheduler output is moving. The response heartbeat is the /// signal rather than this probe's own result, matching Python's /// `last_receive_tstamp` behavior on a busy server. @@ -200,7 +277,7 @@ impl FrontendHandle { } let baseline = self.response_activity(); - let probe = GenerateRequest { + let probe = FrontendRequest { rid: Rid::new_health_check(), input_ids: Some(vec![0]), sampling_params: SamplingParams { @@ -238,7 +315,7 @@ impl FrontendHandle { /// The async send preserves TokenizerManager inbox backpressure. Once the /// request is accepted, the returned call owns cancellation until it /// observes a terminal response. - async fn submit(&self, kind: RequestKind) -> Result { + async fn submit(&self, kind: RequestKind) -> Result { let rid = request_rid(&kind); let expected_response = ExpectedResponse::for_request(&kind); let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity); @@ -246,7 +323,7 @@ impl FrontendHandle { // Arm ownership before awaiting the bounded handoff. If this future is // cancelled after the event is queued but before the await observes // success, dropping `call` still cancels the request. - let mut call = FrontendCall { + let mut call = RuntimeCall { rid: rid.clone(), response_rx, abort_tx: self.inner.abort_tx.clone(), @@ -279,41 +356,74 @@ impl FrontendHandle { } } -/// One accepted request and its response stream. +fn translate_runtime_error(error: Error) -> FrontendError { + match error { + Error::Validation(message) => { + FrontendError::InvalidArgument(format!("validation failed: {message}")) + } + Error::QueueFull => FrontendError::Overloaded("to_scheduler channel full".into()), + Error::Disconnected => FrontendError::Cancelled("client disconnected".into()), + error => FrontendError::Internal(error.to_string()), + } +} + +/// One accepted generation request and its semantic response stream. /// -/// Dropping an unfinished call either cancels a request that Intake has not yet -/// claimed or notifies the existing abort lane after admission. Reading a -/// terminal item disarms that cleanup automatically, so transports cannot -/// accidentally abort completed work or forget disconnect cleanup. +/// Runtime response variants are translated here so adapters cannot depend on +/// scheduler/control encodings. Dropping an unfinished call retains the +/// cancellation guarantees owned by [`RuntimeCall`]. pub(crate) struct FrontendCall { - rid: Rid, - response_rx: mpsc::Receiver, - abort_tx: flume::Sender, - admission: RequestAdmission, - expected_response: ExpectedResponse, - in_flight: bool, + runtime: RuntimeCall, + events_finished: bool, } impl FrontendCall { - pub(crate) fn rid(&self) -> &Rid { - &self.rid + fn new(runtime: RuntimeCall) -> Self { + Self { + runtime, + events_finished: false, + } } - pub(crate) async fn recv(&mut self) -> Option { - let item = self.response_rx.recv().await; - self.observe(item.as_ref()); - item + /// ID that adapters should return to their client. + pub(crate) fn public_id(&self) -> &str { + self.runtime.rid.client_facing() } - pub(crate) fn try_recv(&mut self) -> Result { - let item = self.response_rx.try_recv()?; - self.observe(Some(&item)); - Ok(item) + /// Receive the next semantic event. A runtime channel that closes before a + /// terminal response is converted into one terminal failure event; adapters + /// never need to infer whether an empty runtime channel means success. + pub(crate) async fn recv(&mut self) -> Option { + if self.events_finished { + return None; + } + + let event = match self.runtime.recv().await { + Some(item) => generation_event(item), + None => FrontendEvent::Failed(FrontendError::Internal( + "response truncated before completion".into(), + )), + }; + self.events_finished = event.is_terminal(); + Some(event) } - fn observe(&mut self, item: Option<&ResponseItem>) { - if item.is_some_and(|item| self.expected_response.is_terminal(item)) { - self.in_flight = false; + /// Append every semantic event that is ready without waiting. Channel + /// emptiness and closure remain private runtime details. + pub(crate) fn drain_ready(&mut self, events: &mut Vec) { + while !self.events_finished { + let event = match self.runtime.try_recv() { + Ok(item) => generation_event(item), + // Preserve every queued event before reporting a premature + // close. The next awaited `recv` translates that close into + // one terminal failure, maintaining stream order while still + // hiding channel mechanics from adapters. + Err(mpsc::error::TryRecvError::Disconnected | mpsc::error::TryRecvError::Empty) => { + break; + } + }; + self.events_finished = event.is_terminal(); + events.push(event); } } @@ -325,18 +435,84 @@ impl FrontendCall { ) -> Self { let admission = RequestAdmission::pending(); assert!(admission.try_accept(), "test calls start already admitted"); - Self { + Self::new(RuntimeCall { rid, response_rx, abort_tx, admission, expected_response: ExpectedResponse::Generate, in_flight: true, + }) + } +} + +fn generation_event(item: ResponseItem) -> FrontendEvent { + match item { + ResponseItem::Frame(output) => FrontendEvent::Delta(output.into()), + ResponseItem::Done(output) => { + if let Some((legacy_http_status, message)) = output + .finish_reason + .as_ref() + .and_then(|reason| reason.abort_status()) + { + FrontendEvent::Failed(FrontendError::from_runtime_rejection( + message.to_owned(), + legacy_http_status, + )) + } else { + FrontendEvent::Finished(output.into()) + } + } + ResponseItem::Error(error) => FrontendEvent::Failed(translate_runtime_error(error)), + ResponseItem::Control(_) | ResponseItem::Data(_) => FrontendEvent::Failed( + FrontendError::InvalidResponse("unexpected non-generation response".into()), + ), + } +} + +/// Private scheduler response envelope. This MessagePack shape is decoded at +/// the runtime boundary and deliberately is not part of the frontend contract. +#[derive(serde::Deserialize)] +struct InternalStateEnvelope { + #[serde(default)] + internal_state: InternalState, +} + +/// Runtime-facing request guard kept private behind [`FrontendCall`]. +/// +/// This type is the only owner of the raw response channel and admission/abort +/// capabilities. It is also used directly by typed unary operations such as +/// detokenization and internal-state lookup. +struct RuntimeCall { + rid: Rid, + response_rx: mpsc::Receiver, + abort_tx: flume::Sender, + admission: RequestAdmission, + expected_response: ExpectedResponse, + in_flight: bool, +} + +impl RuntimeCall { + async fn recv(&mut self) -> Option { + let item = self.response_rx.recv().await; + self.observe(item.as_ref()); + item + } + + fn try_recv(&mut self) -> Result { + let item = self.response_rx.try_recv()?; + self.observe(Some(&item)); + Ok(item) + } + + fn observe(&mut self, item: Option<&ResponseItem>) { + if item.is_some_and(|item| self.expected_response.is_terminal(item)) { + self.in_flight = false; } } } -impl Drop for FrontendCall { +impl Drop for RuntimeCall { fn drop(&mut self) { if self.in_flight { // Pending cancellation is consumed by Intake before it starts work. @@ -396,6 +572,7 @@ mod tests { use crate::message::io_struct::GetInternalStateReq; use crate::message::multimodal::MmItem; use crate::message::request::{GenerateRequest, MmData}; + use crate::message::response::ChunkEvent; struct Harness { handle: FrontendHandle, @@ -449,6 +626,7 @@ mod tests { startup_ready, is_disaggregation, mm_limits, + metadata: FrontendMetadata::default(), }, ) } @@ -518,6 +696,136 @@ mod tests { assert!(abort_rx.try_recv().is_err()); } + #[test] + fn runtime_errors_become_transport_neutral_categories() { + assert!(matches!( + translate_runtime_error(Error::Validation("bad input".into())), + FrontendError::InvalidArgument(message) + if message == "validation failed: bad input" + )); + assert!(matches!( + translate_runtime_error(Error::QueueFull), + FrontendError::Overloaded(message) if message == "to_scheduler channel full" + )); + assert!(matches!( + translate_runtime_error(Error::Disconnected), + FrontendError::Cancelled(message) if message == "client disconnected" + )); + + assert!(matches!( + translate_runtime_error(Error::Internal("bug".into())), + FrontendError::Internal(message) if message == "internal error: bug" + )); + } + + #[test] + fn scheduler_status_is_classified_once_for_non_http_adapters() { + for (status, expected) in [ + (400, FrontendErrorKind::InvalidArgument), + (404, FrontendErrorKind::NotFound), + (408, FrontendErrorKind::DeadlineExceeded), + (412, FrontendErrorKind::FailedPrecondition), + (413, FrontendErrorKind::ResourceExhausted), + (429, FrontendErrorKind::ResourceExhausted), + (432, FrontendErrorKind::InvalidArgument), + (499, FrontendErrorKind::Cancelled), + (500, FrontendErrorKind::Internal), + (503, FrontendErrorKind::Unavailable), + (504, FrontendErrorKind::DeadlineExceeded), + ] { + let error = FrontendError::from_runtime_rejection("rejected".into(), status); + assert_eq!(error.kind(), expected, "legacy status {status}"); + assert!(matches!( + error, + FrontendError::RuntimeRejected { + legacy_http_status, + .. + } if legacy_http_status == status + )); + } + } + + #[tokio::test] + async fn generation_exposes_public_id_and_semantic_events() { + let harness = Harness::unbounded(8); + let mut call = harness + .handle + .generate(GenerateRequest { + rid: Rid::from_client("public-id"), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(call.public_id(), "public-id"); + assert_ne!(call.runtime.rid.as_str(), "public-id"); + + let request = accept_intake(harness.intake_rx.recv().unwrap()); + request + .sink + .try_send(ResponseItem::Frame(ChunkEvent { + text: "delta".into(), + ..Default::default() + })) + .unwrap(); + request + .sink + .try_send(ResponseItem::Done(ChunkEvent { + text: "final".into(), + ..Default::default() + })) + .unwrap(); + + assert!(matches!( + call.recv().await, + Some(FrontendEvent::Delta(FrontendOutput { text, .. })) if text == "delta" + )); + assert!(matches!( + call.recv().await, + Some(FrontendEvent::Finished(FrontendOutput { text, .. })) if text == "final" + )); + assert!(call.recv().await.is_none()); + drop(call); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn plain_scheduler_abort_remains_a_terminal_output() { + let harness = Harness::unbounded(8); + let mut call = harness + .handle + .generate(GenerateRequest { + rid: "plain-abort".into(), + ..Default::default() + }) + .await + .unwrap(); + let request = accept_intake(harness.intake_rx.recv().unwrap()); + request + .sink + .try_send(ResponseItem::Done(ChunkEvent { + finish_reason: serde_json::from_value(serde_json::json!({ + "type": "abort", + "message": "Aborted", + "status_code": null, + "err_type": null + })) + .unwrap(), + ..Default::default() + })) + .unwrap(); + + assert!(matches!( + call.recv().await, + Some(FrontendEvent::Finished(FrontendOutput { + finish_reason: Some(_), + .. + })) + )); + assert!(call.recv().await.is_none()); + drop(call); + assert!(harness.abort_rx.try_recv().is_err()); + } + #[tokio::test] async fn generate_batch_failure_aborts_already_accepted_calls() { let (intake_tx, intake_rx) = flume::bounded(0); @@ -686,7 +994,7 @@ mod tests { .sink .try_send(ResponseItem::Data(Bytes::from_static(b"decoded"))) .unwrap(); - assert_eq!(task.await.unwrap().unwrap(), Bytes::from_static(b"decoded")); + assert_eq!(task.await.unwrap().unwrap(), "decoded"); assert!(harness.abort_rx.try_recv().is_err()); } } diff --git a/rust/sglang-server/src/frontend/contract.rs b/rust/sglang-server/src/frontend/contract.rs new file mode 100644 index 0000000000..b8b544f408 --- /dev/null +++ b/rust/sglang-server/src/frontend/contract.rs @@ -0,0 +1,272 @@ +//! Transport-neutral types shared by frontend adapters. +//! +//! These types describe what an inference operation means after a wire adapter +//! has normalized its request. They intentionally contain no Axum, HTTP, SSE, +//! Tonic, protobuf, or scheduler-channel details. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::message::config::{DisaggregationMode, PreferredSamplingParams}; +use crate::message::finish_reason::FinishReason; +use crate::message::request::GenerateRequest; +use crate::message::response::{ChunkEvent, ChunkExtras}; +use crate::message::types::TokenIds; + +/// Canonical transport-neutral input for one generation operation. +/// +/// The existing in-process request already contains semantic generation fields +/// and has no Serde, Axum, Tonic, or protobuf representation attached to it. +/// Re-exporting it through the frontend contract gives every adapter one public +/// boundary without duplicating the runtime's request model. +pub(crate) type FrontendRequest = GenerateRequest; + +/// Canonical transport-neutral output payload for generation events. +/// +/// The runtime correlation ID is deliberately absent: it is an implementation +/// detail used for scheduler routing, while adapters obtain the client-visible +/// identity from [`crate::frontend::FrontendCall::public_id`]. Runtime-only +/// response variants such as raw control bytes are likewise hidden by +/// [`crate::frontend::FrontendCall`]. +#[derive(Clone, Debug, Default)] +pub(crate) struct FrontendOutput { + pub(crate) token_ids: TokenIds, + pub(crate) finish_reason: Option, + pub(crate) prompt_tokens: u32, + pub(crate) text: String, + pub(crate) completion_tokens: u64, + pub(crate) extras: Option>, +} + +impl From for FrontendOutput { + fn from(output: ChunkEvent) -> Self { + let ChunkEvent { + rid: _, + token_ids, + finish_reason, + prompt_tokens, + text, + completion_tokens, + extras, + } = output; + Self { + token_ids, + finish_reason, + prompt_tokens, + text, + completion_tokens, + extras, + } + } +} + +/// Semantic result of a deep frontend health check. +/// +/// Expected lifecycle states are values rather than transport errors so each +/// adapter can render them according to its own protocol. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum HealthStatus { + Healthy, + NotReady, + Stalled, +} + +/// A semantic generation event before any transport-specific framing. +#[derive(Debug)] +pub(crate) enum FrontendEvent { + /// One incremental model-output update. + Delta(FrontendOutput), + /// The final model-output update for this request. + Finished(FrontendOutput), + /// A request-scoped failure. Other requests in a batch may continue. + Failed(FrontendError), +} + +/// Protocol-independent error category used by transport adapters. +/// +/// The set intentionally follows operation semantics rather than HTTP or gRPC +/// status spaces. An adapter maps this category into its own wire status. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum FrontendErrorKind { + InvalidArgument, + NotFound, + FailedPrecondition, + ResourceExhausted, + Cancelled, + DeadlineExceeded, + Unavailable, + Internal, +} + +impl FrontendEvent { + pub(crate) fn is_terminal(&self) -> bool { + matches!(self, Self::Finished(_) | Self::Failed(_)) + } +} + +/// A transport-neutral failure category. +/// +/// HTTP and gRPC adapters map these categories into their own status spaces; +/// neither adapter needs to understand errors emitted by runtime stages. +#[derive(Clone, Debug, thiserror::Error)] +pub(crate) enum FrontendError { + #[error("{0}")] + InvalidArgument(String), + + /// The frontend intake loop has shut down. A caller may retry elsewhere. + #[error("service unavailable")] + Unavailable, + + /// Runtime capacity was exhausted before the request could make progress. + #[error("{0}")] + Overloaded(String), + + #[error("{0}")] + Cancelled(String), + + /// The internal producer disappeared before returning a terminal result. + /// + /// This is a server-side failure, not evidence that the caller cancelled. + #[error("response truncated before completion")] + ResponseTruncated, + + /// A scheduler-side abort that carried an error status. + /// + /// The current Python scheduler communicates this category as an HTTP + /// status code inside its finish reason. The semantic event is still + /// `Failed`; retaining the legacy code only lets the HTTP adapter preserve + /// existing behavior while another adapter maps it into its own status + /// space. New frontend failures should use the semantic variants above. + #[error("{message}")] + RuntimeRejected { + kind: FrontendErrorKind, + message: String, + legacy_http_status: u16, + }, + + /// The runtime replied with an unexpected or malformed result. + #[error("{0}")] + InvalidResponse(String), + + #[error("{0}")] + Internal(String), +} + +impl FrontendError { + /// Semantic status for adapters that do not use the scheduler's legacy + /// HTTP code space (notably the future gRPC adapter). + pub(crate) fn kind(&self) -> FrontendErrorKind { + match self { + Self::InvalidArgument(_) => FrontendErrorKind::InvalidArgument, + Self::Unavailable => FrontendErrorKind::Unavailable, + Self::Overloaded(_) => FrontendErrorKind::ResourceExhausted, + Self::Cancelled(_) => FrontendErrorKind::Cancelled, + Self::ResponseTruncated => FrontendErrorKind::Internal, + Self::RuntimeRejected { kind, .. } => *kind, + Self::InvalidResponse(_) | Self::Internal(_) => FrontendErrorKind::Internal, + } + } + + /// Translate the scheduler's inherited HTTP-coded failure into a semantic + /// category once, at the runtime boundary. HTTP can retain the exact legacy + /// status while other adapters consume [`Self::kind`]. + pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self { + let kind = match legacy_http_status { + 404 => FrontendErrorKind::NotFound, + 408 | 504 => FrontendErrorKind::DeadlineExceeded, + 412 => FrontendErrorKind::FailedPrecondition, + 413 | 429 => FrontendErrorKind::ResourceExhausted, + 499 => FrontendErrorKind::Cancelled, + 502 | 503 => FrontendErrorKind::Unavailable, + 400..=499 => FrontendErrorKind::InvalidArgument, + _ => FrontendErrorKind::Internal, + }; + Self::RuntimeRejected { + kind, + message, + legacy_http_status, + } + } +} + +/// Static model metadata shared by every public transport. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct ModelInfo { + pub(crate) model_path: String, + pub(crate) served_model_name: String, + pub(crate) tokenizer_path: String, + pub(crate) is_generation: bool, + pub(crate) preferred_sampling_params: Option, + pub(crate) weight_version: Option, + pub(crate) load_format: Option, + pub(crate) reasoning_parser: Option, + pub(crate) tool_call_parser: Option, + pub(crate) disaggregation_mode: DisaggregationMode, +} + +/// Public server metadata plus scheduler-owned runtime metrics. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct ServerInfo { + pub(crate) model_path: String, + pub(crate) served_model_name: String, + pub(crate) tokenizer_path: String, + pub(crate) max_context_length: u64, + pub(crate) max_total_num_tokens: u64, + pub(crate) version: String, + pub(crate) frontend: &'static str, + pub(crate) internal_states: Vec, +} + +/// Scheduler-owned runtime metrics used by the public server-info operation. +/// +/// Deserializing into this allowlisted shape is intentional: the scheduler's +/// raw state also contains its full launch arguments, including credentials. +/// Unknown fields are discarded before any transport receives the result. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub(crate) struct InternalState { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) last_gen_throughput: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) memory_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) effective_max_running_requests_per_dp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) avg_spec_accept_length: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) step_time_dict: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) rust_mm_transport: Option>, +} + +/// Public subset of scheduler memory metrics. +/// +/// This nested allowlist is intentional: adding a scheduler field does not +/// silently add different public data to HTTP before the gRPC contract can add +/// the same field. New metrics should be promoted here deliberately. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub(crate) struct MemoryUsage { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) weight: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) kvcache: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) startup_available: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) token_capacity: Option, + pub(crate) token_capacity_swa: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) graph: Option>, +} + +/// The scheduler's KV-cache measurement can be a native float or a NumPy +/// scalar stringified by the MessagePack bridge. Preserve that representation +/// so extracting the typed frontend contract does not change public responses. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(untagged)] +pub(crate) enum MemoryMeasurement { + Number(f64), + String(String), +} diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 8fab3d4815..13d0ec5256 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -318,10 +318,11 @@ impl<'py> pyo3::FromPyObject<'_, 'py> for PreferredSamplingParams { module = "sglang.srt.rust_extensions._server" )] // Lowercase to match the values Python reports for the same field. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum DisaggregationMode { /// Unified prefill + decode. + #[default] Null, Prefill, Decode, diff --git a/rust/sglang-server/src/message/finish_reason.rs b/rust/sglang-server/src/message/finish_reason.rs index f517aa36fc..8050cd6b87 100644 --- a/rust/sglang-server/src/message/finish_reason.rs +++ b/rust/sglang-server/src/message/finish_reason.rs @@ -97,9 +97,9 @@ impl FinishReason { } } - /// `Some((status, message))` when this is an abort carrying a `status_code` — - /// a scheduler-side request error the API surfaces as that HTTP status instead - /// of as a normal completion. A plain abort (no code) reads as `None`. + /// `Some((status, message))` when this is an abort carrying the scheduler's + /// legacy HTTP status — a request failure the frontend boundary converts + /// into a semantic error event. A plain abort (no code) reads as `None`. pub fn abort_status(&self) -> Option<(u16, &str)> { match self { FinishReason::Known(FinishKind::Abort(a)) => Some(( @@ -128,9 +128,9 @@ mod tests { Some((code, message.to_string())) } - /// The classifier both API paths use: a validation abort yields its - /// `(code, message)` (the streaming path turns this into an SSE error event - /// instead of a normal `Done` frame); anything else yields `None`. + /// The classifier the frontend boundary uses: a validation abort yields its + /// `(code, message)` so it can become a semantic failure event; anything + /// else yields `None`. #[test] fn abort_status_extracts_code_and_message() { let (code, msg) = abort_status(&fr(serde_json::json!({ diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index af7e8c4a5c..d1e2fe1763 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -260,8 +260,7 @@ impl GenerateBody { /// prompt + `is_batch` (list form — a 1-element list is still a batch → JSON /// array response). The Rust counterpart of Python /// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent - /// batch is [`Error::Validation`], which the handler surfaces with the - /// variant's own status (400). + /// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400. pub fn into_requests(self) -> Result<(Vec, bool), Error> { let GenerateBody { rid, @@ -653,8 +652,8 @@ fn extension_value_present(value: &rmpv::Value) -> bool { /// mutated lock-free). Common fields here; variant data in [`RequestKind`]. #[derive(Debug)] pub struct Request { - /// Client-visible request id (uuid hex) — what the scheduler wire and - /// `meta_info.id` carry. + /// Runtime correlation ID. Client-provided IDs carry a private incarnation + /// suffix; adapters recover the public ID through `FrontendCall::public_id`. pub rid: Rid, pub state: RequestState, /// Back-channel to the client connection for response frames. @@ -677,10 +676,10 @@ pub struct SchedulerRequest { /// response shape. Each owns its body, so generate/control fields stay type-separate. #[derive(Debug)] pub enum RequestKind { - /// `/generate`: tokenize (if needed) then push a `TokenizedGenerateReqInput`. + /// Generation: tokenize (if needed) then push a `TokenizedGenerateReqInput`. Generate(Box), - /// A control endpoint (e.g. `/server_info`, `/health`): no tokenization, and - /// the response is a single non-streamed JSON result. + /// Internal runtime control operation: no tokenization, with one serialized + /// result that the frontend boundary converts into a typed response. Control(Box), /// Internal service call: decode a complete token-id sequence to text. Walks /// the same FSM as every request (validate → register → Queued), but the @@ -691,11 +690,10 @@ pub enum RequestKind { Detokenize { token_ids: TokenIds }, } -/// A single in-flight `/generate` request (per-item from -/// [`GenerateBody::into_requests`]), +/// A single in-flight generation request (one item after adapter fan-out), /// serialized to the scheduler wire once tokenized (see `to_header_msgpack`). Not a -/// wire type — built by `into_requests`/handlers, never (de)serialized; `input_ids` is -/// client-supplied or filled by the Tokenizer stage. +/// wire type — built by frontend adapters, never (de)serialized; `input_ids` is +/// supplied by an adapter or filled by the Tokenizer stage. #[derive(Debug, Default)] pub struct GenerateRequest { /// This item's final rid: the client's (normalized per item by `into_requests`) or a @@ -726,7 +724,7 @@ pub struct GenerateRequest { /// Sampling params (defaults when the client sent none, as in Python); /// normalized + verified, then serialized into the header. pub sampling_params: SamplingParams, - /// Whether the client asked for SSE streaming. + /// Whether the caller requested incremental output. pub stream: bool, /// Logprob / hidden-state options. This path bypasses the Python /// `TokenizerManager`, so `into_requests` replicates its scalar diff --git a/rust/sglang-server/src/message/response.rs b/rust/sglang-server/src/message/response.rs index c2b6f3147c..807389b5a1 100644 --- a/rust/sglang-server/src/message/response.rs +++ b/rust/sglang-server/src/message/response.rs @@ -1,7 +1,6 @@ -//! The response direction: the per-request back-channel the API handler -//! drains ([`ResponseSink`] / [`ResponseItem`]), the response frame encodings -//! (batch / control result / error), and the columnar batch decode into -//! per-request [`ChunkEvent`]s. +//! The runtime response direction: per-request channels into the frontend +//! boundary ([`ResponseSink`] / [`ResponseItem`]), response frame encodings, +//! and columnar batch decoding into per-request [`ChunkEvent`]s. use bytes::Bytes; use serde::{Deserialize, Serialize}; @@ -12,8 +11,8 @@ use crate::message::ids::Rid; use crate::message::types::TokenIds; use crate::utils::error::Error; -/// Per-request back-channel the detok shard writes decode frames to and the API -/// handler drains for SSE; bounded, and receiver-drop (disconnect) = stream end. +/// Per-request back-channel the detok shard writes runtime outputs to and the +/// frontend boundary drains; bounded, and receiver-drop cancels the producer. #[derive(Clone, Debug)] pub enum ResponseSink { Local(mpsc::Sender), @@ -39,26 +38,26 @@ impl ResponseSink { } } -#[allow(dead_code)] // the receiver half is owned by frontend::FrontendCall. +#[allow(dead_code)] // the receiver half is owned inside the frontend boundary. pub type ResponseSource = mpsc::Receiver; -/// What the connection handler receives on the decode stream: a detok-decoded -/// [`ChunkEvent`] (handler formats it), a verbatim control payload, or an error. +/// Runtime-facing output delivered to the frontend boundary: a detok-decoded +/// [`ChunkEvent`], a serialized internal-operation result, or a stage error. +/// [`crate::frontend::FrontendHandle`] translates these variants before a +/// transport adapter sees them. #[derive(Debug)] pub enum ResponseItem { /// An intermediate streamed generation step (only sent for streaming reqs). Frame(ChunkEvent), /// The final generation step. Done(ChunkEvent), - /// A control-request result: one verbatim payload (e.g. `/server_info`), - /// delivered as-is with no per-protocol formatting. + /// A control-request result encoded by the Python scheduler. Control(Bytes), /// Reply to an internal service request (`RequestKind::Detokenize`): raw - /// bytes for the SUBMITTER to consume (e.g. the decoded prompt text), not - /// client-bound JSON like `Control` and not a generation frame. Generation - /// and control drains never see it. + /// bytes for the frontend to decode (e.g. detokenized text), not a + /// client-facing wire payload and not a generation frame. Data(Bytes), - /// Terminal failure: handler emits an error frame (stream) or status (unary). + /// Terminal runtime failure; the frontend maps it to a semantic error. Error(Error), } @@ -536,8 +535,8 @@ pub fn frame_control_result(rid: &str, payload: &[u8]) -> Bytes { Bytes::from(buf) } -/// Frame a per-request failure `[rid, message]` for the response — routes a -/// terminal error back to the owning request (→ HTTP 400) instead of crashing. +/// Frame a per-request validation failure `[rid, message]` for the response — +/// routes a terminal error back to the owning request instead of crashing. pub fn frame_error(rid: &str, message: &str) -> Bytes { use rmpv::Value; let arr = Value::Array(vec![Value::from(rid), Value::from(message)]); @@ -558,10 +557,9 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes { /// frame and moved between stages in-process (never serialized), so no serde. #[derive(Debug, Clone, Default)] pub struct ChunkEvent { - /// Client-visible rid — the request's IDENTITY. Moved out of the frame header - /// (which owns it and drops it), so carrying it costs no allocation. The shard - /// is still chosen by `Rid::shard`, but a hash collision there now only - /// co-locates two requests instead of merging them. + /// Runtime correlation ID. Moved out of the frame header (which owns it and + /// drops it), so carrying it costs no allocation. The frontend strips this + /// field before exposing semantic output; the shard routes it via `Rid::shard`. pub rid: Rid, /// New token ids for this step, widened from the scheduler's int32 wire /// width at parse time. Empty allowed (e.g. metadata-only frames). diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs index dc90181651..1898305a8d 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs @@ -122,8 +122,8 @@ impl Intake { /// terminal frame and a hung connection. Python cannot hit this because it /// validates before `rid_to_state[obj.rid] = state`. fn fail(&self, req: &mut Request, err: Error, registered: bool) { - // Log only server faults (500); 4xx/499/503 are expected and would spam. - if err.http_status() == 500 { + // Expected request, capacity, and cancellation outcomes would spam. + if err.is_server_fault() { tracing::error!(rid = %req.rid, error = %err, "intake rejected request"); } let _ = req.state.apply(Event::Error(err.clone())); diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs index 27cefed508..1c4e9734f2 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs @@ -202,7 +202,7 @@ fn total_tokens_over_context_is_rejected() { }; let err = check_total_tokens(&mut g, &limits).unwrap_err(); let msg = err.to_string(); - assert_eq!(err.http_status(), 400); + assert!(matches!(err, Error::Validation(_))); assert!(msg.contains("total of 103 tokens"), "{msg}"); assert!(msg.contains("3 tokens from the input"), "{msg}"); assert!(msg.contains("100 tokens for the completion"), "{msg}"); @@ -291,7 +291,7 @@ fn input_length_is_checked_unconditionally() { for max_new_tokens in [None, Some(1)] { let err = check_total_tokens(&mut req(max_new_tokens), &limits) .expect_err("input == context_len must be rejected (Python uses >=)"); - assert_eq!(err.http_status(), 400); + assert!(matches!(err, Error::Validation(_))); assert!(err.to_string().contains("longer than the model's context")); } // One token shorter fits, with or without a cap. @@ -338,7 +338,7 @@ fn auto_truncate_cannot_invert_min_and_max_new_tokens() { }; let err = check_total_tokens(&mut g, &limits) .expect_err("a clamp that inverts min/max must 400, not ride the wire"); - assert_eq!(err.http_status(), 400); + assert!(matches!(err, Error::Validation(_))); assert!(err.to_string().contains("min_new_tokens"), "{err}"); // A clamp that keeps the invariant still clamps. @@ -362,7 +362,7 @@ fn hidden_states_gated_on_server_support() { }; let disabled = test_limits(); let err = validate(&mut req(true), &disabled).unwrap_err(); - assert_eq!(err.http_status(), 400); + assert!(matches!(err, Error::Validation(_))); assert!( err.to_string().contains("--enable-return-hidden-states"), "message must name the flag: {err}" @@ -466,7 +466,7 @@ fn detokenize_negative_ids_reject_before_registration() { let Ok(ResponseItem::Error(err)) = rx.try_recv() else { panic!("sink must receive the validation error"); }; - assert_eq!(err.http_status(), 400); + assert!(matches!(err, Error::Validation(_))); assert!(err.to_string().contains("out of range"), "{err}"); assert!(detok_rx.try_recv().is_err(), "shard never hears of it"); assert!(consumer.drain(16).is_empty()); @@ -525,7 +525,7 @@ fn oversized_rid_is_rejected() { let mut req = generate_req(51, SamplingParams::default()); req.rid = "x".repeat(MAX_RID_LEN + 1).into(); let err = validate(&mut req, &test_limits()).expect_err("must be rejected"); - assert_eq!(err.http_status(), 400); + assert!(matches!(err, Error::Validation(_))); assert!(err.to_string().contains("over the"), "{err}"); // A uuid-sized rid — what Python mints — is nowhere near the cap. @@ -780,7 +780,7 @@ fn mm_pool_gone_deregisters() { let Ok(ResponseItem::Error(err)) = rx.try_recv() else { panic!("expected a terminal error frame"); }; - assert_eq!(err.http_status(), 500, "{err}"); + assert!(matches!(err, Error::Internal(_)), "{err}"); assert!(err.to_string().contains("mm worker pool gone"), "{err}"); assert!(matches!(detok_rx.try_recv(), Ok(DetokMsg::Register { .. }))); assert!(matches!( @@ -1140,7 +1140,7 @@ fn mm_text_prompt_under_skip_tokenizer_init_is_a_400() { let Ok(ResponseItem::Error(err)) = rx.try_recv() else { panic!("expected a terminal error frame"); }; - assert_eq!(err.http_status(), 400, "{err}"); + assert!(matches!(err, Error::Validation(_)), "{err}"); assert!(err.to_string().contains("must provide input_ids"), "{err}"); assert!(detok_rx.try_recv().is_err(), "rejected before registration"); assert!(mm_rx.try_recv().is_err(), "never reaches the mm pool"); diff --git a/rust/sglang-server/src/utils/error.rs b/rust/sglang-server/src/utils/error.rs index 44667b19ce..16341ef956 100644 --- a/rust/sglang-server/src/utils/error.rs +++ b/rust/sglang-server/src/utils/error.rs @@ -36,15 +36,17 @@ pub enum Error { } impl Error { - /// HTTP status to surface for the non-streaming error path. Mirrors the - /// codes used in the Python `_create_error_response`. - pub fn http_status(&self) -> u16 { - match self { - Error::Validation(_) => 400, - Error::Disconnected => 499, // nginx-style client closed request - Error::QueueFull => 503, - _ => 500, - } + /// Whether this failure represents a server defect rather than an expected + /// request, capacity, or cancellation outcome. + pub fn is_server_fault(&self) -> bool { + matches!( + self, + Error::Tokenize(_) + | Error::Encode(_) + | Error::Detokenize(_) + | Error::Codec(_) + | Error::Internal(_) + ) } } diff --git a/rust/sglang-server/src/utils/runtime.rs b/rust/sglang-server/src/utils/runtime.rs index d9e9ef3999..fc938ba35f 100644 --- a/rust/sglang-server/src/utils/runtime.rs +++ b/rust/sglang-server/src/utils/runtime.rs @@ -294,6 +294,7 @@ pub fn start(cfg: RuntimeConfig) -> Result { startup_ready: cfg.server_args.skip_server_warmup, is_disaggregation: cfg.server_args.is_disaggregation(), mm_limits: cfg.server_args.limit_mm_data_per_request.clone(), + metadata: crate::frontend::FrontendMetadata::from(cfg.server_args.as_ref()), }, );