[grpc rust] Define semantic frontend contract (#41709)

Signed-off-by: jain-ria <riajain@NVIDIA.com>
This commit is contained in:
jain-ria
2026-09-29 13:40:04 -07:00
committed by GitHub
parent 77a81996ce
commit 46beacf13d
20 changed files with 1267 additions and 549 deletions
+73 -3
View File
@@ -1,9 +1,8 @@
//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. Only
//! this module knows HTTP. Its handlers use the transport-neutral
//! [`crate::frontend::FrontendHandle`] to enter the shared runtime pipeline.
//! `/generate` submits a request then awaits one `Done` (unary) or relays SSE
//! frames (`data: {json}` … `[DONE]`), byte-compatible with Python
//! `http_server.generate_request`; `/server_info` reuses it for one control result.
//! Generation handlers render semantic frontend events as unary JSON or SSE;
//! control handlers serialize typed frontend results such as server metadata.
pub mod app;
mod common;
mod disaggregation;
@@ -11,3 +10,74 @@ mod frame;
mod log;
mod native_api;
mod openai;
use axum::http::StatusCode;
use crate::frontend::{FrontendError, FrontendErrorKind};
/// Map transport-neutral failures into HTTP status codes. Response bodies and
/// stream framing remain the responsibility of each HTTP API surface.
pub(crate) fn frontend_error_status(error: &FrontendError) -> StatusCode {
if let FrontendError::RuntimeRejected {
legacy_http_status, ..
} = error
{
return StatusCode::from_u16(*legacy_http_status)
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
}
match error.kind() {
FrontendErrorKind::InvalidArgument => StatusCode::BAD_REQUEST,
FrontendErrorKind::NotFound => StatusCode::NOT_FOUND,
FrontendErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED,
// Existing intake backpressure has historically surfaced as 503.
FrontendErrorKind::ResourceExhausted | FrontendErrorKind::Unavailable => {
StatusCode::SERVICE_UNAVAILABLE
}
FrontendErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"),
FrontendErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT,
FrontendErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn frontend_error_categories_map_at_the_http_boundary() {
assert_eq!(
frontend_error_status(&FrontendError::InvalidArgument("bad".into())),
StatusCode::BAD_REQUEST
);
assert_eq!(
frontend_error_status(&FrontendError::Unavailable),
StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(
frontend_error_status(&FrontendError::Overloaded("full".into())),
StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(
frontend_error_status(&FrontendError::Cancelled("gone".into())).as_u16(),
499
);
assert_eq!(
frontend_error_status(&FrontendError::RuntimeRejected {
kind: FrontendErrorKind::InvalidArgument,
message: "runtime rejected request".into(),
legacy_http_status: 432,
})
.as_u16(),
432
);
assert_eq!(
frontend_error_status(&FrontendError::InvalidResponse("bad reply".into())),
StatusCode::INTERNAL_SERVER_ERROR
);
assert_eq!(
frontend_error_status(&FrontendError::Internal("bug".into())),
StatusCode::INTERNAL_SERVER_ERROR
);
}
}
+1
View File
@@ -148,6 +148,7 @@ mod tests {
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::default(),
},
)
}
+270 -153
View File
@@ -1,5 +1,5 @@
//! Common control-plane endpoints — `/server_info`, `/get_model_info`
//! (+ `/model_info` alias), plus the control-request path through
//! (+ `/model_info` alias), backed by typed operations on
//! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`,
//! which round-trips a generate probe) live in the sibling `native_api` and
//! `openai` modules; the shared `AppState` lives in the parent
@@ -15,17 +15,15 @@ use axum::{
use std::sync::Arc;
use super::app::AppState;
use super::frontend_error_status;
use super::native_api::native_error;
use crate::frontend::FrontendError;
use crate::message::config::ServerArgs;
use crate::message::ids::Rid;
use crate::message::io_struct::{ControlRequest, GetInternalStateReq};
use crate::frontend::{FrontendError, ServerInfo};
/// The routes this module owns, mounted by `api_server::serve`.
pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new()
// Control-plane: reuses the request FSM (no tokenization), returns one
// non-streamed JSON result. Adding one = a route line + its struct tag.
// Control-plane: the frontend performs the typed operation and this
// adapter renders its result as one non-streamed JSON response.
.route("/server_info", get(server_info))
// Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info`
// alias).
@@ -33,85 +31,57 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
.route("/model_info", get(model_info))
}
/// Submit a control request through the request FSM (no tokenization) and await the
/// scheduler's single msgpack result (a `structs.asdict` named map). Returns the
/// raw bytes, or an error `Response` to return as-is.
async fn await_control_result(
state: &AppState,
control: ControlRequest,
) -> Result<bytes::Bytes, Response> {
match state.frontend.control(control).await {
Ok(bytes) => Ok(bytes),
/// Await the frontend's complete semantic server-info result. HTTP status and
/// serialization stay here; scheduler control bytes stay behind the contract.
async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
match state.frontend.server_info().await {
Ok(server_info) => Ok(server_info),
Err(FrontendError::Unavailable) => Err(native_error(
StatusCode::SERVICE_UNAVAILABLE,
"service unavailable",
false,
)),
Err(FrontendError::Pipeline(error)) => {
let code = StatusCode::from_u16(error.http_status())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
Err((code, error.to_string()).into_response())
}
Err(FrontendError::ResponseClosed) => {
// Preserve the established HTTP behavior for a scheduler-side control
// stream that disappears, while non-HTTP adapters see Internal via
// FrontendError::kind().
Err(FrontendError::ResponseTruncated) => {
Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response())
}
Err(FrontendError::UnexpectedResponse(message)) => {
Err((StatusCode::INTERNAL_SERVER_ERROR, message).into_response())
}
Err(FrontendError::InvalidArgument(error)) => {
Err((StatusCode::INTERNAL_SERVER_ERROR, error).into_response())
Err(error @ FrontendError::InvalidResponse(_)) => {
tracing::error!(%error, "server_info: invalid runtime response");
Err((
StatusCode::INTERNAL_SERVER_ERROR,
"bad server_info response",
)
.into_response())
}
Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()),
}
}
/// `GET /get_model_info` (+ `/model_info` alias) — static model metadata from
/// `server_args` (no scheduler round-trip); `is_generation` always true.
/// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static
/// model metadata (no scheduler round-trip).
async fn model_info(State(state): State<Arc<AppState>>) -> Response {
let sa = &state.server_args;
let body = serde_json::json!({
"model_path": sa.model_path,
"served_model_name": sa.served_model_name,
"tokenizer_path": sa.tokenizer_path,
"is_generation": true,
// Python's `TokenizerManager` merges this into every request
// (`{**preferred, **client}`); this server has no equivalent yet, so
// `RustServer.launch` REFUSES to start when it is set. It can therefore
// only be null here — echoing it keeps the field's shape.
"preferred_sampling_params": sa.preferred_sampling_params,
// Python answers this through `config_value`, so a control-plane write
// moves it there; here it is the launch value.
"weight_version": sa.weight_version,
"load_format": sa.load_format,
// `auto` never reaches the blob: `resolve_auto_parsers` writes the
// selected parser into `server_args` before the scheduler forks.
"reasoning_parser": sa.reasoning_parser,
"tool_call_parser": sa.tool_call_parser,
"disaggregation_mode": sa.disaggregation_mode,
});
let info = state.frontend.model_info();
(
StatusCode::OK,
[("content-type", "application/json")],
serde_json::to_vec(&body).unwrap_or_default(),
serde_json::to_vec(&info).unwrap_or_default(),
)
.into_response()
}
/// `GET /server_info` — surface only an allowlist ([`INTERNAL_STATE_ALLOWLIST`] +
/// curated [`ServerArgs`] accessors), never the raw server-args dump (embeds
/// `api_key`/`admin_api_key`; see [`shape_server_info`]).
/// `GET /server_info` — serialize typed, allowlisted runtime metrics plus
/// curated launch metadata. The raw scheduler state embeds
/// `api_key`/`admin_api_key` and is never visible to this HTTP adapter.
///
/// TODO(server_info): Python also includes `kv_events`; add once plumbed.
async fn server_info(State(state): State<Arc<AppState>>) -> Response {
let bytes = match await_control_result(
&state,
ControlRequest::GetInternalStateReq(GetInternalStateReq::new(Rid::new().to_string())),
)
.await
{
Ok(b) => b,
let info = match await_server_info(&state).await {
Ok(info) => info,
Err(resp) => return resp,
};
match shape_server_info(&bytes, &state.server_args) {
match serde_json::to_vec(&info) {
Ok(json) => (StatusCode::OK, [("content-type", "application/json")], json).into_response(),
Err(e) => {
tracing::error!(error = %e, "server_info: shaping failed");
@@ -124,116 +94,263 @@ async fn server_info(State(state): State<Arc<AppState>>) -> Response {
}
}
/// Runtime-metric keys `get_internal_state` adds atop the server-args dump. We copy
/// ONLY these out of `internal_state` (an allowlist), so the co-mingled
/// `api_key`/`admin_api_key` can never reach the response.
const INTERNAL_STATE_ALLOWLIST: &[&str] = &[
"last_gen_throughput",
"memory_usage",
"effective_max_running_requests_per_dp",
"avg_spec_accept_length",
"step_time_dict",
"rust_mm_transport",
];
fn shape_server_info(msgpack: &[u8], server_args: &ServerArgs) -> Result<Vec<u8>, String> {
// GetInternalStateReqOutput asdict → `{ "internal_state": { server-args dump +
// metrics }, ... }`. Pull that inner map out (it is NOT safe to expose whole).
let mut obj: serde_json::Map<String, serde_json::Value> =
rmp_serde::from_slice(msgpack).map_err(|e| e.to_string())?;
let internal = match obj.remove("internal_state") {
Some(serde_json::Value::Object(m)) => m,
_ => serde_json::Map::new(),
#[cfg(test)]
mod tests {
use axum::{
body::{Body, to_bytes},
http::{Request as HttpRequest, header::CONTENT_TYPE},
};
use bytes::Bytes;
use tower::ServiceExt;
// Copy only the allowlisted runtime metrics — never the raw server-args dump.
let mut state_out = serde_json::Map::new();
for &k in INTERNAL_STATE_ALLOWLIST {
match internal.get(k) {
Some(v) if !v.is_null() => {
state_out.insert(k.to_string(), v.clone());
}
_ => {}
use super::*;
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs};
use crate::message::request::RequestKind;
use crate::message::response::ResponseItem;
use crate::tokenizer_manager::wiring::TmEvent;
fn test_state(
server_args: ServerArgs,
) -> (
Arc<AppState>,
flume::Receiver<TmEvent>,
flume::Receiver<crate::tokenizer_manager::wiring::AbortSource>,
) {
let server_args = Arc::new(server_args);
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::from(server_args.as_ref()),
},
);
(
Arc::new(AppState {
frontend,
server_args,
chat_formatter: None,
}),
intake_rx,
abort_rx,
)
}
async fn response_json(response: Response) -> serde_json::Value {
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers().get(CONTENT_TYPE).unwrap(),
"application/json"
);
let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
serde_json::from_slice(&body).unwrap()
}
#[tokio::test]
async fn model_info_routes_render_the_typed_frontend_contract() {
let (state, _, _) = test_state(ServerArgs {
model_path: "/model".into(),
served_model_name: "served".into(),
tokenizer_path: "/tokenizer".into(),
weight_version: Some("weights-v1".into()),
load_format: Some("safetensors".into()),
reasoning_parser: Some("reasoner".into()),
tool_call_parser: Some("tools".into()),
disaggregation_mode: DisaggregationMode::Prefill,
..Default::default()
});
let app = routes().with_state(state);
for path in ["/get_model_info", "/model_info"] {
let response = app
.clone()
.oneshot(
HttpRequest::builder()
.uri(path)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json(response).await;
assert_eq!(body["model_path"], "/model");
assert_eq!(body["served_model_name"], "served");
assert_eq!(body["tokenizer_path"], "/tokenizer");
assert_eq!(body["is_generation"], true);
assert_eq!(body["weight_version"], "weights-v1");
assert_eq!(body["load_format"], "safetensors");
assert_eq!(body["reasoning_parser"], "reasoner");
assert_eq!(body["tool_call_parser"], "tools");
assert_eq!(body["disaggregation_mode"], "prefill");
assert!(body["preferred_sampling_params"].is_null());
}
}
// Top-level non-secret config from typed accessors (structurally can't surface
// a key field, unlike the raw dump).
let response = serde_json::json!({
"model_path": server_args.model_path,
"served_model_name": server_args.served_model_name,
"tokenizer_path": server_args.tokenizer_path,
"max_context_length": server_args.model_config.context_len,
"max_total_num_tokens": server_args.max_total_num_tokens,
"version": server_args.version,
"frontend": "rust",
"internal_states": [serde_json::Value::Object(state_out)],
});
serde_json::to_vec(&response).map_err(|e| e.to_string())
}
#[tokio::test]
async fn server_info_route_serializes_only_the_typed_public_subset() {
// Native floats stay JSON numbers; NumPy scalars encoded by Python's
// msgspec enc_hook=str must remain JSON strings, as before the refactor.
assert_server_info_response(rmpv::Value::from(23.616), serde_json::json!(23.616)).await;
assert_server_info_response(rmpv::Value::from("23.616"), serde_json::json!("23.616")).await;
}
#[cfg(test)]
mod tests {
use super::*;
async fn assert_server_info_response(
kvcache: rmpv::Value,
expected_kvcache: serde_json::Value,
) {
let (state, intake_rx, abort_rx) = test_state(ServerArgs {
model_path: "/model".into(),
served_model_name: "served".into(),
tokenizer_path: "/tokenizer".into(),
model_config: ModelConfig {
context_len: 4096,
..Default::default()
},
max_total_num_tokens: 8192,
version: "1.2.3".into(),
..Default::default()
});
let request = tokio::spawn(
routes().with_state(state).oneshot(
HttpRequest::builder()
.uri("/server_info")
.body(Body::empty())
.unwrap(),
),
);
/// The scheduler's `internal_state` embeds the full server-args dump (incl.
/// `api_key`/`admin_api_key`). `/server_info` must surface only the allowlisted
/// runtime metrics + curated config — never the secrets — and must not re-nest
/// the dump under `internal_states[].internal_state`.
#[test]
fn shape_server_info_excludes_secrets_and_dump() {
// GetInternalStateReqOutput.asdict → { "internal_state": { …dump+metrics… } }.
let internal = rmpv::Value::Map(vec![
let TmEvent::Intake {
request: runtime_request,
admission,
} = intake_rx.recv_async().await.unwrap()
else {
panic!("server_info must submit one control request");
};
assert!(admission.try_accept());
assert!(matches!(runtime_request.kind, RequestKind::Control(_)));
let internal_state = rmpv::Value::Map(vec![
(
rmpv::Value::from("rust_mm_transport"),
rmpv::Value::Map(vec![(
rmpv::Value::from("inline_features"),
rmpv::Value::from(3),
)]),
),
(
rmpv::Value::from("api_key"),
rmpv::Value::from("secret-token"),
rmpv::Value::from("must-not-leak"),
),
(
rmpv::Value::from("admin_api_key"),
rmpv::Value::from("admin-token"),
),
(rmpv::Value::from("model_path"), rmpv::Value::from("/m")),
(
rmpv::Value::from("last_gen_throughput"),
rmpv::Value::from(1.5),
),
(
rmpv::Value::from("effective_max_running_requests_per_dp"),
rmpv::Value::from(32),
rmpv::Value::from("memory_usage"),
rmpv::Value::Map(vec![
(rmpv::Value::from("kvcache"), kvcache),
(rmpv::Value::from("token_capacity"), rmpv::Value::from(8192)),
(rmpv::Value::from("token_capacity_swa"), rmpv::Value::Nil),
(
rmpv::Value::from("future_uncontracted_metric"),
rmpv::Value::from(99),
),
]),
),
(
rmpv::Value::from("step_time_dict"),
rmpv::Value::Map(vec![(
rmpv::Value::from(4),
rmpv::Value::Array(vec![rmpv::Value::from(0.01)]),
)]),
),
]);
let outer = rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal)]);
let mut msgpack = Vec::new();
rmpv::encode::write_value(&mut msgpack, &outer).unwrap();
let envelope =
rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal_state)]);
let mut payload = Vec::new();
rmpv::encode::write_value(&mut payload, &envelope).unwrap();
runtime_request
.sink
.try_send(ResponseItem::Control(Bytes::from(payload)))
.unwrap();
// `api_key` is deliberately NOT a `ServerArgs` field — the typed schema
// cannot carry it — so the only place it could leak from is the raw
// scheduler dump shaped above.
let sa = ServerArgs {
model_path: "/m".into(),
..Default::default()
let response = request.await.unwrap().unwrap();
let body = response_json(response).await;
assert_eq!(body["model_path"], "/model");
assert_eq!(body["served_model_name"], "served");
assert_eq!(body["tokenizer_path"], "/tokenizer");
assert_eq!(body["max_context_length"], 4096);
assert_eq!(body["max_total_num_tokens"], 8192);
assert_eq!(body["version"], "1.2.3");
assert_eq!(body["frontend"], "rust");
assert_eq!(
body["internal_states"][0]["rust_mm_transport"]["inline_features"],
3
);
assert_eq!(body["internal_states"][0]["last_gen_throughput"], 1.5);
assert_eq!(
body["internal_states"][0]["memory_usage"]["kvcache"],
expected_kvcache
);
assert_eq!(
body["internal_states"][0]["memory_usage"]["token_capacity"],
8192
);
assert_eq!(
body["internal_states"][0]["memory_usage"]["token_capacity_swa"],
serde_json::Value::Null
);
assert_eq!(
body["internal_states"][0]["step_time_dict"]["4"],
serde_json::json!([0.01])
);
assert!(!body.to_string().contains("must-not-leak"));
assert!(
body["internal_states"][0]["memory_usage"]
.get("future_uncontracted_metric")
.is_none(),
"uncontracted scheduler fields must not silently become public API"
);
assert!(abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn server_info_preserves_legacy_499_for_a_truncated_control_response() {
let (state, intake_rx, abort_rx) = test_state(ServerArgs::default());
let response = tokio::spawn(
routes().with_state(state).oneshot(
HttpRequest::builder()
.uri("/server_info")
.body(Body::empty())
.unwrap(),
),
);
let TmEvent::Intake {
request: runtime_request,
admission,
} = intake_rx.recv_async().await.unwrap()
else {
panic!("server_info must submit one control request");
};
let out = shape_server_info(&msgpack, &sa).unwrap();
let text = String::from_utf8(out.clone()).unwrap();
// No secret leaks anywhere in the serialized response.
assert!(!text.contains("secret-token"), "api_key leaked: {text}");
assert!(
!text.contains("admin-token"),
"admin_api_key leaked: {text}"
);
assert!(admission.try_accept());
let rid = runtime_request.rid.clone();
drop(runtime_request);
let v: serde_json::Value = serde_json::from_slice(&out).unwrap();
// Allowlisted metric surfaced; the whole dump did not.
let state0 = &v["internal_states"][0];
assert_eq!(state0["last_gen_throughput"], 1.5);
assert_eq!(state0["effective_max_running_requests_per_dp"], 32);
assert!(
state0.get("internal_state").is_none(),
"must not re-nest the dump under internal_state"
);
assert!(state0.get("api_key").is_none());
// Curated top-level config comes from typed accessors, not the dump.
assert_eq!(v["model_path"], "/m");
assert_eq!(v["frontend"], "rust");
let response = response.await.unwrap().unwrap();
assert_eq!(response.status().as_u16(), 499);
let body = to_bytes(response.into_body(), 1024).await.unwrap();
assert_eq!(body.as_ref(), b"request aborted");
assert!(matches!(
abort_rx.recv_async().await.unwrap(),
crate::tokenizer_manager::wiring::AbortSource::Guard(aborted) if aborted == rid
));
}
}
+26 -35
View File
@@ -1,10 +1,11 @@
//! Frame shaping for the native `/generate` protocol: the cumulative
//! [`OutputAccumulator`] plus the functions that render [`ChunkEvent`]s /
//! [`OutputAccumulator`] plus the functions that render [`FrontendOutput`]s /
//! accumulated state into wire JSON (`meta_info`, logprob tuples, error and
//! abort frames). No HTTP here — the sibling `native_api` module owns the handlers
//! and streams; it calls these per frame.
use crate::message::response::{ChunkEvent, ChunkExtras};
use crate::frontend::FrontendOutput;
use crate::message::response::ChunkExtras;
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
@@ -159,10 +160,10 @@ fn hidden_states_rows(vals: &[f32], lens: &[u32]) -> serde_json::Value {
serde_json::Value::Array(rows)
}
/// Format a decoded [`ChunkEvent`] as one SGLang `/generate` frame's JSON. `rid`
/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid`
/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is
/// just the shard routing key.
pub(super) fn frame_value(out: &ChunkEvent, rid: &str) -> serde_json::Value {
pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
let mut v = serde_json::json!({
"text": out.text,
"meta_info": {
@@ -318,7 +319,7 @@ pub(super) fn tag_value(mut v: serde_json::Value, index: Option<usize>) -> Strin
/// One streaming frame's JSON: cumulative ignores `delta`, incremental ships it.
pub(super) fn stream_frame_string(
delta: ChunkEvent,
delta: FrontendOutput,
acc: &OutputAccumulator,
incremental: bool,
rid_str: &str,
@@ -344,7 +345,7 @@ pub(super) fn cumulative_frame_string(
/// Format one streaming frame: the accumulator's cumulative view (default), or this
/// step's delta with the cumulative token count in `meta_info` (matching Python).
pub(super) fn stream_frame_value(
delta: ChunkEvent,
delta: FrontendOutput,
acc: &OutputAccumulator,
incremental: bool,
rid_str: &str,
@@ -358,15 +359,15 @@ pub(super) fn stream_frame_value(
}
}
/// Folds per-chunk [`ChunkEvent`] deltas into a cumulative view — used by the drain
/// Folds per-chunk [`FrontendOutput`] deltas into a cumulative view — used by the drain
/// loops needing cumulative output (every unary response + the cumulative SGLang
/// stream; OpenAI streaming forwards deltas and skips this). Holds a single
/// [`ChunkEvent`] so `snapshot` hands back a **borrow** per frame — no per-frame
/// [`FrontendOutput`] so `snapshot` hands back a **borrow** per frame — no per-frame
/// clone of the growing buffers (that added O(T²) atop the wire's inherent O(T²)).
/// Shared with the [`openai`] submodule.
#[derive(Default)]
pub(super) struct OutputAccumulator {
out: ChunkEvent,
out: FrontendOutput,
/// Serialized cumulative `output_ids` body (`"1,2,3"`, no brackets), appended per
/// delta so a frame memcpy's it instead of rebuilding the array — O(T), not O(T²).
ids_json: String,
@@ -414,7 +415,7 @@ impl OutputAccumulator {
/// Fold one delta frame in. Output families concatenate; input families and
/// hidden states are set-once / last-writer-wins (they ride the prefill/final
/// chunk), matching the Python `meta_info` assignment.
pub(super) fn fold(&mut self, d: &ChunkEvent) {
pub(super) fn fold(&mut self, d: &FrontendOutput) {
use std::fmt::Write;
// Grow the memoized serializations alongside the raw cumulative buffers.
@@ -427,7 +428,6 @@ impl OutputAccumulator {
}
let o = &mut self.out;
o.rid.clone_from(&d.rid); // constant across the request; keeps the accumulated view coherent
o.text.push_str(&d.text);
o.token_ids.extend_from_slice(&d.token_ids); // token_ids doubles as output_ids
o.completion_tokens += d.completion_tokens;
@@ -532,12 +532,12 @@ impl OutputAccumulator {
}
/// Borrow the cumulative output for an intermediate streaming frame.
pub(super) fn snapshot(&self) -> &ChunkEvent {
pub(super) fn snapshot(&self) -> &FrontendOutput {
&self.out
}
/// Consume into the final cumulative output.
pub(super) fn into_output(self) -> ChunkEvent {
pub(super) fn into_output(self) -> FrontendOutput {
self.out
}
}
@@ -606,12 +606,12 @@ mod tests {
);
}
/// End-to-end: a `ChunkEvent` carrying a prompt-logprob request (first input
/// End-to-end: a `FrontendOutput` carrying a prompt-logprob request (first input
/// logprob is the `NaN` sentinel) formats without panicking and emits
/// `input_token_logprobs` with a leading `[null, token_id, text]`.
#[test]
fn prompt_logprob_frame_emits_null_first() {
let out = ChunkEvent {
let out = FrontendOutput {
extras: Some(Box::new(ChunkExtras {
in_lp_val: vec![f32::NAN, -0.5],
in_lp_idx: vec![10, 20],
@@ -632,7 +632,7 @@ mod tests {
#[test]
fn accumulator_snapshot_is_cumulative() {
let mut acc = OutputAccumulator::default();
acc.fold(&ChunkEvent {
acc.fold(&FrontendOutput {
text: "he".into(),
token_ids: vec![1, 2],
completion_tokens: 2,
@@ -643,7 +643,7 @@ mod tests {
assert_eq!(s.text, "he");
assert_eq!(s.token_ids, vec![1, 2]);
}
acc.fold(&ChunkEvent {
acc.fold(&FrontendOutput {
text: "llo".into(),
token_ids: vec![3],
completion_tokens: 1,
@@ -681,32 +681,28 @@ mod tests {
#[test]
fn cumulative_frame_json_matches_serde() {
let deltas = [
ChunkEvent {
rid: "7".into(),
FrontendOutput {
text: String::new(),
token_ids: vec![],
completion_tokens: 0,
prompt_tokens: 128,
..Default::default()
},
ChunkEvent {
rid: "7".into(),
FrontendOutput {
text: "He\"llo\n\t".into(),
token_ids: vec![1000],
completion_tokens: 1,
prompt_tokens: 128,
..Default::default()
},
ChunkEvent {
rid: "7".into(),
FrontendOutput {
text: " 世界 🌍 \\".into(),
token_ids: vec![-2, 3],
completion_tokens: 2,
prompt_tokens: 128,
..Default::default()
},
ChunkEvent {
rid: "7".into(),
FrontendOutput {
text: "!".into(),
token_ids: vec![9],
completion_tokens: 1,
@@ -752,8 +748,7 @@ mod tests {
};
let deltas = [
// Prefill: the set-once input families and a null top-k position.
ChunkEvent {
rid: "9".into(),
FrontendOutput {
prompt_tokens: 4,
extras: Some(Box::new(ChunkExtras {
in_lp_val: vec![f32::NAN, -1.5],
@@ -771,8 +766,7 @@ mod tests {
})),
..Default::default()
},
ChunkEvent {
rid: "9".into(),
FrontendOutput {
text: "He\"llo".into(),
token_ids: vec![100],
completion_tokens: 1,
@@ -793,8 +787,7 @@ mod tests {
})),
..Default::default()
},
ChunkEvent {
rid: "9".into(),
FrontendOutput {
text: " 世界".into(),
token_ids: vec![-2, 3],
completion_tokens: 2,
@@ -847,8 +840,7 @@ mod tests {
#[test]
fn mismatched_logprob_texts_fall_back_to_the_value_path() {
let mut acc = OutputAccumulator::default();
acc.fold(&ChunkEvent {
rid: "1".into(),
acc.fold(&FrontendOutput {
extras: Some(Box::new(ChunkExtras {
out_lp_val: vec![-0.5],
out_lp_idx: vec![5],
@@ -857,8 +849,7 @@ mod tests {
..Default::default()
});
assert!(cumulative_frame_json(&acc, "1", None).is_some());
acc.fold(&ChunkEvent {
rid: "1".into(),
acc.fold(&FrontendOutput {
extras: Some(Box::new(ChunkExtras {
out_lp_val: vec![-0.25],
out_lp_idx: vec![6],
+109 -81
View File
@@ -14,10 +14,13 @@ use super::app::AppState;
use super::frame::{
OutputAccumulator, cumulative_frame_string, frame_value, stream_frame_string, tag_value,
};
use crate::frontend::{FrontendCall, FrontendError, HealthStatus};
use super::frontend_error_status;
use crate::frontend::{
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
};
#[cfg(test)]
use crate::message::ids::Rid;
use crate::message::request::{GenerateBody, GenerateRequest};
use crate::message::response::{ChunkEvent, ResponseItem};
use crate::message::request::GenerateBody;
use crate::utils::{
environ,
response::{error_response, error_value},
@@ -87,19 +90,7 @@ pub(super) fn native_error(code: StatusCode, message: &str, stream: bool) -> Res
}
fn native_frontend_error(error: FrontendError, stream: bool) -> Response {
match error {
FrontendError::Unavailable => {
native_error(StatusCode::SERVICE_UNAVAILABLE, &error.to_string(), stream)
}
FrontendError::InvalidArgument(_) => {
native_error(StatusCode::BAD_REQUEST, &error.to_string(), stream)
}
_ => native_error(
StatusCode::INTERNAL_SERVER_ERROR,
&error.to_string(),
stream,
),
}
native_error(frontend_error_status(&error), &error.to_string(), stream)
}
/// `/health` + `/health_generate`. Both env knobs are resolved ONCE here, at
@@ -196,10 +187,9 @@ async fn generate(
// payloads. `is_batch` = list form → the response is a JSON array.
let (payloads, is_batch) = match body.into_requests() {
Ok(v) => v,
// The error carries its own status (a bad batch is `Validation` → 400).
// Request normalization only returns validation failures.
Err(e) => {
let code = StatusCode::from_u16(e.http_status()).unwrap_or(StatusCode::BAD_REQUEST);
return native_error(code, &e.to_string(), stream);
return native_error(StatusCode::BAD_REQUEST, &e.to_string(), stream);
}
};
// Python starts APIServerReqTimeStats after request normalization and before
@@ -225,7 +215,7 @@ async fn generate(
/// SSE frames or fold to one unary response.
async fn generate_single(
state: &AppState,
req: GenerateRequest,
req: FrontendRequest,
stream: bool,
timing: RequestTiming,
) -> Response {
@@ -235,7 +225,7 @@ async fn generate_single(
Ok(call) => call,
Err(error) => return native_frontend_error(error, stream),
};
let rid = call.rid().clone();
let rid = call.public_id().to_owned();
// Cumulative frames (SGLang default) vs per-step deltas.
let incremental = state.server_args.incremental_streaming_output;
@@ -249,7 +239,7 @@ async fn generate_single(
} else {
// Unary: fold to the terminal and respond once. `FrontendCall` disarms
// itself on a terminal item; truncation leaves it armed for drop cleanup.
let (status, value) = drain_unary(&mut call, rid.client_facing(), timing).await;
let (status, value) = drain_unary(&mut call, &rid, timing).await;
(status, Json(value)).into_response()
}
}
@@ -263,41 +253,28 @@ async fn drain_unary(
let mut acc = OutputAccumulator::default();
while let Some(item) = call.recv().await {
match item {
ResponseItem::Frame(out) => {
FrontendEvent::Delta(out) => {
timing.observe_first_output();
acc.fold(&out);
}
ResponseItem::Done(out) => {
FrontendEvent::Finished(out) => {
timing.observe_first_output();
timing.finish();
acc.fold(&out);
let final_out = acc.into_output();
// A validation abort carries its own HTTP status + diagnostic.
if let Some((code, message)) = final_out
.finish_reason
.as_ref()
.and_then(|f| f.abort_status())
{
let status =
StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
return (status, error_value(code, message));
}
let mut value = frame_value(&final_out, rid_str);
add_e2e_latency(&mut value, &timing);
return (StatusCode::OK, value);
}
ResponseItem::Error(e) => {
FrontendEvent::Failed(error) => {
timing.finish();
let code = e.http_status();
let status =
StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
return (status, error_value(code, &e.to_string()));
let status = frontend_error_status(&error);
return (status, error_value(status.as_u16(), &error.to_string()));
}
ResponseItem::Control(_) | ResponseItem::Data(_) => continue, // never on `/generate`
}
}
// Sender dropped without a terminal item: the shard dropped this request (a
// truncation — a client disconnect would have dropped the handler future).
// Defensive fallback: FrontendCall normally turns premature runtime
// termination into a terminal Failed event before exposing stream end.
(
StatusCode::INTERNAL_SERVER_ERROR,
error_value(500, "response truncated before completion"),
@@ -312,7 +289,7 @@ async fn drain_unary(
/// own `{ "error": … }` entry; the batch response is 200.
async fn generate_batch(
state: &AppState,
requests: Vec<GenerateRequest>,
requests: Vec<FrontendRequest>,
stream: bool,
timing: RequestTiming,
) -> Response {
@@ -341,7 +318,7 @@ async fn generate_batch(
// its own terminal output promptly (important for per-item e2e_latency).
let drained = futures::future::join_all(calls.into_iter().map(
|(mut call, request_timing)| async move {
let client_rid = call.rid().client_facing().to_owned();
let client_rid = call.public_id().to_owned();
let (_status, value) = drain_unary(&mut call, &client_rid, request_timing).await;
value
},
@@ -351,21 +328,20 @@ async fn generate_batch(
}
}
/// Await the next item from `call`, then drain whatever queued behind it (so the caller
/// can coalesce a backlog, as Python's `state.out_list` does), handing the call
/// back for `FuturesUnordered` to re-poll. Empty result = channel closed.
/// Await the next event from `call`, then drain whatever queued behind it (so the
/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the
/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic
/// stream was already exhausted.
async fn recv_indexed(
index: usize,
mut call: FrontendCall,
) -> (usize, FrontendCall, Vec<ResponseItem>) {
) -> (usize, FrontendCall, Vec<FrontendEvent>) {
let mut items = Vec::new();
match call.recv().await {
Some(item) => items.push(item),
None => return (index, call, items), // closed
}
while let Ok(item) = call.try_recv() {
items.push(item);
None => return (index, call, items), // already exhausted
}
call.drain_ready(&mut items);
(index, call, items)
}
@@ -381,9 +357,9 @@ fn generation_event_stream(
use futures::StreamExt;
let n = calls.len();
let rid_strs: Vec<Rid> = calls
let rid_strs: Vec<String> = calls
.iter()
.map(|(call, _)| call.rid().clone())
.map(|(call, _)| call.public_id().to_owned())
.collect();
let mut timings: Vec<RequestTiming> = calls
.iter()
@@ -404,8 +380,9 @@ fn generation_event_stream(
while let Some((i, call, items)) = futs.next().await {
if items.is_empty() {
// Channel closed with no terminal → truncation for this item;
// leave its rid armed so the scheduler work is aborted.
// Defensive fallback: a premature runtime close is normally a
// Failed event produced by FrontendCall. Keep this item's call
// armed if its semantic stream somehow ends unexpectedly.
yield tag_value(error_value(500, "response truncated before completion"), idx(i));
continue;
}
@@ -418,48 +395,45 @@ fn generation_event_stream(
for item in items {
match item {
ResponseItem::Frame(out) => {
FrontendEvent::Delta(out) => {
timings[i].observe_first_output();
accs[i].fold(&out);
if incremental {
yield stream_frame_string(out, &accs[i], true, rid_strs[i].client_facing(), idx(i));
yield stream_frame_string(out, &accs[i], true, &rid_strs[i], idx(i));
} else {
coalesced = true;
}
}
ResponseItem::Done(out) => {
FrontendEvent::Finished(out) => {
timings[i].observe_first_output();
timings[i].finish();
accs[i].fold(&out);
terminal = Some(out);
}
ResponseItem::Error(e) => {
FrontendEvent::Failed(error) => {
timings[i].finish();
failed = Some(e);
failed = Some(error);
}
ResponseItem::Control(_) | ResponseItem::Data(_) => {} // never on /generate
}
}
if let Some(e) = failed {
yield tag_value(error_value(e.http_status(), &e.to_string()), idx(i));
if let Some(error) = failed {
let status = frontend_error_status(&error);
yield tag_value(error_value(status.as_u16(), &error.to_string()), idx(i));
} else if let Some(out) = terminal {
// A validation abort → an error object, not a frame. The final frame
// carries the full cumulative state, so any coalesced ones are moot.
yield match out.finish_reason.as_ref().and_then(|f| f.abort_status()) {
Some((code, message)) => tag_value(error_value(code, message), idx(i)),
None => terminal_stream_frame_string(
out,
&accs[i],
incremental,
rid_strs[i].client_facing(),
idx(i),
&timings[i],
),
};
// The final frame carries the full cumulative state, so any
// coalesced non-terminal frames are moot.
yield terminal_stream_frame_string(
out,
&accs[i],
incremental,
&rid_strs[i],
idx(i),
&timings[i],
);
} else {
if coalesced {
yield cumulative_frame_string(&accs[i], rid_strs[i].client_facing(), idx(i));
yield cumulative_frame_string(&accs[i], &rid_strs[i], idx(i));
}
futs.push(recv_indexed(i, call)); // keep this item flowing
}
@@ -471,7 +445,7 @@ fn generation_event_stream(
/// Python's `e2e_latency` is `finished_time - created_time`, in seconds, and is
/// attached only when the request finishes. The Rust native API owns the same
/// lifecycle boundary, so it adds the value while handling the terminal egress
/// item rather than putting API-only timing onto every scheduler `ChunkEvent`.
/// item rather than putting API-only timing onto every shared frontend output.
fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) {
let (time_to_first_token, e2e_latency) = timing
.terminal_latencies()
@@ -484,7 +458,7 @@ fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) {
/// memoized fast path; the one terminal frame uses the Value path so it can carry
/// the request-local `e2e_latency`, exactly as Python does.
fn terminal_stream_frame_string(
out: ChunkEvent,
out: FrontendOutput,
acc: &OutputAccumulator,
incremental: bool,
rid_str: &str,
@@ -500,7 +474,7 @@ fn terminal_stream_frame_string(
mod tests {
use super::*;
use crate::message::request::RequestKind;
use crate::message::response::ChunkEvent;
use crate::message::response::{ChunkEvent, ResponseItem};
use crate::tokenizer_manager::wiring::TmEvent;
use crate::utils::error::Error;
use axum::body::Body;
@@ -568,6 +542,7 @@ mod tests {
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::default(),
},
),
server_args: Arc::new(crate::message::config::ServerArgs::default()),
@@ -593,6 +568,7 @@ mod tests {
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::default(),
},
);
let state = Arc::new(AppState {
@@ -825,6 +801,32 @@ mod tests {
assert_eq!(stream.next().await.unwrap(), "[DONE]");
}
#[tokio::test]
async fn scheduler_rejection_preserves_legacy_http_status() {
let (tx, rx) = mpsc::channel(8);
let calls = vec![timed_call(10, rx)];
let stream = generation_event_stream(calls, false, false);
futures::pin_mut!(stream);
tx.send(ResponseItem::Done(ChunkEvent {
finish_reason: serde_json::from_value(serde_json::json!({
"type": "abort",
"message": "media decode failed",
"status_code": 432,
"err_type": "BadRequestError"
}))
.unwrap(),
..Default::default()
}))
.await
.unwrap();
let failure = parse(&stream.next().await.unwrap());
assert_eq!(failure["error"]["code"], 432);
assert_eq!(failure["error"]["message"], "media decode failed");
assert_eq!(stream.next().await.unwrap(), "[DONE]");
}
/// `incremental=true`: each frame carries this step's **delta** text/output_ids,
/// but `meta_info.completion_tokens` stays cumulative (matching Python).
#[tokio::test]
@@ -905,6 +907,32 @@ mod tests {
assert_eq!(stream.next().await.unwrap(), "[DONE]");
}
/// A runtime close must not overtake output that was already queued. The
/// cumulative adapter first exposes the produced text, then reports the
/// truncation on the next event, preserving the Step 2 HTTP behavior.
#[tokio::test]
async fn cumulative_output_precedes_truncation_error() {
let (tx, rx) = mpsc::channel(8);
let calls = vec![timed_call(10, rx)];
let stream = generation_event_stream(calls, false, false);
futures::pin_mut!(stream);
tx.send(frame(10, "already-produced")).await.unwrap();
drop(tx);
let output = parse(&stream.next().await.unwrap());
assert_eq!(output["text"], "already-produced");
let failure = parse(&stream.next().await.unwrap());
assert_eq!(failure["error"]["code"], 500);
assert_eq!(
failure["error"]["message"],
"response truncated before completion"
);
assert_eq!(stream.next().await.unwrap(), "[DONE]");
assert!(stream.next().await.is_none());
}
/// Incremental frames are *deltas*, so a backlog must emit every one — dropping
/// any would silently lose tokens. Only the cumulative protocol may coalesce.
#[tokio::test]
+18 -33
View File
@@ -1,8 +1,8 @@
//! OpenAI-compatible generation endpoints.
//!
//! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI
//! request and response primitives. Scheduler [`ChunkEvent`] values remain the one
//! backend output type for both unary and streaming responses.
//! request and response primitives. [`FrontendOutput`] remains the one backend
//! output type for both unary and streaming responses.
use axum::{Router, http::StatusCode, response::Response};
use futures::StreamExt;
@@ -22,10 +22,11 @@ pub(super) use template::{ChatFormatter, ChatTemplateKwargs};
use super::app::AppState;
use super::frame::OutputAccumulator;
use crate::frontend::{FrontendCall, FrontendError};
use super::frontend_error_status;
use crate::frontend::{
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
};
use crate::message::config::ServerArgs;
use crate::message::request::GenerateRequest;
use crate::message::response::{ChunkEvent, ResponseItem};
use crate::tokenizer_manager::tokenizer;
use crate::utils::response::error_response;
@@ -115,23 +116,21 @@ pub(super) fn openai_error(code: StatusCode, message: impl Into<String>, stream:
}
/// Drain one submitted request to its terminal output, fold frames, and map
/// errors / validation aborts / truncation to `(status, message)` for the
/// OpenAI error shape. The call owns cancellation and disarms itself.
async fn collect_output(mut call: FrontendCall) -> Result<ChunkEvent, (StatusCode, String)> {
/// semantic failures / truncation to `(status, message)` for the OpenAI error
/// shape. The call owns cancellation and disarms itself.
async fn collect_output(mut call: FrontendCall) -> Result<FrontendOutput, (StatusCode, String)> {
let mut accumulator = OutputAccumulator::default();
let output = loop {
match call.recv().await {
Some(ResponseItem::Frame(output)) => accumulator.fold(&output),
Some(ResponseItem::Done(output)) => {
Some(FrontendEvent::Delta(output)) => accumulator.fold(&output),
Some(FrontendEvent::Finished(output)) => {
accumulator.fold(&output);
break accumulator.into_output();
}
Some(ResponseItem::Error(error)) => {
let status = StatusCode::from_u16(error.http_status())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
Some(FrontendEvent::Failed(error)) => {
let status = frontend_error_status(&error);
return Err((status, error.to_string()));
}
Some(ResponseItem::Control(_)) | Some(ResponseItem::Data(_)) => {}
None => {
return Err((
StatusCode::INTERNAL_SERVER_ERROR,
@@ -140,22 +139,12 @@ async fn collect_output(mut call: FrontendCall) -> Result<ChunkEvent, (StatusCod
}
}
};
if let Some((code, message)) = output
.finish_reason
.as_ref()
.and_then(|reason| reason.abort_status())
{
return Err((
StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
message.to_owned(),
));
}
Ok(output)
}
async fn submit_generation(
state: &AppState,
request: GenerateRequest,
request: FrontendRequest,
stream: bool,
) -> Result<FrontendCall, Response> {
match state.frontend.generate(request).await {
@@ -179,18 +168,14 @@ async fn submit_generation(
fn indexed_decode_stream(
index: usize,
call: FrontendCall,
) -> futures::stream::BoxStream<'static, (usize, Option<ResponseItem>)> {
) -> futures::stream::BoxStream<'static, (usize, FrontendEvent)> {
futures::stream::unfold((call, false), move |(mut call, finished)| async move {
if finished {
return None;
}
match call.recv().await {
Some(item) => {
let finished = matches!(item, ResponseItem::Done(_) | ResponseItem::Error(_));
Some(((index, Some(item)), (call, finished)))
}
None => Some(((index, None), (call, true))),
}
let event = call.recv().await?;
let finished = event.is_terminal();
Some(((index, event), (call, finished)))
})
.boxed()
}
@@ -36,11 +36,11 @@ use super::{
AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload,
indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32,
};
use crate::frontend::FrontendCall;
use crate::api_server::frontend_error_status;
use crate::frontend::{FrontendCall, FrontendEvent, FrontendRequest};
use crate::message::config::{DefaultSamplingParams, ServerArgs};
use crate::message::ids::Rid;
use crate::message::request::GenerateRequest;
use crate::message::response::{ChunkExtras, ResponseItem};
use crate::message::response::ChunkExtras;
use crate::message::sampling::SamplingParams;
use crate::message::types::OneOrMany;
@@ -204,7 +204,7 @@ async fn chat_completions(
.expect("chat prompt exists until the last choice")
.clone()
};
let native = GenerateRequest {
let native = FrontendRequest {
rid: rid.clone(),
text: Some(choice_prompt),
// Rendered templates own their special tokens — the pool must not
@@ -580,47 +580,20 @@ pub(super) fn chat_event_stream(
}
let mut events = futures::stream::select_all(streams);
while let Some((index, item)) = events.next().await {
let Some(item) = item else {
yield Annotated {
data: None,
id: None,
event: None,
comment: None,
error: Some(error_payload(StatusCode::INTERNAL_SERVER_ERROR, "response truncated before completion").to_string()),
};
continue;
};
let output = match item {
ResponseItem::Frame(output) => output,
ResponseItem::Done(output) => output,
ResponseItem::Error(error) => {
while let Some((index, event)) = events.next().await {
let output = match event {
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
FrontendEvent::Failed(error) => {
yield Annotated {
data: None,
id: None,
event: None,
comment: None,
error: Some(error_payload(StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), error.to_string()).to_string()),
error: Some(error_payload(frontend_error_status(&error), error.to_string()).to_string()),
};
continue;
}
ResponseItem::Control(_) | ResponseItem::Data(_) => continue,
};
if let Some((code, message)) = output
.finish_reason
.as_ref()
.and_then(|reason| reason.abort_status())
{
yield Annotated {
data: None,
id: None,
event: None,
comment: None,
error: Some(error_payload(StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), message).to_string()),
};
continue;
}
if prompt_tokens == 0 {
prompt_tokens = output.prompt_tokens;
}
@@ -24,14 +24,15 @@ use super::{
AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream,
openai_error, submit_generation, unix_seconds_u32,
};
use crate::frontend::{FrontendCall, FrontendError};
use crate::api_server::frontend_error_status;
use crate::frontend::{
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
};
use crate::message::finish_reason::Matched;
use crate::message::ids::Rid;
use crate::message::request::GenerateRequest;
use crate::message::response::{ChunkEvent, ChunkExtras, ResponseItem};
use crate::message::response::ChunkExtras;
use crate::message::sampling::SamplingParams;
use crate::message::types::{OneOrMany, TokenIds};
use crate::utils::error::Error;
pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new().route("/v1/completions", post(completions))
@@ -163,7 +164,7 @@ async fn completions(
Err(response) => return response,
};
}
let native = GenerateRequest {
let native = FrontendRequest {
rid: rid.clone(),
text: text.clone(),
input_ids: input_ids.clone(),
@@ -234,13 +235,7 @@ async fn completions(
/// or an `Error` (e.g. out-of-range ids → `Validation` → 400).
async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result<String, Response> {
match state.frontend.detokenize(token_ids).await {
Ok(payload) => String::from_utf8(payload.to_vec()).map_err(|_| {
openai_error(
StatusCode::INTERNAL_SERVER_ERROR,
"detokenized prompt is not valid UTF-8",
false,
)
}),
Ok(text) => Ok(text),
// Same rule as `submit_generation`: build the refusal in the OpenAI
// error shape rather than forwarding the native-shaped response.
Err(FrontendError::Unavailable) => Err(openai_error(
@@ -248,30 +243,22 @@ async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result<Str
"service unavailable",
false,
)),
Err(FrontendError::Pipeline(Error::Validation(message))) => {
Err(FrontendError::InvalidArgument(message)) => {
Err(openai_error(StatusCode::BAD_REQUEST, &message, false))
}
Err(FrontendError::Pipeline(error)) => {
let status = StatusCode::from_u16(error.http_status())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
Err(FrontendError::InvalidResponse(message)) => Err(openai_error(
StatusCode::INTERNAL_SERVER_ERROR,
message,
false,
)),
Err(error) => {
let status = frontend_error_status(&error);
Err(openai_error(
status,
format!("failed to decode prompt for echo: {error}"),
false,
))
}
Err(FrontendError::ResponseClosed | FrontendError::UnexpectedResponse(_)) => {
Err(openai_error(
StatusCode::INTERNAL_SERVER_ERROR,
"failed to decode prompt for echo: reply channel closed",
false,
))
}
Err(error @ FrontendError::InvalidArgument(_)) => Err(openai_error(
StatusCode::INTERNAL_SERVER_ERROR,
format!("failed to decode prompt for echo: {error}"),
false,
)),
}
}
@@ -419,7 +406,7 @@ pub(super) async fn unary_completion(
fn completion_choice(
index: usize,
text: String,
output: &ChunkEvent,
output: &FrontendOutput,
want_logprobs: bool,
include_input_logprobs: bool,
) -> (Choice, ChoiceExtensions) {
@@ -532,30 +519,15 @@ pub(super) fn completion_event_stream(
}
let mut events = futures::stream::select_all(streams);
while let Some((index, item)) = events.next().await {
let Some(item) = item else {
yield error_payload(StatusCode::INTERNAL_SERVER_ERROR, "response truncated before completion").to_string();
continue;
};
let output = match item {
ResponseItem::Frame(output) => output,
ResponseItem::Done(output) => output,
ResponseItem::Error(error) => {
yield error_payload(StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), error.to_string()).to_string();
while let Some((index, event)) = events.next().await {
let output = match event {
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
FrontendEvent::Failed(error) => {
yield error_payload(frontend_error_status(&error), error.to_string()).to_string();
continue;
}
ResponseItem::Control(_) | ResponseItem::Data(_) => continue,
};
if let Some((code, message)) = output
.finish_reason
.as_ref()
.and_then(|reason| reason.abort_status())
{
yield error_payload(StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), message).to_string();
continue;
}
prompt_tokens_by_prompt
.entry(prompt_indexes[index])
.or_insert(output.prompt_tokens);
@@ -20,7 +20,7 @@ use serde_json::json;
use tower::util::ServiceExt;
use super::{indexed_decode_stream, openai_error, routes};
use crate::frontend::{FrontendCall, FrontendHandle};
use crate::frontend::{FrontendCall, FrontendEvent, FrontendHandle};
use crate::message::config::ServerArgs;
use crate::message::response::{ChunkEvent, ResponseItem};
pub(super) fn frontend() -> FrontendHandle {
@@ -33,6 +33,7 @@ pub(super) fn frontend() -> FrontendHandle {
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
},
)
}
@@ -130,6 +131,7 @@ pub(super) fn frontend_closed() -> FrontendHandle {
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
},
)
}
@@ -170,7 +172,7 @@ async fn dropping_indexed_stream_aborts_its_live_call() {
response_tx.send(chunk("live", "x", false)).await.unwrap();
assert!(matches!(
stream.next().await,
Some((0, Some(ResponseItem::Frame(_))))
Some((0, FrontendEvent::Delta(_)))
));
drop(stream);
@@ -36,7 +36,7 @@ use dynamo_protocols::types::{
Role,
};
use crate::message::response::ChunkEvent;
use crate::frontend::FrontendOutput;
use crate::message::sampling::SamplingParams;
/// Canonicalize a tool-call parser name onto the dynamo-parsers registry keys.
@@ -246,7 +246,7 @@ pub(super) async fn parse_chat_tool_calls(
/// Map the scheduler's finish kind onto the OpenAI wire values. Length and
/// content-filter keep their names; everything else (including a bare abort)
/// reports as `stop`, matching Python's fallback.
pub(super) fn chat_finish_reason(output: &ChunkEvent) -> Option<OpenAIFinishReason> {
pub(super) fn chat_finish_reason(output: &FrontendOutput) -> Option<OpenAIFinishReason> {
let kind = output
.finish_reason
.as_ref()
@@ -264,7 +264,7 @@ mod tests {
apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name,
dynamo_tool_choice, parse_chat_tool_calls,
};
use crate::message::response::ChunkEvent;
use crate::frontend::FrontendOutput;
use crate::message::sampling::SamplingParams;
use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail};
use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition};
@@ -766,8 +766,7 @@ mod tests {
#[test]
fn chat_finish_reason_maps_scheduler_kinds() {
let output = |finish: serde_json::Value| ChunkEvent {
rid: "r".into(),
let output = |finish: serde_json::Value| FrontendOutput {
text: "x".into(),
token_ids: vec![1],
prompt_tokens: 1,
@@ -795,7 +794,7 @@ mod tests {
Some(OpenAIFinishReason::Stop)
);
assert_eq!(
chat_finish_reason(&ChunkEvent {
chat_finish_reason(&FrontendOutput {
finish_reason: None,
..Default::default()
}),
+400 -92
View File
@@ -1,10 +1,10 @@
//! Transport-neutral entry point into the Rust frontend pipeline.
//!
//! Wire adapters normalize their protocol into the existing in-process request
//! types, submit them through [`FrontendHandle`], and render the returned
//! [`ResponseItem`] values for their own transport. This module owns shared
//! preprocessing, runtime capabilities, and request lifetime; it deliberately
//! knows nothing about Axum, HTTP response shapes, Tonic, or protobuf.
//! Wire adapters normalize their protocol into [`FrontendRequest`], submit it
//! through [`FrontendHandle`], and render semantic [`FrontendEvent`]s for their
//! own transport. This module owns shared preprocessing, runtime translation,
//! capabilities, and request lifetime; it deliberately knows nothing about
//! Axum, HTTP response shapes, Tonic, or protobuf.
use std::collections::BTreeMap;
use std::sync::{
@@ -13,12 +13,12 @@ use std::sync::{
};
use std::time::Duration;
use bytes::Bytes;
use tokio::sync::mpsc;
use crate::message::config::{DisaggregationMode, ServerArgs};
use crate::message::ids::Rid;
use crate::message::io_struct::ControlRequest;
use crate::message::request::{GenerateRequest, Request, RequestKind};
use crate::message::io_struct::{ControlRequest, GetInternalStateReq};
use crate::message::request::{Request, RequestKind};
use crate::message::response::{ResponseItem, ResponseSink};
use crate::message::sampling::SamplingParams;
use crate::message::types::TokenIds;
@@ -27,50 +27,63 @@ use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
use crate::utils::error::Error;
use crate::utils::fsm::RequestState;
mod contract;
mod prefetch;
pub(crate) use contract::{
FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
InternalState, ModelInfo, ServerInfo,
};
/// Sentinel host that makes the KV connector no-op. Parity with
/// `sglang.srt.disaggregation.utils.FAKE_BOOTSTRAP_HOST`.
const FAKE_BOOTSTRAP_HOST: &str = "2.2.2.2";
/// A transport-neutral failure while executing a frontend operation.
#[derive(Clone, Debug, thiserror::Error)]
pub(crate) enum FrontendError {
/// The TokenizerManager intake loop has shut down, so callers may retry on
/// another healthy server.
#[error("service unavailable")]
Unavailable,
#[error("{0}")]
InvalidArgument(String),
#[error(transparent)]
Pipeline(#[from] Error),
#[error("response channel closed before terminal output")]
ResponseClosed,
#[error("{0}")]
UnexpectedResponse(&'static str),
}
/// Semantic result of a deep frontend health check.
///
/// Expected lifecycle states are values rather than transport errors so each
/// adapter can render them according to its own protocol.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum HealthStatus {
Healthy,
NotReady,
Stalled,
}
pub(crate) struct FrontendConfig {
pub(crate) response_capacity: usize,
pub(crate) response_activity: ActivityCounter,
pub(crate) startup_ready: bool,
pub(crate) is_disaggregation: bool,
pub(crate) mm_limits: BTreeMap<String, usize>,
pub(crate) metadata: FrontendMetadata,
}
/// Immutable, transport-independent metadata retained by the frontend.
/// Keeping this snapshot narrow avoids making adapters or the shared handle
/// depend on the full launch-configuration object.
#[derive(Clone, Debug, Default)]
pub(crate) struct FrontendMetadata {
model_path: String,
served_model_name: String,
tokenizer_path: String,
preferred_sampling_params: Option<crate::message::config::PreferredSamplingParams>,
weight_version: Option<String>,
load_format: Option<String>,
reasoning_parser: Option<String>,
tool_call_parser: Option<String>,
disaggregation_mode: DisaggregationMode,
max_context_length: u64,
max_total_num_tokens: u64,
version: String,
}
impl From<&ServerArgs> for FrontendMetadata {
fn from(args: &ServerArgs) -> Self {
Self {
model_path: args.model_path.clone(),
served_model_name: args.served_model_name.clone(),
tokenizer_path: args.tokenizer_path.clone(),
preferred_sampling_params: args.preferred_sampling_params.clone(),
weight_version: args.weight_version.clone(),
load_format: args.load_format.clone(),
reasoning_parser: args.reasoning_parser.clone(),
tool_call_parser: args.tool_call_parser.clone(),
disaggregation_mode: args.disaggregation_mode,
max_context_length: args.model_config.context_len,
max_total_num_tokens: args.max_total_num_tokens,
version: args.version.clone(),
}
}
}
/// Cloneable capability for submitting work to the existing frontend runtime.
@@ -90,6 +103,7 @@ struct FrontendInner {
startup_ready: AtomicBool,
is_disaggregation: bool,
mm_limits: BTreeMap<String, usize>,
metadata: FrontendMetadata,
}
impl FrontendHandle {
@@ -107,6 +121,7 @@ impl FrontendHandle {
startup_ready: AtomicBool::new(config.startup_ready),
is_disaggregation: config.is_disaggregation,
mm_limits: config.mm_limits,
metadata: config.metadata,
}),
}
}
@@ -122,17 +137,19 @@ impl FrontendHandle {
self.inner.startup_ready.store(true, Ordering::Release);
}
pub(crate) fn response_activity(&self) -> u64 {
fn response_activity(&self) -> u64 {
self.inner.response_activity.load(Ordering::Relaxed)
}
pub(crate) async fn generate(
&self,
mut request: GenerateRequest,
mut request: FrontendRequest,
) -> Result<FrontendCall, FrontendError> {
self.prepare_generations(std::slice::from_mut(&mut request))
.await?;
self.submit(RequestKind::Generate(Box::new(request))).await
self.submit(RequestKind::Generate(Box::new(request)))
.await
.map(FrontendCall::new)
}
/// Preprocess and submit a group atomically from the adapter's perspective:
@@ -141,7 +158,7 @@ impl FrontendHandle {
/// the already-created calls aborts those accepted requests.
pub(crate) async fn generate_batch(
&self,
mut requests: Vec<GenerateRequest>,
mut requests: Vec<FrontendRequest>,
) -> Result<Vec<FrontendCall>, FrontendError> {
self.prepare_generations(&mut requests).await?;
@@ -149,7 +166,8 @@ impl FrontendHandle {
for request in requests {
calls.push(
self.submit(RequestKind::Generate(Box::new(request)))
.await?,
.await
.map(FrontendCall::new)?,
);
}
Ok(calls)
@@ -157,37 +175,96 @@ impl FrontendHandle {
async fn prepare_generations(
&self,
requests: &mut [GenerateRequest],
requests: &mut [FrontendRequest],
) -> Result<(), FrontendError> {
prefetch::prefetch_all(requests, &self.inner.mm_limits)
.await
.map_err(FrontendError::InvalidArgument)
}
pub(crate) async fn control(&self, request: ControlRequest) -> Result<Bytes, FrontendError> {
let mut call = self.submit(RequestKind::Control(Box::new(request))).await?;
match call.recv().await {
Some(ResponseItem::Control(bytes)) => Ok(bytes),
Some(ResponseItem::Error(error)) => Err(FrontendError::Pipeline(error)),
Some(_) => Err(FrontendError::UnexpectedResponse(
"unexpected generation output for control request",
)),
None => Err(FrontendError::ResponseClosed),
/// Return static model metadata without exposing the full launch config.
pub(crate) fn model_info(&self) -> ModelInfo {
let metadata = &self.inner.metadata;
ModelInfo {
model_path: metadata.model_path.clone(),
served_model_name: metadata.served_model_name.clone(),
tokenizer_path: metadata.tokenizer_path.clone(),
is_generation: true,
preferred_sampling_params: metadata.preferred_sampling_params.clone(),
weight_version: metadata.weight_version.clone(),
load_format: metadata.load_format.clone(),
reasoning_parser: metadata.reasoning_parser.clone(),
tool_call_parser: metadata.tool_call_parser.clone(),
disaggregation_mode: metadata.disaggregation_mode,
}
}
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<Bytes, FrontendError> {
let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?;
/// Return public server metadata with the current scheduler metrics.
/// Raw control bytes and the scheduler's full launch-argument dump never
/// cross the frontend boundary.
pub(crate) async fn server_info(&self) -> Result<ServerInfo, FrontendError> {
let internal_state = self.internal_state().await?;
let metadata = &self.inner.metadata;
Ok(ServerInfo {
model_path: metadata.model_path.clone(),
served_model_name: metadata.served_model_name.clone(),
tokenizer_path: metadata.tokenizer_path.clone(),
max_context_length: metadata.max_context_length,
max_total_num_tokens: metadata.max_total_num_tokens,
version: metadata.version.clone(),
frontend: "rust",
internal_states: vec![internal_state],
})
}
async fn internal_state(&self) -> Result<InternalState, FrontendError> {
let bytes = self
.control(ControlRequest::GetInternalStateReq(
GetInternalStateReq::new(Rid::new().to_string()),
))
.await?;
rmp_serde::from_slice::<InternalStateEnvelope>(&bytes)
.map(|response| response.internal_state)
.map_err(|error| {
FrontendError::InvalidResponse(format!("invalid internal-state response: {error}"))
})
}
async fn control(&self, request: ControlRequest) -> Result<bytes::Bytes, FrontendError> {
let mut call = self.submit(RequestKind::Control(Box::new(request))).await?;
match call.recv().await {
Some(ResponseItem::Data(bytes)) => Ok(bytes),
Some(ResponseItem::Error(error)) => Err(FrontendError::Pipeline(error)),
Some(_) => Err(FrontendError::UnexpectedResponse(
"unexpected output for detokenize request",
Some(ResponseItem::Control(bytes)) => Ok(bytes),
Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)),
Some(_) => Err(FrontendError::InvalidResponse(
"unexpected generation output for control request".into(),
)),
None => Err(FrontendError::ResponseClosed),
None => Err(FrontendError::ResponseTruncated),
}
}
/// Decode token IDs into text without exposing the runtime's byte payload.
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<String, FrontendError> {
let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?;
let bytes = match call.recv().await {
Some(ResponseItem::Data(bytes)) => bytes,
Some(ResponseItem::Error(Error::Validation(message))) => {
return Err(FrontendError::InvalidArgument(message));
}
Some(ResponseItem::Error(error)) => return Err(translate_runtime_error(error)),
Some(_) => {
return Err(FrontendError::InvalidResponse(
"unexpected output for detokenize request".into(),
));
}
None => {
return Err(FrontendError::Internal("reply channel closed".into()));
}
};
String::from_utf8(bytes.to_vec()).map_err(|_| {
FrontendError::InvalidResponse("detokenized prompt is not valid UTF-8".into())
})
}
/// Confirm that scheduler output is moving. The response heartbeat is the
/// signal rather than this probe's own result, matching Python's
/// `last_receive_tstamp` behavior on a busy server.
@@ -200,7 +277,7 @@ impl FrontendHandle {
}
let baseline = self.response_activity();
let probe = GenerateRequest {
let probe = FrontendRequest {
rid: Rid::new_health_check(),
input_ids: Some(vec![0]),
sampling_params: SamplingParams {
@@ -238,7 +315,7 @@ impl FrontendHandle {
/// The async send preserves TokenizerManager inbox backpressure. Once the
/// request is accepted, the returned call owns cancellation until it
/// observes a terminal response.
async fn submit(&self, kind: RequestKind) -> Result<FrontendCall, FrontendError> {
async fn submit(&self, kind: RequestKind) -> Result<RuntimeCall, FrontendError> {
let rid = request_rid(&kind);
let expected_response = ExpectedResponse::for_request(&kind);
let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity);
@@ -246,7 +323,7 @@ impl FrontendHandle {
// Arm ownership before awaiting the bounded handoff. If this future is
// cancelled after the event is queued but before the await observes
// success, dropping `call` still cancels the request.
let mut call = FrontendCall {
let mut call = RuntimeCall {
rid: rid.clone(),
response_rx,
abort_tx: self.inner.abort_tx.clone(),
@@ -279,41 +356,74 @@ impl FrontendHandle {
}
}
/// One accepted request and its response stream.
fn translate_runtime_error(error: Error) -> FrontendError {
match error {
Error::Validation(message) => {
FrontendError::InvalidArgument(format!("validation failed: {message}"))
}
Error::QueueFull => FrontendError::Overloaded("to_scheduler channel full".into()),
Error::Disconnected => FrontendError::Cancelled("client disconnected".into()),
error => FrontendError::Internal(error.to_string()),
}
}
/// One accepted generation request and its semantic response stream.
///
/// Dropping an unfinished call either cancels a request that Intake has not yet
/// claimed or notifies the existing abort lane after admission. Reading a
/// terminal item disarms that cleanup automatically, so transports cannot
/// accidentally abort completed work or forget disconnect cleanup.
/// Runtime response variants are translated here so adapters cannot depend on
/// scheduler/control encodings. Dropping an unfinished call retains the
/// cancellation guarantees owned by [`RuntimeCall`].
pub(crate) struct FrontendCall {
rid: Rid,
response_rx: mpsc::Receiver<ResponseItem>,
abort_tx: flume::Sender<AbortSource>,
admission: RequestAdmission,
expected_response: ExpectedResponse,
in_flight: bool,
runtime: RuntimeCall,
events_finished: bool,
}
impl FrontendCall {
pub(crate) fn rid(&self) -> &Rid {
&self.rid
fn new(runtime: RuntimeCall) -> Self {
Self {
runtime,
events_finished: false,
}
}
pub(crate) async fn recv(&mut self) -> Option<ResponseItem> {
let item = self.response_rx.recv().await;
self.observe(item.as_ref());
item
/// ID that adapters should return to their client.
pub(crate) fn public_id(&self) -> &str {
self.runtime.rid.client_facing()
}
pub(crate) fn try_recv(&mut self) -> Result<ResponseItem, mpsc::error::TryRecvError> {
let item = self.response_rx.try_recv()?;
self.observe(Some(&item));
Ok(item)
/// Receive the next semantic event. A runtime channel that closes before a
/// terminal response is converted into one terminal failure event; adapters
/// never need to infer whether an empty runtime channel means success.
pub(crate) async fn recv(&mut self) -> Option<FrontendEvent> {
if self.events_finished {
return None;
}
let event = match self.runtime.recv().await {
Some(item) => generation_event(item),
None => FrontendEvent::Failed(FrontendError::Internal(
"response truncated before completion".into(),
)),
};
self.events_finished = event.is_terminal();
Some(event)
}
fn observe(&mut self, item: Option<&ResponseItem>) {
if item.is_some_and(|item| self.expected_response.is_terminal(item)) {
self.in_flight = false;
/// Append every semantic event that is ready without waiting. Channel
/// emptiness and closure remain private runtime details.
pub(crate) fn drain_ready(&mut self, events: &mut Vec<FrontendEvent>) {
while !self.events_finished {
let event = match self.runtime.try_recv() {
Ok(item) => generation_event(item),
// Preserve every queued event before reporting a premature
// close. The next awaited `recv` translates that close into
// one terminal failure, maintaining stream order while still
// hiding channel mechanics from adapters.
Err(mpsc::error::TryRecvError::Disconnected | mpsc::error::TryRecvError::Empty) => {
break;
}
};
self.events_finished = event.is_terminal();
events.push(event);
}
}
@@ -325,18 +435,84 @@ impl FrontendCall {
) -> Self {
let admission = RequestAdmission::pending();
assert!(admission.try_accept(), "test calls start already admitted");
Self {
Self::new(RuntimeCall {
rid,
response_rx,
abort_tx,
admission,
expected_response: ExpectedResponse::Generate,
in_flight: true,
})
}
}
fn generation_event(item: ResponseItem) -> FrontendEvent {
match item {
ResponseItem::Frame(output) => FrontendEvent::Delta(output.into()),
ResponseItem::Done(output) => {
if let Some((legacy_http_status, message)) = output
.finish_reason
.as_ref()
.and_then(|reason| reason.abort_status())
{
FrontendEvent::Failed(FrontendError::from_runtime_rejection(
message.to_owned(),
legacy_http_status,
))
} else {
FrontendEvent::Finished(output.into())
}
}
ResponseItem::Error(error) => FrontendEvent::Failed(translate_runtime_error(error)),
ResponseItem::Control(_) | ResponseItem::Data(_) => FrontendEvent::Failed(
FrontendError::InvalidResponse("unexpected non-generation response".into()),
),
}
}
/// Private scheduler response envelope. This MessagePack shape is decoded at
/// the runtime boundary and deliberately is not part of the frontend contract.
#[derive(serde::Deserialize)]
struct InternalStateEnvelope {
#[serde(default)]
internal_state: InternalState,
}
/// Runtime-facing request guard kept private behind [`FrontendCall`].
///
/// This type is the only owner of the raw response channel and admission/abort
/// capabilities. It is also used directly by typed unary operations such as
/// detokenization and internal-state lookup.
struct RuntimeCall {
rid: Rid,
response_rx: mpsc::Receiver<ResponseItem>,
abort_tx: flume::Sender<AbortSource>,
admission: RequestAdmission,
expected_response: ExpectedResponse,
in_flight: bool,
}
impl RuntimeCall {
async fn recv(&mut self) -> Option<ResponseItem> {
let item = self.response_rx.recv().await;
self.observe(item.as_ref());
item
}
fn try_recv(&mut self) -> Result<ResponseItem, mpsc::error::TryRecvError> {
let item = self.response_rx.try_recv()?;
self.observe(Some(&item));
Ok(item)
}
fn observe(&mut self, item: Option<&ResponseItem>) {
if item.is_some_and(|item| self.expected_response.is_terminal(item)) {
self.in_flight = false;
}
}
}
impl Drop for FrontendCall {
impl Drop for RuntimeCall {
fn drop(&mut self) {
if self.in_flight {
// Pending cancellation is consumed by Intake before it starts work.
@@ -396,6 +572,7 @@ mod tests {
use crate::message::io_struct::GetInternalStateReq;
use crate::message::multimodal::MmItem;
use crate::message::request::{GenerateRequest, MmData};
use crate::message::response::ChunkEvent;
struct Harness {
handle: FrontendHandle,
@@ -449,6 +626,7 @@ mod tests {
startup_ready,
is_disaggregation,
mm_limits,
metadata: FrontendMetadata::default(),
},
)
}
@@ -518,6 +696,136 @@ mod tests {
assert!(abort_rx.try_recv().is_err());
}
#[test]
fn runtime_errors_become_transport_neutral_categories() {
assert!(matches!(
translate_runtime_error(Error::Validation("bad input".into())),
FrontendError::InvalidArgument(message)
if message == "validation failed: bad input"
));
assert!(matches!(
translate_runtime_error(Error::QueueFull),
FrontendError::Overloaded(message) if message == "to_scheduler channel full"
));
assert!(matches!(
translate_runtime_error(Error::Disconnected),
FrontendError::Cancelled(message) if message == "client disconnected"
));
assert!(matches!(
translate_runtime_error(Error::Internal("bug".into())),
FrontendError::Internal(message) if message == "internal error: bug"
));
}
#[test]
fn scheduler_status_is_classified_once_for_non_http_adapters() {
for (status, expected) in [
(400, FrontendErrorKind::InvalidArgument),
(404, FrontendErrorKind::NotFound),
(408, FrontendErrorKind::DeadlineExceeded),
(412, FrontendErrorKind::FailedPrecondition),
(413, FrontendErrorKind::ResourceExhausted),
(429, FrontendErrorKind::ResourceExhausted),
(432, FrontendErrorKind::InvalidArgument),
(499, FrontendErrorKind::Cancelled),
(500, FrontendErrorKind::Internal),
(503, FrontendErrorKind::Unavailable),
(504, FrontendErrorKind::DeadlineExceeded),
] {
let error = FrontendError::from_runtime_rejection("rejected".into(), status);
assert_eq!(error.kind(), expected, "legacy status {status}");
assert!(matches!(
error,
FrontendError::RuntimeRejected {
legacy_http_status,
..
} if legacy_http_status == status
));
}
}
#[tokio::test]
async fn generation_exposes_public_id_and_semantic_events() {
let harness = Harness::unbounded(8);
let mut call = harness
.handle
.generate(GenerateRequest {
rid: Rid::from_client("public-id"),
..Default::default()
})
.await
.unwrap();
assert_eq!(call.public_id(), "public-id");
assert_ne!(call.runtime.rid.as_str(), "public-id");
let request = accept_intake(harness.intake_rx.recv().unwrap());
request
.sink
.try_send(ResponseItem::Frame(ChunkEvent {
text: "delta".into(),
..Default::default()
}))
.unwrap();
request
.sink
.try_send(ResponseItem::Done(ChunkEvent {
text: "final".into(),
..Default::default()
}))
.unwrap();
assert!(matches!(
call.recv().await,
Some(FrontendEvent::Delta(FrontendOutput { text, .. })) if text == "delta"
));
assert!(matches!(
call.recv().await,
Some(FrontendEvent::Finished(FrontendOutput { text, .. })) if text == "final"
));
assert!(call.recv().await.is_none());
drop(call);
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn plain_scheduler_abort_remains_a_terminal_output() {
let harness = Harness::unbounded(8);
let mut call = harness
.handle
.generate(GenerateRequest {
rid: "plain-abort".into(),
..Default::default()
})
.await
.unwrap();
let request = accept_intake(harness.intake_rx.recv().unwrap());
request
.sink
.try_send(ResponseItem::Done(ChunkEvent {
finish_reason: serde_json::from_value(serde_json::json!({
"type": "abort",
"message": "Aborted",
"status_code": null,
"err_type": null
}))
.unwrap(),
..Default::default()
}))
.unwrap();
assert!(matches!(
call.recv().await,
Some(FrontendEvent::Finished(FrontendOutput {
finish_reason: Some(_),
..
}))
));
assert!(call.recv().await.is_none());
drop(call);
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn generate_batch_failure_aborts_already_accepted_calls() {
let (intake_tx, intake_rx) = flume::bounded(0);
@@ -686,7 +994,7 @@ mod tests {
.sink
.try_send(ResponseItem::Data(Bytes::from_static(b"decoded")))
.unwrap();
assert_eq!(task.await.unwrap().unwrap(), Bytes::from_static(b"decoded"));
assert_eq!(task.await.unwrap().unwrap(), "decoded");
assert!(harness.abort_rx.try_recv().is_err());
}
}
+272
View File
@@ -0,0 +1,272 @@
//! Transport-neutral types shared by frontend adapters.
//!
//! These types describe what an inference operation means after a wire adapter
//! has normalized its request. They intentionally contain no Axum, HTTP, SSE,
//! Tonic, protobuf, or scheduler-channel details.
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::message::config::{DisaggregationMode, PreferredSamplingParams};
use crate::message::finish_reason::FinishReason;
use crate::message::request::GenerateRequest;
use crate::message::response::{ChunkEvent, ChunkExtras};
use crate::message::types::TokenIds;
/// Canonical transport-neutral input for one generation operation.
///
/// The existing in-process request already contains semantic generation fields
/// and has no Serde, Axum, Tonic, or protobuf representation attached to it.
/// Re-exporting it through the frontend contract gives every adapter one public
/// boundary without duplicating the runtime's request model.
pub(crate) type FrontendRequest = GenerateRequest;
/// Canonical transport-neutral output payload for generation events.
///
/// The runtime correlation ID is deliberately absent: it is an implementation
/// detail used for scheduler routing, while adapters obtain the client-visible
/// identity from [`crate::frontend::FrontendCall::public_id`]. Runtime-only
/// response variants such as raw control bytes are likewise hidden by
/// [`crate::frontend::FrontendCall`].
#[derive(Clone, Debug, Default)]
pub(crate) struct FrontendOutput {
pub(crate) token_ids: TokenIds,
pub(crate) finish_reason: Option<FinishReason>,
pub(crate) prompt_tokens: u32,
pub(crate) text: String,
pub(crate) completion_tokens: u64,
pub(crate) extras: Option<Box<ChunkExtras>>,
}
impl From<ChunkEvent> for FrontendOutput {
fn from(output: ChunkEvent) -> Self {
let ChunkEvent {
rid: _,
token_ids,
finish_reason,
prompt_tokens,
text,
completion_tokens,
extras,
} = output;
Self {
token_ids,
finish_reason,
prompt_tokens,
text,
completion_tokens,
extras,
}
}
}
/// Semantic result of a deep frontend health check.
///
/// Expected lifecycle states are values rather than transport errors so each
/// adapter can render them according to its own protocol.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum HealthStatus {
Healthy,
NotReady,
Stalled,
}
/// A semantic generation event before any transport-specific framing.
#[derive(Debug)]
pub(crate) enum FrontendEvent {
/// One incremental model-output update.
Delta(FrontendOutput),
/// The final model-output update for this request.
Finished(FrontendOutput),
/// A request-scoped failure. Other requests in a batch may continue.
Failed(FrontendError),
}
/// Protocol-independent error category used by transport adapters.
///
/// The set intentionally follows operation semantics rather than HTTP or gRPC
/// status spaces. An adapter maps this category into its own wire status.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum FrontendErrorKind {
InvalidArgument,
NotFound,
FailedPrecondition,
ResourceExhausted,
Cancelled,
DeadlineExceeded,
Unavailable,
Internal,
}
impl FrontendEvent {
pub(crate) fn is_terminal(&self) -> bool {
matches!(self, Self::Finished(_) | Self::Failed(_))
}
}
/// A transport-neutral failure category.
///
/// HTTP and gRPC adapters map these categories into their own status spaces;
/// neither adapter needs to understand errors emitted by runtime stages.
#[derive(Clone, Debug, thiserror::Error)]
pub(crate) enum FrontendError {
#[error("{0}")]
InvalidArgument(String),
/// The frontend intake loop has shut down. A caller may retry elsewhere.
#[error("service unavailable")]
Unavailable,
/// Runtime capacity was exhausted before the request could make progress.
#[error("{0}")]
Overloaded(String),
#[error("{0}")]
Cancelled(String),
/// The internal producer disappeared before returning a terminal result.
///
/// This is a server-side failure, not evidence that the caller cancelled.
#[error("response truncated before completion")]
ResponseTruncated,
/// A scheduler-side abort that carried an error status.
///
/// The current Python scheduler communicates this category as an HTTP
/// status code inside its finish reason. The semantic event is still
/// `Failed`; retaining the legacy code only lets the HTTP adapter preserve
/// existing behavior while another adapter maps it into its own status
/// space. New frontend failures should use the semantic variants above.
#[error("{message}")]
RuntimeRejected {
kind: FrontendErrorKind,
message: String,
legacy_http_status: u16,
},
/// The runtime replied with an unexpected or malformed result.
#[error("{0}")]
InvalidResponse(String),
#[error("{0}")]
Internal(String),
}
impl FrontendError {
/// Semantic status for adapters that do not use the scheduler's legacy
/// HTTP code space (notably the future gRPC adapter).
pub(crate) fn kind(&self) -> FrontendErrorKind {
match self {
Self::InvalidArgument(_) => FrontendErrorKind::InvalidArgument,
Self::Unavailable => FrontendErrorKind::Unavailable,
Self::Overloaded(_) => FrontendErrorKind::ResourceExhausted,
Self::Cancelled(_) => FrontendErrorKind::Cancelled,
Self::ResponseTruncated => FrontendErrorKind::Internal,
Self::RuntimeRejected { kind, .. } => *kind,
Self::InvalidResponse(_) | Self::Internal(_) => FrontendErrorKind::Internal,
}
}
/// Translate the scheduler's inherited HTTP-coded failure into a semantic
/// category once, at the runtime boundary. HTTP can retain the exact legacy
/// status while other adapters consume [`Self::kind`].
pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self {
let kind = match legacy_http_status {
404 => FrontendErrorKind::NotFound,
408 | 504 => FrontendErrorKind::DeadlineExceeded,
412 => FrontendErrorKind::FailedPrecondition,
413 | 429 => FrontendErrorKind::ResourceExhausted,
499 => FrontendErrorKind::Cancelled,
502 | 503 => FrontendErrorKind::Unavailable,
400..=499 => FrontendErrorKind::InvalidArgument,
_ => FrontendErrorKind::Internal,
};
Self::RuntimeRejected {
kind,
message,
legacy_http_status,
}
}
}
/// Static model metadata shared by every public transport.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ModelInfo {
pub(crate) model_path: String,
pub(crate) served_model_name: String,
pub(crate) tokenizer_path: String,
pub(crate) is_generation: bool,
pub(crate) preferred_sampling_params: Option<PreferredSamplingParams>,
pub(crate) weight_version: Option<String>,
pub(crate) load_format: Option<String>,
pub(crate) reasoning_parser: Option<String>,
pub(crate) tool_call_parser: Option<String>,
pub(crate) disaggregation_mode: DisaggregationMode,
}
/// Public server metadata plus scheduler-owned runtime metrics.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ServerInfo {
pub(crate) model_path: String,
pub(crate) served_model_name: String,
pub(crate) tokenizer_path: String,
pub(crate) max_context_length: u64,
pub(crate) max_total_num_tokens: u64,
pub(crate) version: String,
pub(crate) frontend: &'static str,
pub(crate) internal_states: Vec<InternalState>,
}
/// Scheduler-owned runtime metrics used by the public server-info operation.
///
/// Deserializing into this allowlisted shape is intentional: the scheduler's
/// raw state also contains its full launch arguments, including credentials.
/// Unknown fields are discarded before any transport receives the result.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub(crate) struct InternalState {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) last_gen_throughput: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) memory_usage: Option<MemoryUsage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) effective_max_running_requests_per_dp: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) avg_spec_accept_length: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) step_time_dict: Option<BTreeMap<u64, Vec<f64>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) rust_mm_transport: Option<BTreeMap<String, u64>>,
}
/// Public subset of scheduler memory metrics.
///
/// This nested allowlist is intentional: adding a scheduler field does not
/// silently add different public data to HTTP before the gRPC contract can add
/// the same field. New metrics should be promoted here deliberately.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub(crate) struct MemoryUsage {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) weight: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) kvcache: Option<MemoryMeasurement>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) startup_available: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) token_capacity: Option<u64>,
pub(crate) token_capacity_swa: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) graph: Option<BTreeMap<String, f64>>,
}
/// The scheduler's KV-cache measurement can be a native float or a NumPy
/// scalar stringified by the MessagePack bridge. Preserve that representation
/// so extracting the typed frontend contract does not change public responses.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(untagged)]
pub(crate) enum MemoryMeasurement {
Number(f64),
String(String),
}
+2 -1
View File
@@ -318,10 +318,11 @@ impl<'py> pyo3::FromPyObject<'_, 'py> for PreferredSamplingParams {
module = "sglang.srt.rust_extensions._server"
)]
// Lowercase to match the values Python reports for the same field.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum DisaggregationMode {
/// Unified prefill + decode.
#[default]
Null,
Prefill,
Decode,
@@ -97,9 +97,9 @@ impl FinishReason {
}
}
/// `Some((status, message))` when this is an abort carrying a `status_code` —
/// a scheduler-side request error the API surfaces as that HTTP status instead
/// of as a normal completion. A plain abort (no code) reads as `None`.
/// `Some((status, message))` when this is an abort carrying the scheduler's
/// legacy HTTP status — a request failure the frontend boundary converts
/// into a semantic error event. A plain abort (no code) reads as `None`.
pub fn abort_status(&self) -> Option<(u16, &str)> {
match self {
FinishReason::Known(FinishKind::Abort(a)) => Some((
@@ -128,9 +128,9 @@ mod tests {
Some((code, message.to_string()))
}
/// The classifier both API paths use: a validation abort yields its
/// `(code, message)` (the streaming path turns this into an SSE error event
/// instead of a normal `Done` frame); anything else yields `None`.
/// The classifier the frontend boundary uses: a validation abort yields its
/// `(code, message)` so it can become a semantic failure event; anything
/// else yields `None`.
#[test]
fn abort_status_extracts_code_and_message() {
let (code, msg) = abort_status(&fr(serde_json::json!({
+10 -12
View File
@@ -260,8 +260,7 @@ impl GenerateBody {
/// prompt + `is_batch` (list form — a 1-element list is still a batch → JSON
/// array response). The Rust counterpart of Python
/// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent
/// batch is [`Error::Validation`], which the handler surfaces with the
/// variant's own status (400).
/// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400.
pub fn into_requests(self) -> Result<(Vec<GenerateRequest>, bool), Error> {
let GenerateBody {
rid,
@@ -653,8 +652,8 @@ fn extension_value_present(value: &rmpv::Value) -> bool {
/// mutated lock-free). Common fields here; variant data in [`RequestKind`].
#[derive(Debug)]
pub struct Request {
/// Client-visible request id (uuid hex) — what the scheduler wire and
/// `meta_info.id` carry.
/// Runtime correlation ID. Client-provided IDs carry a private incarnation
/// suffix; adapters recover the public ID through `FrontendCall::public_id`.
pub rid: Rid,
pub state: RequestState,
/// Back-channel to the client connection for response frames.
@@ -677,10 +676,10 @@ pub struct SchedulerRequest {
/// response shape. Each owns its body, so generate/control fields stay type-separate.
#[derive(Debug)]
pub enum RequestKind {
/// `/generate`: tokenize (if needed) then push a `TokenizedGenerateReqInput`.
/// Generation: tokenize (if needed) then push a `TokenizedGenerateReqInput`.
Generate(Box<GenerateRequest>),
/// A control endpoint (e.g. `/server_info`, `/health`): no tokenization, and
/// the response is a single non-streamed JSON result.
/// Internal runtime control operation: no tokenization, with one serialized
/// result that the frontend boundary converts into a typed response.
Control(Box<ControlRequest>),
/// Internal service call: decode a complete token-id sequence to text. Walks
/// the same FSM as every request (validate → register → Queued), but the
@@ -691,11 +690,10 @@ pub enum RequestKind {
Detokenize { token_ids: TokenIds },
}
/// A single in-flight `/generate` request (per-item from
/// [`GenerateBody::into_requests`]),
/// A single in-flight generation request (one item after adapter fan-out),
/// serialized to the scheduler wire once tokenized (see `to_header_msgpack`). Not a
/// wire type — built by `into_requests`/handlers, never (de)serialized; `input_ids` is
/// client-supplied or filled by the Tokenizer stage.
/// wire type — built by frontend adapters, never (de)serialized; `input_ids` is
/// supplied by an adapter or filled by the Tokenizer stage.
#[derive(Debug, Default)]
pub struct GenerateRequest {
/// This item's final rid: the client's (normalized per item by `into_requests`) or a
@@ -726,7 +724,7 @@ pub struct GenerateRequest {
/// Sampling params (defaults when the client sent none, as in Python);
/// normalized + verified, then serialized into the header.
pub sampling_params: SamplingParams,
/// Whether the client asked for SSE streaming.
/// Whether the caller requested incremental output.
pub stream: bool,
/// Logprob / hidden-state options. This path bypasses the Python
/// `TokenizerManager`, so `into_requests` replicates its scalar
+19 -21
View File
@@ -1,7 +1,6 @@
//! The response direction: the per-request back-channel the API handler
//! drains ([`ResponseSink`] / [`ResponseItem`]), the response frame encodings
//! (batch / control result / error), and the columnar batch decode into
//! per-request [`ChunkEvent`]s.
//! The runtime response direction: per-request channels into the frontend
//! boundary ([`ResponseSink`] / [`ResponseItem`]), response frame encodings,
//! and columnar batch decoding into per-request [`ChunkEvent`]s.
use bytes::Bytes;
use serde::{Deserialize, Serialize};
@@ -12,8 +11,8 @@ use crate::message::ids::Rid;
use crate::message::types::TokenIds;
use crate::utils::error::Error;
/// Per-request back-channel the detok shard writes decode frames to and the API
/// handler drains for SSE; bounded, and receiver-drop (disconnect) = stream end.
/// Per-request back-channel the detok shard writes runtime outputs to and the
/// frontend boundary drains; bounded, and receiver-drop cancels the producer.
#[derive(Clone, Debug)]
pub enum ResponseSink {
Local(mpsc::Sender<ResponseItem>),
@@ -39,26 +38,26 @@ impl ResponseSink {
}
}
#[allow(dead_code)] // the receiver half is owned by frontend::FrontendCall.
#[allow(dead_code)] // the receiver half is owned inside the frontend boundary.
pub type ResponseSource = mpsc::Receiver<ResponseItem>;
/// What the connection handler receives on the decode stream: a detok-decoded
/// [`ChunkEvent`] (handler formats it), a verbatim control payload, or an error.
/// Runtime-facing output delivered to the frontend boundary: a detok-decoded
/// [`ChunkEvent`], a serialized internal-operation result, or a stage error.
/// [`crate::frontend::FrontendHandle`] translates these variants before a
/// transport adapter sees them.
#[derive(Debug)]
pub enum ResponseItem {
/// An intermediate streamed generation step (only sent for streaming reqs).
Frame(ChunkEvent),
/// The final generation step.
Done(ChunkEvent),
/// A control-request result: one verbatim payload (e.g. `/server_info`),
/// delivered as-is with no per-protocol formatting.
/// A control-request result encoded by the Python scheduler.
Control(Bytes),
/// Reply to an internal service request (`RequestKind::Detokenize`): raw
/// bytes for the SUBMITTER to consume (e.g. the decoded prompt text), not
/// client-bound JSON like `Control` and not a generation frame. Generation
/// and control drains never see it.
/// bytes for the frontend to decode (e.g. detokenized text), not a
/// client-facing wire payload and not a generation frame.
Data(Bytes),
/// Terminal failure: handler emits an error frame (stream) or status (unary).
/// Terminal runtime failure; the frontend maps it to a semantic error.
Error(Error),
}
@@ -536,8 +535,8 @@ pub fn frame_control_result(rid: &str, payload: &[u8]) -> Bytes {
Bytes::from(buf)
}
/// Frame a per-request failure `[rid, message]` for the response — routes a
/// terminal error back to the owning request (→ HTTP 400) instead of crashing.
/// Frame a per-request validation failure `[rid, message]` for the response —
/// routes a terminal error back to the owning request instead of crashing.
pub fn frame_error(rid: &str, message: &str) -> Bytes {
use rmpv::Value;
let arr = Value::Array(vec![Value::from(rid), Value::from(message)]);
@@ -558,10 +557,9 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes {
/// frame and moved between stages in-process (never serialized), so no serde.
#[derive(Debug, Clone, Default)]
pub struct ChunkEvent {
/// Client-visible rid — the request's IDENTITY. Moved out of the frame header
/// (which owns it and drops it), so carrying it costs no allocation. The shard
/// is still chosen by `Rid::shard`, but a hash collision there now only
/// co-locates two requests instead of merging them.
/// Runtime correlation ID. Moved out of the frame header (which owns it and
/// drops it), so carrying it costs no allocation. The frontend strips this
/// field before exposing semantic output; the shard routes it via `Rid::shard`.
pub rid: Rid,
/// New token ids for this step, widened from the scheduler's int32 wire
/// width at parse time. Empty allowed (e.g. metadata-only frames).
@@ -122,8 +122,8 @@ impl Intake {
/// terminal frame and a hung connection. Python cannot hit this because it
/// validates before `rid_to_state[obj.rid] = state`.
fn fail(&self, req: &mut Request, err: Error, registered: bool) {
// Log only server faults (500); 4xx/499/503 are expected and would spam.
if err.http_status() == 500 {
// Expected request, capacity, and cancellation outcomes would spam.
if err.is_server_fault() {
tracing::error!(rid = %req.rid, error = %err, "intake rejected request");
}
let _ = req.state.apply(Event::Error(err.clone()));
@@ -202,7 +202,7 @@ fn total_tokens_over_context_is_rejected() {
};
let err = check_total_tokens(&mut g, &limits).unwrap_err();
let msg = err.to_string();
assert_eq!(err.http_status(), 400);
assert!(matches!(err, Error::Validation(_)));
assert!(msg.contains("total of 103 tokens"), "{msg}");
assert!(msg.contains("3 tokens from the input"), "{msg}");
assert!(msg.contains("100 tokens for the completion"), "{msg}");
@@ -291,7 +291,7 @@ fn input_length_is_checked_unconditionally() {
for max_new_tokens in [None, Some(1)] {
let err = check_total_tokens(&mut req(max_new_tokens), &limits)
.expect_err("input == context_len must be rejected (Python uses >=)");
assert_eq!(err.http_status(), 400);
assert!(matches!(err, Error::Validation(_)));
assert!(err.to_string().contains("longer than the model's context"));
}
// One token shorter fits, with or without a cap.
@@ -338,7 +338,7 @@ fn auto_truncate_cannot_invert_min_and_max_new_tokens() {
};
let err = check_total_tokens(&mut g, &limits)
.expect_err("a clamp that inverts min/max must 400, not ride the wire");
assert_eq!(err.http_status(), 400);
assert!(matches!(err, Error::Validation(_)));
assert!(err.to_string().contains("min_new_tokens"), "{err}");
// A clamp that keeps the invariant still clamps.
@@ -362,7 +362,7 @@ fn hidden_states_gated_on_server_support() {
};
let disabled = test_limits();
let err = validate(&mut req(true), &disabled).unwrap_err();
assert_eq!(err.http_status(), 400);
assert!(matches!(err, Error::Validation(_)));
assert!(
err.to_string().contains("--enable-return-hidden-states"),
"message must name the flag: {err}"
@@ -466,7 +466,7 @@ fn detokenize_negative_ids_reject_before_registration() {
let Ok(ResponseItem::Error(err)) = rx.try_recv() else {
panic!("sink must receive the validation error");
};
assert_eq!(err.http_status(), 400);
assert!(matches!(err, Error::Validation(_)));
assert!(err.to_string().contains("out of range"), "{err}");
assert!(detok_rx.try_recv().is_err(), "shard never hears of it");
assert!(consumer.drain(16).is_empty());
@@ -525,7 +525,7 @@ fn oversized_rid_is_rejected() {
let mut req = generate_req(51, SamplingParams::default());
req.rid = "x".repeat(MAX_RID_LEN + 1).into();
let err = validate(&mut req, &test_limits()).expect_err("must be rejected");
assert_eq!(err.http_status(), 400);
assert!(matches!(err, Error::Validation(_)));
assert!(err.to_string().contains("over the"), "{err}");
// A uuid-sized rid — what Python mints — is nowhere near the cap.
@@ -780,7 +780,7 @@ fn mm_pool_gone_deregisters() {
let Ok(ResponseItem::Error(err)) = rx.try_recv() else {
panic!("expected a terminal error frame");
};
assert_eq!(err.http_status(), 500, "{err}");
assert!(matches!(err, Error::Internal(_)), "{err}");
assert!(err.to_string().contains("mm worker pool gone"), "{err}");
assert!(matches!(detok_rx.try_recv(), Ok(DetokMsg::Register { .. })));
assert!(matches!(
@@ -1140,7 +1140,7 @@ fn mm_text_prompt_under_skip_tokenizer_init_is_a_400() {
let Ok(ResponseItem::Error(err)) = rx.try_recv() else {
panic!("expected a terminal error frame");
};
assert_eq!(err.http_status(), 400, "{err}");
assert!(matches!(err, Error::Validation(_)), "{err}");
assert!(err.to_string().contains("must provide input_ids"), "{err}");
assert!(detok_rx.try_recv().is_err(), "rejected before registration");
assert!(mm_rx.try_recv().is_err(), "never reaches the mm pool");
+11 -9
View File
@@ -36,15 +36,17 @@ pub enum Error {
}
impl Error {
/// HTTP status to surface for the non-streaming error path. Mirrors the
/// codes used in the Python `_create_error_response`.
pub fn http_status(&self) -> u16 {
match self {
Error::Validation(_) => 400,
Error::Disconnected => 499, // nginx-style client closed request
Error::QueueFull => 503,
_ => 500,
}
/// Whether this failure represents a server defect rather than an expected
/// request, capacity, or cancellation outcome.
pub fn is_server_fault(&self) -> bool {
matches!(
self,
Error::Tokenize(_)
| Error::Encode(_)
| Error::Detokenize(_)
| Error::Codec(_)
| Error::Internal(_)
)
}
}
+1
View File
@@ -294,6 +294,7 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
startup_ready: cfg.server_args.skip_server_warmup,
is_disaggregation: cfg.server_args.is_disaggregation(),
mm_limits: cfg.server_args.limit_mm_data_per_request.clone(),
metadata: crate::frontend::FrontendMetadata::from(cfg.server_args.as_ref()),
},
);