[grpc rust] Define semantic frontend contract (#41709)
Signed-off-by: jain-ria <riajain@NVIDIA.com>
This commit is contained in:
@@ -1,9 +1,8 @@
|
||||
//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. Only
|
||||
//! this module knows HTTP. Its handlers use the transport-neutral
|
||||
//! [`crate::frontend::FrontendHandle`] to enter the shared runtime pipeline.
|
||||
//! `/generate` submits a request then awaits one `Done` (unary) or relays SSE
|
||||
//! frames (`data: {json}` … `[DONE]`), byte-compatible with Python
|
||||
//! `http_server.generate_request`; `/server_info` reuses it for one control result.
|
||||
//! Generation handlers render semantic frontend events as unary JSON or SSE;
|
||||
//! control handlers serialize typed frontend results such as server metadata.
|
||||
pub mod app;
|
||||
mod common;
|
||||
mod disaggregation;
|
||||
@@ -11,3 +10,74 @@ mod frame;
|
||||
mod log;
|
||||
mod native_api;
|
||||
mod openai;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
|
||||
use crate::frontend::{FrontendError, FrontendErrorKind};
|
||||
|
||||
/// Map transport-neutral failures into HTTP status codes. Response bodies and
|
||||
/// stream framing remain the responsibility of each HTTP API surface.
|
||||
pub(crate) fn frontend_error_status(error: &FrontendError) -> StatusCode {
|
||||
if let FrontendError::RuntimeRejected {
|
||||
legacy_http_status, ..
|
||||
} = error
|
||||
{
|
||||
return StatusCode::from_u16(*legacy_http_status)
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
|
||||
match error.kind() {
|
||||
FrontendErrorKind::InvalidArgument => StatusCode::BAD_REQUEST,
|
||||
FrontendErrorKind::NotFound => StatusCode::NOT_FOUND,
|
||||
FrontendErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED,
|
||||
// Existing intake backpressure has historically surfaced as 503.
|
||||
FrontendErrorKind::ResourceExhausted | FrontendErrorKind::Unavailable => {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
FrontendErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"),
|
||||
FrontendErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT,
|
||||
FrontendErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frontend_error_categories_map_at_the_http_boundary() {
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::InvalidArgument("bad".into())),
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Unavailable),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Overloaded("full".into())),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Cancelled("gone".into())).as_u16(),
|
||||
499
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::InvalidArgument,
|
||||
message: "runtime rejected request".into(),
|
||||
legacy_http_status: 432,
|
||||
})
|
||||
.as_u16(),
|
||||
432
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::InvalidResponse("bad reply".into())),
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Internal("bug".into())),
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,6 +148,7 @@ mod tests {
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::default(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! Common control-plane endpoints — `/server_info`, `/get_model_info`
|
||||
//! (+ `/model_info` alias), plus the control-request path through
|
||||
//! (+ `/model_info` alias), backed by typed operations on
|
||||
//! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`,
|
||||
//! which round-trips a generate probe) live in the sibling `native_api` and
|
||||
//! `openai` modules; the shared `AppState` lives in the parent
|
||||
@@ -15,17 +15,15 @@ use axum::{
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::app::AppState;
|
||||
use super::frontend_error_status;
|
||||
use super::native_api::native_error;
|
||||
use crate::frontend::FrontendError;
|
||||
use crate::message::config::ServerArgs;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::io_struct::{ControlRequest, GetInternalStateReq};
|
||||
use crate::frontend::{FrontendError, ServerInfo};
|
||||
|
||||
/// The routes this module owns, mounted by `api_server::serve`.
|
||||
pub(super) fn routes() -> Router<Arc<AppState>> {
|
||||
Router::new()
|
||||
// Control-plane: reuses the request FSM (no tokenization), returns one
|
||||
// non-streamed JSON result. Adding one = a route line + its struct tag.
|
||||
// Control-plane: the frontend performs the typed operation and this
|
||||
// adapter renders its result as one non-streamed JSON response.
|
||||
.route("/server_info", get(server_info))
|
||||
// Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info`
|
||||
// alias).
|
||||
@@ -33,85 +31,57 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
|
||||
.route("/model_info", get(model_info))
|
||||
}
|
||||
|
||||
/// Submit a control request through the request FSM (no tokenization) and await the
|
||||
/// scheduler's single msgpack result (a `structs.asdict` named map). Returns the
|
||||
/// raw bytes, or an error `Response` to return as-is.
|
||||
async fn await_control_result(
|
||||
state: &AppState,
|
||||
control: ControlRequest,
|
||||
) -> Result<bytes::Bytes, Response> {
|
||||
match state.frontend.control(control).await {
|
||||
Ok(bytes) => Ok(bytes),
|
||||
/// Await the frontend's complete semantic server-info result. HTTP status and
|
||||
/// serialization stay here; scheduler control bytes stay behind the contract.
|
||||
async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
|
||||
match state.frontend.server_info().await {
|
||||
Ok(server_info) => Ok(server_info),
|
||||
Err(FrontendError::Unavailable) => Err(native_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"service unavailable",
|
||||
false,
|
||||
)),
|
||||
Err(FrontendError::Pipeline(error)) => {
|
||||
let code = StatusCode::from_u16(error.http_status())
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
Err((code, error.to_string()).into_response())
|
||||
}
|
||||
Err(FrontendError::ResponseClosed) => {
|
||||
// Preserve the established HTTP behavior for a scheduler-side control
|
||||
// stream that disappears, while non-HTTP adapters see Internal via
|
||||
// FrontendError::kind().
|
||||
Err(FrontendError::ResponseTruncated) => {
|
||||
Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response())
|
||||
}
|
||||
Err(FrontendError::UnexpectedResponse(message)) => {
|
||||
Err((StatusCode::INTERNAL_SERVER_ERROR, message).into_response())
|
||||
}
|
||||
Err(FrontendError::InvalidArgument(error)) => {
|
||||
Err((StatusCode::INTERNAL_SERVER_ERROR, error).into_response())
|
||||
Err(error @ FrontendError::InvalidResponse(_)) => {
|
||||
tracing::error!(%error, "server_info: invalid runtime response");
|
||||
Err((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"bad server_info response",
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /get_model_info` (+ `/model_info` alias) — static model metadata from
|
||||
/// `server_args` (no scheduler round-trip); `is_generation` always true.
|
||||
/// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static
|
||||
/// model metadata (no scheduler round-trip).
|
||||
async fn model_info(State(state): State<Arc<AppState>>) -> Response {
|
||||
let sa = &state.server_args;
|
||||
let body = serde_json::json!({
|
||||
"model_path": sa.model_path,
|
||||
"served_model_name": sa.served_model_name,
|
||||
"tokenizer_path": sa.tokenizer_path,
|
||||
"is_generation": true,
|
||||
// Python's `TokenizerManager` merges this into every request
|
||||
// (`{**preferred, **client}`); this server has no equivalent yet, so
|
||||
// `RustServer.launch` REFUSES to start when it is set. It can therefore
|
||||
// only be null here — echoing it keeps the field's shape.
|
||||
"preferred_sampling_params": sa.preferred_sampling_params,
|
||||
// Python answers this through `config_value`, so a control-plane write
|
||||
// moves it there; here it is the launch value.
|
||||
"weight_version": sa.weight_version,
|
||||
"load_format": sa.load_format,
|
||||
// `auto` never reaches the blob: `resolve_auto_parsers` writes the
|
||||
// selected parser into `server_args` before the scheduler forks.
|
||||
"reasoning_parser": sa.reasoning_parser,
|
||||
"tool_call_parser": sa.tool_call_parser,
|
||||
"disaggregation_mode": sa.disaggregation_mode,
|
||||
});
|
||||
let info = state.frontend.model_info();
|
||||
(
|
||||
StatusCode::OK,
|
||||
[("content-type", "application/json")],
|
||||
serde_json::to_vec(&body).unwrap_or_default(),
|
||||
serde_json::to_vec(&info).unwrap_or_default(),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// `GET /server_info` — surface only an allowlist ([`INTERNAL_STATE_ALLOWLIST`] +
|
||||
/// curated [`ServerArgs`] accessors), never the raw server-args dump (embeds
|
||||
/// `api_key`/`admin_api_key`; see [`shape_server_info`]).
|
||||
/// `GET /server_info` — serialize typed, allowlisted runtime metrics plus
|
||||
/// curated launch metadata. The raw scheduler state embeds
|
||||
/// `api_key`/`admin_api_key` and is never visible to this HTTP adapter.
|
||||
///
|
||||
/// TODO(server_info): Python also includes `kv_events`; add once plumbed.
|
||||
async fn server_info(State(state): State<Arc<AppState>>) -> Response {
|
||||
let bytes = match await_control_result(
|
||||
&state,
|
||||
ControlRequest::GetInternalStateReq(GetInternalStateReq::new(Rid::new().to_string())),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(b) => b,
|
||||
let info = match await_server_info(&state).await {
|
||||
Ok(info) => info,
|
||||
Err(resp) => return resp,
|
||||
};
|
||||
match shape_server_info(&bytes, &state.server_args) {
|
||||
match serde_json::to_vec(&info) {
|
||||
Ok(json) => (StatusCode::OK, [("content-type", "application/json")], json).into_response(),
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "server_info: shaping failed");
|
||||
@@ -124,116 +94,263 @@ async fn server_info(State(state): State<Arc<AppState>>) -> Response {
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime-metric keys `get_internal_state` adds atop the server-args dump. We copy
|
||||
/// ONLY these out of `internal_state` (an allowlist), so the co-mingled
|
||||
/// `api_key`/`admin_api_key` can never reach the response.
|
||||
const INTERNAL_STATE_ALLOWLIST: &[&str] = &[
|
||||
"last_gen_throughput",
|
||||
"memory_usage",
|
||||
"effective_max_running_requests_per_dp",
|
||||
"avg_spec_accept_length",
|
||||
"step_time_dict",
|
||||
"rust_mm_transport",
|
||||
];
|
||||
|
||||
fn shape_server_info(msgpack: &[u8], server_args: &ServerArgs) -> Result<Vec<u8>, String> {
|
||||
// GetInternalStateReqOutput asdict → `{ "internal_state": { server-args dump +
|
||||
// metrics }, ... }`. Pull that inner map out (it is NOT safe to expose whole).
|
||||
let mut obj: serde_json::Map<String, serde_json::Value> =
|
||||
rmp_serde::from_slice(msgpack).map_err(|e| e.to_string())?;
|
||||
let internal = match obj.remove("internal_state") {
|
||||
Some(serde_json::Value::Object(m)) => m,
|
||||
_ => serde_json::Map::new(),
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use axum::{
|
||||
body::{Body, to_bytes},
|
||||
http::{Request as HttpRequest, header::CONTENT_TYPE},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use tower::ServiceExt;
|
||||
|
||||
// Copy only the allowlisted runtime metrics — never the raw server-args dump.
|
||||
let mut state_out = serde_json::Map::new();
|
||||
for &k in INTERNAL_STATE_ALLOWLIST {
|
||||
match internal.get(k) {
|
||||
Some(v) if !v.is_null() => {
|
||||
state_out.insert(k.to_string(), v.clone());
|
||||
}
|
||||
_ => {}
|
||||
use super::*;
|
||||
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
|
||||
use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs};
|
||||
use crate::message::request::RequestKind;
|
||||
use crate::message::response::ResponseItem;
|
||||
use crate::tokenizer_manager::wiring::TmEvent;
|
||||
|
||||
fn test_state(
|
||||
server_args: ServerArgs,
|
||||
) -> (
|
||||
Arc<AppState>,
|
||||
flume::Receiver<TmEvent>,
|
||||
flume::Receiver<crate::tokenizer_manager::wiring::AbortSource>,
|
||||
) {
|
||||
let server_args = Arc::new(server_args);
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let frontend = FrontendHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: FrontendMetadata::from(server_args.as_ref()),
|
||||
},
|
||||
);
|
||||
(
|
||||
Arc::new(AppState {
|
||||
frontend,
|
||||
server_args,
|
||||
chat_formatter: None,
|
||||
}),
|
||||
intake_rx,
|
||||
abort_rx,
|
||||
)
|
||||
}
|
||||
|
||||
async fn response_json(response: Response) -> serde_json::Value {
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.headers().get(CONTENT_TYPE).unwrap(),
|
||||
"application/json"
|
||||
);
|
||||
let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
|
||||
serde_json::from_slice(&body).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_info_routes_render_the_typed_frontend_contract() {
|
||||
let (state, _, _) = test_state(ServerArgs {
|
||||
model_path: "/model".into(),
|
||||
served_model_name: "served".into(),
|
||||
tokenizer_path: "/tokenizer".into(),
|
||||
weight_version: Some("weights-v1".into()),
|
||||
load_format: Some("safetensors".into()),
|
||||
reasoning_parser: Some("reasoner".into()),
|
||||
tool_call_parser: Some("tools".into()),
|
||||
disaggregation_mode: DisaggregationMode::Prefill,
|
||||
..Default::default()
|
||||
});
|
||||
let app = routes().with_state(state);
|
||||
|
||||
for path in ["/get_model_info", "/model_info"] {
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
HttpRequest::builder()
|
||||
.uri(path)
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["model_path"], "/model");
|
||||
assert_eq!(body["served_model_name"], "served");
|
||||
assert_eq!(body["tokenizer_path"], "/tokenizer");
|
||||
assert_eq!(body["is_generation"], true);
|
||||
assert_eq!(body["weight_version"], "weights-v1");
|
||||
assert_eq!(body["load_format"], "safetensors");
|
||||
assert_eq!(body["reasoning_parser"], "reasoner");
|
||||
assert_eq!(body["tool_call_parser"], "tools");
|
||||
assert_eq!(body["disaggregation_mode"], "prefill");
|
||||
assert!(body["preferred_sampling_params"].is_null());
|
||||
}
|
||||
}
|
||||
|
||||
// Top-level non-secret config from typed accessors (structurally can't surface
|
||||
// a key field, unlike the raw dump).
|
||||
let response = serde_json::json!({
|
||||
"model_path": server_args.model_path,
|
||||
"served_model_name": server_args.served_model_name,
|
||||
"tokenizer_path": server_args.tokenizer_path,
|
||||
"max_context_length": server_args.model_config.context_len,
|
||||
"max_total_num_tokens": server_args.max_total_num_tokens,
|
||||
"version": server_args.version,
|
||||
"frontend": "rust",
|
||||
"internal_states": [serde_json::Value::Object(state_out)],
|
||||
});
|
||||
serde_json::to_vec(&response).map_err(|e| e.to_string())
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn server_info_route_serializes_only_the_typed_public_subset() {
|
||||
// Native floats stay JSON numbers; NumPy scalars encoded by Python's
|
||||
// msgspec enc_hook=str must remain JSON strings, as before the refactor.
|
||||
assert_server_info_response(rmpv::Value::from(23.616), serde_json::json!(23.616)).await;
|
||||
assert_server_info_response(rmpv::Value::from("23.616"), serde_json::json!("23.616")).await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
async fn assert_server_info_response(
|
||||
kvcache: rmpv::Value,
|
||||
expected_kvcache: serde_json::Value,
|
||||
) {
|
||||
let (state, intake_rx, abort_rx) = test_state(ServerArgs {
|
||||
model_path: "/model".into(),
|
||||
served_model_name: "served".into(),
|
||||
tokenizer_path: "/tokenizer".into(),
|
||||
model_config: ModelConfig {
|
||||
context_len: 4096,
|
||||
..Default::default()
|
||||
},
|
||||
max_total_num_tokens: 8192,
|
||||
version: "1.2.3".into(),
|
||||
..Default::default()
|
||||
});
|
||||
let request = tokio::spawn(
|
||||
routes().with_state(state).oneshot(
|
||||
HttpRequest::builder()
|
||||
.uri("/server_info")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
),
|
||||
);
|
||||
|
||||
/// The scheduler's `internal_state` embeds the full server-args dump (incl.
|
||||
/// `api_key`/`admin_api_key`). `/server_info` must surface only the allowlisted
|
||||
/// runtime metrics + curated config — never the secrets — and must not re-nest
|
||||
/// the dump under `internal_states[].internal_state`.
|
||||
#[test]
|
||||
fn shape_server_info_excludes_secrets_and_dump() {
|
||||
// GetInternalStateReqOutput.asdict → { "internal_state": { …dump+metrics… } }.
|
||||
let internal = rmpv::Value::Map(vec![
|
||||
let TmEvent::Intake {
|
||||
request: runtime_request,
|
||||
admission,
|
||||
} = intake_rx.recv_async().await.unwrap()
|
||||
else {
|
||||
panic!("server_info must submit one control request");
|
||||
};
|
||||
assert!(admission.try_accept());
|
||||
assert!(matches!(runtime_request.kind, RequestKind::Control(_)));
|
||||
|
||||
let internal_state = rmpv::Value::Map(vec![
|
||||
(
|
||||
rmpv::Value::from("rust_mm_transport"),
|
||||
rmpv::Value::Map(vec![(
|
||||
rmpv::Value::from("inline_features"),
|
||||
rmpv::Value::from(3),
|
||||
)]),
|
||||
),
|
||||
(
|
||||
rmpv::Value::from("api_key"),
|
||||
rmpv::Value::from("secret-token"),
|
||||
rmpv::Value::from("must-not-leak"),
|
||||
),
|
||||
(
|
||||
rmpv::Value::from("admin_api_key"),
|
||||
rmpv::Value::from("admin-token"),
|
||||
),
|
||||
(rmpv::Value::from("model_path"), rmpv::Value::from("/m")),
|
||||
(
|
||||
rmpv::Value::from("last_gen_throughput"),
|
||||
rmpv::Value::from(1.5),
|
||||
),
|
||||
(
|
||||
rmpv::Value::from("effective_max_running_requests_per_dp"),
|
||||
rmpv::Value::from(32),
|
||||
rmpv::Value::from("memory_usage"),
|
||||
rmpv::Value::Map(vec![
|
||||
(rmpv::Value::from("kvcache"), kvcache),
|
||||
(rmpv::Value::from("token_capacity"), rmpv::Value::from(8192)),
|
||||
(rmpv::Value::from("token_capacity_swa"), rmpv::Value::Nil),
|
||||
(
|
||||
rmpv::Value::from("future_uncontracted_metric"),
|
||||
rmpv::Value::from(99),
|
||||
),
|
||||
]),
|
||||
),
|
||||
(
|
||||
rmpv::Value::from("step_time_dict"),
|
||||
rmpv::Value::Map(vec![(
|
||||
rmpv::Value::from(4),
|
||||
rmpv::Value::Array(vec![rmpv::Value::from(0.01)]),
|
||||
)]),
|
||||
),
|
||||
]);
|
||||
let outer = rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal)]);
|
||||
let mut msgpack = Vec::new();
|
||||
rmpv::encode::write_value(&mut msgpack, &outer).unwrap();
|
||||
let envelope =
|
||||
rmpv::Value::Map(vec![(rmpv::Value::from("internal_state"), internal_state)]);
|
||||
let mut payload = Vec::new();
|
||||
rmpv::encode::write_value(&mut payload, &envelope).unwrap();
|
||||
runtime_request
|
||||
.sink
|
||||
.try_send(ResponseItem::Control(Bytes::from(payload)))
|
||||
.unwrap();
|
||||
|
||||
// `api_key` is deliberately NOT a `ServerArgs` field — the typed schema
|
||||
// cannot carry it — so the only place it could leak from is the raw
|
||||
// scheduler dump shaped above.
|
||||
let sa = ServerArgs {
|
||||
model_path: "/m".into(),
|
||||
..Default::default()
|
||||
let response = request.await.unwrap().unwrap();
|
||||
let body = response_json(response).await;
|
||||
assert_eq!(body["model_path"], "/model");
|
||||
assert_eq!(body["served_model_name"], "served");
|
||||
assert_eq!(body["tokenizer_path"], "/tokenizer");
|
||||
assert_eq!(body["max_context_length"], 4096);
|
||||
assert_eq!(body["max_total_num_tokens"], 8192);
|
||||
assert_eq!(body["version"], "1.2.3");
|
||||
assert_eq!(body["frontend"], "rust");
|
||||
assert_eq!(
|
||||
body["internal_states"][0]["rust_mm_transport"]["inline_features"],
|
||||
3
|
||||
);
|
||||
assert_eq!(body["internal_states"][0]["last_gen_throughput"], 1.5);
|
||||
assert_eq!(
|
||||
body["internal_states"][0]["memory_usage"]["kvcache"],
|
||||
expected_kvcache
|
||||
);
|
||||
assert_eq!(
|
||||
body["internal_states"][0]["memory_usage"]["token_capacity"],
|
||||
8192
|
||||
);
|
||||
assert_eq!(
|
||||
body["internal_states"][0]["memory_usage"]["token_capacity_swa"],
|
||||
serde_json::Value::Null
|
||||
);
|
||||
assert_eq!(
|
||||
body["internal_states"][0]["step_time_dict"]["4"],
|
||||
serde_json::json!([0.01])
|
||||
);
|
||||
assert!(!body.to_string().contains("must-not-leak"));
|
||||
assert!(
|
||||
body["internal_states"][0]["memory_usage"]
|
||||
.get("future_uncontracted_metric")
|
||||
.is_none(),
|
||||
"uncontracted scheduler fields must not silently become public API"
|
||||
);
|
||||
assert!(abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn server_info_preserves_legacy_499_for_a_truncated_control_response() {
|
||||
let (state, intake_rx, abort_rx) = test_state(ServerArgs::default());
|
||||
let response = tokio::spawn(
|
||||
routes().with_state(state).oneshot(
|
||||
HttpRequest::builder()
|
||||
.uri("/server_info")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
),
|
||||
);
|
||||
|
||||
let TmEvent::Intake {
|
||||
request: runtime_request,
|
||||
admission,
|
||||
} = intake_rx.recv_async().await.unwrap()
|
||||
else {
|
||||
panic!("server_info must submit one control request");
|
||||
};
|
||||
let out = shape_server_info(&msgpack, &sa).unwrap();
|
||||
let text = String::from_utf8(out.clone()).unwrap();
|
||||
// No secret leaks anywhere in the serialized response.
|
||||
assert!(!text.contains("secret-token"), "api_key leaked: {text}");
|
||||
assert!(
|
||||
!text.contains("admin-token"),
|
||||
"admin_api_key leaked: {text}"
|
||||
);
|
||||
assert!(admission.try_accept());
|
||||
let rid = runtime_request.rid.clone();
|
||||
drop(runtime_request);
|
||||
|
||||
let v: serde_json::Value = serde_json::from_slice(&out).unwrap();
|
||||
// Allowlisted metric surfaced; the whole dump did not.
|
||||
let state0 = &v["internal_states"][0];
|
||||
assert_eq!(state0["last_gen_throughput"], 1.5);
|
||||
assert_eq!(state0["effective_max_running_requests_per_dp"], 32);
|
||||
assert!(
|
||||
state0.get("internal_state").is_none(),
|
||||
"must not re-nest the dump under internal_state"
|
||||
);
|
||||
assert!(state0.get("api_key").is_none());
|
||||
// Curated top-level config comes from typed accessors, not the dump.
|
||||
assert_eq!(v["model_path"], "/m");
|
||||
assert_eq!(v["frontend"], "rust");
|
||||
let response = response.await.unwrap().unwrap();
|
||||
assert_eq!(response.status().as_u16(), 499);
|
||||
let body = to_bytes(response.into_body(), 1024).await.unwrap();
|
||||
assert_eq!(body.as_ref(), b"request aborted");
|
||||
assert!(matches!(
|
||||
abort_rx.recv_async().await.unwrap(),
|
||||
crate::tokenizer_manager::wiring::AbortSource::Guard(aborted) if aborted == rid
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
//! Frame shaping for the native `/generate` protocol: the cumulative
|
||||
//! [`OutputAccumulator`] plus the functions that render [`ChunkEvent`]s /
|
||||
//! [`OutputAccumulator`] plus the functions that render [`FrontendOutput`]s /
|
||||
//! accumulated state into wire JSON (`meta_info`, logprob tuples, error and
|
||||
//! abort frames). No HTTP here — the sibling `native_api` module owns the handlers
|
||||
//! and streams; it calls these per frame.
|
||||
|
||||
use crate::message::response::{ChunkEvent, ChunkExtras};
|
||||
use crate::frontend::FrontendOutput;
|
||||
use crate::message::response::ChunkExtras;
|
||||
|
||||
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
|
||||
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
|
||||
@@ -159,10 +160,10 @@ fn hidden_states_rows(vals: &[f32], lens: &[u32]) -> serde_json::Value {
|
||||
serde_json::Value::Array(rows)
|
||||
}
|
||||
|
||||
/// Format a decoded [`ChunkEvent`] as one SGLang `/generate` frame's JSON. `rid`
|
||||
/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid`
|
||||
/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is
|
||||
/// just the shard routing key.
|
||||
pub(super) fn frame_value(out: &ChunkEvent, rid: &str) -> serde_json::Value {
|
||||
pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
|
||||
let mut v = serde_json::json!({
|
||||
"text": out.text,
|
||||
"meta_info": {
|
||||
@@ -318,7 +319,7 @@ pub(super) fn tag_value(mut v: serde_json::Value, index: Option<usize>) -> Strin
|
||||
|
||||
/// One streaming frame's JSON: cumulative ignores `delta`, incremental ships it.
|
||||
pub(super) fn stream_frame_string(
|
||||
delta: ChunkEvent,
|
||||
delta: FrontendOutput,
|
||||
acc: &OutputAccumulator,
|
||||
incremental: bool,
|
||||
rid_str: &str,
|
||||
@@ -344,7 +345,7 @@ pub(super) fn cumulative_frame_string(
|
||||
/// Format one streaming frame: the accumulator's cumulative view (default), or this
|
||||
/// step's delta with the cumulative token count in `meta_info` (matching Python).
|
||||
pub(super) fn stream_frame_value(
|
||||
delta: ChunkEvent,
|
||||
delta: FrontendOutput,
|
||||
acc: &OutputAccumulator,
|
||||
incremental: bool,
|
||||
rid_str: &str,
|
||||
@@ -358,15 +359,15 @@ pub(super) fn stream_frame_value(
|
||||
}
|
||||
}
|
||||
|
||||
/// Folds per-chunk [`ChunkEvent`] deltas into a cumulative view — used by the drain
|
||||
/// Folds per-chunk [`FrontendOutput`] deltas into a cumulative view — used by the drain
|
||||
/// loops needing cumulative output (every unary response + the cumulative SGLang
|
||||
/// stream; OpenAI streaming forwards deltas and skips this). Holds a single
|
||||
/// [`ChunkEvent`] so `snapshot` hands back a **borrow** per frame — no per-frame
|
||||
/// [`FrontendOutput`] so `snapshot` hands back a **borrow** per frame — no per-frame
|
||||
/// clone of the growing buffers (that added O(T²) atop the wire's inherent O(T²)).
|
||||
/// Shared with the [`openai`] submodule.
|
||||
#[derive(Default)]
|
||||
pub(super) struct OutputAccumulator {
|
||||
out: ChunkEvent,
|
||||
out: FrontendOutput,
|
||||
/// Serialized cumulative `output_ids` body (`"1,2,3"`, no brackets), appended per
|
||||
/// delta so a frame memcpy's it instead of rebuilding the array — O(T), not O(T²).
|
||||
ids_json: String,
|
||||
@@ -414,7 +415,7 @@ impl OutputAccumulator {
|
||||
/// Fold one delta frame in. Output families concatenate; input families and
|
||||
/// hidden states are set-once / last-writer-wins (they ride the prefill/final
|
||||
/// chunk), matching the Python `meta_info` assignment.
|
||||
pub(super) fn fold(&mut self, d: &ChunkEvent) {
|
||||
pub(super) fn fold(&mut self, d: &FrontendOutput) {
|
||||
use std::fmt::Write;
|
||||
|
||||
// Grow the memoized serializations alongside the raw cumulative buffers.
|
||||
@@ -427,7 +428,6 @@ impl OutputAccumulator {
|
||||
}
|
||||
|
||||
let o = &mut self.out;
|
||||
o.rid.clone_from(&d.rid); // constant across the request; keeps the accumulated view coherent
|
||||
o.text.push_str(&d.text);
|
||||
o.token_ids.extend_from_slice(&d.token_ids); // token_ids doubles as output_ids
|
||||
o.completion_tokens += d.completion_tokens;
|
||||
@@ -532,12 +532,12 @@ impl OutputAccumulator {
|
||||
}
|
||||
|
||||
/// Borrow the cumulative output for an intermediate streaming frame.
|
||||
pub(super) fn snapshot(&self) -> &ChunkEvent {
|
||||
pub(super) fn snapshot(&self) -> &FrontendOutput {
|
||||
&self.out
|
||||
}
|
||||
|
||||
/// Consume into the final cumulative output.
|
||||
pub(super) fn into_output(self) -> ChunkEvent {
|
||||
pub(super) fn into_output(self) -> FrontendOutput {
|
||||
self.out
|
||||
}
|
||||
}
|
||||
@@ -606,12 +606,12 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// End-to-end: a `ChunkEvent` carrying a prompt-logprob request (first input
|
||||
/// End-to-end: a `FrontendOutput` carrying a prompt-logprob request (first input
|
||||
/// logprob is the `NaN` sentinel) formats without panicking and emits
|
||||
/// `input_token_logprobs` with a leading `[null, token_id, text]`.
|
||||
#[test]
|
||||
fn prompt_logprob_frame_emits_null_first() {
|
||||
let out = ChunkEvent {
|
||||
let out = FrontendOutput {
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
in_lp_val: vec![f32::NAN, -0.5],
|
||||
in_lp_idx: vec![10, 20],
|
||||
@@ -632,7 +632,7 @@ mod tests {
|
||||
#[test]
|
||||
fn accumulator_snapshot_is_cumulative() {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
acc.fold(&ChunkEvent {
|
||||
acc.fold(&FrontendOutput {
|
||||
text: "he".into(),
|
||||
token_ids: vec![1, 2],
|
||||
completion_tokens: 2,
|
||||
@@ -643,7 +643,7 @@ mod tests {
|
||||
assert_eq!(s.text, "he");
|
||||
assert_eq!(s.token_ids, vec![1, 2]);
|
||||
}
|
||||
acc.fold(&ChunkEvent {
|
||||
acc.fold(&FrontendOutput {
|
||||
text: "llo".into(),
|
||||
token_ids: vec![3],
|
||||
completion_tokens: 1,
|
||||
@@ -681,32 +681,28 @@ mod tests {
|
||||
#[test]
|
||||
fn cumulative_frame_json_matches_serde() {
|
||||
let deltas = [
|
||||
ChunkEvent {
|
||||
rid: "7".into(),
|
||||
FrontendOutput {
|
||||
text: String::new(),
|
||||
token_ids: vec![],
|
||||
completion_tokens: 0,
|
||||
prompt_tokens: 128,
|
||||
..Default::default()
|
||||
},
|
||||
ChunkEvent {
|
||||
rid: "7".into(),
|
||||
FrontendOutput {
|
||||
text: "He\"llo\n\t".into(),
|
||||
token_ids: vec![1000],
|
||||
completion_tokens: 1,
|
||||
prompt_tokens: 128,
|
||||
..Default::default()
|
||||
},
|
||||
ChunkEvent {
|
||||
rid: "7".into(),
|
||||
FrontendOutput {
|
||||
text: " 世界 🌍 \\".into(),
|
||||
token_ids: vec![-2, 3],
|
||||
completion_tokens: 2,
|
||||
prompt_tokens: 128,
|
||||
..Default::default()
|
||||
},
|
||||
ChunkEvent {
|
||||
rid: "7".into(),
|
||||
FrontendOutput {
|
||||
text: "!".into(),
|
||||
token_ids: vec![9],
|
||||
completion_tokens: 1,
|
||||
@@ -752,8 +748,7 @@ mod tests {
|
||||
};
|
||||
let deltas = [
|
||||
// Prefill: the set-once input families and a null top-k position.
|
||||
ChunkEvent {
|
||||
rid: "9".into(),
|
||||
FrontendOutput {
|
||||
prompt_tokens: 4,
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
in_lp_val: vec![f32::NAN, -1.5],
|
||||
@@ -771,8 +766,7 @@ mod tests {
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
ChunkEvent {
|
||||
rid: "9".into(),
|
||||
FrontendOutput {
|
||||
text: "He\"llo".into(),
|
||||
token_ids: vec![100],
|
||||
completion_tokens: 1,
|
||||
@@ -793,8 +787,7 @@ mod tests {
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
ChunkEvent {
|
||||
rid: "9".into(),
|
||||
FrontendOutput {
|
||||
text: " 世界".into(),
|
||||
token_ids: vec![-2, 3],
|
||||
completion_tokens: 2,
|
||||
@@ -847,8 +840,7 @@ mod tests {
|
||||
#[test]
|
||||
fn mismatched_logprob_texts_fall_back_to_the_value_path() {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
acc.fold(&ChunkEvent {
|
||||
rid: "1".into(),
|
||||
acc.fold(&FrontendOutput {
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
out_lp_val: vec![-0.5],
|
||||
out_lp_idx: vec![5],
|
||||
@@ -857,8 +849,7 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
assert!(cumulative_frame_json(&acc, "1", None).is_some());
|
||||
acc.fold(&ChunkEvent {
|
||||
rid: "1".into(),
|
||||
acc.fold(&FrontendOutput {
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
out_lp_val: vec![-0.25],
|
||||
out_lp_idx: vec![6],
|
||||
|
||||
@@ -14,10 +14,13 @@ use super::app::AppState;
|
||||
use super::frame::{
|
||||
OutputAccumulator, cumulative_frame_string, frame_value, stream_frame_string, tag_value,
|
||||
};
|
||||
use crate::frontend::{FrontendCall, FrontendError, HealthStatus};
|
||||
use super::frontend_error_status;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::{GenerateBody, GenerateRequest};
|
||||
use crate::message::response::{ChunkEvent, ResponseItem};
|
||||
use crate::message::request::GenerateBody;
|
||||
use crate::utils::{
|
||||
environ,
|
||||
response::{error_response, error_value},
|
||||
@@ -87,19 +90,7 @@ pub(super) fn native_error(code: StatusCode, message: &str, stream: bool) -> Res
|
||||
}
|
||||
|
||||
fn native_frontend_error(error: FrontendError, stream: bool) -> Response {
|
||||
match error {
|
||||
FrontendError::Unavailable => {
|
||||
native_error(StatusCode::SERVICE_UNAVAILABLE, &error.to_string(), stream)
|
||||
}
|
||||
FrontendError::InvalidArgument(_) => {
|
||||
native_error(StatusCode::BAD_REQUEST, &error.to_string(), stream)
|
||||
}
|
||||
_ => native_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&error.to_string(),
|
||||
stream,
|
||||
),
|
||||
}
|
||||
native_error(frontend_error_status(&error), &error.to_string(), stream)
|
||||
}
|
||||
|
||||
/// `/health` + `/health_generate`. Both env knobs are resolved ONCE here, at
|
||||
@@ -196,10 +187,9 @@ async fn generate(
|
||||
// payloads. `is_batch` = list form → the response is a JSON array.
|
||||
let (payloads, is_batch) = match body.into_requests() {
|
||||
Ok(v) => v,
|
||||
// The error carries its own status (a bad batch is `Validation` → 400).
|
||||
// Request normalization only returns validation failures.
|
||||
Err(e) => {
|
||||
let code = StatusCode::from_u16(e.http_status()).unwrap_or(StatusCode::BAD_REQUEST);
|
||||
return native_error(code, &e.to_string(), stream);
|
||||
return native_error(StatusCode::BAD_REQUEST, &e.to_string(), stream);
|
||||
}
|
||||
};
|
||||
// Python starts APIServerReqTimeStats after request normalization and before
|
||||
@@ -225,7 +215,7 @@ async fn generate(
|
||||
/// SSE frames or fold to one unary response.
|
||||
async fn generate_single(
|
||||
state: &AppState,
|
||||
req: GenerateRequest,
|
||||
req: FrontendRequest,
|
||||
stream: bool,
|
||||
timing: RequestTiming,
|
||||
) -> Response {
|
||||
@@ -235,7 +225,7 @@ async fn generate_single(
|
||||
Ok(call) => call,
|
||||
Err(error) => return native_frontend_error(error, stream),
|
||||
};
|
||||
let rid = call.rid().clone();
|
||||
let rid = call.public_id().to_owned();
|
||||
// Cumulative frames (SGLang default) vs per-step deltas.
|
||||
let incremental = state.server_args.incremental_streaming_output;
|
||||
|
||||
@@ -249,7 +239,7 @@ async fn generate_single(
|
||||
} else {
|
||||
// Unary: fold to the terminal and respond once. `FrontendCall` disarms
|
||||
// itself on a terminal item; truncation leaves it armed for drop cleanup.
|
||||
let (status, value) = drain_unary(&mut call, rid.client_facing(), timing).await;
|
||||
let (status, value) = drain_unary(&mut call, &rid, timing).await;
|
||||
(status, Json(value)).into_response()
|
||||
}
|
||||
}
|
||||
@@ -263,41 +253,28 @@ async fn drain_unary(
|
||||
let mut acc = OutputAccumulator::default();
|
||||
while let Some(item) = call.recv().await {
|
||||
match item {
|
||||
ResponseItem::Frame(out) => {
|
||||
FrontendEvent::Delta(out) => {
|
||||
timing.observe_first_output();
|
||||
acc.fold(&out);
|
||||
}
|
||||
ResponseItem::Done(out) => {
|
||||
FrontendEvent::Finished(out) => {
|
||||
timing.observe_first_output();
|
||||
timing.finish();
|
||||
acc.fold(&out);
|
||||
let final_out = acc.into_output();
|
||||
// A validation abort carries its own HTTP status + diagnostic.
|
||||
if let Some((code, message)) = final_out
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
.and_then(|f| f.abort_status())
|
||||
{
|
||||
let status =
|
||||
StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
return (status, error_value(code, message));
|
||||
}
|
||||
let mut value = frame_value(&final_out, rid_str);
|
||||
add_e2e_latency(&mut value, &timing);
|
||||
return (StatusCode::OK, value);
|
||||
}
|
||||
ResponseItem::Error(e) => {
|
||||
FrontendEvent::Failed(error) => {
|
||||
timing.finish();
|
||||
let code = e.http_status();
|
||||
let status =
|
||||
StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
return (status, error_value(code, &e.to_string()));
|
||||
let status = frontend_error_status(&error);
|
||||
return (status, error_value(status.as_u16(), &error.to_string()));
|
||||
}
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => continue, // never on `/generate`
|
||||
}
|
||||
}
|
||||
// Sender dropped without a terminal item: the shard dropped this request (a
|
||||
// truncation — a client disconnect would have dropped the handler future).
|
||||
// Defensive fallback: FrontendCall normally turns premature runtime
|
||||
// termination into a terminal Failed event before exposing stream end.
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error_value(500, "response truncated before completion"),
|
||||
@@ -312,7 +289,7 @@ async fn drain_unary(
|
||||
/// own `{ "error": … }` entry; the batch response is 200.
|
||||
async fn generate_batch(
|
||||
state: &AppState,
|
||||
requests: Vec<GenerateRequest>,
|
||||
requests: Vec<FrontendRequest>,
|
||||
stream: bool,
|
||||
timing: RequestTiming,
|
||||
) -> Response {
|
||||
@@ -341,7 +318,7 @@ async fn generate_batch(
|
||||
// its own terminal output promptly (important for per-item e2e_latency).
|
||||
let drained = futures::future::join_all(calls.into_iter().map(
|
||||
|(mut call, request_timing)| async move {
|
||||
let client_rid = call.rid().client_facing().to_owned();
|
||||
let client_rid = call.public_id().to_owned();
|
||||
let (_status, value) = drain_unary(&mut call, &client_rid, request_timing).await;
|
||||
value
|
||||
},
|
||||
@@ -351,21 +328,20 @@ async fn generate_batch(
|
||||
}
|
||||
}
|
||||
|
||||
/// Await the next item from `call`, then drain whatever queued behind it (so the caller
|
||||
/// can coalesce a backlog, as Python's `state.out_list` does), handing the call
|
||||
/// back for `FuturesUnordered` to re-poll. Empty result = channel closed.
|
||||
/// Await the next event from `call`, then drain whatever queued behind it (so the
|
||||
/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the
|
||||
/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic
|
||||
/// stream was already exhausted.
|
||||
async fn recv_indexed(
|
||||
index: usize,
|
||||
mut call: FrontendCall,
|
||||
) -> (usize, FrontendCall, Vec<ResponseItem>) {
|
||||
) -> (usize, FrontendCall, Vec<FrontendEvent>) {
|
||||
let mut items = Vec::new();
|
||||
match call.recv().await {
|
||||
Some(item) => items.push(item),
|
||||
None => return (index, call, items), // closed
|
||||
}
|
||||
while let Ok(item) = call.try_recv() {
|
||||
items.push(item);
|
||||
None => return (index, call, items), // already exhausted
|
||||
}
|
||||
call.drain_ready(&mut items);
|
||||
(index, call, items)
|
||||
}
|
||||
|
||||
@@ -381,9 +357,9 @@ fn generation_event_stream(
|
||||
use futures::StreamExt;
|
||||
|
||||
let n = calls.len();
|
||||
let rid_strs: Vec<Rid> = calls
|
||||
let rid_strs: Vec<String> = calls
|
||||
.iter()
|
||||
.map(|(call, _)| call.rid().clone())
|
||||
.map(|(call, _)| call.public_id().to_owned())
|
||||
.collect();
|
||||
let mut timings: Vec<RequestTiming> = calls
|
||||
.iter()
|
||||
@@ -404,8 +380,9 @@ fn generation_event_stream(
|
||||
|
||||
while let Some((i, call, items)) = futs.next().await {
|
||||
if items.is_empty() {
|
||||
// Channel closed with no terminal → truncation for this item;
|
||||
// leave its rid armed so the scheduler work is aborted.
|
||||
// Defensive fallback: a premature runtime close is normally a
|
||||
// Failed event produced by FrontendCall. Keep this item's call
|
||||
// armed if its semantic stream somehow ends unexpectedly.
|
||||
yield tag_value(error_value(500, "response truncated before completion"), idx(i));
|
||||
continue;
|
||||
}
|
||||
@@ -418,48 +395,45 @@ fn generation_event_stream(
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
ResponseItem::Frame(out) => {
|
||||
FrontendEvent::Delta(out) => {
|
||||
timings[i].observe_first_output();
|
||||
accs[i].fold(&out);
|
||||
if incremental {
|
||||
yield stream_frame_string(out, &accs[i], true, rid_strs[i].client_facing(), idx(i));
|
||||
yield stream_frame_string(out, &accs[i], true, &rid_strs[i], idx(i));
|
||||
} else {
|
||||
coalesced = true;
|
||||
}
|
||||
}
|
||||
ResponseItem::Done(out) => {
|
||||
FrontendEvent::Finished(out) => {
|
||||
timings[i].observe_first_output();
|
||||
timings[i].finish();
|
||||
accs[i].fold(&out);
|
||||
terminal = Some(out);
|
||||
}
|
||||
ResponseItem::Error(e) => {
|
||||
FrontendEvent::Failed(error) => {
|
||||
timings[i].finish();
|
||||
failed = Some(e);
|
||||
failed = Some(error);
|
||||
}
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => {} // never on /generate
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(e) = failed {
|
||||
yield tag_value(error_value(e.http_status(), &e.to_string()), idx(i));
|
||||
if let Some(error) = failed {
|
||||
let status = frontend_error_status(&error);
|
||||
yield tag_value(error_value(status.as_u16(), &error.to_string()), idx(i));
|
||||
} else if let Some(out) = terminal {
|
||||
// A validation abort → an error object, not a frame. The final frame
|
||||
// carries the full cumulative state, so any coalesced ones are moot.
|
||||
yield match out.finish_reason.as_ref().and_then(|f| f.abort_status()) {
|
||||
Some((code, message)) => tag_value(error_value(code, message), idx(i)),
|
||||
None => terminal_stream_frame_string(
|
||||
out,
|
||||
&accs[i],
|
||||
incremental,
|
||||
rid_strs[i].client_facing(),
|
||||
idx(i),
|
||||
&timings[i],
|
||||
),
|
||||
};
|
||||
// The final frame carries the full cumulative state, so any
|
||||
// coalesced non-terminal frames are moot.
|
||||
yield terminal_stream_frame_string(
|
||||
out,
|
||||
&accs[i],
|
||||
incremental,
|
||||
&rid_strs[i],
|
||||
idx(i),
|
||||
&timings[i],
|
||||
);
|
||||
} else {
|
||||
if coalesced {
|
||||
yield cumulative_frame_string(&accs[i], rid_strs[i].client_facing(), idx(i));
|
||||
yield cumulative_frame_string(&accs[i], &rid_strs[i], idx(i));
|
||||
}
|
||||
futs.push(recv_indexed(i, call)); // keep this item flowing
|
||||
}
|
||||
@@ -471,7 +445,7 @@ fn generation_event_stream(
|
||||
/// Python's `e2e_latency` is `finished_time - created_time`, in seconds, and is
|
||||
/// attached only when the request finishes. The Rust native API owns the same
|
||||
/// lifecycle boundary, so it adds the value while handling the terminal egress
|
||||
/// item rather than putting API-only timing onto every scheduler `ChunkEvent`.
|
||||
/// item rather than putting API-only timing onto every shared frontend output.
|
||||
fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) {
|
||||
let (time_to_first_token, e2e_latency) = timing
|
||||
.terminal_latencies()
|
||||
@@ -484,7 +458,7 @@ fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) {
|
||||
/// memoized fast path; the one terminal frame uses the Value path so it can carry
|
||||
/// the request-local `e2e_latency`, exactly as Python does.
|
||||
fn terminal_stream_frame_string(
|
||||
out: ChunkEvent,
|
||||
out: FrontendOutput,
|
||||
acc: &OutputAccumulator,
|
||||
incremental: bool,
|
||||
rid_str: &str,
|
||||
@@ -500,7 +474,7 @@ fn terminal_stream_frame_string(
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::message::request::RequestKind;
|
||||
use crate::message::response::ChunkEvent;
|
||||
use crate::message::response::{ChunkEvent, ResponseItem};
|
||||
use crate::tokenizer_manager::wiring::TmEvent;
|
||||
use crate::utils::error::Error;
|
||||
use axum::body::Body;
|
||||
@@ -568,6 +542,7 @@ mod tests {
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::default(),
|
||||
},
|
||||
),
|
||||
server_args: Arc::new(crate::message::config::ServerArgs::default()),
|
||||
@@ -593,6 +568,7 @@ mod tests {
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::default(),
|
||||
},
|
||||
);
|
||||
let state = Arc::new(AppState {
|
||||
@@ -825,6 +801,32 @@ mod tests {
|
||||
assert_eq!(stream.next().await.unwrap(), "[DONE]");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_rejection_preserves_legacy_http_status() {
|
||||
let (tx, rx) = mpsc::channel(8);
|
||||
let calls = vec![timed_call(10, rx)];
|
||||
let stream = generation_event_stream(calls, false, false);
|
||||
futures::pin_mut!(stream);
|
||||
|
||||
tx.send(ResponseItem::Done(ChunkEvent {
|
||||
finish_reason: serde_json::from_value(serde_json::json!({
|
||||
"type": "abort",
|
||||
"message": "media decode failed",
|
||||
"status_code": 432,
|
||||
"err_type": "BadRequestError"
|
||||
}))
|
||||
.unwrap(),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let failure = parse(&stream.next().await.unwrap());
|
||||
assert_eq!(failure["error"]["code"], 432);
|
||||
assert_eq!(failure["error"]["message"], "media decode failed");
|
||||
assert_eq!(stream.next().await.unwrap(), "[DONE]");
|
||||
}
|
||||
|
||||
/// `incremental=true`: each frame carries this step's **delta** text/output_ids,
|
||||
/// but `meta_info.completion_tokens` stays cumulative (matching Python).
|
||||
#[tokio::test]
|
||||
@@ -905,6 +907,32 @@ mod tests {
|
||||
assert_eq!(stream.next().await.unwrap(), "[DONE]");
|
||||
}
|
||||
|
||||
/// A runtime close must not overtake output that was already queued. The
|
||||
/// cumulative adapter first exposes the produced text, then reports the
|
||||
/// truncation on the next event, preserving the Step 2 HTTP behavior.
|
||||
#[tokio::test]
|
||||
async fn cumulative_output_precedes_truncation_error() {
|
||||
let (tx, rx) = mpsc::channel(8);
|
||||
let calls = vec![timed_call(10, rx)];
|
||||
let stream = generation_event_stream(calls, false, false);
|
||||
futures::pin_mut!(stream);
|
||||
|
||||
tx.send(frame(10, "already-produced")).await.unwrap();
|
||||
drop(tx);
|
||||
|
||||
let output = parse(&stream.next().await.unwrap());
|
||||
assert_eq!(output["text"], "already-produced");
|
||||
|
||||
let failure = parse(&stream.next().await.unwrap());
|
||||
assert_eq!(failure["error"]["code"], 500);
|
||||
assert_eq!(
|
||||
failure["error"]["message"],
|
||||
"response truncated before completion"
|
||||
);
|
||||
assert_eq!(stream.next().await.unwrap(), "[DONE]");
|
||||
assert!(stream.next().await.is_none());
|
||||
}
|
||||
|
||||
/// Incremental frames are *deltas*, so a backlog must emit every one — dropping
|
||||
/// any would silently lose tokens. Only the cumulative protocol may coalesce.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
//! OpenAI-compatible generation endpoints.
|
||||
//!
|
||||
//! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI
|
||||
//! request and response primitives. Scheduler [`ChunkEvent`] values remain the one
|
||||
//! backend output type for both unary and streaming responses.
|
||||
//! request and response primitives. [`FrontendOutput`] remains the one backend
|
||||
//! output type for both unary and streaming responses.
|
||||
|
||||
use axum::{Router, http::StatusCode, response::Response};
|
||||
use futures::StreamExt;
|
||||
@@ -22,10 +22,11 @@ pub(super) use template::{ChatFormatter, ChatTemplateKwargs};
|
||||
|
||||
use super::app::AppState;
|
||||
use super::frame::OutputAccumulator;
|
||||
use crate::frontend::{FrontendCall, FrontendError};
|
||||
use super::frontend_error_status;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
|
||||
};
|
||||
use crate::message::config::ServerArgs;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::{ChunkEvent, ResponseItem};
|
||||
use crate::tokenizer_manager::tokenizer;
|
||||
use crate::utils::response::error_response;
|
||||
|
||||
@@ -115,23 +116,21 @@ pub(super) fn openai_error(code: StatusCode, message: impl Into<String>, stream:
|
||||
}
|
||||
|
||||
/// Drain one submitted request to its terminal output, fold frames, and map
|
||||
/// errors / validation aborts / truncation to `(status, message)` for the
|
||||
/// OpenAI error shape. The call owns cancellation and disarms itself.
|
||||
async fn collect_output(mut call: FrontendCall) -> Result<ChunkEvent, (StatusCode, String)> {
|
||||
/// semantic failures / truncation to `(status, message)` for the OpenAI error
|
||||
/// shape. The call owns cancellation and disarms itself.
|
||||
async fn collect_output(mut call: FrontendCall) -> Result<FrontendOutput, (StatusCode, String)> {
|
||||
let mut accumulator = OutputAccumulator::default();
|
||||
let output = loop {
|
||||
match call.recv().await {
|
||||
Some(ResponseItem::Frame(output)) => accumulator.fold(&output),
|
||||
Some(ResponseItem::Done(output)) => {
|
||||
Some(FrontendEvent::Delta(output)) => accumulator.fold(&output),
|
||||
Some(FrontendEvent::Finished(output)) => {
|
||||
accumulator.fold(&output);
|
||||
break accumulator.into_output();
|
||||
}
|
||||
Some(ResponseItem::Error(error)) => {
|
||||
let status = StatusCode::from_u16(error.http_status())
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
Some(FrontendEvent::Failed(error)) => {
|
||||
let status = frontend_error_status(&error);
|
||||
return Err((status, error.to_string()));
|
||||
}
|
||||
Some(ResponseItem::Control(_)) | Some(ResponseItem::Data(_)) => {}
|
||||
None => {
|
||||
return Err((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -140,22 +139,12 @@ async fn collect_output(mut call: FrontendCall) -> Result<ChunkEvent, (StatusCod
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some((code, message)) = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
.and_then(|reason| reason.abort_status())
|
||||
{
|
||||
return Err((
|
||||
StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
|
||||
message.to_owned(),
|
||||
));
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
async fn submit_generation(
|
||||
state: &AppState,
|
||||
request: GenerateRequest,
|
||||
request: FrontendRequest,
|
||||
stream: bool,
|
||||
) -> Result<FrontendCall, Response> {
|
||||
match state.frontend.generate(request).await {
|
||||
@@ -179,18 +168,14 @@ async fn submit_generation(
|
||||
fn indexed_decode_stream(
|
||||
index: usize,
|
||||
call: FrontendCall,
|
||||
) -> futures::stream::BoxStream<'static, (usize, Option<ResponseItem>)> {
|
||||
) -> futures::stream::BoxStream<'static, (usize, FrontendEvent)> {
|
||||
futures::stream::unfold((call, false), move |(mut call, finished)| async move {
|
||||
if finished {
|
||||
return None;
|
||||
}
|
||||
match call.recv().await {
|
||||
Some(item) => {
|
||||
let finished = matches!(item, ResponseItem::Done(_) | ResponseItem::Error(_));
|
||||
Some(((index, Some(item)), (call, finished)))
|
||||
}
|
||||
None => Some(((index, None), (call, true))),
|
||||
}
|
||||
let event = call.recv().await?;
|
||||
let finished = event.is_terminal();
|
||||
Some(((index, event), (call, finished)))
|
||||
})
|
||||
.boxed()
|
||||
}
|
||||
|
||||
@@ -36,11 +36,11 @@ use super::{
|
||||
AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload,
|
||||
indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32,
|
||||
};
|
||||
use crate::frontend::FrontendCall;
|
||||
use crate::api_server::frontend_error_status;
|
||||
use crate::frontend::{FrontendCall, FrontendEvent, FrontendRequest};
|
||||
use crate::message::config::{DefaultSamplingParams, ServerArgs};
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::{ChunkExtras, ResponseItem};
|
||||
use crate::message::response::ChunkExtras;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use crate::message::types::OneOrMany;
|
||||
|
||||
@@ -204,7 +204,7 @@ async fn chat_completions(
|
||||
.expect("chat prompt exists until the last choice")
|
||||
.clone()
|
||||
};
|
||||
let native = GenerateRequest {
|
||||
let native = FrontendRequest {
|
||||
rid: rid.clone(),
|
||||
text: Some(choice_prompt),
|
||||
// Rendered templates own their special tokens — the pool must not
|
||||
@@ -580,47 +580,20 @@ pub(super) fn chat_event_stream(
|
||||
}
|
||||
|
||||
let mut events = futures::stream::select_all(streams);
|
||||
while let Some((index, item)) = events.next().await {
|
||||
let Some(item) = item else {
|
||||
yield Annotated {
|
||||
data: None,
|
||||
id: None,
|
||||
event: None,
|
||||
comment: None,
|
||||
error: Some(error_payload(StatusCode::INTERNAL_SERVER_ERROR, "response truncated before completion").to_string()),
|
||||
};
|
||||
continue;
|
||||
};
|
||||
let output = match item {
|
||||
ResponseItem::Frame(output) => output,
|
||||
ResponseItem::Done(output) => output,
|
||||
ResponseItem::Error(error) => {
|
||||
while let Some((index, event)) = events.next().await {
|
||||
let output = match event {
|
||||
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
|
||||
FrontendEvent::Failed(error) => {
|
||||
yield Annotated {
|
||||
data: None,
|
||||
id: None,
|
||||
event: None,
|
||||
comment: None,
|
||||
error: Some(error_payload(StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), error.to_string()).to_string()),
|
||||
error: Some(error_payload(frontend_error_status(&error), error.to_string()).to_string()),
|
||||
};
|
||||
continue;
|
||||
}
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => continue,
|
||||
};
|
||||
if let Some((code, message)) = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
.and_then(|reason| reason.abort_status())
|
||||
{
|
||||
yield Annotated {
|
||||
data: None,
|
||||
id: None,
|
||||
event: None,
|
||||
comment: None,
|
||||
error: Some(error_payload(StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), message).to_string()),
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if prompt_tokens == 0 {
|
||||
prompt_tokens = output.prompt_tokens;
|
||||
}
|
||||
|
||||
@@ -24,14 +24,15 @@ use super::{
|
||||
AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream,
|
||||
openai_error, submit_generation, unix_seconds_u32,
|
||||
};
|
||||
use crate::frontend::{FrontendCall, FrontendError};
|
||||
use crate::api_server::frontend_error_status;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
|
||||
};
|
||||
use crate::message::finish_reason::Matched;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::{ChunkEvent, ChunkExtras, ResponseItem};
|
||||
use crate::message::response::ChunkExtras;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use crate::message::types::{OneOrMany, TokenIds};
|
||||
use crate::utils::error::Error;
|
||||
|
||||
pub(super) fn routes() -> Router<Arc<AppState>> {
|
||||
Router::new().route("/v1/completions", post(completions))
|
||||
@@ -163,7 +164,7 @@ async fn completions(
|
||||
Err(response) => return response,
|
||||
};
|
||||
}
|
||||
let native = GenerateRequest {
|
||||
let native = FrontendRequest {
|
||||
rid: rid.clone(),
|
||||
text: text.clone(),
|
||||
input_ids: input_ids.clone(),
|
||||
@@ -234,13 +235,7 @@ async fn completions(
|
||||
/// or an `Error` (e.g. out-of-range ids → `Validation` → 400).
|
||||
async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result<String, Response> {
|
||||
match state.frontend.detokenize(token_ids).await {
|
||||
Ok(payload) => String::from_utf8(payload.to_vec()).map_err(|_| {
|
||||
openai_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"detokenized prompt is not valid UTF-8",
|
||||
false,
|
||||
)
|
||||
}),
|
||||
Ok(text) => Ok(text),
|
||||
// Same rule as `submit_generation`: build the refusal in the OpenAI
|
||||
// error shape rather than forwarding the native-shaped response.
|
||||
Err(FrontendError::Unavailable) => Err(openai_error(
|
||||
@@ -248,30 +243,22 @@ async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result<Str
|
||||
"service unavailable",
|
||||
false,
|
||||
)),
|
||||
Err(FrontendError::Pipeline(Error::Validation(message))) => {
|
||||
Err(FrontendError::InvalidArgument(message)) => {
|
||||
Err(openai_error(StatusCode::BAD_REQUEST, &message, false))
|
||||
}
|
||||
Err(FrontendError::Pipeline(error)) => {
|
||||
let status = StatusCode::from_u16(error.http_status())
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
Err(FrontendError::InvalidResponse(message)) => Err(openai_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
message,
|
||||
false,
|
||||
)),
|
||||
Err(error) => {
|
||||
let status = frontend_error_status(&error);
|
||||
Err(openai_error(
|
||||
status,
|
||||
format!("failed to decode prompt for echo: {error}"),
|
||||
false,
|
||||
))
|
||||
}
|
||||
Err(FrontendError::ResponseClosed | FrontendError::UnexpectedResponse(_)) => {
|
||||
Err(openai_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"failed to decode prompt for echo: reply channel closed",
|
||||
false,
|
||||
))
|
||||
}
|
||||
Err(error @ FrontendError::InvalidArgument(_)) => Err(openai_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to decode prompt for echo: {error}"),
|
||||
false,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -419,7 +406,7 @@ pub(super) async fn unary_completion(
|
||||
fn completion_choice(
|
||||
index: usize,
|
||||
text: String,
|
||||
output: &ChunkEvent,
|
||||
output: &FrontendOutput,
|
||||
want_logprobs: bool,
|
||||
include_input_logprobs: bool,
|
||||
) -> (Choice, ChoiceExtensions) {
|
||||
@@ -532,30 +519,15 @@ pub(super) fn completion_event_stream(
|
||||
}
|
||||
let mut events = futures::stream::select_all(streams);
|
||||
|
||||
while let Some((index, item)) = events.next().await {
|
||||
let Some(item) = item else {
|
||||
yield error_payload(StatusCode::INTERNAL_SERVER_ERROR, "response truncated before completion").to_string();
|
||||
continue;
|
||||
};
|
||||
let output = match item {
|
||||
ResponseItem::Frame(output) => output,
|
||||
ResponseItem::Done(output) => output,
|
||||
ResponseItem::Error(error) => {
|
||||
yield error_payload(StatusCode::from_u16(error.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), error.to_string()).to_string();
|
||||
while let Some((index, event)) = events.next().await {
|
||||
let output = match event {
|
||||
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
|
||||
FrontendEvent::Failed(error) => {
|
||||
yield error_payload(frontend_error_status(&error), error.to_string()).to_string();
|
||||
continue;
|
||||
}
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => continue,
|
||||
};
|
||||
|
||||
if let Some((code, message)) = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
.and_then(|reason| reason.abort_status())
|
||||
{
|
||||
yield error_payload(StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), message).to_string();
|
||||
continue;
|
||||
}
|
||||
|
||||
prompt_tokens_by_prompt
|
||||
.entry(prompt_indexes[index])
|
||||
.or_insert(output.prompt_tokens);
|
||||
|
||||
@@ -20,7 +20,7 @@ use serde_json::json;
|
||||
use tower::util::ServiceExt;
|
||||
|
||||
use super::{indexed_decode_stream, openai_error, routes};
|
||||
use crate::frontend::{FrontendCall, FrontendHandle};
|
||||
use crate::frontend::{FrontendCall, FrontendEvent, FrontendHandle};
|
||||
use crate::message::config::ServerArgs;
|
||||
use crate::message::response::{ChunkEvent, ResponseItem};
|
||||
pub(super) fn frontend() -> FrontendHandle {
|
||||
@@ -33,6 +33,7 @@ pub(super) fn frontend() -> FrontendHandle {
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -130,6 +131,7 @@ pub(super) fn frontend_closed() -> FrontendHandle {
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -170,7 +172,7 @@ async fn dropping_indexed_stream_aborts_its_live_call() {
|
||||
response_tx.send(chunk("live", "x", false)).await.unwrap();
|
||||
assert!(matches!(
|
||||
stream.next().await,
|
||||
Some((0, Some(ResponseItem::Frame(_))))
|
||||
Some((0, FrontendEvent::Delta(_)))
|
||||
));
|
||||
|
||||
drop(stream);
|
||||
|
||||
@@ -36,7 +36,7 @@ use dynamo_protocols::types::{
|
||||
Role,
|
||||
};
|
||||
|
||||
use crate::message::response::ChunkEvent;
|
||||
use crate::frontend::FrontendOutput;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
|
||||
/// Canonicalize a tool-call parser name onto the dynamo-parsers registry keys.
|
||||
@@ -246,7 +246,7 @@ pub(super) async fn parse_chat_tool_calls(
|
||||
/// Map the scheduler's finish kind onto the OpenAI wire values. Length and
|
||||
/// content-filter keep their names; everything else (including a bare abort)
|
||||
/// reports as `stop`, matching Python's fallback.
|
||||
pub(super) fn chat_finish_reason(output: &ChunkEvent) -> Option<OpenAIFinishReason> {
|
||||
pub(super) fn chat_finish_reason(output: &FrontendOutput) -> Option<OpenAIFinishReason> {
|
||||
let kind = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
@@ -264,7 +264,7 @@ mod tests {
|
||||
apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name,
|
||||
dynamo_tool_choice, parse_chat_tool_calls,
|
||||
};
|
||||
use crate::message::response::ChunkEvent;
|
||||
use crate::frontend::FrontendOutput;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail};
|
||||
use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition};
|
||||
@@ -766,8 +766,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chat_finish_reason_maps_scheduler_kinds() {
|
||||
let output = |finish: serde_json::Value| ChunkEvent {
|
||||
rid: "r".into(),
|
||||
let output = |finish: serde_json::Value| FrontendOutput {
|
||||
text: "x".into(),
|
||||
token_ids: vec![1],
|
||||
prompt_tokens: 1,
|
||||
@@ -795,7 +794,7 @@ mod tests {
|
||||
Some(OpenAIFinishReason::Stop)
|
||||
);
|
||||
assert_eq!(
|
||||
chat_finish_reason(&ChunkEvent {
|
||||
chat_finish_reason(&FrontendOutput {
|
||||
finish_reason: None,
|
||||
..Default::default()
|
||||
}),
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
//! Transport-neutral entry point into the Rust frontend pipeline.
|
||||
//!
|
||||
//! Wire adapters normalize their protocol into the existing in-process request
|
||||
//! types, submit them through [`FrontendHandle`], and render the returned
|
||||
//! [`ResponseItem`] values for their own transport. This module owns shared
|
||||
//! preprocessing, runtime capabilities, and request lifetime; it deliberately
|
||||
//! knows nothing about Axum, HTTP response shapes, Tonic, or protobuf.
|
||||
//! Wire adapters normalize their protocol into [`FrontendRequest`], submit it
|
||||
//! through [`FrontendHandle`], and render semantic [`FrontendEvent`]s for their
|
||||
//! own transport. This module owns shared preprocessing, runtime translation,
|
||||
//! capabilities, and request lifetime; it deliberately knows nothing about
|
||||
//! Axum, HTTP response shapes, Tonic, or protobuf.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{
|
||||
@@ -13,12 +13,12 @@ use std::sync::{
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::message::config::{DisaggregationMode, ServerArgs};
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::io_struct::ControlRequest;
|
||||
use crate::message::request::{GenerateRequest, Request, RequestKind};
|
||||
use crate::message::io_struct::{ControlRequest, GetInternalStateReq};
|
||||
use crate::message::request::{Request, RequestKind};
|
||||
use crate::message::response::{ResponseItem, ResponseSink};
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use crate::message::types::TokenIds;
|
||||
@@ -27,50 +27,63 @@ use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
|
||||
use crate::utils::error::Error;
|
||||
use crate::utils::fsm::RequestState;
|
||||
|
||||
mod contract;
|
||||
mod prefetch;
|
||||
|
||||
pub(crate) use contract::{
|
||||
FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
|
||||
InternalState, ModelInfo, ServerInfo,
|
||||
};
|
||||
|
||||
/// Sentinel host that makes the KV connector no-op. Parity with
|
||||
/// `sglang.srt.disaggregation.utils.FAKE_BOOTSTRAP_HOST`.
|
||||
const FAKE_BOOTSTRAP_HOST: &str = "2.2.2.2";
|
||||
|
||||
/// A transport-neutral failure while executing a frontend operation.
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
pub(crate) enum FrontendError {
|
||||
/// The TokenizerManager intake loop has shut down, so callers may retry on
|
||||
/// another healthy server.
|
||||
#[error("service unavailable")]
|
||||
Unavailable,
|
||||
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
|
||||
#[error(transparent)]
|
||||
Pipeline(#[from] Error),
|
||||
|
||||
#[error("response channel closed before terminal output")]
|
||||
ResponseClosed,
|
||||
|
||||
#[error("{0}")]
|
||||
UnexpectedResponse(&'static str),
|
||||
}
|
||||
|
||||
/// Semantic result of a deep frontend health check.
|
||||
///
|
||||
/// Expected lifecycle states are values rather than transport errors so each
|
||||
/// adapter can render them according to its own protocol.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum HealthStatus {
|
||||
Healthy,
|
||||
NotReady,
|
||||
Stalled,
|
||||
}
|
||||
|
||||
pub(crate) struct FrontendConfig {
|
||||
pub(crate) response_capacity: usize,
|
||||
pub(crate) response_activity: ActivityCounter,
|
||||
pub(crate) startup_ready: bool,
|
||||
pub(crate) is_disaggregation: bool,
|
||||
pub(crate) mm_limits: BTreeMap<String, usize>,
|
||||
pub(crate) metadata: FrontendMetadata,
|
||||
}
|
||||
|
||||
/// Immutable, transport-independent metadata retained by the frontend.
|
||||
/// Keeping this snapshot narrow avoids making adapters or the shared handle
|
||||
/// depend on the full launch-configuration object.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct FrontendMetadata {
|
||||
model_path: String,
|
||||
served_model_name: String,
|
||||
tokenizer_path: String,
|
||||
preferred_sampling_params: Option<crate::message::config::PreferredSamplingParams>,
|
||||
weight_version: Option<String>,
|
||||
load_format: Option<String>,
|
||||
reasoning_parser: Option<String>,
|
||||
tool_call_parser: Option<String>,
|
||||
disaggregation_mode: DisaggregationMode,
|
||||
max_context_length: u64,
|
||||
max_total_num_tokens: u64,
|
||||
version: String,
|
||||
}
|
||||
|
||||
impl From<&ServerArgs> for FrontendMetadata {
|
||||
fn from(args: &ServerArgs) -> Self {
|
||||
Self {
|
||||
model_path: args.model_path.clone(),
|
||||
served_model_name: args.served_model_name.clone(),
|
||||
tokenizer_path: args.tokenizer_path.clone(),
|
||||
preferred_sampling_params: args.preferred_sampling_params.clone(),
|
||||
weight_version: args.weight_version.clone(),
|
||||
load_format: args.load_format.clone(),
|
||||
reasoning_parser: args.reasoning_parser.clone(),
|
||||
tool_call_parser: args.tool_call_parser.clone(),
|
||||
disaggregation_mode: args.disaggregation_mode,
|
||||
max_context_length: args.model_config.context_len,
|
||||
max_total_num_tokens: args.max_total_num_tokens,
|
||||
version: args.version.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable capability for submitting work to the existing frontend runtime.
|
||||
@@ -90,6 +103,7 @@ struct FrontendInner {
|
||||
startup_ready: AtomicBool,
|
||||
is_disaggregation: bool,
|
||||
mm_limits: BTreeMap<String, usize>,
|
||||
metadata: FrontendMetadata,
|
||||
}
|
||||
|
||||
impl FrontendHandle {
|
||||
@@ -107,6 +121,7 @@ impl FrontendHandle {
|
||||
startup_ready: AtomicBool::new(config.startup_ready),
|
||||
is_disaggregation: config.is_disaggregation,
|
||||
mm_limits: config.mm_limits,
|
||||
metadata: config.metadata,
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -122,17 +137,19 @@ impl FrontendHandle {
|
||||
self.inner.startup_ready.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn response_activity(&self) -> u64 {
|
||||
fn response_activity(&self) -> u64 {
|
||||
self.inner.response_activity.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) async fn generate(
|
||||
&self,
|
||||
mut request: GenerateRequest,
|
||||
mut request: FrontendRequest,
|
||||
) -> Result<FrontendCall, FrontendError> {
|
||||
self.prepare_generations(std::slice::from_mut(&mut request))
|
||||
.await?;
|
||||
self.submit(RequestKind::Generate(Box::new(request))).await
|
||||
self.submit(RequestKind::Generate(Box::new(request)))
|
||||
.await
|
||||
.map(FrontendCall::new)
|
||||
}
|
||||
|
||||
/// Preprocess and submit a group atomically from the adapter's perspective:
|
||||
@@ -141,7 +158,7 @@ impl FrontendHandle {
|
||||
/// the already-created calls aborts those accepted requests.
|
||||
pub(crate) async fn generate_batch(
|
||||
&self,
|
||||
mut requests: Vec<GenerateRequest>,
|
||||
mut requests: Vec<FrontendRequest>,
|
||||
) -> Result<Vec<FrontendCall>, FrontendError> {
|
||||
self.prepare_generations(&mut requests).await?;
|
||||
|
||||
@@ -149,7 +166,8 @@ impl FrontendHandle {
|
||||
for request in requests {
|
||||
calls.push(
|
||||
self.submit(RequestKind::Generate(Box::new(request)))
|
||||
.await?,
|
||||
.await
|
||||
.map(FrontendCall::new)?,
|
||||
);
|
||||
}
|
||||
Ok(calls)
|
||||
@@ -157,37 +175,96 @@ impl FrontendHandle {
|
||||
|
||||
async fn prepare_generations(
|
||||
&self,
|
||||
requests: &mut [GenerateRequest],
|
||||
requests: &mut [FrontendRequest],
|
||||
) -> Result<(), FrontendError> {
|
||||
prefetch::prefetch_all(requests, &self.inner.mm_limits)
|
||||
.await
|
||||
.map_err(FrontendError::InvalidArgument)
|
||||
}
|
||||
|
||||
pub(crate) async fn control(&self, request: ControlRequest) -> Result<Bytes, FrontendError> {
|
||||
let mut call = self.submit(RequestKind::Control(Box::new(request))).await?;
|
||||
match call.recv().await {
|
||||
Some(ResponseItem::Control(bytes)) => Ok(bytes),
|
||||
Some(ResponseItem::Error(error)) => Err(FrontendError::Pipeline(error)),
|
||||
Some(_) => Err(FrontendError::UnexpectedResponse(
|
||||
"unexpected generation output for control request",
|
||||
)),
|
||||
None => Err(FrontendError::ResponseClosed),
|
||||
/// Return static model metadata without exposing the full launch config.
|
||||
pub(crate) fn model_info(&self) -> ModelInfo {
|
||||
let metadata = &self.inner.metadata;
|
||||
ModelInfo {
|
||||
model_path: metadata.model_path.clone(),
|
||||
served_model_name: metadata.served_model_name.clone(),
|
||||
tokenizer_path: metadata.tokenizer_path.clone(),
|
||||
is_generation: true,
|
||||
preferred_sampling_params: metadata.preferred_sampling_params.clone(),
|
||||
weight_version: metadata.weight_version.clone(),
|
||||
load_format: metadata.load_format.clone(),
|
||||
reasoning_parser: metadata.reasoning_parser.clone(),
|
||||
tool_call_parser: metadata.tool_call_parser.clone(),
|
||||
disaggregation_mode: metadata.disaggregation_mode,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<Bytes, FrontendError> {
|
||||
let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?;
|
||||
/// Return public server metadata with the current scheduler metrics.
|
||||
/// Raw control bytes and the scheduler's full launch-argument dump never
|
||||
/// cross the frontend boundary.
|
||||
pub(crate) async fn server_info(&self) -> Result<ServerInfo, FrontendError> {
|
||||
let internal_state = self.internal_state().await?;
|
||||
let metadata = &self.inner.metadata;
|
||||
Ok(ServerInfo {
|
||||
model_path: metadata.model_path.clone(),
|
||||
served_model_name: metadata.served_model_name.clone(),
|
||||
tokenizer_path: metadata.tokenizer_path.clone(),
|
||||
max_context_length: metadata.max_context_length,
|
||||
max_total_num_tokens: metadata.max_total_num_tokens,
|
||||
version: metadata.version.clone(),
|
||||
frontend: "rust",
|
||||
internal_states: vec![internal_state],
|
||||
})
|
||||
}
|
||||
|
||||
async fn internal_state(&self) -> Result<InternalState, FrontendError> {
|
||||
let bytes = self
|
||||
.control(ControlRequest::GetInternalStateReq(
|
||||
GetInternalStateReq::new(Rid::new().to_string()),
|
||||
))
|
||||
.await?;
|
||||
rmp_serde::from_slice::<InternalStateEnvelope>(&bytes)
|
||||
.map(|response| response.internal_state)
|
||||
.map_err(|error| {
|
||||
FrontendError::InvalidResponse(format!("invalid internal-state response: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
async fn control(&self, request: ControlRequest) -> Result<bytes::Bytes, FrontendError> {
|
||||
let mut call = self.submit(RequestKind::Control(Box::new(request))).await?;
|
||||
match call.recv().await {
|
||||
Some(ResponseItem::Data(bytes)) => Ok(bytes),
|
||||
Some(ResponseItem::Error(error)) => Err(FrontendError::Pipeline(error)),
|
||||
Some(_) => Err(FrontendError::UnexpectedResponse(
|
||||
"unexpected output for detokenize request",
|
||||
Some(ResponseItem::Control(bytes)) => Ok(bytes),
|
||||
Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)),
|
||||
Some(_) => Err(FrontendError::InvalidResponse(
|
||||
"unexpected generation output for control request".into(),
|
||||
)),
|
||||
None => Err(FrontendError::ResponseClosed),
|
||||
None => Err(FrontendError::ResponseTruncated),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode token IDs into text without exposing the runtime's byte payload.
|
||||
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<String, FrontendError> {
|
||||
let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?;
|
||||
let bytes = match call.recv().await {
|
||||
Some(ResponseItem::Data(bytes)) => bytes,
|
||||
Some(ResponseItem::Error(Error::Validation(message))) => {
|
||||
return Err(FrontendError::InvalidArgument(message));
|
||||
}
|
||||
Some(ResponseItem::Error(error)) => return Err(translate_runtime_error(error)),
|
||||
Some(_) => {
|
||||
return Err(FrontendError::InvalidResponse(
|
||||
"unexpected output for detokenize request".into(),
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(FrontendError::Internal("reply channel closed".into()));
|
||||
}
|
||||
};
|
||||
String::from_utf8(bytes.to_vec()).map_err(|_| {
|
||||
FrontendError::InvalidResponse("detokenized prompt is not valid UTF-8".into())
|
||||
})
|
||||
}
|
||||
|
||||
/// Confirm that scheduler output is moving. The response heartbeat is the
|
||||
/// signal rather than this probe's own result, matching Python's
|
||||
/// `last_receive_tstamp` behavior on a busy server.
|
||||
@@ -200,7 +277,7 @@ impl FrontendHandle {
|
||||
}
|
||||
|
||||
let baseline = self.response_activity();
|
||||
let probe = GenerateRequest {
|
||||
let probe = FrontendRequest {
|
||||
rid: Rid::new_health_check(),
|
||||
input_ids: Some(vec![0]),
|
||||
sampling_params: SamplingParams {
|
||||
@@ -238,7 +315,7 @@ impl FrontendHandle {
|
||||
/// The async send preserves TokenizerManager inbox backpressure. Once the
|
||||
/// request is accepted, the returned call owns cancellation until it
|
||||
/// observes a terminal response.
|
||||
async fn submit(&self, kind: RequestKind) -> Result<FrontendCall, FrontendError> {
|
||||
async fn submit(&self, kind: RequestKind) -> Result<RuntimeCall, FrontendError> {
|
||||
let rid = request_rid(&kind);
|
||||
let expected_response = ExpectedResponse::for_request(&kind);
|
||||
let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity);
|
||||
@@ -246,7 +323,7 @@ impl FrontendHandle {
|
||||
// Arm ownership before awaiting the bounded handoff. If this future is
|
||||
// cancelled after the event is queued but before the await observes
|
||||
// success, dropping `call` still cancels the request.
|
||||
let mut call = FrontendCall {
|
||||
let mut call = RuntimeCall {
|
||||
rid: rid.clone(),
|
||||
response_rx,
|
||||
abort_tx: self.inner.abort_tx.clone(),
|
||||
@@ -279,41 +356,74 @@ impl FrontendHandle {
|
||||
}
|
||||
}
|
||||
|
||||
/// One accepted request and its response stream.
|
||||
fn translate_runtime_error(error: Error) -> FrontendError {
|
||||
match error {
|
||||
Error::Validation(message) => {
|
||||
FrontendError::InvalidArgument(format!("validation failed: {message}"))
|
||||
}
|
||||
Error::QueueFull => FrontendError::Overloaded("to_scheduler channel full".into()),
|
||||
Error::Disconnected => FrontendError::Cancelled("client disconnected".into()),
|
||||
error => FrontendError::Internal(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// One accepted generation request and its semantic response stream.
|
||||
///
|
||||
/// Dropping an unfinished call either cancels a request that Intake has not yet
|
||||
/// claimed or notifies the existing abort lane after admission. Reading a
|
||||
/// terminal item disarms that cleanup automatically, so transports cannot
|
||||
/// accidentally abort completed work or forget disconnect cleanup.
|
||||
/// Runtime response variants are translated here so adapters cannot depend on
|
||||
/// scheduler/control encodings. Dropping an unfinished call retains the
|
||||
/// cancellation guarantees owned by [`RuntimeCall`].
|
||||
pub(crate) struct FrontendCall {
|
||||
rid: Rid,
|
||||
response_rx: mpsc::Receiver<ResponseItem>,
|
||||
abort_tx: flume::Sender<AbortSource>,
|
||||
admission: RequestAdmission,
|
||||
expected_response: ExpectedResponse,
|
||||
in_flight: bool,
|
||||
runtime: RuntimeCall,
|
||||
events_finished: bool,
|
||||
}
|
||||
|
||||
impl FrontendCall {
|
||||
pub(crate) fn rid(&self) -> &Rid {
|
||||
&self.rid
|
||||
fn new(runtime: RuntimeCall) -> Self {
|
||||
Self {
|
||||
runtime,
|
||||
events_finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn recv(&mut self) -> Option<ResponseItem> {
|
||||
let item = self.response_rx.recv().await;
|
||||
self.observe(item.as_ref());
|
||||
item
|
||||
/// ID that adapters should return to their client.
|
||||
pub(crate) fn public_id(&self) -> &str {
|
||||
self.runtime.rid.client_facing()
|
||||
}
|
||||
|
||||
pub(crate) fn try_recv(&mut self) -> Result<ResponseItem, mpsc::error::TryRecvError> {
|
||||
let item = self.response_rx.try_recv()?;
|
||||
self.observe(Some(&item));
|
||||
Ok(item)
|
||||
/// Receive the next semantic event. A runtime channel that closes before a
|
||||
/// terminal response is converted into one terminal failure event; adapters
|
||||
/// never need to infer whether an empty runtime channel means success.
|
||||
pub(crate) async fn recv(&mut self) -> Option<FrontendEvent> {
|
||||
if self.events_finished {
|
||||
return None;
|
||||
}
|
||||
|
||||
let event = match self.runtime.recv().await {
|
||||
Some(item) => generation_event(item),
|
||||
None => FrontendEvent::Failed(FrontendError::Internal(
|
||||
"response truncated before completion".into(),
|
||||
)),
|
||||
};
|
||||
self.events_finished = event.is_terminal();
|
||||
Some(event)
|
||||
}
|
||||
|
||||
fn observe(&mut self, item: Option<&ResponseItem>) {
|
||||
if item.is_some_and(|item| self.expected_response.is_terminal(item)) {
|
||||
self.in_flight = false;
|
||||
/// Append every semantic event that is ready without waiting. Channel
|
||||
/// emptiness and closure remain private runtime details.
|
||||
pub(crate) fn drain_ready(&mut self, events: &mut Vec<FrontendEvent>) {
|
||||
while !self.events_finished {
|
||||
let event = match self.runtime.try_recv() {
|
||||
Ok(item) => generation_event(item),
|
||||
// Preserve every queued event before reporting a premature
|
||||
// close. The next awaited `recv` translates that close into
|
||||
// one terminal failure, maintaining stream order while still
|
||||
// hiding channel mechanics from adapters.
|
||||
Err(mpsc::error::TryRecvError::Disconnected | mpsc::error::TryRecvError::Empty) => {
|
||||
break;
|
||||
}
|
||||
};
|
||||
self.events_finished = event.is_terminal();
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,18 +435,84 @@ impl FrontendCall {
|
||||
) -> Self {
|
||||
let admission = RequestAdmission::pending();
|
||||
assert!(admission.try_accept(), "test calls start already admitted");
|
||||
Self {
|
||||
Self::new(RuntimeCall {
|
||||
rid,
|
||||
response_rx,
|
||||
abort_tx,
|
||||
admission,
|
||||
expected_response: ExpectedResponse::Generate,
|
||||
in_flight: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn generation_event(item: ResponseItem) -> FrontendEvent {
|
||||
match item {
|
||||
ResponseItem::Frame(output) => FrontendEvent::Delta(output.into()),
|
||||
ResponseItem::Done(output) => {
|
||||
if let Some((legacy_http_status, message)) = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
.and_then(|reason| reason.abort_status())
|
||||
{
|
||||
FrontendEvent::Failed(FrontendError::from_runtime_rejection(
|
||||
message.to_owned(),
|
||||
legacy_http_status,
|
||||
))
|
||||
} else {
|
||||
FrontendEvent::Finished(output.into())
|
||||
}
|
||||
}
|
||||
ResponseItem::Error(error) => FrontendEvent::Failed(translate_runtime_error(error)),
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => FrontendEvent::Failed(
|
||||
FrontendError::InvalidResponse("unexpected non-generation response".into()),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Private scheduler response envelope. This MessagePack shape is decoded at
|
||||
/// the runtime boundary and deliberately is not part of the frontend contract.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct InternalStateEnvelope {
|
||||
#[serde(default)]
|
||||
internal_state: InternalState,
|
||||
}
|
||||
|
||||
/// Runtime-facing request guard kept private behind [`FrontendCall`].
|
||||
///
|
||||
/// This type is the only owner of the raw response channel and admission/abort
|
||||
/// capabilities. It is also used directly by typed unary operations such as
|
||||
/// detokenization and internal-state lookup.
|
||||
struct RuntimeCall {
|
||||
rid: Rid,
|
||||
response_rx: mpsc::Receiver<ResponseItem>,
|
||||
abort_tx: flume::Sender<AbortSource>,
|
||||
admission: RequestAdmission,
|
||||
expected_response: ExpectedResponse,
|
||||
in_flight: bool,
|
||||
}
|
||||
|
||||
impl RuntimeCall {
|
||||
async fn recv(&mut self) -> Option<ResponseItem> {
|
||||
let item = self.response_rx.recv().await;
|
||||
self.observe(item.as_ref());
|
||||
item
|
||||
}
|
||||
|
||||
fn try_recv(&mut self) -> Result<ResponseItem, mpsc::error::TryRecvError> {
|
||||
let item = self.response_rx.try_recv()?;
|
||||
self.observe(Some(&item));
|
||||
Ok(item)
|
||||
}
|
||||
|
||||
fn observe(&mut self, item: Option<&ResponseItem>) {
|
||||
if item.is_some_and(|item| self.expected_response.is_terminal(item)) {
|
||||
self.in_flight = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FrontendCall {
|
||||
impl Drop for RuntimeCall {
|
||||
fn drop(&mut self) {
|
||||
if self.in_flight {
|
||||
// Pending cancellation is consumed by Intake before it starts work.
|
||||
@@ -396,6 +572,7 @@ mod tests {
|
||||
use crate::message::io_struct::GetInternalStateReq;
|
||||
use crate::message::multimodal::MmItem;
|
||||
use crate::message::request::{GenerateRequest, MmData};
|
||||
use crate::message::response::ChunkEvent;
|
||||
|
||||
struct Harness {
|
||||
handle: FrontendHandle,
|
||||
@@ -449,6 +626,7 @@ mod tests {
|
||||
startup_ready,
|
||||
is_disaggregation,
|
||||
mm_limits,
|
||||
metadata: FrontendMetadata::default(),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -518,6 +696,136 @@ mod tests {
|
||||
assert!(abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_errors_become_transport_neutral_categories() {
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::Validation("bad input".into())),
|
||||
FrontendError::InvalidArgument(message)
|
||||
if message == "validation failed: bad input"
|
||||
));
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::QueueFull),
|
||||
FrontendError::Overloaded(message) if message == "to_scheduler channel full"
|
||||
));
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::Disconnected),
|
||||
FrontendError::Cancelled(message) if message == "client disconnected"
|
||||
));
|
||||
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::Internal("bug".into())),
|
||||
FrontendError::Internal(message) if message == "internal error: bug"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scheduler_status_is_classified_once_for_non_http_adapters() {
|
||||
for (status, expected) in [
|
||||
(400, FrontendErrorKind::InvalidArgument),
|
||||
(404, FrontendErrorKind::NotFound),
|
||||
(408, FrontendErrorKind::DeadlineExceeded),
|
||||
(412, FrontendErrorKind::FailedPrecondition),
|
||||
(413, FrontendErrorKind::ResourceExhausted),
|
||||
(429, FrontendErrorKind::ResourceExhausted),
|
||||
(432, FrontendErrorKind::InvalidArgument),
|
||||
(499, FrontendErrorKind::Cancelled),
|
||||
(500, FrontendErrorKind::Internal),
|
||||
(503, FrontendErrorKind::Unavailable),
|
||||
(504, FrontendErrorKind::DeadlineExceeded),
|
||||
] {
|
||||
let error = FrontendError::from_runtime_rejection("rejected".into(), status);
|
||||
assert_eq!(error.kind(), expected, "legacy status {status}");
|
||||
assert!(matches!(
|
||||
error,
|
||||
FrontendError::RuntimeRejected {
|
||||
legacy_http_status,
|
||||
..
|
||||
} if legacy_http_status == status
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generation_exposes_public_id_and_semantic_events() {
|
||||
let harness = Harness::unbounded(8);
|
||||
let mut call = harness
|
||||
.handle
|
||||
.generate(GenerateRequest {
|
||||
rid: Rid::from_client("public-id"),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(call.public_id(), "public-id");
|
||||
assert_ne!(call.runtime.rid.as_str(), "public-id");
|
||||
|
||||
let request = accept_intake(harness.intake_rx.recv().unwrap());
|
||||
request
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(ChunkEvent {
|
||||
text: "delta".into(),
|
||||
..Default::default()
|
||||
}))
|
||||
.unwrap();
|
||||
request
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(ChunkEvent {
|
||||
text: "final".into(),
|
||||
..Default::default()
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
call.recv().await,
|
||||
Some(FrontendEvent::Delta(FrontendOutput { text, .. })) if text == "delta"
|
||||
));
|
||||
assert!(matches!(
|
||||
call.recv().await,
|
||||
Some(FrontendEvent::Finished(FrontendOutput { text, .. })) if text == "final"
|
||||
));
|
||||
assert!(call.recv().await.is_none());
|
||||
drop(call);
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plain_scheduler_abort_remains_a_terminal_output() {
|
||||
let harness = Harness::unbounded(8);
|
||||
let mut call = harness
|
||||
.handle
|
||||
.generate(GenerateRequest {
|
||||
rid: "plain-abort".into(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let request = accept_intake(harness.intake_rx.recv().unwrap());
|
||||
request
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(ChunkEvent {
|
||||
finish_reason: serde_json::from_value(serde_json::json!({
|
||||
"type": "abort",
|
||||
"message": "Aborted",
|
||||
"status_code": null,
|
||||
"err_type": null
|
||||
}))
|
||||
.unwrap(),
|
||||
..Default::default()
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
call.recv().await,
|
||||
Some(FrontendEvent::Finished(FrontendOutput {
|
||||
finish_reason: Some(_),
|
||||
..
|
||||
}))
|
||||
));
|
||||
assert!(call.recv().await.is_none());
|
||||
drop(call);
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_batch_failure_aborts_already_accepted_calls() {
|
||||
let (intake_tx, intake_rx) = flume::bounded(0);
|
||||
@@ -686,7 +994,7 @@ mod tests {
|
||||
.sink
|
||||
.try_send(ResponseItem::Data(Bytes::from_static(b"decoded")))
|
||||
.unwrap();
|
||||
assert_eq!(task.await.unwrap().unwrap(), Bytes::from_static(b"decoded"));
|
||||
assert_eq!(task.await.unwrap().unwrap(), "decoded");
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
//! Transport-neutral types shared by frontend adapters.
|
||||
//!
|
||||
//! These types describe what an inference operation means after a wire adapter
|
||||
//! has normalized its request. They intentionally contain no Axum, HTTP, SSE,
|
||||
//! Tonic, protobuf, or scheduler-channel details.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::message::config::{DisaggregationMode, PreferredSamplingParams};
|
||||
use crate::message::finish_reason::FinishReason;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::{ChunkEvent, ChunkExtras};
|
||||
use crate::message::types::TokenIds;
|
||||
|
||||
/// Canonical transport-neutral input for one generation operation.
|
||||
///
|
||||
/// The existing in-process request already contains semantic generation fields
|
||||
/// and has no Serde, Axum, Tonic, or protobuf representation attached to it.
|
||||
/// Re-exporting it through the frontend contract gives every adapter one public
|
||||
/// boundary without duplicating the runtime's request model.
|
||||
pub(crate) type FrontendRequest = GenerateRequest;
|
||||
|
||||
/// Canonical transport-neutral output payload for generation events.
|
||||
///
|
||||
/// The runtime correlation ID is deliberately absent: it is an implementation
|
||||
/// detail used for scheduler routing, while adapters obtain the client-visible
|
||||
/// identity from [`crate::frontend::FrontendCall::public_id`]. Runtime-only
|
||||
/// response variants such as raw control bytes are likewise hidden by
|
||||
/// [`crate::frontend::FrontendCall`].
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct FrontendOutput {
|
||||
pub(crate) token_ids: TokenIds,
|
||||
pub(crate) finish_reason: Option<FinishReason>,
|
||||
pub(crate) prompt_tokens: u32,
|
||||
pub(crate) text: String,
|
||||
pub(crate) completion_tokens: u64,
|
||||
pub(crate) extras: Option<Box<ChunkExtras>>,
|
||||
}
|
||||
|
||||
impl From<ChunkEvent> for FrontendOutput {
|
||||
fn from(output: ChunkEvent) -> Self {
|
||||
let ChunkEvent {
|
||||
rid: _,
|
||||
token_ids,
|
||||
finish_reason,
|
||||
prompt_tokens,
|
||||
text,
|
||||
completion_tokens,
|
||||
extras,
|
||||
} = output;
|
||||
Self {
|
||||
token_ids,
|
||||
finish_reason,
|
||||
prompt_tokens,
|
||||
text,
|
||||
completion_tokens,
|
||||
extras,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Semantic result of a deep frontend health check.
|
||||
///
|
||||
/// Expected lifecycle states are values rather than transport errors so each
|
||||
/// adapter can render them according to its own protocol.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum HealthStatus {
|
||||
Healthy,
|
||||
NotReady,
|
||||
Stalled,
|
||||
}
|
||||
|
||||
/// A semantic generation event before any transport-specific framing.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum FrontendEvent {
|
||||
/// One incremental model-output update.
|
||||
Delta(FrontendOutput),
|
||||
/// The final model-output update for this request.
|
||||
Finished(FrontendOutput),
|
||||
/// A request-scoped failure. Other requests in a batch may continue.
|
||||
Failed(FrontendError),
|
||||
}
|
||||
|
||||
/// Protocol-independent error category used by transport adapters.
|
||||
///
|
||||
/// The set intentionally follows operation semantics rather than HTTP or gRPC
|
||||
/// status spaces. An adapter maps this category into its own wire status.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum FrontendErrorKind {
|
||||
InvalidArgument,
|
||||
NotFound,
|
||||
FailedPrecondition,
|
||||
ResourceExhausted,
|
||||
Cancelled,
|
||||
DeadlineExceeded,
|
||||
Unavailable,
|
||||
Internal,
|
||||
}
|
||||
|
||||
impl FrontendEvent {
|
||||
pub(crate) fn is_terminal(&self) -> bool {
|
||||
matches!(self, Self::Finished(_) | Self::Failed(_))
|
||||
}
|
||||
}
|
||||
|
||||
/// A transport-neutral failure category.
|
||||
///
|
||||
/// HTTP and gRPC adapters map these categories into their own status spaces;
|
||||
/// neither adapter needs to understand errors emitted by runtime stages.
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
pub(crate) enum FrontendError {
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
|
||||
/// The frontend intake loop has shut down. A caller may retry elsewhere.
|
||||
#[error("service unavailable")]
|
||||
Unavailable,
|
||||
|
||||
/// Runtime capacity was exhausted before the request could make progress.
|
||||
#[error("{0}")]
|
||||
Overloaded(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Cancelled(String),
|
||||
|
||||
/// The internal producer disappeared before returning a terminal result.
|
||||
///
|
||||
/// This is a server-side failure, not evidence that the caller cancelled.
|
||||
#[error("response truncated before completion")]
|
||||
ResponseTruncated,
|
||||
|
||||
/// A scheduler-side abort that carried an error status.
|
||||
///
|
||||
/// The current Python scheduler communicates this category as an HTTP
|
||||
/// status code inside its finish reason. The semantic event is still
|
||||
/// `Failed`; retaining the legacy code only lets the HTTP adapter preserve
|
||||
/// existing behavior while another adapter maps it into its own status
|
||||
/// space. New frontend failures should use the semantic variants above.
|
||||
#[error("{message}")]
|
||||
RuntimeRejected {
|
||||
kind: FrontendErrorKind,
|
||||
message: String,
|
||||
legacy_http_status: u16,
|
||||
},
|
||||
|
||||
/// The runtime replied with an unexpected or malformed result.
|
||||
#[error("{0}")]
|
||||
InvalidResponse(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl FrontendError {
|
||||
/// Semantic status for adapters that do not use the scheduler's legacy
|
||||
/// HTTP code space (notably the future gRPC adapter).
|
||||
pub(crate) fn kind(&self) -> FrontendErrorKind {
|
||||
match self {
|
||||
Self::InvalidArgument(_) => FrontendErrorKind::InvalidArgument,
|
||||
Self::Unavailable => FrontendErrorKind::Unavailable,
|
||||
Self::Overloaded(_) => FrontendErrorKind::ResourceExhausted,
|
||||
Self::Cancelled(_) => FrontendErrorKind::Cancelled,
|
||||
Self::ResponseTruncated => FrontendErrorKind::Internal,
|
||||
Self::RuntimeRejected { kind, .. } => *kind,
|
||||
Self::InvalidResponse(_) | Self::Internal(_) => FrontendErrorKind::Internal,
|
||||
}
|
||||
}
|
||||
|
||||
/// Translate the scheduler's inherited HTTP-coded failure into a semantic
|
||||
/// category once, at the runtime boundary. HTTP can retain the exact legacy
|
||||
/// status while other adapters consume [`Self::kind`].
|
||||
pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self {
|
||||
let kind = match legacy_http_status {
|
||||
404 => FrontendErrorKind::NotFound,
|
||||
408 | 504 => FrontendErrorKind::DeadlineExceeded,
|
||||
412 => FrontendErrorKind::FailedPrecondition,
|
||||
413 | 429 => FrontendErrorKind::ResourceExhausted,
|
||||
499 => FrontendErrorKind::Cancelled,
|
||||
502 | 503 => FrontendErrorKind::Unavailable,
|
||||
400..=499 => FrontendErrorKind::InvalidArgument,
|
||||
_ => FrontendErrorKind::Internal,
|
||||
};
|
||||
Self::RuntimeRejected {
|
||||
kind,
|
||||
message,
|
||||
legacy_http_status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Static model metadata shared by every public transport.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct ModelInfo {
|
||||
pub(crate) model_path: String,
|
||||
pub(crate) served_model_name: String,
|
||||
pub(crate) tokenizer_path: String,
|
||||
pub(crate) is_generation: bool,
|
||||
pub(crate) preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
pub(crate) weight_version: Option<String>,
|
||||
pub(crate) load_format: Option<String>,
|
||||
pub(crate) reasoning_parser: Option<String>,
|
||||
pub(crate) tool_call_parser: Option<String>,
|
||||
pub(crate) disaggregation_mode: DisaggregationMode,
|
||||
}
|
||||
|
||||
/// Public server metadata plus scheduler-owned runtime metrics.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct ServerInfo {
|
||||
pub(crate) model_path: String,
|
||||
pub(crate) served_model_name: String,
|
||||
pub(crate) tokenizer_path: String,
|
||||
pub(crate) max_context_length: u64,
|
||||
pub(crate) max_total_num_tokens: u64,
|
||||
pub(crate) version: String,
|
||||
pub(crate) frontend: &'static str,
|
||||
pub(crate) internal_states: Vec<InternalState>,
|
||||
}
|
||||
|
||||
/// Scheduler-owned runtime metrics used by the public server-info operation.
|
||||
///
|
||||
/// Deserializing into this allowlisted shape is intentional: the scheduler's
|
||||
/// raw state also contains its full launch arguments, including credentials.
|
||||
/// Unknown fields are discarded before any transport receives the result.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct InternalState {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) last_gen_throughput: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) memory_usage: Option<MemoryUsage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) effective_max_running_requests_per_dp: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) avg_spec_accept_length: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) step_time_dict: Option<BTreeMap<u64, Vec<f64>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) rust_mm_transport: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// Public subset of scheduler memory metrics.
|
||||
///
|
||||
/// This nested allowlist is intentional: adding a scheduler field does not
|
||||
/// silently add different public data to HTTP before the gRPC contract can add
|
||||
/// the same field. New metrics should be promoted here deliberately.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct MemoryUsage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) weight: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) kvcache: Option<MemoryMeasurement>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) startup_available: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) token_capacity: Option<u64>,
|
||||
pub(crate) token_capacity_swa: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) graph: Option<BTreeMap<String, f64>>,
|
||||
}
|
||||
|
||||
/// The scheduler's KV-cache measurement can be a native float or a NumPy
|
||||
/// scalar stringified by the MessagePack bridge. Preserve that representation
|
||||
/// so extracting the typed frontend contract does not change public responses.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum MemoryMeasurement {
|
||||
Number(f64),
|
||||
String(String),
|
||||
}
|
||||
@@ -318,10 +318,11 @@ impl<'py> pyo3::FromPyObject<'_, 'py> for PreferredSamplingParams {
|
||||
module = "sglang.srt.rust_extensions._server"
|
||||
)]
|
||||
// Lowercase to match the values Python reports for the same field.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DisaggregationMode {
|
||||
/// Unified prefill + decode.
|
||||
#[default]
|
||||
Null,
|
||||
Prefill,
|
||||
Decode,
|
||||
|
||||
@@ -97,9 +97,9 @@ impl FinishReason {
|
||||
}
|
||||
}
|
||||
|
||||
/// `Some((status, message))` when this is an abort carrying a `status_code` —
|
||||
/// a scheduler-side request error the API surfaces as that HTTP status instead
|
||||
/// of as a normal completion. A plain abort (no code) reads as `None`.
|
||||
/// `Some((status, message))` when this is an abort carrying the scheduler's
|
||||
/// legacy HTTP status — a request failure the frontend boundary converts
|
||||
/// into a semantic error event. A plain abort (no code) reads as `None`.
|
||||
pub fn abort_status(&self) -> Option<(u16, &str)> {
|
||||
match self {
|
||||
FinishReason::Known(FinishKind::Abort(a)) => Some((
|
||||
@@ -128,9 +128,9 @@ mod tests {
|
||||
Some((code, message.to_string()))
|
||||
}
|
||||
|
||||
/// The classifier both API paths use: a validation abort yields its
|
||||
/// `(code, message)` (the streaming path turns this into an SSE error event
|
||||
/// instead of a normal `Done` frame); anything else yields `None`.
|
||||
/// The classifier the frontend boundary uses: a validation abort yields its
|
||||
/// `(code, message)` so it can become a semantic failure event; anything
|
||||
/// else yields `None`.
|
||||
#[test]
|
||||
fn abort_status_extracts_code_and_message() {
|
||||
let (code, msg) = abort_status(&fr(serde_json::json!({
|
||||
|
||||
@@ -260,8 +260,7 @@ impl GenerateBody {
|
||||
/// prompt + `is_batch` (list form — a 1-element list is still a batch → JSON
|
||||
/// array response). The Rust counterpart of Python
|
||||
/// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent
|
||||
/// batch is [`Error::Validation`], which the handler surfaces with the
|
||||
/// variant's own status (400).
|
||||
/// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400.
|
||||
pub fn into_requests(self) -> Result<(Vec<GenerateRequest>, bool), Error> {
|
||||
let GenerateBody {
|
||||
rid,
|
||||
@@ -653,8 +652,8 @@ fn extension_value_present(value: &rmpv::Value) -> bool {
|
||||
/// mutated lock-free). Common fields here; variant data in [`RequestKind`].
|
||||
#[derive(Debug)]
|
||||
pub struct Request {
|
||||
/// Client-visible request id (uuid hex) — what the scheduler wire and
|
||||
/// `meta_info.id` carry.
|
||||
/// Runtime correlation ID. Client-provided IDs carry a private incarnation
|
||||
/// suffix; adapters recover the public ID through `FrontendCall::public_id`.
|
||||
pub rid: Rid,
|
||||
pub state: RequestState,
|
||||
/// Back-channel to the client connection for response frames.
|
||||
@@ -677,10 +676,10 @@ pub struct SchedulerRequest {
|
||||
/// response shape. Each owns its body, so generate/control fields stay type-separate.
|
||||
#[derive(Debug)]
|
||||
pub enum RequestKind {
|
||||
/// `/generate`: tokenize (if needed) then push a `TokenizedGenerateReqInput`.
|
||||
/// Generation: tokenize (if needed) then push a `TokenizedGenerateReqInput`.
|
||||
Generate(Box<GenerateRequest>),
|
||||
/// A control endpoint (e.g. `/server_info`, `/health`): no tokenization, and
|
||||
/// the response is a single non-streamed JSON result.
|
||||
/// Internal runtime control operation: no tokenization, with one serialized
|
||||
/// result that the frontend boundary converts into a typed response.
|
||||
Control(Box<ControlRequest>),
|
||||
/// Internal service call: decode a complete token-id sequence to text. Walks
|
||||
/// the same FSM as every request (validate → register → Queued), but the
|
||||
@@ -691,11 +690,10 @@ pub enum RequestKind {
|
||||
Detokenize { token_ids: TokenIds },
|
||||
}
|
||||
|
||||
/// A single in-flight `/generate` request (per-item from
|
||||
/// [`GenerateBody::into_requests`]),
|
||||
/// A single in-flight generation request (one item after adapter fan-out),
|
||||
/// serialized to the scheduler wire once tokenized (see `to_header_msgpack`). Not a
|
||||
/// wire type — built by `into_requests`/handlers, never (de)serialized; `input_ids` is
|
||||
/// client-supplied or filled by the Tokenizer stage.
|
||||
/// wire type — built by frontend adapters, never (de)serialized; `input_ids` is
|
||||
/// supplied by an adapter or filled by the Tokenizer stage.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct GenerateRequest {
|
||||
/// This item's final rid: the client's (normalized per item by `into_requests`) or a
|
||||
@@ -726,7 +724,7 @@ pub struct GenerateRequest {
|
||||
/// Sampling params (defaults when the client sent none, as in Python);
|
||||
/// normalized + verified, then serialized into the header.
|
||||
pub sampling_params: SamplingParams,
|
||||
/// Whether the client asked for SSE streaming.
|
||||
/// Whether the caller requested incremental output.
|
||||
pub stream: bool,
|
||||
/// Logprob / hidden-state options. This path bypasses the Python
|
||||
/// `TokenizerManager`, so `into_requests` replicates its scalar
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//! The response direction: the per-request back-channel the API handler
|
||||
//! drains ([`ResponseSink`] / [`ResponseItem`]), the response frame encodings
|
||||
//! (batch / control result / error), and the columnar batch decode into
|
||||
//! per-request [`ChunkEvent`]s.
|
||||
//! The runtime response direction: per-request channels into the frontend
|
||||
//! boundary ([`ResponseSink`] / [`ResponseItem`]), response frame encodings,
|
||||
//! and columnar batch decoding into per-request [`ChunkEvent`]s.
|
||||
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -12,8 +11,8 @@ use crate::message::ids::Rid;
|
||||
use crate::message::types::TokenIds;
|
||||
use crate::utils::error::Error;
|
||||
|
||||
/// Per-request back-channel the detok shard writes decode frames to and the API
|
||||
/// handler drains for SSE; bounded, and receiver-drop (disconnect) = stream end.
|
||||
/// Per-request back-channel the detok shard writes runtime outputs to and the
|
||||
/// frontend boundary drains; bounded, and receiver-drop cancels the producer.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum ResponseSink {
|
||||
Local(mpsc::Sender<ResponseItem>),
|
||||
@@ -39,26 +38,26 @@ impl ResponseSink {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // the receiver half is owned by frontend::FrontendCall.
|
||||
#[allow(dead_code)] // the receiver half is owned inside the frontend boundary.
|
||||
pub type ResponseSource = mpsc::Receiver<ResponseItem>;
|
||||
|
||||
/// What the connection handler receives on the decode stream: a detok-decoded
|
||||
/// [`ChunkEvent`] (handler formats it), a verbatim control payload, or an error.
|
||||
/// Runtime-facing output delivered to the frontend boundary: a detok-decoded
|
||||
/// [`ChunkEvent`], a serialized internal-operation result, or a stage error.
|
||||
/// [`crate::frontend::FrontendHandle`] translates these variants before a
|
||||
/// transport adapter sees them.
|
||||
#[derive(Debug)]
|
||||
pub enum ResponseItem {
|
||||
/// An intermediate streamed generation step (only sent for streaming reqs).
|
||||
Frame(ChunkEvent),
|
||||
/// The final generation step.
|
||||
Done(ChunkEvent),
|
||||
/// A control-request result: one verbatim payload (e.g. `/server_info`),
|
||||
/// delivered as-is with no per-protocol formatting.
|
||||
/// A control-request result encoded by the Python scheduler.
|
||||
Control(Bytes),
|
||||
/// Reply to an internal service request (`RequestKind::Detokenize`): raw
|
||||
/// bytes for the SUBMITTER to consume (e.g. the decoded prompt text), not
|
||||
/// client-bound JSON like `Control` and not a generation frame. Generation
|
||||
/// and control drains never see it.
|
||||
/// bytes for the frontend to decode (e.g. detokenized text), not a
|
||||
/// client-facing wire payload and not a generation frame.
|
||||
Data(Bytes),
|
||||
/// Terminal failure: handler emits an error frame (stream) or status (unary).
|
||||
/// Terminal runtime failure; the frontend maps it to a semantic error.
|
||||
Error(Error),
|
||||
}
|
||||
|
||||
@@ -536,8 +535,8 @@ pub fn frame_control_result(rid: &str, payload: &[u8]) -> Bytes {
|
||||
Bytes::from(buf)
|
||||
}
|
||||
|
||||
/// Frame a per-request failure `[rid, message]` for the response — routes a
|
||||
/// terminal error back to the owning request (→ HTTP 400) instead of crashing.
|
||||
/// Frame a per-request validation failure `[rid, message]` for the response —
|
||||
/// routes a terminal error back to the owning request instead of crashing.
|
||||
pub fn frame_error(rid: &str, message: &str) -> Bytes {
|
||||
use rmpv::Value;
|
||||
let arr = Value::Array(vec![Value::from(rid), Value::from(message)]);
|
||||
@@ -558,10 +557,9 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes {
|
||||
/// frame and moved between stages in-process (never serialized), so no serde.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ChunkEvent {
|
||||
/// Client-visible rid — the request's IDENTITY. Moved out of the frame header
|
||||
/// (which owns it and drops it), so carrying it costs no allocation. The shard
|
||||
/// is still chosen by `Rid::shard`, but a hash collision there now only
|
||||
/// co-locates two requests instead of merging them.
|
||||
/// Runtime correlation ID. Moved out of the frame header (which owns it and
|
||||
/// drops it), so carrying it costs no allocation. The frontend strips this
|
||||
/// field before exposing semantic output; the shard routes it via `Rid::shard`.
|
||||
pub rid: Rid,
|
||||
/// New token ids for this step, widened from the scheduler's int32 wire
|
||||
/// width at parse time. Empty allowed (e.g. metadata-only frames).
|
||||
|
||||
@@ -122,8 +122,8 @@ impl Intake {
|
||||
/// terminal frame and a hung connection. Python cannot hit this because it
|
||||
/// validates before `rid_to_state[obj.rid] = state`.
|
||||
fn fail(&self, req: &mut Request, err: Error, registered: bool) {
|
||||
// Log only server faults (500); 4xx/499/503 are expected and would spam.
|
||||
if err.http_status() == 500 {
|
||||
// Expected request, capacity, and cancellation outcomes would spam.
|
||||
if err.is_server_fault() {
|
||||
tracing::error!(rid = %req.rid, error = %err, "intake rejected request");
|
||||
}
|
||||
let _ = req.state.apply(Event::Error(err.clone()));
|
||||
|
||||
@@ -202,7 +202,7 @@ fn total_tokens_over_context_is_rejected() {
|
||||
};
|
||||
let err = check_total_tokens(&mut g, &limits).unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert_eq!(err.http_status(), 400);
|
||||
assert!(matches!(err, Error::Validation(_)));
|
||||
assert!(msg.contains("total of 103 tokens"), "{msg}");
|
||||
assert!(msg.contains("3 tokens from the input"), "{msg}");
|
||||
assert!(msg.contains("100 tokens for the completion"), "{msg}");
|
||||
@@ -291,7 +291,7 @@ fn input_length_is_checked_unconditionally() {
|
||||
for max_new_tokens in [None, Some(1)] {
|
||||
let err = check_total_tokens(&mut req(max_new_tokens), &limits)
|
||||
.expect_err("input == context_len must be rejected (Python uses >=)");
|
||||
assert_eq!(err.http_status(), 400);
|
||||
assert!(matches!(err, Error::Validation(_)));
|
||||
assert!(err.to_string().contains("longer than the model's context"));
|
||||
}
|
||||
// One token shorter fits, with or without a cap.
|
||||
@@ -338,7 +338,7 @@ fn auto_truncate_cannot_invert_min_and_max_new_tokens() {
|
||||
};
|
||||
let err = check_total_tokens(&mut g, &limits)
|
||||
.expect_err("a clamp that inverts min/max must 400, not ride the wire");
|
||||
assert_eq!(err.http_status(), 400);
|
||||
assert!(matches!(err, Error::Validation(_)));
|
||||
assert!(err.to_string().contains("min_new_tokens"), "{err}");
|
||||
|
||||
// A clamp that keeps the invariant still clamps.
|
||||
@@ -362,7 +362,7 @@ fn hidden_states_gated_on_server_support() {
|
||||
};
|
||||
let disabled = test_limits();
|
||||
let err = validate(&mut req(true), &disabled).unwrap_err();
|
||||
assert_eq!(err.http_status(), 400);
|
||||
assert!(matches!(err, Error::Validation(_)));
|
||||
assert!(
|
||||
err.to_string().contains("--enable-return-hidden-states"),
|
||||
"message must name the flag: {err}"
|
||||
@@ -466,7 +466,7 @@ fn detokenize_negative_ids_reject_before_registration() {
|
||||
let Ok(ResponseItem::Error(err)) = rx.try_recv() else {
|
||||
panic!("sink must receive the validation error");
|
||||
};
|
||||
assert_eq!(err.http_status(), 400);
|
||||
assert!(matches!(err, Error::Validation(_)));
|
||||
assert!(err.to_string().contains("out of range"), "{err}");
|
||||
assert!(detok_rx.try_recv().is_err(), "shard never hears of it");
|
||||
assert!(consumer.drain(16).is_empty());
|
||||
@@ -525,7 +525,7 @@ fn oversized_rid_is_rejected() {
|
||||
let mut req = generate_req(51, SamplingParams::default());
|
||||
req.rid = "x".repeat(MAX_RID_LEN + 1).into();
|
||||
let err = validate(&mut req, &test_limits()).expect_err("must be rejected");
|
||||
assert_eq!(err.http_status(), 400);
|
||||
assert!(matches!(err, Error::Validation(_)));
|
||||
assert!(err.to_string().contains("over the"), "{err}");
|
||||
|
||||
// A uuid-sized rid — what Python mints — is nowhere near the cap.
|
||||
@@ -780,7 +780,7 @@ fn mm_pool_gone_deregisters() {
|
||||
let Ok(ResponseItem::Error(err)) = rx.try_recv() else {
|
||||
panic!("expected a terminal error frame");
|
||||
};
|
||||
assert_eq!(err.http_status(), 500, "{err}");
|
||||
assert!(matches!(err, Error::Internal(_)), "{err}");
|
||||
assert!(err.to_string().contains("mm worker pool gone"), "{err}");
|
||||
assert!(matches!(detok_rx.try_recv(), Ok(DetokMsg::Register { .. })));
|
||||
assert!(matches!(
|
||||
@@ -1140,7 +1140,7 @@ fn mm_text_prompt_under_skip_tokenizer_init_is_a_400() {
|
||||
let Ok(ResponseItem::Error(err)) = rx.try_recv() else {
|
||||
panic!("expected a terminal error frame");
|
||||
};
|
||||
assert_eq!(err.http_status(), 400, "{err}");
|
||||
assert!(matches!(err, Error::Validation(_)), "{err}");
|
||||
assert!(err.to_string().contains("must provide input_ids"), "{err}");
|
||||
assert!(detok_rx.try_recv().is_err(), "rejected before registration");
|
||||
assert!(mm_rx.try_recv().is_err(), "never reaches the mm pool");
|
||||
|
||||
@@ -36,15 +36,17 @@ pub enum Error {
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// HTTP status to surface for the non-streaming error path. Mirrors the
|
||||
/// codes used in the Python `_create_error_response`.
|
||||
pub fn http_status(&self) -> u16 {
|
||||
match self {
|
||||
Error::Validation(_) => 400,
|
||||
Error::Disconnected => 499, // nginx-style client closed request
|
||||
Error::QueueFull => 503,
|
||||
_ => 500,
|
||||
}
|
||||
/// Whether this failure represents a server defect rather than an expected
|
||||
/// request, capacity, or cancellation outcome.
|
||||
pub fn is_server_fault(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Error::Tokenize(_)
|
||||
| Error::Encode(_)
|
||||
| Error::Detokenize(_)
|
||||
| Error::Codec(_)
|
||||
| Error::Internal(_)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -294,6 +294,7 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
|
||||
startup_ready: cfg.server_args.skip_server_warmup,
|
||||
is_disaggregation: cfg.server_args.is_disaggregation(),
|
||||
mm_limits: cfg.server_args.limit_mm_data_per_request.clone(),
|
||||
metadata: crate::frontend::FrontendMetadata::from(cfg.server_args.as_ref()),
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user