diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index 4624372085..17ea4e7270 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -6,7 +6,7 @@ import json import logging import os from types import ModuleType -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.entrypoints.v1_loads import ( @@ -126,6 +126,9 @@ def _build_server_args( # Not a `server_args` field: `TokenizerManager` derives it, and the # rust ingress needs the same number for its total-token check. num_reserved_tokens=compute_num_reserved_tokens(), + tokenizer_model_max_length=_tokenizer_model_max_length( + scheduler.tokenizer, mc.context_len + ), # Launch-time facts Python's /server_info reports from # scheduler_info / the package — stamped here so the rust endpoint # can serve them statically (no scheduler round-trip). @@ -170,6 +173,17 @@ def _launch_server_info(scheduler: Scheduler) -> str: return json.dumps(info, default=str) +def _tokenizer_model_max_length(tokenizer: Any, context_len: int) -> int: + """`/v1/tokenize`'s `max_model_len`, as `OpenAIServingTokenize` reports + it: the tokenizer's `model_max_length`, or the context length when that is + not a 64-bit integer (an unset HF value is a 1e30 sentinel). The Rust + field is signed, so the bound stops at 2**63.""" + max_model_len = getattr(tokenizer, "model_max_length", -1) + if isinstance(max_model_len, int) and -(2**63) <= max_model_len < 2**63: + return max_model_len + return context_len + + def _partition_cores( mm_workers: int = 0, server_core_budget: Optional[Callable[[int, int], int]] = None, diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index 2a625f8e6c..190bd0245f 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -277,8 +277,17 @@ impl CoreHandle { } /// Decode token IDs into text without exposing the runtime's byte payload. - pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result { - let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?; + pub(crate) async fn detokenize( + &self, + token_ids: TokenIds, + skip_special_tokens: bool, + ) -> Result { + let mut call = self + .submit(RequestKind::Detokenize { + token_ids, + skip_special_tokens, + }) + .await?; let bytes = match call.recv().await { Some(ResponseItem::Data(bytes)) => bytes, Some(ResponseItem::Error(Error::Validation(message))) => { @@ -298,6 +307,31 @@ impl CoreHandle { .map_err(|_| CoreError::InvalidResponse("detokenized prompt is not valid UTF-8".into())) } + /// Encode one text into token IDs on the tokenizer pool. + pub(crate) async fn tokenize( + &self, + text: String, + add_special_tokens: bool, + ) -> Result { + let mut call = self + .submit(RequestKind::Tokenize { + text, + add_special_tokens, + }) + .await?; + match call.recv().await { + Some(ResponseItem::TokenIds(ids)) => Ok(ids), + Some(ResponseItem::Error(Error::Validation(message))) => { + Err(CoreError::InvalidArgument(message)) + } + Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)), + Some(_) => Err(CoreError::InvalidResponse( + "unexpected output for tokenize request".into(), + )), + None => Err(CoreError::Internal("reply channel closed".into())), + } + } + /// Confirm that scheduler output is moving. The response heartbeat is the /// signal rather than this probe's own result, matching Python's /// `last_receive_tstamp` behavior on a busy server. @@ -548,9 +582,11 @@ fn generation_event(item: ResponseItem) -> CoreEvent { } } ResponseItem::Error(error) => CoreEvent::Failed(translate_runtime_error(error)), - ResponseItem::Control(_) | ResponseItem::Data(_) => CoreEvent::Failed( - CoreError::InvalidResponse("unexpected non-generation response".into()), - ), + ResponseItem::Control(_) | ResponseItem::Data(_) | ResponseItem::TokenIds(_) => { + CoreEvent::Failed(CoreError::InvalidResponse( + "unexpected non-generation response".into(), + )) + } } } @@ -621,7 +657,16 @@ impl Drop for RuntimeCall { // Pending cancellation is consumed by Intake before it starts work. // Once Intake has accepted the request, use the normal abort lane. if self.admission.cancel_requires_abort() { - let _ = self.abort_tx.send(AbortSource::Guard(self.rid.clone())); + let rid = self.rid.clone(); + let source = match self.expected_response { + ExpectedResponse::Tokenize | ExpectedResponse::Detokenize => { + AbortSource::Local(rid) + } + ExpectedResponse::Generate | ExpectedResponse::Control => { + AbortSource::Guard(rid) + } + }; + let _ = self.abort_tx.send(source); } self.in_flight = false; } @@ -634,7 +679,7 @@ fn request_rid(kind: &RequestKind) -> Rid { RequestKind::Generate(request) => request.rid.clone(), RequestKind::Control(request) => request.rid().into(), // Internal service calls are not client-addressable. - RequestKind::Detokenize { .. } => Rid::new(), + RequestKind::Detokenize { .. } | RequestKind::Tokenize { .. } => Rid::new(), } } @@ -643,6 +688,7 @@ enum ExpectedResponse { Generate, Control, Detokenize, + Tokenize, } impl ExpectedResponse { @@ -651,6 +697,7 @@ impl ExpectedResponse { RequestKind::Generate(_) => Self::Generate, RequestKind::Control(_) => Self::Control, RequestKind::Detokenize { .. } => Self::Detokenize, + RequestKind::Tokenize { .. } => Self::Tokenize, } } @@ -661,6 +708,7 @@ impl ExpectedResponse { (Self::Generate, ResponseItem::Done(_)) | (Self::Control, ResponseItem::Control(_)) | (Self::Detokenize, ResponseItem::Data(_)) + | (Self::Tokenize, ResponseItem::TokenIds(_)) ) } } @@ -1140,19 +1188,50 @@ mod tests { } #[tokio::test] - async fn detokenize_operation_mints_an_internal_rid() { + async fn token_operations_mint_internal_rids() { let harness = Harness::unbounded(8); let handle = harness.handle.clone(); - let task = tokio::spawn(async move { handle.detokenize(vec![1, 2]).await }); + let task = tokio::spawn(async move { + handle + .detokenize(vec![1, 2], false /* skip_special_tokens */) + .await + }); let request = accept_intake(harness.intake_rx.recv_async().await.unwrap()); assert!(!request.rid.as_str().is_empty()); - assert!(matches!(request.kind, RequestKind::Detokenize { .. })); + assert!(matches!( + request.kind, + RequestKind::Detokenize { + skip_special_tokens: false, + .. + } + )); request .sink .try_send(ResponseItem::Data(Bytes::from_static(b"decoded"))) .unwrap(); assert_eq!(task.await.unwrap().unwrap(), "decoded"); + + let handle = harness.handle.clone(); + let task = tokio::spawn(async move { + handle + .tokenize("hi".into(), true /* add_special_tokens */) + .await + }); + let request = accept_intake(harness.intake_rx.recv_async().await.unwrap()); + assert!(!request.rid.as_str().is_empty()); + assert!(matches!( + &request.kind, + RequestKind::Tokenize { + text, + add_special_tokens: true, + } if text == "hi" + )); + request + .sink + .try_send(ResponseItem::TokenIds(vec![1, 2])) + .unwrap(); + assert_eq!(task.await.unwrap().unwrap(), vec![1, 2]); assert!(harness.abort_rx.try_recv().is_err()); } } diff --git a/rust/sglang-server/src/api_server/http/openai.rs b/rust/sglang-server/src/api_server/http/openai.rs index f9f92371f3..cbdc48ce8a 100644 --- a/rust/sglang-server/src/api_server/http/openai.rs +++ b/rust/sglang-server/src/api_server/http/openai.rs @@ -16,6 +16,7 @@ mod template; mod template_builtins; mod template_legacy; mod template_loader; +mod tokenize; mod tools; pub(super) use template::{ChatFormatter, ChatTemplateKwargs}; @@ -35,6 +36,7 @@ pub(super) fn routes() -> Router> { .merge(models::routes()) .merge(completions::routes()) .merge(chat::routes()) + .merge(tokenize::routes()) } /// Resolve the chat formatter, or `None` to disable the OpenAI chat-completions diff --git a/rust/sglang-server/src/api_server/http/openai/chat.rs b/rust/sglang-server/src/api_server/http/openai/chat.rs index 8f1fe38b98..08e0abff68 100644 --- a/rust/sglang-server/src/api_server/http/openai/chat.rs +++ b/rust/sglang-server/src/api_server/http/openai/chat.rs @@ -50,10 +50,10 @@ pub(super) fn routes() -> Router> { } #[derive(Deserialize)] -struct ChatRequest { +pub(super) struct ChatRequest { #[serde(flatten)] - request: CreateChatCompletionRequest, - chat_template_kwargs: Option, + pub(super) request: CreateChatCompletionRequest, + pub(super) chat_template_kwargs: Option, #[serde(flatten)] routing: PDRoutingFields, } diff --git a/rust/sglang-server/src/api_server/http/openai/completions.rs b/rust/sglang-server/src/api_server/http/openai/completions.rs index 8be51e33b5..8662315f87 100644 --- a/rust/sglang-server/src/api_server/http/openai/completions.rs +++ b/rust/sglang-server/src/api_server/http/openai/completions.rs @@ -252,7 +252,11 @@ async fn completions( /// detok stage answers it with a single `Data` payload (the raw UTF-8 text), /// or an `Error` (e.g. out-of-range ids → `Validation` → 400). async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result { - match state.core.detokenize(token_ids).await { + match state + .core + .detokenize(token_ids, true /* skip_special_tokens */) + .await + { Ok(text) => Ok(text), // Same rule as `submit_generation`: build the refusal in the OpenAI // error shape rather than forwarding the native-shaped response. diff --git a/rust/sglang-server/src/api_server/http/openai/tokenize.rs b/rust/sglang-server/src/api_server/http/openai/tokenize.rs new file mode 100644 index 0000000000..b7a6457087 --- /dev/null +++ b/rust/sglang-server/src/api_server/http/openai/tokenize.rs @@ -0,0 +1,368 @@ +//! `/v1/tokenize` and `/v1/detokenize` (+ the unversioned aliases), Python's +//! `OpenAIServingTokenize` / `OpenAIServingDetokenize`: the same request and +//! response shapes and the same flat error body. Encoding runs on the +//! tokenizer pool and decoding on a detok shard, never on these I/O threads. + +use std::sync::Arc; + +use axum::{ + Json, Router, + extract::{State, rejection::JsonRejection}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::post, +}; +use futures::future::try_join_all; +use serde::Deserialize; +use serde_json::{Map, Value, json}; + +use super::chat::ChatRequest; +use super::{AppState, contains_media}; +use crate::api_server::core::CoreError; +use crate::message::types::{OneOrMany, TokenIds}; + +pub(super) fn routes() -> Router> { + Router::new() + .route("/v1/tokenize", post(tokenize)) + .route("/tokenize", post(tokenize)) + .route("/v1/detokenize", post(detokenize)) + .route("/detokenize", post(detokenize)) +} + +fn default_true() -> bool { + true +} + +/// Python's `TokenizeRequest`. Unknown keys are kept for the chat-template +/// path, as its `extra="allow"` does. +#[derive(Deserialize)] +struct TokenizeBody { + #[serde(default)] + prompt: Option>, + #[serde(default)] + messages: Option, + #[serde(default = "default_true")] + add_special_tokens: bool, + #[serde(flatten)] + rest: Map, +} + +/// Python's `DetokenizeRequest`. +#[derive(Deserialize)] +struct DetokenizeBody { + tokens: OneOrMany, + #[serde(default = "default_true")] + skip_special_tokens: bool, +} + +/// Python's `ErrorResponse`, which these endpoints return flat rather than +/// under an `error` key. +fn error_body(status: StatusCode, error_type: &str, message: impl Into) -> Response { + let body = json!({ + "object": "error", + "message": message.into(), + "type": error_type, + "param": null, + "code": status.as_u16(), + }); + (status, Json(body)).into_response() +} + +/// A body that fails schema validation: what Python's +/// `validation_exception_handler` answers for a pydantic error. +fn invalid_body(message: impl Into) -> Response { + error_body(StatusCode::BAD_REQUEST, "Bad Request", message) +} + +fn bad_request(message: impl Into) -> Response { + error_body(StatusCode::BAD_REQUEST, "BadRequestError", message) +} + +/// A failed core operation: a rejected input is the client's 400; anything +/// else is the 500 Python's catch-all returns. +fn core_error(error: CoreError, operation: &str) -> Response { + match error { + CoreError::InvalidArgument(message) => bad_request(message), + other => error_body( + StatusCode::INTERNAL_SERVER_ERROR, + "InternalServerError", + format!("Internal server error during {operation}: {other}"), + ), + } +} + +async fn tokenize( + State(state): State>, + body: Result, JsonRejection>, +) -> Response { + let body = match body { + Ok(Json(body)) => body, + Err(rejection) => return invalid_body(rejection.body_text()), + }; + let body: TokenizeBody = match serde_json::from_value(body) { + Ok(body) => body, + Err(error) => return invalid_body(error.to_string()), + }; + let (texts, add_special_tokens) = match (body.prompt, body.messages) { + (Some(prompt), None) => (prompt, body.add_special_tokens), + // The rendered template owns its special tokens, as on the chat path + // (Python encodes it with `add_special_tokens=False`). + (None, Some(messages)) => match render_chat_prompt(&state, messages, body.rest) { + Ok(prompt) => (OneOrMany::One(prompt), false /* add_special_tokens */), + Err((error_type, message)) => { + return error_body(StatusCode::BAD_REQUEST, error_type, message); + } + }, + _ => return invalid_body("Exactly one of 'prompt' or 'messages' must be provided."), + }; + let max_model_len = state.server_args.tokenizer_model_max_length; + let response = match texts { + OneOrMany::One(text) => match state.core.tokenize(text, add_special_tokens).await { + Ok(tokens) => json!({ + "tokens": tokens, + "count": tokens.len(), + "max_model_len": max_model_len, + }), + Err(error) => return core_error(error, "tokenization"), + }, + OneOrMany::Many(texts) => { + let calls = texts + .into_iter() + .map(|text| state.core.tokenize(text, add_special_tokens)); + match try_join_all(calls).await { + Ok(tokens) => { + let count: Vec = tokens.iter().map(Vec::len).collect(); + json!({ "tokens": tokens, "count": count, "max_model_len": max_model_len }) + } + Err(error) => return core_error(error, "tokenization"), + } + } + }; + Json(response).into_response() +} + +/// Render `messages` through the chat template, like Python's +/// `TokenizeRequest.to_chat_completion_request`: every other tokenize field +/// except `prompt` and `add_special_tokens` carries over to the chat request. +/// A failure is a 400, returned as its error type and message. +fn render_chat_prompt( + state: &AppState, + messages: Value, + mut fields: Map, +) -> Result { + const INVALID_BODY: &str = "Bad Request"; + const BAD_REQUEST: &str = "BadRequestError"; + fields.insert("messages".into(), messages); + // Python's `TokenizeRequest.model` default; the chat request requires one. + fields + .entry("model") + .or_insert_with(|| Value::from("default")); + let ChatRequest { + request, + chat_template_kwargs, + .. + } = serde_json::from_value(Value::Object(fields)) + .map_err(|error| (INVALID_BODY, error.to_string()))?; + if request.messages.is_empty() { + return Err((BAD_REQUEST, "messages cannot be empty".into())); + } + if serde_json::to_value(&request.messages).is_ok_and(|messages| contains_media(&messages)) { + return Err(( + BAD_REQUEST, + "image, audio, video, and file message content is not supported".into(), + )); + } + let Some(formatter) = &state.chat_formatter else { + return Err((BAD_REQUEST, "this model has no usable chat template".into())); + }; + formatter + .render(&request, chat_template_kwargs.as_ref()) + .map_err(|error| (BAD_REQUEST, format!("chat template render failed: {error}"))) +} + +async fn detokenize( + State(state): State>, + body: Result, JsonRejection>, +) -> Response { + let body = match body { + Ok(Json(body)) => body, + Err(rejection) => return invalid_body(rejection.body_text()), + }; + let body: DetokenizeBody = match serde_json::from_value(body) { + Ok(body) => body, + Err(error) => return invalid_body(error.to_string()), + }; + let text = match body.tokens { + // Python answers an empty list without decoding. + OneOrMany::One(tokens) if tokens.is_empty() => Value::from(""), + OneOrMany::One(tokens) => match state + .core + .detokenize(tokens, body.skip_special_tokens) + .await + { + Ok(text) => Value::from(text), + Err(error) => return core_error(error, "detokenization"), + }, + OneOrMany::Many(lists) => { + let calls = lists + .into_iter() + .map(|tokens| state.core.detokenize(tokens, body.skip_special_tokens)); + match try_join_all(calls).await { + Ok(texts) => Value::from(texts), + Err(error) => return core_error(error, "detokenization"), + } + } + }; + Json(json!({ "text": text })).into_response() +} + +#[cfg(test)] +mod tests { + use axum::http::StatusCode; + use serde_json::json; + + use super::super::template_loader::load_chat_formatter; + use super::super::test_utils::{body_json, post_json}; + use super::*; + use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata}; + use crate::message::config::ServerArgs; + use crate::message::request::RequestKind; + use crate::message::response::ResponseItem; + use crate::tokenizer_manager::wiring::TmEvent; + + /// A core whose runtime answers tokenize with `[0?, len(word)...]` (the 0 + /// standing in for an auto-added BOS) and detokenize with a description + /// of what it was asked, so the tests see exactly what reached it. + fn answering_state() -> Arc { + let (intake_tx, intake_rx) = flume::unbounded::(); + tokio::spawn(async move { + while let Ok(TmEvent::Intake { request, admission }) = intake_rx.recv_async().await { + assert!(admission.try_accept()); + let item = match &request.kind { + RequestKind::Tokenize { + text, + add_special_tokens, + } => { + let bos = add_special_tokens.then_some(0); + let words = text.split_whitespace().map(|w| w.len() as i64); + ResponseItem::TokenIds(bos.into_iter().chain(words).collect()) + } + RequestKind::Detokenize { + token_ids, + skip_special_tokens, + } => ResponseItem::Data(format!("{token_ids:?} {skip_special_tokens}").into()), + _ => panic!("unexpected request kind"), + }; + request.sink.try_send(item).unwrap(); + } + }); + let server_args = Arc::new(ServerArgs { + tokenizer_model_max_length: 131072, + ..Default::default() + }); + Arc::new(AppState { + core: CoreHandle::new( + intake_tx, + flume::unbounded().0, + CoreConfig { + response_capacity: 8, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: CoreMetadata::from(server_args.as_ref()), + }, + ), + server_args, + chat_formatter: Some( + load_chat_formatter( + None, /* config_file */ + None, /* model_path */ + None, /* model_type */ + Some("chatml"), /* chat_template_arg */ + ) + .unwrap(), + ), + }) + } + + async fn call(path: &str, body: serde_json::Value) -> (StatusCode, serde_json::Value) { + let app = routes().with_state(answering_state()); + let response = post_json(app, path, body).await; + (response.status(), body_json(response).await) + } + + #[tokio::test] + async fn tokenize_returns_python_response_shape() { + for path in ["/v1/tokenize", "/tokenize"] { + let (status, body) = call(path, json!({"model": "m", "prompt": "a bcd"})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + body.to_string(), + r#"{"tokens":[0,1,3],"count":3,"max_model_len":131072}"# + ); + } + let (_, body) = call( + "/v1/tokenize", + json!({"prompt": ["a", "bb cc"], "add_special_tokens": false}), + ) + .await; + assert_eq!(body["tokens"], json!([[1], [2, 2]])); + assert_eq!(body["count"], json!([1, 2])); + let (_, body) = call("/v1/tokenize", json!({"prompt": []})).await; + assert_eq!((&body["tokens"], &body["count"]), (&json!([]), &json!([]))); + + // `messages` render through the chat template, whose output keeps + // its own specials: encoded without auto-added ones. + let (status, body) = call( + "/v1/tokenize", + json!({"messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_ne!(body["tokens"][0], 0, "no auto BOS on a rendered template"); + assert_eq!(body["count"], body["tokens"].as_array().unwrap().len()); + } + + #[tokio::test] + async fn detokenize_returns_python_response_shape() { + for path in ["/v1/detokenize", "/detokenize"] { + let (status, body) = call(path, json!({"tokens": [1, 2]})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, json!({"text": "[1, 2] true"})); + } + let (_, body) = call( + "/v1/detokenize", + json!({"tokens": [[1], []], "skip_special_tokens": false}), + ) + .await; + assert_eq!(body, json!({"text": ["[1] false", "[] false"]})); + let (_, body) = call("/v1/detokenize", json!({"tokens": []})).await; + assert_eq!(body, json!({"text": ""})); + } + + /// Bad bodies get Python's flat `ErrorResponse` with a 400. + #[tokio::test] + async fn invalid_requests_get_the_flat_python_error() { + for (path, body) in [ + ("/v1/tokenize", json!({"model": "m"})), + ( + "/v1/tokenize", + json!({"prompt": "a", "messages": [{"role": "user", "content": "hi"}]}), + ), + ("/v1/tokenize", json!({"prompt": 3})), + ("/v1/detokenize", json!({"tokens": [1, [2]]})), + ("/v1/detokenize", json!({})), + ] { + let (status, error) = call(path, body.clone()).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + assert_eq!(error["object"], "error", "{body}"); + assert_eq!(error["type"], "Bad Request", "{body}"); + assert_eq!(error["code"], 400, "{body}"); + assert!(error["param"].is_null(), "{body}"); + } + let (status, error) = call("/v1/tokenize", json!({"messages": []})).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error["type"], "BadRequestError"); + } +} diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index c1c76c6485..f6d6e2aec6 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -170,6 +170,10 @@ pub struct ServerArgs { /// tokens there). Not a `server_args` field — `TokenizerManager` derives it and /// `RustServer._build_server_args` stamps it in, so both sides count alike. pub num_reserved_tokens: u64, + /// `/v1/tokenize`'s `max_model_len`: the Python tokenizer's + /// `model_max_length` (`-1` when it has none), or the context length when + /// that is not a 64-bit integer. + pub tokenizer_model_max_length: i64, /// Launch-time stamps (not `server_args` fields): sglang package version /// and the scheduler-derived KV token capacity, reported by `/server_info`. pub version: String, @@ -224,6 +228,7 @@ impl ServerArgs { allow_auto_truncate, enable_return_hidden_states, num_reserved_tokens, + tokenizer_model_max_length, version, max_total_num_tokens, launch_server_info, @@ -267,6 +272,7 @@ impl ServerArgs { allow_auto_truncate: bool, enable_return_hidden_states: bool, num_reserved_tokens: u64, + tokenizer_model_max_length: i64, version: String, max_total_num_tokens: u64, launch_server_info: LaunchServerInfo, @@ -308,6 +314,7 @@ impl ServerArgs { allow_auto_truncate, enable_return_hidden_states, num_reserved_tokens, + tokenizer_model_max_length, version, max_total_num_tokens, launch_server_info, @@ -357,6 +364,7 @@ impl Default for ServerArgs { allow_auto_truncate: false, enable_return_hidden_states: false, num_reserved_tokens: 0, + tokenizer_model_max_length: -1, version: String::new(), max_total_num_tokens: 0, launch_server_info: LaunchServerInfo::default(), diff --git a/rust/sglang-server/src/message/detok.rs b/rust/sglang-server/src/message/detok.rs index af18c12d62..67cc9f8cc8 100644 --- a/rust/sglang-server/src/message/detok.rs +++ b/rust/sglang-server/src/message/detok.rs @@ -30,7 +30,11 @@ pub enum DetokMsg { /// scheduler ring). Sent by to-scheduler right after the same rid's `Register` /// on the same channel (FIFO), so the shard delivers the text through the /// registered sink like a control `Result` and drops the entry. - Decode { rid: Rid, token_ids: Vec }, + Decode { + rid: Rid, + token_ids: Vec, + skip_special_tokens: bool, + }, /// Control result: one already-serialized payload delivered to the sink verbatim. Result { rid: Rid, payload: bytes::Bytes }, /// Terminal per-request failure → an `Error` to the sink (a 400, not a crash). diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index d1c78eb3d9..24e25293a0 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -472,9 +472,20 @@ pub enum RequestKind { /// the same FSM as every request (validate → register → Queued), but the /// stage that answers it is the detok shard itself, never the scheduler /// ring; the result arrives on the registered sink as one `Data` payload - /// (the raw UTF-8 text). First caller: `/v1/completions` `echo` for - /// token-id prompts; a future `/detokenize` parity endpoint maps 1:1. - Detokenize { token_ids: TokenIds }, + /// (the raw UTF-8 text). Callers: `/v1/completions` `echo` for token-id + /// prompts, and `/v1/detokenize`. + Detokenize { + token_ids: TokenIds, + skip_special_tokens: bool, + }, + /// Internal service call: encode one text to token ids (`/v1/tokenize`). + /// Walks the same FSM to `Queued`, then the tokenizer pool answers it + /// through the request's own sink with one `TokenIds` payload. It never + /// reaches a detok shard, so it is not registered with one. + Tokenize { + text: String, + add_special_tokens: bool, + }, } /// A single in-flight generation request (one item after adapter fan-out), diff --git a/rust/sglang-server/src/message/response.rs b/rust/sglang-server/src/message/response.rs index 34d7a02d6f..2ed77702f5 100644 --- a/rust/sglang-server/src/message/response.rs +++ b/rust/sglang-server/src/message/response.rs @@ -57,6 +57,8 @@ pub enum ResponseItem { /// bytes for the core to decode (e.g. detokenized text), not a /// client-facing wire payload and not a generation frame. Data(Bytes), + /// Reply to `RequestKind::Tokenize`: the encoded token ids. + TokenIds(TokenIds), /// Terminal runtime failure; the core maps it to a semantic error. Error(Error), } diff --git a/rust/sglang-server/src/multi_modality/worker.rs b/rust/sglang-server/src/multi_modality/worker.rs index 59b78ac75c..bb909fe725 100644 --- a/rust/sglang-server/src/multi_modality/worker.rs +++ b/rust/sglang-server/src/multi_modality/worker.rs @@ -832,6 +832,7 @@ mod tests { sink: ResponseSink::Local(sink_tx), kind: RequestKind::Detokenize { token_ids: vec![1, 2], + skip_special_tokens: true, }, }) .unwrap(); diff --git a/rust/sglang-server/src/tokenizer_manager/detokenizer.rs b/rust/sglang-server/src/tokenizer_manager/detokenizer.rs index 6270747275..a588469ae7 100644 --- a/rust/sglang-server/src/tokenizer_manager/detokenizer.rs +++ b/rust/sglang-server/src/tokenizer_manager/detokenizer.rs @@ -107,14 +107,14 @@ impl DetokenizerBackend { /// Decode one complete sequence without creating request-scoped streaming /// state. This runs on a pinned detokenizer worker, never on an API runtime /// thread. - fn decode_once(&self, token_ids: &[u32]) -> Result { + fn decode_once(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { match self { DetokenizerBackend::Dynamo(tokenizer) => tokenizer - .decode(token_ids, true) + .decode(token_ids, skip_special_tokens) .map(String::from) .map_err(|error| Error::Detokenize(error.to_string())), DetokenizerBackend::Skip => Err(Error::Validation( - "echo for token-ID prompts is unavailable when skip_tokenizer_init=True".into(), + "detokenization is unavailable when skip_tokenizer_init=True".into(), )), } } @@ -220,9 +220,17 @@ impl Runnable for DetokenizerWorker { handle_chunk(&mut table, ev, &self.backend, &self.abort); } } - DetokMsg::Decode { rid, token_ids } => { - handle_decode(&mut table, &rid, &token_ids, &self.backend) - } + DetokMsg::Decode { + rid, + token_ids, + skip_special_tokens, + } => handle_decode( + &mut table, + &rid, + &token_ids, + skip_special_tokens, + &self.backend, + ), DetokMsg::Result { rid, payload } => handle_result(&mut table, &rid, payload), DetokMsg::Fail { rid, message } => { handle_fail(&mut table, &rid, message, &self.abort) @@ -244,10 +252,11 @@ fn handle_decode( table: &mut HashMap, rid: &Rid, token_ids: &[u32], + skip_special_tokens: bool, backend: &DetokenizerBackend, ) { if let Some(mut st) = table.remove(rid) { - let item = match backend.decode_once(token_ids) { + let item = match backend.decode_once(token_ids, skip_special_tokens) { Ok(text) => ResponseItem::Data(text.into()), Err(e) => ResponseItem::Error(e), }; @@ -515,7 +524,9 @@ mod tests { #[test] fn decode_once_rejects_skip_mode() { - let error = DetokenizerBackend::Skip.decode_once(&[1]).unwrap_err(); + let error = DetokenizerBackend::Skip + .decode_once(&[1], true /* skip_special_tokens */) + .unwrap_err(); assert!(matches!(error, Error::Validation(_))); assert!(error.to_string().contains("skip_tokenizer_init=True")); } @@ -541,6 +552,7 @@ mod tests { &mut table, &Rid::from("d1"), &[1], + true, /* skip_special_tokens */ &DetokenizerBackend::Skip, ); @@ -556,6 +568,7 @@ mod tests { &mut table, &Rid::from("d2"), &[1], + true, /* skip_special_tokens */ &DetokenizerBackend::Skip, ); assert!(rx.try_recv().is_err()); diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs index 1898305a8d..ec59bc803d 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs @@ -166,19 +166,26 @@ impl Intake { let _ = req.state.apply(Event::Error(e)); // → Failed continue; } - if !self.register_detok(&req) { - let _ = req - .state - .apply(Event::Error(Error::Internal("detok shard gone".into()))); - continue; + // A tokenize request is answered by the tokenizer pool + // through its own sink, never by a detok shard. + if !matches!(req.kind, RequestKind::Tokenize { .. }) { + if !self.register_detok(&req) { + let _ = req + .state + .apply(Event::Error(Error::Internal("detok shard gone".into()))); + continue; + } + registered = true; } - registered = true; // `validate` advanced Received → Validating; keep driving. } - // Control and detokenize skip normalization (no sampling params) - // straight to the pre-send checks; generate goes to Normalizing. + // Control, detokenize and tokenize skip normalization (no + // sampling params) straight to the pre-send checks; generate + // goes to Normalizing. RequestState::Validating => match &req.kind { - RequestKind::Control(_) | RequestKind::Detokenize { .. } => { + RequestKind::Control(_) + | RequestKind::Detokenize { .. } + | RequestKind::Tokenize { .. } => { let _ = req .state .apply(Event::Validated(ValidationOutcome::AlreadyTokenized)); @@ -308,7 +315,7 @@ impl Intake { } // Hand the request to the stage that answers it: the scheduler // channel (generate payload or control frame), or -- for detokenize - // — the detok shard itself. + // and tokenize — the detok shard or the tokenizer pool itself. RequestState::Queued => { // The patterns bind nothing, so the match reads only the // discriminant and `req` can be moved into each push. @@ -316,6 +323,7 @@ impl Intake { RequestKind::Generate(_) => self.push_to_channel(req), RequestKind::Control(_) => self.push_control_to_channel(req), RequestKind::Detokenize { .. } => self.push_detokenize_to_shard(req), + RequestKind::Tokenize { .. } => self.push_tokenize_to_pool(req), } return; } @@ -349,7 +357,12 @@ impl Intake { g.return_text_in_logprobs.unwrap_or(false), g.sampling_params.no_stop_trim, ), - RequestKind::Control(_) | RequestKind::Detokenize { .. } => (false, false), + RequestKind::Control(_) + | RequestKind::Detokenize { .. } + | RequestKind::Tokenize { .. } => ( + false, /* decode_logprob_text */ + false, /* no_stop_trim */ + ), }; self.senders .detok_for(&req.rid) @@ -367,7 +380,11 @@ impl Intake { /// already holds this rid's sink: `register_detok` queued `Register` on the /// same channel from this same thread, so FIFO gives Register → Decode. fn push_detokenize_to_shard(&self, mut req: Request) { - let RequestKind::Detokenize { token_ids } = &req.kind else { + let RequestKind::Detokenize { + token_ids, + skip_special_tokens, + } = &req.kind + else { self.fail( &mut req, Error::Internal("non-detokenize request reached push_detokenize_to_shard".into()), @@ -383,6 +400,7 @@ impl Intake { .send(DetokMsg::Decode { rid: req.rid.clone(), token_ids, + skip_special_tokens: *skip_special_tokens, }) .is_err() { @@ -390,6 +408,21 @@ impl Intake { } } + /// Hand a `Tokenize` request to the tokenizer pool, which answers it + /// through the request's own sink. It was never registered with a detok + /// shard, so a failure here has nothing to deregister. + fn push_tokenize_to_pool(&self, req: Request) { + if let Err(err) = self.senders.tokenizer_tx.send(req) { + // Pool gone (workers exited); flume hands the request back. + let mut req = err.into_inner(); + self.fail( + &mut req, + Error::Internal("tokenizer pool gone".into()), + false, /* registered */ + ); + } + } + /// Push a bare control request (`[tag, rid, nil]`) onto the to_scheduler channel. The /// scheduler dispatches it (e.g. `GetInternalStateReq`) and replies via the /// from_scheduler channel as a single `Result`. @@ -438,6 +471,9 @@ impl Intake { tracing::debug!(rid = %rid, "abort recorded for request"); return; } + if matches!(source, AbortSource::Local(_)) { + return; + } // The channel is BOUNDED and drops pushes under exactly the load this matters // for, so report the miss rather than assuming the scheduler was told. diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs index 61a1efa50b..d2d1b22c46 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs @@ -107,11 +107,16 @@ fn ids_len(req: &SchedulerRequest) -> usize { /// the same rid could `Register` ahead of the stale abort and be torn down by /// it. `Rid::from_client` removes the premise: a retry carries a different /// `Rid`, so no abort in flight can name it. +/// +/// A dropped tokenize or detokenize call (`Local`) never reached the +/// scheduler, so it only deregisters: thousands of cancelled items must not +/// queue scheduler aborts ahead of generation requests. #[test] fn every_abort_source_deregisters_and_stops_the_scheduler() { - for source in [ - AbortSource::Guard("x".into()), - AbortSource::Detok("x".into()), + for (source, scheduler_aborts) in [ + (AbortSource::Guard("x".into()), 1), + (AbortSource::Detok("x".into()), 1), + (AbortSource::Local("x".into()), 0), ] { let (detok_tx, detok_rx) = flume::unbounded::(); let (to_scheduler_tx, consumer) = to_scheduler(16); @@ -140,8 +145,8 @@ fn every_abort_source_deregisters_and_stops_the_scheduler() { ); assert_eq!( consumer.drain(8).len(), - 1, - "{source:?} must push an AbortReq so the scheduler stops", + scheduler_aborts, + "{source:?}: AbortReqs pushed to the scheduler", ); } } @@ -446,6 +451,7 @@ fn detokenize_flows_register_then_decode_and_skips_the_channel() { sink: ResponseSink::Local(tx), kind: RequestKind::Detokenize { token_ids: vec![7, 8, 9], + skip_special_tokens: false, }, }); assert!( @@ -455,10 +461,10 @@ fn detokenize_flows_register_then_decode_and_skips_the_channel() { assert!( matches!( detok_rx.try_recv(), - Ok(DetokMsg::Decode { rid, token_ids }) + Ok(DetokMsg::Decode { rid, token_ids, skip_special_tokens: false }) if rid.as_str() == "41" && token_ids == [7, 8, 9] ), - "the decode job follows, ids intact", + "the decode job follows, ids and flag intact", ); assert!( consumer.drain(16).is_empty(), @@ -484,6 +490,7 @@ fn detokenize_negative_ids_reject_before_registration() { sink: ResponseSink::Local(tx), kind: RequestKind::Detokenize { token_ids: vec![1, -1], + skip_special_tokens: true, }, }); let Ok(ResponseItem::Error(err)) = rx.try_recv() else { @@ -495,6 +502,71 @@ fn detokenize_negative_ids_reject_before_registration() { assert!(consumer.drain(16).is_empty()); } +/// A `Tokenize` request goes to the tokenizer pool, which answers it through +/// its own sink: no detok shard hears of it and nothing reaches the +/// scheduler channel. Without a tokenizer (`skip_tokenizer_init`) it is a +/// validation error instead. +#[test] +fn tokenize_goes_to_the_pool_without_detok_registration() { + for skip_tokenizer_init in [false, true] { + let (tok_tx, tok_rx) = flume::unbounded(); + let (detok_tx, detok_rx) = flume::unbounded(); + let (to_scheduler_tx, consumer) = to_scheduler(16); + let (sd_tx, sd_rx) = flume::unbounded::<()>(); + std::mem::forget(sd_tx); + let mut intake = Intake::new( + flume::unbounded().1, + flume::unbounded().1, + Senders { + tok_manager_tx: flume::unbounded().0, + abort_tx: flume::unbounded().0, + tokenizer_tx: tok_tx, + detokenizer_tx: vec![detok_tx], + }, + to_scheduler_tx, + Limits { + skip_tokenizer_init, + ..test_limits() + }, + test_mm(flume::unbounded().0, true /* enabled */), + sd_rx, + ); + let (tx, mut rx) = mpsc::channel(8); + intake.drive(Request { + rid: "45".into(), + state: RequestState::Received, + sink: ResponseSink::Local(tx), + kind: RequestKind::Tokenize { + text: "hi".into(), + add_special_tokens: false, + }, + }); + if skip_tokenizer_init { + let Ok(ResponseItem::Error(err)) = rx.try_recv() else { + panic!("sink must receive the validation error"); + }; + assert!(matches!(err, Error::Validation(_))); + assert!(tok_rx.try_recv().is_err(), "the pool never sees it"); + } else { + let pooled = tok_rx.try_recv().expect("handed to the tokenizer pool"); + assert_eq!(pooled.rid.as_str(), "45"); + assert!(matches!( + pooled.kind, + RequestKind::Tokenize { + add_special_tokens: false, + .. + } + )); + assert!(rx.try_recv().is_err(), "no response until the pool answers"); + } + assert!(detok_rx.try_recv().is_err(), "no detok registration"); + assert!( + consumer.drain(16).is_empty(), + "must never reach the scheduler" + ); + } +} + /// A dropped channel push is survivable, and this pins WHY. The channel is bounded, /// so under load the scheduler never learns to stop and keeps generating; its /// chunks then arrive for a rid the detok table no longer holds and are diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs index 8cf2199acd..4161373109 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_validation.rs @@ -40,6 +40,12 @@ pub(super) fn validate(req: &mut Request, limits: &Limits) -> Result<(), Error> "skip_tokenizer_init is set: request must provide input_ids".into(), )); } + // No tokenizer pool runs under `skip_tokenizer_init` to answer it. + if skip_tokenizer_init && matches!(&req.kind, RequestKind::Tokenize { .. }) { + return Err(Error::Validation( + "tokenization is unavailable when skip_tokenizer_init=True".into(), + )); + } // Python `_set_default_priority`: an unset priority takes the configured // default instead of ranking as the scheduler's lowest. @@ -72,7 +78,7 @@ pub(super) fn validate(req: &mut Request, limits: &Limits) -> Result<(), Error> // bound — parity with the retired direct decode service: an unknown id is // the tokenizer's error to report, and nothing here reaches the scheduler's // embedding lookup. - if let RequestKind::Detokenize { token_ids } = &req.kind { + if let RequestKind::Detokenize { token_ids, .. } = &req.kind { for &id in token_ids { if u32::try_from(id).is_err() { return Err(Error::Validation(format!("Token ID {id} is out of range"))); diff --git a/rust/sglang-server/src/tokenizer_manager/tokenizer.rs b/rust/sglang-server/src/tokenizer_manager/tokenizer.rs index 99a948bfcf..7e9c58ce1a 100644 --- a/rust/sglang-server/src/tokenizer_manager/tokenizer.rs +++ b/rust/sglang-server/src/tokenizer_manager/tokenizer.rs @@ -15,6 +15,7 @@ use std::path::Path; use std::sync::Arc; use crate::message::request::{Request, RequestKind}; +use crate::message::response::ResponseItem; use crate::message::types::TokenIds; use crate::runtime::Runnable; use crate::tokenizer_manager::wiring::TmEvent; @@ -193,10 +194,42 @@ impl TokenizerWorker { } } +impl TokenizerWorker { + /// `RequestKind::Tokenize`: Python's `tokenizer.encode(text, + /// add_special_tokens=...)`. An empty text is not an error here, unlike a + /// generate prompt: it encodes to just the auto-added specials. + fn tokenize(&self, text: &str, add_special_tokens: bool) -> Result { + let ids = if text.is_empty() { + self.auto_specials.clone() + } else { + self.tokenizer.encode(text)? + }; + Ok(if add_special_tokens { + ids + } else { + strip_auto_specials(ids, &self.auto_specials) + }) + } +} + impl Runnable for TokenizerWorker { fn run(self) { while let Ok(mut req) = self.rx.recv() { - // The tokenizer pool only ever receives generate requests. Encode, + // A tokenize request is answered here, through its own sink: it is + // not registered with a detok shard and does not return to intake. + if let RequestKind::Tokenize { + text, + add_special_tokens, + } = &req.kind + { + let item = match self.tokenize(text, *add_special_tokens) { + Ok(ids) => ResponseItem::TokenIds(ids), + Err(err) => ResponseItem::Error(err), + }; + let _ = req.sink.try_send(item); // client may be gone + continue; + } + // Every other request is a generate request. Encode, // then advance the FSM: `TokenizeDone` on success (-> PreSendValidating, // or -> Encoding for a multimodal prompt; the state's `then` says which). // A request is never dropped here: a kind this pool cannot serve @@ -327,6 +360,7 @@ mod tests { sink: ResponseSink::Local(sink_tx), kind: RequestKind::Detokenize { token_ids: vec![1, 2], + skip_special_tokens: true, }, }) .expect("send"); @@ -404,4 +438,39 @@ mod tests { assert_eq!(run(false), vec![0, 2], "plain text prompts keep specials"); assert_eq!(run(true), vec![2], "rendered prompts lose the auto BOS"); } + + /// A tokenize request is answered through its own sink and never returns + /// to intake. `add_special_tokens` keeps or strips the auto specials, and + /// an empty text encodes to just those specials, as Python's + /// `encode("")` does. + #[test] + fn tokenize_answers_through_the_sink_without_returning_to_intake() { + let run = |text: &str, add_special_tokens: bool| { + let (req_tx, req_rx) = flume::unbounded::(); + let (tm_tx, tm_rx) = flume::unbounded::(); + let (sink_tx, mut sink_rx) = tokio::sync::mpsc::channel(4); + req_tx + .send(Request { + rid: "t".into(), + state: RequestState::Queued, + sink: ResponseSink::Local(sink_tx), + kind: RequestKind::Tokenize { + text: text.into(), + add_special_tokens, + }, + }) + .expect("send"); + drop(req_tx); + TokenizerWorker::new(req_rx, tm_tx, Arc::new(MarkedTokenizer)).run(); + assert!(tm_rx.try_recv().is_err(), "answered here, not by intake"); + match sink_rx.try_recv() { + Ok(ResponseItem::TokenIds(ids)) => ids, + other => panic!("expected token ids, got {other:?}"), + } + }; + assert_eq!(run("hi", true /* add_special_tokens */), vec![0, 2]); + assert_eq!(run("hi", false /* add_special_tokens */), vec![2]); + assert_eq!(run("", true /* add_special_tokens */), vec![0]); + assert_eq!(run("", false /* add_special_tokens */), Vec::::new()); + } } diff --git a/rust/sglang-server/src/tokenizer_manager/wiring.rs b/rust/sglang-server/src/tokenizer_manager/wiring.rs index 353b29c6cc..bc7f1b9290 100644 --- a/rust/sglang-server/src/tokenizer_manager/wiring.rs +++ b/rust/sglang-server/src/tokenizer_manager/wiring.rs @@ -81,9 +81,9 @@ impl RequestAdmission { } } -/// The source of the abort request. Both variants do the same work in -/// [`Intake::on_abort`] — deregister the detok entry, tell the scheduler to -/// stop — and the source is kept for diagnostics. +/// The source of the abort request. [`Intake::on_abort`] deregisters the +/// detok entry for every source and tells the scheduler to stop for all but +/// [`Self::Local`]; the source is otherwise kept for diagnostics. /// /// There is no in-flight rid registry to keep consistent, and so no release /// ordering to get wrong: [`Rid::from_client`] makes every client-supplied rid @@ -95,12 +95,15 @@ pub enum AbortSource { Guard(Rid), /// From a detokenizer terminal path. Aborts the scheduler work. Detok(Rid), + /// From a dropped call that never reaches the scheduler (tokenize, + /// detokenize): only its Rust-side state is released. + Local(Rid), } impl AbortSource { pub fn rid(&self) -> &Rid { match self { - Self::Guard(rid) | Self::Detok(rid) => rid, + Self::Guard(rid) | Self::Detok(rid) | Self::Local(rid) => rid, } } } diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index 723bfea2e3..85c47c8a03 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -24,6 +24,7 @@ def _scheduler_for_typed_config(): get_default_sampling_params=lambda: {}, ), rust_server_tokenizer_path=lambda: "tokenizer", + tokenizer=None, max_total_num_tokens=1024, load_snapshot_writer=None, ) @@ -93,6 +94,21 @@ def test_typed_config_points_metrics_at_the_multiproc_dir(enable_metrics): assert metrics is None +@pytest.mark.parametrize( + "tokenizer,expected", + [ + (None, -1), + (SimpleNamespace(), -1), + (SimpleNamespace(model_max_length=4096), 4096), + # HF's unset sentinel, int(1e30), is not a 64-bit integer. + (SimpleNamespace(model_max_length=int(1e30)), 2048), + ], + ids=["no-tokenizer", "no-attribute", "configured", "unset-sentinel"], +) +def test_tokenizer_model_max_length_matches_python_tokenize(tokenizer, expected): + assert rust_config._tokenizer_model_max_length(tokenizer, 2048) == expected + + @pytest.mark.parametrize( "nnodes,tp_size,attn_dp_size,ep_join_mode,ranks,expected", [