diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index 7eccfd8a2..ec42e93b0 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -69,6 +69,7 @@ def _build_server_args(scheduler: Scheduler) -> ServerArgs: context_len=mc.context_len, vocab_size=mc.vocab_size, is_multimodal=mc.is_multimodal, + model_type=getattr(mc.hf_config, "model_type", None), # Resolved default sampling params (generation_config.json when # `--sampling-defaults model`, {} otherwise). The rust server # consumes these for omitted temperature/top_p in chat diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 93e1a560d..8b627ffe2 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1005,9 +1005,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "5.3.1" +version = "5.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fd951c32c033f4f220b6087db91d7f7d475f1bf1f886c8e60798ac1cf4cc9a" +checksum = "b1cdacfc1398779cf173d5ff54438d9f14682f76875b6879853a84cc7d646228" dependencies = [ "async-openai", "derive_builder", @@ -1022,9 +1022,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "5.0.1" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe04753d666e3462e3eed4abcf2a4d05997cf2dea45858733f2bdea5b61f733" +checksum = "3ae2eaa139651c535aeaad8856c5546709608931ccd4d24d3a529f6c5233c4b6" dependencies = [ "anyhow", "chrono", @@ -1040,9 +1040,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "821c767e896f1f225b6411a5ce41dba7f114c2c97db862a13fd962195665075e" +checksum = "ed232a9d254149a4e2e14f0a7d77f59bd64161964c0ba55267beea3955650faa" dependencies = [ "aho-corasick", "anyhow", @@ -3715,6 +3715,7 @@ dependencies = [ "hf-hub", "itertools", "libc", + "minijinja", "numpy", "pyo3", "regex-syntax", diff --git a/rust/sglang-server/Cargo.toml b/rust/sglang-server/Cargo.toml index 2a11b3ecc..b98a796a6 100644 --- a/rust/sglang-server/Cargo.toml +++ b/rust/sglang-server/Cargo.toml @@ -44,7 +44,8 @@ dynamo-tokenizers = "1.7.0" # small service helpers. dynamo-parsers = "7.0.1" dynamo-protocols = "5.1.0" -dynamo-renderer = "5.0.0" +dynamo-renderer = "5.1.2" +minijinja = "2.24" flume = "0.12.0" hf-hub = { version = "0.4", default-features = false } itertools = "0.14" diff --git a/rust/sglang-server/src/api_server/openai.rs b/rust/sglang-server/src/api_server/openai.rs index f3b2b0a14..0d3a153eb 100644 --- a/rust/sglang-server/src/api_server/openai.rs +++ b/rust/sglang-server/src/api_server/openai.rs @@ -19,7 +19,7 @@ mod template_legacy; mod template_loader; mod tools; -pub(super) use template::ChatFormatter; +pub(super) use template::{ChatFormatter, ChatTemplateKwargs}; use super::app::AppState; use super::frame::OutputAccumulator; @@ -62,6 +62,7 @@ pub(super) fn load_chat_support(server_args: &ServerArgs) -> Option { diff --git a/rust/sglang-server/src/api_server/openai/chat.rs b/rust/sglang-server/src/api_server/openai/chat.rs index 6483e35aa..8bc40c69e 100644 --- a/rust/sglang-server/src/api_server/openai/chat.rs +++ b/rust/sglang-server/src/api_server/openai/chat.rs @@ -24,6 +24,7 @@ use dynamo_protocols::types::{ TopLogprobs, }; use futures::StreamExt; +use serde::Deserialize; use tokio::sync::mpsc; use super::super::guard::AbortGuard; @@ -34,8 +35,8 @@ use super::tools::{ parse_chat_tool_calls, }; use super::{ - AppState, ChatFormatter, collect_output, contains_media, error_payload, indexed_decode_stream, - openai_error, submit_generation, unix_seconds_u32, + AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload, + indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32, }; use crate::message::config::{DefaultSamplingParams, ServerArgs}; use crate::message::ids::Rid; @@ -48,11 +49,21 @@ pub(super) fn routes() -> Router> { Router::new().route("/v1/chat/completions", post(chat_completions)) } +#[derive(Deserialize)] +struct ChatRequest { + #[serde(flatten)] + request: CreateChatCompletionRequest, + chat_template_kwargs: Option, +} + async fn chat_completions( State(state): State>, - body: Result, JsonRejection>, + body: Result, JsonRejection>, ) -> Response { - let request = match body { + let ChatRequest { + request, + chat_template_kwargs, + } = match body { Ok(Json(request)) => request, Err(rejection) => { return openai_error(StatusCode::BAD_REQUEST, rejection.body_text(), false); @@ -141,10 +152,11 @@ async fn chat_completions( }); let tools_slice = tools.as_deref().unwrap_or_default(); - let (request, prompt) = match prepare_chat_request(&state, request).await { - Ok(prepared) => prepared, - Err(response) => return response, - }; + let (request, prompt) = + match prepare_chat_request(&state, request, chat_template_kwargs.as_ref()).await { + Ok(prepared) => prepared, + Err(response) => return response, + }; let sampling = match chat_sampling( &request, @@ -178,6 +190,11 @@ async fn chat_completions( let mut guard = AbortGuard::new_empty(state.senders.clone()); let mut submitted = Vec::with_capacity(n); + // V4 prefills , so the generated stream has no opening marker. + let starts_in_reasoning = matches!( + reasoning_parser.as_deref(), + Some("deepseek-v4" | "deepseek_v4" | "deepseekv4") + ) && prompt.ends_with(""); let mut prompt = Some(prompt); for index in 0..n { let rid = Rid::from_client(&format!("{response_id}-{index}")); @@ -221,6 +238,7 @@ async fn chat_completions( include_usage, parser, reasoning_parser, + starts_in_reasoning, tools, stream_tool_choice, uses_tool_call_structural_tag, @@ -254,6 +272,7 @@ async fn chat_completions( pub(super) async fn prepare_chat_request( state: &AppState, mut request: CreateChatCompletionRequest, + kwargs: Option<&ChatTemplateKwargs>, ) -> Result<(CreateChatCompletionRequest, String), Response> { let Some(formatter) = state.chat_formatter.clone() else { return Err(openai_error( @@ -267,7 +286,7 @@ pub(super) async fn prepare_chat_request( // token-id stop cannot be merged into the string list (Python has no such // field), so it is kept alone. merge_template_stops(&mut request, &formatter); - let prompt = formatter.render(&request).map_err(|error| { + let prompt = formatter.render(&request, kwargs).map_err(|error| { openai_error( StatusCode::BAD_REQUEST, format!("chat template render failed: {error}"), @@ -516,6 +535,7 @@ pub(super) fn chat_event_stream( include_usage: bool, parser: Option, reasoning_parser: Option, + starts_in_reasoning: bool, tools: Option>, tool_choice: Option, uses_tool_call_structural_tag: bool, @@ -534,7 +554,7 @@ pub(super) fn chat_event_stream( let mut reasoning_splitters: Vec = if reasoning_parser.is_some() { (0..count) - .map(|_| ReasoningStreamSplitter::new(reasoning_parser.as_deref())) + .map(|_| ReasoningStreamSplitter::new(reasoning_parser.as_deref(), starts_in_reasoning)) .collect() } else { vec![] @@ -1120,6 +1140,7 @@ mod tests { true, None, Some("deepseek-r1".into()), + false, None, None, false, @@ -1166,6 +1187,7 @@ mod tests { true, None, None, + false, None, None, false, diff --git a/rust/sglang-server/src/api_server/openai/reasoning.rs b/rust/sglang-server/src/api_server/openai/reasoning.rs index 3533e6a6d..95b6df997 100644 --- a/rust/sglang-server/src/api_server/openai/reasoning.rs +++ b/rust/sglang-server/src/api_server/openai/reasoning.rs @@ -71,13 +71,15 @@ pub(super) fn split_reasoning_unary( #[derive(Default)] pub(super) struct ReasoningStreamSplitter { name: Option, + starts_in_reasoning: bool, parser: Option, } impl ReasoningStreamSplitter { - pub(super) fn new(name: Option<&str>) -> Self { + pub(super) fn new(name: Option<&str>, starts_in_reasoning: bool) -> Self { Self { name: name.map(str::to_owned), + starts_in_reasoning, parser: None, } } @@ -87,9 +89,13 @@ impl ReasoningStreamSplitter { let Some(name) = self.name.as_deref() else { return (String::new(), text.to_owned()); }; - let parser = self - .parser - .get_or_insert_with(|| build_reasoning_parser(name)); + let parser = self.parser.get_or_insert_with(|| { + let mut parser = build_reasoning_parser(name); + if self.starts_in_reasoning { + parser.set_in_reasoning(true); + } + parser + }); let token_ids = token_ids .iter() .filter_map(|&id| u32::try_from(id).ok()) @@ -171,13 +177,23 @@ mod tests { assert_eq!(normal, "kept as text"); } + #[test] + fn v4_streaming_separates_prefilled_reasoning() { + let mut splitter = ReasoningStreamSplitter::new(Some("deepseek-v4"), true); + assert_eq!(splitter.split("reason", &[]), ("reason".into(), "".into())); + assert_eq!( + splitter.split("answer", &[]), + ("".into(), "answer".into()) + ); + } + /// REASONING_P1: MiniMax M3's implicit-tool-start recovery buffers the /// answer text until a boundary establishes the mode; with no opener the /// whole buffer is released as normal text only at `finish`. The chat /// terminal flush must emit the normal half of the tail. #[test] fn streaming_tail_releases_normal_text_only_at_finish() { - let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3")); + let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3"), false); let (reasoning, normal) = splitter.split("The answer is", &[]); assert_eq!(reasoning, ""); assert_eq!(normal, "", "M3 holds the ambiguous prefix until a boundary"); @@ -191,7 +207,7 @@ mod tests { #[test] fn streaming_tail_releases_reasoning_after_marker_boundary() { - let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3")); + let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3"), false); let (reasoning, normal) = splitter.split("think", &[]); assert_eq!(reasoning, "think"); assert_eq!(normal, ""); @@ -205,7 +221,7 @@ mod tests { #[test] fn finish_without_a_parser_is_empty() { - let mut splitter = ReasoningStreamSplitter::new(None); + let mut splitter = ReasoningStreamSplitter::new(None, false); let (reasoning, normal) = splitter.split("plain", &[]); assert_eq!(reasoning, ""); assert_eq!(normal, "plain"); diff --git a/rust/sglang-server/src/api_server/openai/template.rs b/rust/sglang-server/src/api_server/openai/template.rs index c9b2467b9..3539d52ae 100644 --- a/rust/sglang-server/src/api_server/openai/template.rs +++ b/rust/sglang-server/src/api_server/openai/template.rs @@ -6,10 +6,11 @@ //! `Conversation.get_prompt()` so there is exactly one implementation of the //! per-style formatting logic (no Jinja translation to drift). +use std::collections::HashMap; use std::path::PathBuf; -use dynamo_protocols::types::CreateChatCompletionRequest; -use dynamo_renderer::PromptFormatter; +use dynamo_protocols::types::{ChatCompletionRequestMessage, CreateChatCompletionRequest}; +use dynamo_renderer::{OAIChatLikeRequest, PromptFormatter, TextInput}; use thiserror::Error; use crate::message::types::OneOrMany; @@ -23,8 +24,12 @@ pub(super) use super::template_legacy::LegacySpec; use super::template_loader::infer_legacy_template_from_model_path; pub(super) use super::template_loader::load_chat_formatter; -/// A chat prompt formatter: either the model's HuggingFace Jinja template or a -/// legacy SGLang conversation template. +/// Extra variables for the chat template (`chat_template_kwargs`). +pub type ChatTemplateKwargs = HashMap; + +/// A chat prompt formatter: either the model's HuggingFace Jinja template (or +/// Dynamo's built-in encoder for models that ship none) or a legacy SGLang +/// conversation template. #[derive(Clone)] pub enum ChatFormatter { HuggingFace(PromptFormatter), @@ -36,16 +41,14 @@ impl ChatFormatter { pub(super) fn render( &self, request: &CreateChatCompletionRequest, + kwargs: Option<&ChatTemplateKwargs>, ) -> Result { match self { - ChatFormatter::HuggingFace(formatter) => { - let PromptFormatter::OAI(formatter) = formatter; - formatter - .render(request) - .map_err(|error| TemplateError::Renderer { - message: error.to_string(), - }) - } + ChatFormatter::HuggingFace(PromptFormatter::OAI(formatter)) => formatter + .render(&TemplateRequest { request, kwargs }) + .map_err(|error| TemplateError::Renderer { + message: error.to_string(), + }), ChatFormatter::Legacy(formatter) => formatter.render(request), } } @@ -62,6 +65,49 @@ impl ChatFormatter { } } +/// The wire request plus its `chat_template_kwargs`, which the protocol type +/// does not carry. +struct TemplateRequest<'a> { + request: &'a CreateChatCompletionRequest, + kwargs: Option<&'a ChatTemplateKwargs>, +} + +impl OAIChatLikeRequest for TemplateRequest<'_> { + fn model(&self) -> String { + self.request.model() + } + fn messages(&self) -> minijinja::Value { + self.request.messages() + } + fn typed_messages(&self) -> Option<&[ChatCompletionRequestMessage]> { + self.request.typed_messages() + } + fn tools(&self) -> Option { + self.request.tools() + } + fn tool_choice(&self) -> Option { + self.request.tool_choice() + } + fn response_format(&self) -> Option { + self.request.response_format() + } + fn reasoning_effort(&self) -> Option { + self.request.reasoning_effort() + } + fn should_add_generation_prompt(&self) -> bool { + self.request.should_add_generation_prompt() + } + fn chat_template_args(&self) -> Option<&ChatTemplateKwargs> { + self.kwargs + } + fn extract_text(&self) -> Option { + self.request.extract_text() + } + fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> { + self.request.mm_processor_kwargs() + } +} + #[derive(Debug, Error)] pub(super) enum TemplateError { #[error("failed to read {kind} `{path}`: {source}")] @@ -134,6 +180,8 @@ pub(super) enum TemplateError { #[cfg(test)] mod tests { + use std::collections::HashMap; + use dynamo_protocols::types::{ ChatCompletionRequestMessage, ChatCompletionRequestMessageContentPartText, ChatCompletionRequestSystemMessage, ChatCompletionRequestSystemMessageContent, @@ -191,7 +239,7 @@ mod tests { let formatter = ChatFormatter::Legacy(Box::new(LegacyFormatter { spec: builtin_template("chatml").unwrap(), })); - let rendered = formatter.render(&request()).unwrap(); + let rendered = formatter.render(&request(), None).unwrap(); assert_eq!( rendered, "<|im_start|>system\nBe concise.<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n" @@ -420,6 +468,7 @@ mod tests { let formatter = load_chat_formatter( Some(base.to_str().unwrap()), None, + None, Some(base.to_str().unwrap()), ) .unwrap(); @@ -456,10 +505,11 @@ mod tests { let formatter = load_chat_formatter( Some(base.to_str().unwrap()), None, + None, Some(legacy.to_str().unwrap()), ) .unwrap(); - let rendered = formatter.render(&request()).unwrap(); + let rendered = formatter.render(&request(), None).unwrap(); assert_eq!(rendered, "System\nBe concise.\nUSER: Hello\nASSISTANT:"); let _ = std::fs::remove_file(base); @@ -557,7 +607,7 @@ mod tests { /// A built-in `--chat-template` name resolves without any tokenizer config. #[test] fn builtin_argument_works_without_tokenizer_config() { - let formatter = load_chat_formatter(None, None, Some("chatml")).unwrap(); + let formatter = load_chat_formatter(None, None, None, Some("chatml")).unwrap(); let ChatFormatter::Legacy(formatter) = &formatter else { panic!("expected a legacy formatter"); }; @@ -585,6 +635,7 @@ mod tests { Some(base.to_str().unwrap()), Some("models/vicuna-7b-v1.5"), None, + None, ) .unwrap(); let ChatFormatter::Legacy(formatter) = &formatter else { @@ -592,7 +643,7 @@ mod tests { }; assert_eq!(formatter.spec.name, "vicuna_v1.1"); // No config at all + path matcher. - let formatter = load_chat_formatter(None, Some("deepseek-vl2-7b"), None).unwrap(); + let formatter = load_chat_formatter(None, Some("deepseek-vl2-7b"), None, None).unwrap(); let ChatFormatter::Legacy(formatter) = &formatter else { panic!("expected a legacy formatter"); }; @@ -609,7 +660,8 @@ mod tests { r#"{"model_type":"phi4mm","architectures":["Phi4MMForCausalLM"]}"#, ) .unwrap(); - let formatter = load_chat_formatter(None, Some(model_dir.to_str().unwrap()), None).unwrap(); + let formatter = + load_chat_formatter(None, Some(model_dir.to_str().unwrap()), None, None).unwrap(); let ChatFormatter::Legacy(formatter) = &formatter else { panic!("expected a legacy formatter"); }; @@ -653,7 +705,66 @@ mod tests { fn minicpm_4_6_skips_legacy_inference() { assert!(infer_legacy_template_from_model_path("minicpm-v-4.6").is_none()); assert!(matches!( - load_chat_formatter(None, Some("minicpm-v-4.6"), None), + load_chat_formatter(None, Some("minicpm-v-4.6"), None, None), + Err(TemplateError::MissingConfig) + )); + } + + fn temp_config(contents: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("sglang-template-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let config = dir.join("tokenizer_config.json"); + std::fs::write(&config, contents).unwrap(); + config + } + + #[test] + fn chat_template_kwargs_reach_the_template() { + let config = temp_config(r#"{"chat_template": "thinking={{ thinking }}"}"#); + let formatter = + load_chat_formatter(Some(config.to_str().unwrap()), None, None, None).unwrap(); + let kwargs = HashMap::from([("thinking".into(), serde_json::json!(true))]); + assert_eq!(formatter.render(&request(), None).unwrap(), "thinking="); + assert_eq!( + formatter.render(&request(), Some(&kwargs)).unwrap(), + "thinking=True" + ); + } + + #[test] + fn missing_template_falls_back_to_native_formatter() { + let config = temp_config("{}"); + let config = config.to_str().unwrap(); + let load = |config, model_type, arg| { + load_chat_formatter(config, Some("/models/x"), model_type, arg) + }; + + let formatter = load(Some(config), Some("deepseek_v4"), None).unwrap(); + assert!(matches!(formatter, ChatFormatter::HuggingFace(_))); + let kwargs = HashMap::from([("thinking".into(), serde_json::json!(false))]); + assert_eq!( + formatter.render(&request(), Some(&kwargs)).unwrap(), + "<｜begin▁of▁sentence｜>Be concise.<｜User｜>Hello<｜Assistant｜>" + ); + assert!(matches!( + load(None, Some("deepseek_v4"), None), + Ok(ChatFormatter::HuggingFace(_)) + )); + + // A template, `--chat-template`, or an unknown architecture wins. + let templated = temp_config(r#"{"chat_template": "Hi"}"#); + let formatter = load(Some(templated.to_str().unwrap()), Some("deepseek_v4"), None).unwrap(); + assert_eq!(formatter.render(&request(), None).unwrap(), "Hi"); + assert!(matches!( + load(Some(config), Some("deepseek_v4"), Some("chatml")), + Ok(ChatFormatter::Legacy(_)) + )); + assert!(matches!( + load(Some(config), Some("llama"), None), + Err(TemplateError::Missing) + )); + assert!(matches!( + load(None, None, None), Err(TemplateError::MissingConfig) )); } diff --git a/rust/sglang-server/src/api_server/openai/template_loader.rs b/rust/sglang-server/src/api_server/openai/template_loader.rs index 99f2403a3..e791dfc67 100644 --- a/rust/sglang-server/src/api_server/openai/template_loader.rs +++ b/rust/sglang-server/src/api_server/openai/template_loader.rs @@ -2,7 +2,9 @@ use std::path::Path; -use dynamo_renderer::{ChatTemplate, ContextMixins, PromptContextMixin, PromptFormatter}; +use dynamo_renderer::{ + ChatTemplate, ContextMixins, PromptContextMixin, PromptFormatter, native_formatter_for, +}; use serde_json::Value; use crate::message::types::OneOrMany; @@ -44,6 +46,7 @@ const SUPPORTED_STYLES: &[&str] = &[ pub(super) fn load_chat_formatter( config_file: Option<&str>, model_path: Option<&str>, + model_type: Option<&str>, chat_template_arg: Option<&str>, ) -> Result { // Python resolves registry names before looking at the filesystem — and @@ -66,10 +69,21 @@ pub(super) fn load_chat_formatter( return Ok(ChatFormatter::Legacy(Box::new(LegacyFormatter { spec }))); } + // Models that ship no template (Python `resolve_chat_encoding_spec`) use + // Dynamo's built-in encoder for their architecture. + let native_formatter = || match chat_template_arg { + None => native_formatter_for( + &model_type.map(str::to_lowercase), + &model_path.unwrap_or_default().to_lowercase(), + ) + .map(ChatFormatter::HuggingFace), + Some(_) => None, + }; + // Every remaining source builds the HF renderer around the tokenizer // config (the template itself, or the argument injected into it). let Some(config_file) = config_file else { - return Err(TemplateError::MissingConfig); + return native_formatter().ok_or(TemplateError::MissingConfig); }; let config_path = Path::new(config_file); @@ -77,7 +91,10 @@ pub(super) fn load_chat_formatter( let mut config = parse_json(&config_text, config_path, "tokenizer config")?; let Some(argument) = chat_template_arg else { - return formatter_from_config(&config); + return match formatter_from_config(&config) { + Err(TemplateError::Missing) => native_formatter().ok_or(TemplateError::Missing), + result => result, + }; }; let path = Path::new(argument); diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 8ecdf4814..e8f102080 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -329,6 +329,8 @@ pub enum DisaggregationMode { #[pyo3::pyclass(frozen, from_py_object, module = "sglang.srt.rust_extensions._server")] #[derive(Clone, Debug)] pub struct ModelConfig { + /// Authoritative HF model type, used to select a native chat formatter. + pub model_type: Option, /// Resolved context length (`max_model_len` in `/v1/models`); the ceiling /// for input + `max_new_tokens`. pub context_len: u64, @@ -351,18 +353,20 @@ pub struct ModelConfig { #[pyo3::pymethods] impl ModelConfig { #[new] - #[pyo3(signature = (*, context_len, vocab_size, is_multimodal, default_sampling_params))] + #[pyo3(signature = (*, context_len, vocab_size, is_multimodal, default_sampling_params, model_type))] fn py_new( context_len: u64, vocab_size: u64, is_multimodal: bool, default_sampling_params: DefaultSamplingParams, + model_type: Option, ) -> Self { Self { context_len, vocab_size, is_multimodal, default_sampling_params, + model_type, } } } @@ -372,6 +376,7 @@ impl Default for ModelConfig { fn default() -> Self { Self { context_len: 2048, + model_type: None, vocab_size: 1000, is_multimodal: false, default_sampling_params: DefaultSamplingParams::default(),