minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[Rust] Use Dynamo native renderers when chat templates are missing (#38939)

Browse Source
This commit is contained in:
Kan Wu
2026-09-14 17:29:10 -07:00
committed by GitHub
parent 8874c51a96
commit 07e1918924
9 changed files with 222 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/Cargo.lock
Generated
+7 -6
View File
@@ -1005,9 +1005,9 @@ dependencies = [
[[package]]
name = "dynamo-protocols"
version = "5.3.1"
version = "5.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97fd951c32c033f4f220b6087db91d7f7d475f1bf1f886c8e60798ac1cf4cc9a"
checksum = "b1cdacfc1398779cf173d5ff54438d9f14682f76875b6879853a84cc7d646228"
dependencies = [
"async-openai",
"derive_builder",
@@ -1022,9 +1022,9 @@ dependencies = [
[[package]]
name = "dynamo-renderer"
version = "5.0.1"
version = "5.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfe04753d666e3462e3eed4abcf2a4d05997cf2dea45858733f2bdea5b61f733"
checksum = "3ae2eaa139651c535aeaad8856c5546709608931ccd4d24d3a529f6c5233c4b6"
dependencies = [
"anyhow",
"chrono",
@@ -1040,9 +1040,9 @@ dependencies = [
[[package]]
name = "dynamo-tokenizers"
version = "1.8.0"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "821c767e896f1f225b6411a5ce41dba7f114c2c97db862a13fd962195665075e"
checksum = "ed232a9d254149a4e2e14f0a7d77f59bd64161964c0ba55267beea3955650faa"
dependencies = [
"aho-corasick",
"anyhow",
@@ -3715,6 +3715,7 @@ dependencies = [
"hf-hub",
"itertools",
"libc",
"minijinja",
"numpy",
"pyo3",
"regex-syntax",
rust/sglang-server/Cargo.toml
+2 -1
View File
@@ -44,7 +44,8 @@ dynamo-tokenizers = "1.7.0"
# small service helpers.
dynamo-parsers = "7.0.1"
dynamo-protocols = "5.1.0"
dynamo-renderer = "5.0.0"
dynamo-renderer = "5.1.2"
minijinja = "2.24"
flume = "0.12.0"
hf-hub = { version = "0.4", default-features = false }
itertools = "0.14"
rust/sglang-server/src/api_server/openai.rs
+2 -1
View File
@@ -19,7 +19,7 @@ mod template_legacy;
mod template_loader;
mod tools;
pub(super) use template::ChatFormatter;
pub(super) use template::{ChatFormatter, ChatTemplateKwargs};
use super::app::AppState;
use super::frame::OutputAccumulator;
@@ -62,6 +62,7 @@ pub(super) fn load_chat_support(server_args: &ServerArgs) -> Option<ChatFormatte
match template::load_chat_formatter(
config_file.as_deref(),
(!server_args.model_path.is_empty()).then_some(server_args.model_path.as_str()),
server_args.model_config.model_type.as_deref(),
server_args.chat_template.as_deref(),
) {
Ok(formatter) => {
rust/sglang-server/src/api_server/openai/chat.rs
+32 -10
View File
@@ -24,6 +24,7 @@ use dynamo_protocols::types::{
TopLogprobs,
};
use futures::StreamExt;
use serde::Deserialize;
use tokio::sync::mpsc;
use super::super::guard::AbortGuard;
@@ -34,8 +35,8 @@ use super::tools::{
parse_chat_tool_calls,
};
use super::{
AppState, ChatFormatter, collect_output, contains_media, error_payload, indexed_decode_stream,
openai_error, submit_generation, unix_seconds_u32,
AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload,
indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32,
};
use crate::message::config::{DefaultSamplingParams, ServerArgs};
use crate::message::ids::Rid;
@@ -48,11 +49,21 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new().route("/v1/chat/completions", post(chat_completions))
}
#[derive(Deserialize)]
struct ChatRequest {
#[serde(flatten)]
request: CreateChatCompletionRequest,
chat_template_kwargs: Option<ChatTemplateKwargs>,
}
async fn chat_completions(
State(state): State<Arc<AppState>>,
body: Result<Json<CreateChatCompletionRequest>, JsonRejection>,
body: Result<Json<ChatRequest>, JsonRejection>,
) -> Response {
let request = match body {
let ChatRequest {
request,
chat_template_kwargs,
} = match body {
Ok(Json(request)) => request,
Err(rejection) => {
return openai_error(StatusCode::BAD_REQUEST, rejection.body_text(), false);
@@ -141,10 +152,11 @@ async fn chat_completions(
});
let tools_slice = tools.as_deref().unwrap_or_default();
let (request, prompt) = match prepare_chat_request(&state, request).await {
Ok(prepared) => prepared,
Err(response) => return response,
};
let (request, prompt) =
match prepare_chat_request(&state, request, chat_template_kwargs.as_ref()).await {
Ok(prepared) => prepared,
Err(response) => return response,
};
let sampling = match chat_sampling(
&request,
@@ -178,6 +190,11 @@ async fn chat_completions(
let mut guard = AbortGuard::new_empty(state.senders.clone());
let mut submitted = Vec::with_capacity(n);
// V4 prefills <think>, so the generated stream has no opening marker.
let starts_in_reasoning = matches!(
reasoning_parser.as_deref(),
Some("deepseek-v4" | "deepseek_v4" | "deepseekv4")
) && prompt.ends_with("<think>");
let mut prompt = Some(prompt);
for index in 0..n {
let rid = Rid::from_client(&format!("{response_id}-{index}"));
@@ -221,6 +238,7 @@ async fn chat_completions(
include_usage,
parser,
reasoning_parser,
starts_in_reasoning,
tools,
stream_tool_choice,
uses_tool_call_structural_tag,
@@ -254,6 +272,7 @@ async fn chat_completions(
pub(super) async fn prepare_chat_request(
state: &AppState,
mut request: CreateChatCompletionRequest,
kwargs: Option<&ChatTemplateKwargs>,
) -> Result<(CreateChatCompletionRequest, String), Response> {
let Some(formatter) = state.chat_formatter.clone() else {
return Err(openai_error(
@@ -267,7 +286,7 @@ pub(super) async fn prepare_chat_request(
// token-id stop cannot be merged into the string list (Python has no such
// field), so it is kept alone.
merge_template_stops(&mut request, &formatter);
let prompt = formatter.render(&request).map_err(|error| {
let prompt = formatter.render(&request, kwargs).map_err(|error| {
openai_error(
StatusCode::BAD_REQUEST,
format!("chat template render failed: {error}"),
@@ -516,6 +535,7 @@ pub(super) fn chat_event_stream(
include_usage: bool,
parser: Option<String>,
reasoning_parser: Option<String>,
starts_in_reasoning: bool,
tools: Option<Vec<ToolDefinition>>,
tool_choice: Option<ChatCompletionToolChoiceOption>,
uses_tool_call_structural_tag: bool,
@@ -534,7 +554,7 @@ pub(super) fn chat_event_stream(
let mut reasoning_splitters: Vec<ReasoningStreamSplitter> =
if reasoning_parser.is_some() {
(0..count)
.map(|_| ReasoningStreamSplitter::new(reasoning_parser.as_deref()))
.map(|_| ReasoningStreamSplitter::new(reasoning_parser.as_deref(), starts_in_reasoning))
.collect()
} else {
vec![]
@@ -1120,6 +1140,7 @@ mod tests {
true,
None,
Some("deepseek-r1".into()),
false,
None,
None,
false,
@@ -1166,6 +1187,7 @@ mod tests {
true,
None,
None,
false,
None,
None,
false,
rust/sglang-server/src/api_server/openai/reasoning.rs
+23 -7
View File
@@ -71,13 +71,15 @@ pub(super) fn split_reasoning_unary(
#[derive(Default)]
pub(super) struct ReasoningStreamSplitter {
name: Option<String>,
starts_in_reasoning: bool,
parser: Option<ReasoningParserWrapper>,
}
impl ReasoningStreamSplitter {
pub(super) fn new(name: Option<&str>) -> Self {
pub(super) fn new(name: Option<&str>, starts_in_reasoning: bool) -> Self {
Self {
name: name.map(str::to_owned),
starts_in_reasoning,
parser: None,
}
}
@@ -87,9 +89,13 @@ impl ReasoningStreamSplitter {
let Some(name) = self.name.as_deref() else {
return (String::new(), text.to_owned());
};
let parser = self
.parser
.get_or_insert_with(|| build_reasoning_parser(name));
let parser = self.parser.get_or_insert_with(|| {
let mut parser = build_reasoning_parser(name);
if self.starts_in_reasoning {
parser.set_in_reasoning(true);
}
parser
});
let token_ids = token_ids
.iter()
.filter_map(|&id| u32::try_from(id).ok())
@@ -171,13 +177,23 @@ mod tests {
assert_eq!(normal, "<think>kept as text</think>");
}
#[test]
fn v4_streaming_separates_prefilled_reasoning() {
let mut splitter = ReasoningStreamSplitter::new(Some("deepseek-v4"), true);
assert_eq!(splitter.split("reason", &[]), ("reason".into(), "".into()));
assert_eq!(
splitter.split("</think>answer", &[]),
("".into(), "answer".into())
);
}
/// REASONING_P1: MiniMax M3's implicit-tool-start recovery buffers the
/// answer text until a boundary establishes the mode; with no opener the
/// whole buffer is released as normal text only at `finish`. The chat
/// terminal flush must emit the normal half of the tail.
#[test]
fn streaming_tail_releases_normal_text_only_at_finish() {
let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3"));
let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3"), false);
let (reasoning, normal) = splitter.split("The answer is", &[]);
assert_eq!(reasoning, "");
assert_eq!(normal, "", "M3 holds the ambiguous prefix until a boundary");
@@ -191,7 +207,7 @@ mod tests {
#[test]
fn streaming_tail_releases_reasoning_after_marker_boundary() {
let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3"));
let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3"), false);
let (reasoning, normal) = splitter.split("<mm:think>think", &[]);
assert_eq!(reasoning, "think");
assert_eq!(normal, "");
@@ -205,7 +221,7 @@ mod tests {
#[test]
fn finish_without_a_parser_is_empty() {
let mut splitter = ReasoningStreamSplitter::new(None);
let mut splitter = ReasoningStreamSplitter::new(None, false);
let (reasoning, normal) = splitter.split("plain", &[]);
assert_eq!(reasoning, "");
assert_eq!(normal, "plain");
rust/sglang-server/src/api_server/openai/template.rs
+129 -18
View File
@@ -6,10 +6,11 @@
//! `Conversation.get_prompt()` so there is exactly one implementation of the
//! per-style formatting logic (no Jinja translation to drift).
use std::collections::HashMap;
use std::path::PathBuf;
use dynamo_protocols::types::CreateChatCompletionRequest;
use dynamo_renderer::PromptFormatter;
use dynamo_protocols::types::{ChatCompletionRequestMessage, CreateChatCompletionRequest};
use dynamo_renderer::{OAIChatLikeRequest, PromptFormatter, TextInput};
use thiserror::Error;
use crate::message::types::OneOrMany;
@@ -23,8 +24,12 @@ pub(super) use super::template_legacy::LegacySpec;
use super::template_loader::infer_legacy_template_from_model_path;
pub(super) use super::template_loader::load_chat_formatter;
/// A chat prompt formatter: either the model's HuggingFace Jinja template or a
/// legacy SGLang conversation template.
/// Extra variables for the chat template (`chat_template_kwargs`).
pub type ChatTemplateKwargs = HashMap<String, serde_json::Value>;
/// A chat prompt formatter: either the model's HuggingFace Jinja template (or
/// Dynamo's built-in encoder for models that ship none) or a legacy SGLang
/// conversation template.
#[derive(Clone)]
pub enum ChatFormatter {
HuggingFace(PromptFormatter),
@@ -36,16 +41,14 @@ impl ChatFormatter {
pub(super) fn render(
&self,
request: &CreateChatCompletionRequest,
kwargs: Option<&ChatTemplateKwargs>,
) -> Result<String, TemplateError> {
match self {
ChatFormatter::HuggingFace(formatter) => {
let PromptFormatter::OAI(formatter) = formatter;
formatter
.render(request)
.map_err(|error| TemplateError::Renderer {
message: error.to_string(),
})
}
ChatFormatter::HuggingFace(PromptFormatter::OAI(formatter)) => formatter
.render(&TemplateRequest { request, kwargs })
.map_err(|error| TemplateError::Renderer {
message: error.to_string(),
}),
ChatFormatter::Legacy(formatter) => formatter.render(request),
}
}
@@ -62,6 +65,49 @@ impl ChatFormatter {
}
}
/// The wire request plus its `chat_template_kwargs`, which the protocol type
/// does not carry.
struct TemplateRequest<'a> {
request: &'a CreateChatCompletionRequest,
kwargs: Option<&'a ChatTemplateKwargs>,
}
impl OAIChatLikeRequest for TemplateRequest<'_> {
fn model(&self) -> String {
self.request.model()
}
fn messages(&self) -> minijinja::Value {
self.request.messages()
}
fn typed_messages(&self) -> Option<&[ChatCompletionRequestMessage]> {
self.request.typed_messages()
}
fn tools(&self) -> Option<minijinja::Value> {
self.request.tools()
}
fn tool_choice(&self) -> Option<minijinja::Value> {
self.request.tool_choice()
}
fn response_format(&self) -> Option<minijinja::Value> {
self.request.response_format()
}
fn reasoning_effort(&self) -> Option<minijinja::Value> {
self.request.reasoning_effort()
}
fn should_add_generation_prompt(&self) -> bool {
self.request.should_add_generation_prompt()
}
fn chat_template_args(&self) -> Option<&ChatTemplateKwargs> {
self.kwargs
}
fn extract_text(&self) -> Option<TextInput> {
self.request.extract_text()
}
fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
self.request.mm_processor_kwargs()
}
}
#[derive(Debug, Error)]
pub(super) enum TemplateError {
#[error("failed to read {kind} `{path}`: {source}")]
@@ -134,6 +180,8 @@ pub(super) enum TemplateError {
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use dynamo_protocols::types::{
ChatCompletionRequestMessage, ChatCompletionRequestMessageContentPartText,
ChatCompletionRequestSystemMessage, ChatCompletionRequestSystemMessageContent,
@@ -191,7 +239,7 @@ mod tests {
let formatter = ChatFormatter::Legacy(Box::new(LegacyFormatter {
spec: builtin_template("chatml").unwrap(),
}));
let rendered = formatter.render(&request()).unwrap();
let rendered = formatter.render(&request(), None).unwrap();
assert_eq!(
rendered,
"<|im_start|>system\nBe concise.<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n"
@@ -420,6 +468,7 @@ mod tests {
let formatter = load_chat_formatter(
Some(base.to_str().unwrap()),
None,
None,
Some(base.to_str().unwrap()),
)
.unwrap();
@@ -456,10 +505,11 @@ mod tests {
let formatter = load_chat_formatter(
Some(base.to_str().unwrap()),
None,
None,
Some(legacy.to_str().unwrap()),
)
.unwrap();
let rendered = formatter.render(&request()).unwrap();
let rendered = formatter.render(&request(), None).unwrap();
assert_eq!(rendered, "System\nBe concise.\nUSER: Hello\nASSISTANT:");
let _ = std::fs::remove_file(base);
@@ -557,7 +607,7 @@ mod tests {
/// A built-in `--chat-template` name resolves without any tokenizer config.
#[test]
fn builtin_argument_works_without_tokenizer_config() {
let formatter = load_chat_formatter(None, None, Some("chatml")).unwrap();
let formatter = load_chat_formatter(None, None, None, Some("chatml")).unwrap();
let ChatFormatter::Legacy(formatter) = &formatter else {
panic!("expected a legacy formatter");
};
@@ -585,6 +635,7 @@ mod tests {
Some(base.to_str().unwrap()),
Some("models/vicuna-7b-v1.5"),
None,
None,
)
.unwrap();
let ChatFormatter::Legacy(formatter) = &formatter else {
@@ -592,7 +643,7 @@ mod tests {
};
assert_eq!(formatter.spec.name, "vicuna_v1.1");
// No config at all + path matcher.
let formatter = load_chat_formatter(None, Some("deepseek-vl2-7b"), None).unwrap();
let formatter = load_chat_formatter(None, Some("deepseek-vl2-7b"), None, None).unwrap();
let ChatFormatter::Legacy(formatter) = &formatter else {
panic!("expected a legacy formatter");
};
@@ -609,7 +660,8 @@ mod tests {
r#"{"model_type":"phi4mm","architectures":["Phi4MMForCausalLM"]}"#,
)
.unwrap();
let formatter = load_chat_formatter(None, Some(model_dir.to_str().unwrap()), None).unwrap();
let formatter =
load_chat_formatter(None, Some(model_dir.to_str().unwrap()), None, None).unwrap();
let ChatFormatter::Legacy(formatter) = &formatter else {
panic!("expected a legacy formatter");
};
@@ -653,7 +705,66 @@ mod tests {
fn minicpm_4_6_skips_legacy_inference() {
assert!(infer_legacy_template_from_model_path("minicpm-v-4.6").is_none());
assert!(matches!(
load_chat_formatter(None, Some("minicpm-v-4.6"), None),
load_chat_formatter(None, Some("minicpm-v-4.6"), None, None),
Err(TemplateError::MissingConfig)
));
}
fn temp_config(contents: &str) -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!("sglang-template-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let config = dir.join("tokenizer_config.json");
std::fs::write(&config, contents).unwrap();
config
}
#[test]
fn chat_template_kwargs_reach_the_template() {
let config = temp_config(r#"{"chat_template": "thinking={{ thinking }}"}"#);
let formatter =
load_chat_formatter(Some(config.to_str().unwrap()), None, None, None).unwrap();
let kwargs = HashMap::from([("thinking".into(), serde_json::json!(true))]);
assert_eq!(formatter.render(&request(), None).unwrap(), "thinking=");
assert_eq!(
formatter.render(&request(), Some(&kwargs)).unwrap(),
"thinking=True"
);
}
#[test]
fn missing_template_falls_back_to_native_formatter() {
let config = temp_config("{}");
let config = config.to_str().unwrap();
let load = |config, model_type, arg| {
load_chat_formatter(config, Some("/models/x"), model_type, arg)
};
let formatter = load(Some(config), Some("deepseek_v4"), None).unwrap();
assert!(matches!(formatter, ChatFormatter::HuggingFace(_)));
let kwargs = HashMap::from([("thinking".into(), serde_json::json!(false))]);
assert_eq!(
formatter.render(&request(), Some(&kwargs)).unwrap(),
"<｜begin▁of▁sentence｜>Be concise.<｜User｜>Hello<｜Assistant｜></think>"
);
assert!(matches!(
load(None, Some("deepseek_v4"), None),
Ok(ChatFormatter::HuggingFace(_))
));
// A template, `--chat-template`, or an unknown architecture wins.
let templated = temp_config(r#"{"chat_template": "Hi"}"#);
let formatter = load(Some(templated.to_str().unwrap()), Some("deepseek_v4"), None).unwrap();
assert_eq!(formatter.render(&request(), None).unwrap(), "Hi");
assert!(matches!(
load(Some(config), Some("deepseek_v4"), Some("chatml")),
Ok(ChatFormatter::Legacy(_))
));
assert!(matches!(
load(Some(config), Some("llama"), None),
Err(TemplateError::Missing)
));
assert!(matches!(
load(None, None, None),
Err(TemplateError::MissingConfig)
));
}
rust/sglang-server/src/api_server/openai/template_loader.rs
+20 -3
View File
@@ -2,7 +2,9 @@
use std::path::Path;
use dynamo_renderer::{ChatTemplate, ContextMixins, PromptContextMixin, PromptFormatter};
use dynamo_renderer::{
ChatTemplate, ContextMixins, PromptContextMixin, PromptFormatter, native_formatter_for,
};
use serde_json::Value;
use crate::message::types::OneOrMany;
@@ -44,6 +46,7 @@ const SUPPORTED_STYLES: &[&str] = &[
pub(super) fn load_chat_formatter(
config_file: Option<&str>,
model_path: Option<&str>,
model_type: Option<&str>,
chat_template_arg: Option<&str>,
) -> Result<ChatFormatter, TemplateError> {
// Python resolves registry names before looking at the filesystem — and
@@ -66,10 +69,21 @@ pub(super) fn load_chat_formatter(
return Ok(ChatFormatter::Legacy(Box::new(LegacyFormatter { spec })));
}
// Models that ship no template (Python `resolve_chat_encoding_spec`) use
// Dynamo's built-in encoder for their architecture.
let native_formatter = || match chat_template_arg {
None => native_formatter_for(
&model_type.map(str::to_lowercase),
&model_path.unwrap_or_default().to_lowercase(),
)
.map(ChatFormatter::HuggingFace),
Some(_) => None,
};
// Every remaining source builds the HF renderer around the tokenizer
// config (the template itself, or the argument injected into it).
let Some(config_file) = config_file else {
return Err(TemplateError::MissingConfig);
return native_formatter().ok_or(TemplateError::MissingConfig);
};
let config_path = Path::new(config_file);
@@ -77,7 +91,10 @@ pub(super) fn load_chat_formatter(
let mut config = parse_json(&config_text, config_path, "tokenizer config")?;
let Some(argument) = chat_template_arg else {
return formatter_from_config(&config);
return match formatter_from_config(&config) {
Err(TemplateError::Missing) => native_formatter().ok_or(TemplateError::Missing),
result => result,
};
};
let path = Path::new(argument);
rust/sglang-server/src/message/config.rs
+6 -1
View File
@@ -329,6 +329,8 @@ pub enum DisaggregationMode {
#[pyo3::pyclass(frozen, from_py_object, module = "sglang.srt.rust_extensions._server")]
#[derive(Clone, Debug)]
pub struct ModelConfig {
/// Authoritative HF model type, used to select a native chat formatter.
pub model_type: Option<String>,
/// Resolved context length (`max_model_len` in `/v1/models`); the ceiling
/// for input + `max_new_tokens`.
pub context_len: u64,
@@ -351,18 +353,20 @@ pub struct ModelConfig {
#[pyo3::pymethods]
impl ModelConfig {
#[new]
#[pyo3(signature = (*, context_len, vocab_size, is_multimodal, default_sampling_params))]
#[pyo3(signature = (*, context_len, vocab_size, is_multimodal, default_sampling_params, model_type))]
fn py_new(
context_len: u64,
vocab_size: u64,
is_multimodal: bool,
default_sampling_params: DefaultSamplingParams,
model_type: Option<String>,
) -> Self {
Self {
context_len,
vocab_size,
is_multimodal,
default_sampling_params,
model_type,
}
}
}
@@ -372,6 +376,7 @@ impl Default for ModelConfig {
fn default() -> Self {
Self {
context_len: 2048,
model_type: None,
vocab_size: 1000,
is_multimodal: false,
default_sampling_params: DefaultSamplingParams::default(),