[sgl-router] Match DeepSeek V4 rendering to SGLang (#40530)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Kan Wu
co-authored by
Claude Fable 5.1
Shangming Cai
Claude Fable 5.1
Shangming Cai
parent
2016f5e7a1
commit
0abb251a20
@@ -211,6 +211,16 @@ Detailed content-format parity coverage follows in #39133.
|
||||
The Dynamo crates are pinned exactly and `Cargo.lock` is committed; CI builds
|
||||
with `--locked`, so rendered bytes cannot change without a reviewed diff.
|
||||
|
||||
## DeepSeek V4
|
||||
|
||||
Native V4 rendering follows SGLang's serving path (`serving_chat.py`), not
|
||||
Dynamo's OpenAI defaults: all declared tools are rendered with SGLang's schema
|
||||
defaults, reasoning effort comes from `reasoning` / `reasoning_effort`, and the
|
||||
official/preview effort profile is detected from the checkpoint's
|
||||
`encoding/encoding_dsv4.py` or overridden by `dsv4_reasoning_effort_profile` in
|
||||
`config.json`, as in SGLang. Reference prompts live in `tests/fixtures/deepseek/`
|
||||
and are regenerated by `tests/scripts/generate_deepseek_parity.py`.
|
||||
|
||||
## Kimi-K3
|
||||
|
||||
Kimi-K3 renders through dynamo-render's native XTML formatter with SGLang's
|
||||
|
||||
@@ -40,7 +40,7 @@ pub struct ChatFormatter {
|
||||
defaults: ChatTemplateKwargs,
|
||||
/// Stripped from a separately tokenized continuation prefix, as SGLang does.
|
||||
bos_token: Option<String>,
|
||||
is_deepseek_v4: bool,
|
||||
deepseek_v4: Option<super::deepseek::V4Profile>,
|
||||
is_kimi_k3: bool,
|
||||
}
|
||||
|
||||
@@ -48,9 +48,8 @@ impl ChatFormatter {
|
||||
/// Load model files and select a template or native formatter from dynamo-render.
|
||||
pub fn load(model_id: &str, tokenizer_path: &str) -> Result<Option<Self>> {
|
||||
let files = super::adapter::ModelFiles::open(tokenizer_path);
|
||||
let model_type = files
|
||||
.json("config.json")?
|
||||
.and_then(|cfg| cfg["model_type"].as_str().map(str::to_owned));
|
||||
let config = files.json("config.json")?.unwrap_or_default();
|
||||
let model_type = config["model_type"].as_str().map(str::to_owned);
|
||||
if let Some(kimi) = Self::kimi_native(model_type.as_deref(), model_id) {
|
||||
return Ok(Some(kimi));
|
||||
}
|
||||
@@ -58,7 +57,12 @@ impl ChatFormatter {
|
||||
// These require tokenization paths not yet supported by this adapter.
|
||||
Some("inkling_mm_model") => return Ok(None),
|
||||
Some(t) if t.starts_with("deepseek_v4") => {
|
||||
return Ok(Self::deepseek_native(model_type.as_deref(), model_id));
|
||||
let mut formatter = Self::deepseek_native(model_type.as_deref(), model_id);
|
||||
if let Some(formatter) = &mut formatter {
|
||||
formatter.deepseek_v4 =
|
||||
Some(super::deepseek::V4Profile::load(&files, &config)?);
|
||||
}
|
||||
return Ok(formatter);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
@@ -66,8 +70,14 @@ impl ChatFormatter {
|
||||
.json("tokenizer_config.json")?
|
||||
.unwrap_or_else(|| serde_json::json!({}));
|
||||
let jinja = files.text("chat_template.jinja")?;
|
||||
Ok(Self::from_tokenizer_config(cfg, jinja.as_deref())?
|
||||
.or_else(|| Self::deepseek_native(model_type.as_deref(), model_id)))
|
||||
let mut formatter = Self::from_tokenizer_config(cfg, jinja.as_deref())?
|
||||
.or_else(|| Self::deepseek_native(model_type.as_deref(), model_id));
|
||||
if let Some(formatter) = &mut formatter {
|
||||
if formatter.deepseek_v4.is_some() {
|
||||
formatter.deepseek_v4 = Some(super::deepseek::V4Profile::load(&files, &config)?);
|
||||
}
|
||||
}
|
||||
Ok(formatter)
|
||||
}
|
||||
|
||||
/// HF Jinja template from `tokenizer_config.json`, overridden by a sibling
|
||||
@@ -136,7 +146,7 @@ impl ChatFormatter {
|
||||
formatter,
|
||||
defaults,
|
||||
bos_token,
|
||||
is_deepseek_v4: false,
|
||||
deepseek_v4: None,
|
||||
is_kimi_k3: false,
|
||||
}))
|
||||
}
|
||||
@@ -151,25 +161,25 @@ impl ChatFormatter {
|
||||
formatter,
|
||||
defaults: HashMap::new(),
|
||||
bos_token: Some("[BOS]".into()),
|
||||
is_deepseek_v4: false,
|
||||
deepseek_v4: None,
|
||||
is_kimi_k3: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// dynamo-render's code-based DeepSeek encoders, for V4 (including variants
|
||||
/// such as V4.1) and V3.2 non-Exp: the only built-in formatters verified
|
||||
/// against the engine. `model_type` (from `config.json`) is authoritative;
|
||||
/// the model id's last path segment is the fallback.
|
||||
/// Native V4 and V3.2 formatters. Other V4-family encoders must be
|
||||
/// supported explicitly; a V4.1 checkpoint cannot use the V4 wire format.
|
||||
pub fn deepseek_native(model_type: Option<&str>, model_id: &str) -> Option<Self> {
|
||||
let name = model_name(model_id);
|
||||
// The engine treats every `deepseek_v4*` variant (e.g. V4.1) as V4.
|
||||
let model_type = model_type.map(str::to_lowercase).map(|t| {
|
||||
if t.starts_with("deepseek_v4") {
|
||||
"deepseek_v4".into()
|
||||
} else {
|
||||
t
|
||||
}
|
||||
});
|
||||
let model_type = model_type.map(str::to_lowercase);
|
||||
if model_type.is_none() && (name.contains("v4.1") || name.contains("v41")) {
|
||||
return None;
|
||||
}
|
||||
if model_type
|
||||
.as_deref()
|
||||
.is_some_and(|t| t.starts_with("deepseek_v4") && t != "deepseek_v4")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let PromptFormatter::OAI(formatter) = deepseek_formatter_for(&model_type, &name)?;
|
||||
// Same rule dynamo-render applies for the name fallback: `deepseek` + one
|
||||
// separator + a `v4` segment.
|
||||
@@ -190,7 +200,7 @@ impl ChatFormatter {
|
||||
formatter,
|
||||
defaults,
|
||||
bos_token: Some("<｜begin▁of▁sentence｜>".into()),
|
||||
is_deepseek_v4,
|
||||
deepseek_v4: is_deepseek_v4.then(Default::default),
|
||||
is_kimi_k3: false,
|
||||
})
|
||||
}
|
||||
@@ -241,12 +251,7 @@ impl ChatFormatter {
|
||||
.entry("enable_thinking".into())
|
||||
.or_insert(thinking.into());
|
||||
}
|
||||
if let Some(mut effort) = effort {
|
||||
// The engine's official V4 profile accepts only these; others map to
|
||||
// no preamble.
|
||||
if self.is_deepseek_v4 && !matches!(effort.as_str(), Some("low" | "high" | "max")) {
|
||||
effort = "low".into();
|
||||
}
|
||||
if let Some(effort) = effort {
|
||||
if self.is_kimi_k3 {
|
||||
if matches!(effort.as_str(), Some("low" | "high" | "max")) {
|
||||
kwargs.entry("thinking_effort".into()).or_insert(effort);
|
||||
@@ -275,6 +280,9 @@ impl ChatFormatter {
|
||||
if self.is_kimi_k3 {
|
||||
super::kimi::normalize(request, &mut messages, &mut kwargs)?;
|
||||
}
|
||||
if self.deepseek_v4.is_some() {
|
||||
super::deepseek::normalize_messages(&mut messages)?;
|
||||
}
|
||||
let mut prefix = String::new();
|
||||
if let Some(last) = messages.last_mut().filter(|m| m["role"] == "assistant") {
|
||||
if let Some(content) = last["content"].as_str() {
|
||||
@@ -286,7 +294,7 @@ impl ChatFormatter {
|
||||
}
|
||||
}
|
||||
}
|
||||
if self.is_deepseek_v4 {
|
||||
if self.deepseek_v4.is_some() {
|
||||
if let Some(task) = request.get("task").filter(|v| !v.is_null()).cloned() {
|
||||
let message = messages
|
||||
.iter_mut()
|
||||
@@ -296,6 +304,12 @@ impl ChatFormatter {
|
||||
message["task"] = task;
|
||||
}
|
||||
}
|
||||
if let Some(profile) = self.deepseek_v4 {
|
||||
return Ok((
|
||||
RenderedPrompt::text(profile.render(request, messages, &kwargs)?),
|
||||
prefix,
|
||||
));
|
||||
}
|
||||
let prompt = self
|
||||
.formatter
|
||||
.render_prompt(&ChatRequest {
|
||||
@@ -643,7 +657,7 @@ mod tests {
|
||||
assert!(ChatFormatter::deepseek_native(None, "deepseek-ai/DeepSeek-V4-Flash").is_some());
|
||||
assert!(ChatFormatter::deepseek_native(None, "deepseek-v4-tiny").is_some());
|
||||
assert!(ChatFormatter::deepseek_native(Some("deepseek_v4"), "alias").is_some());
|
||||
assert!(ChatFormatter::deepseek_native(Some("deepseek_v41"), "alias").is_some());
|
||||
assert!(ChatFormatter::deepseek_native(Some("deepseek_v41"), "alias").is_none());
|
||||
assert!(ChatFormatter::deepseek_native(Some("deepseek_v32"), "DeepSeek-V3.2").is_some());
|
||||
assert!(ChatFormatter::deepseek_native(Some("inkling_mm_model"), "inkling").is_none());
|
||||
assert!(ChatFormatter::deepseek_native(Some("llama"), "deepseek-v4").is_none());
|
||||
@@ -700,27 +714,25 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn request_controls_reach_dynamo() {
|
||||
let jinja = jinja(
|
||||
let formatter = jinja(
|
||||
json!({"chat_template": "{{ tools | tojson }} {{ thinking }} {{ reasoning_effort }}"}),
|
||||
);
|
||||
for formatter in [jinja, deepseek_v4()] {
|
||||
let mut req = request(json!([{"role":"user","content":"hi"}]));
|
||||
req["tools"] = json!([
|
||||
{"type":"function","function":{"name":"first"}},
|
||||
{"type":"function","function":{"name":"second"}}
|
||||
]);
|
||||
req["tool_choice"] = json!({"type":"function","function":{"name":"second"}});
|
||||
req["reasoning"] = json!({"effort":"high"});
|
||||
let out = formatter.render(&req).unwrap();
|
||||
assert!(out.contains("second") && !out.contains("first"));
|
||||
assert!(out.contains("high") || out.contains("Reasoning Effort:"));
|
||||
req["tool_choice"] = json!("none");
|
||||
req["reasoning_effort"] = json!("none");
|
||||
req["reasoning"] = JsonValue::Null;
|
||||
let out = formatter.render(&req).unwrap();
|
||||
assert!(!out.contains("first") && !out.contains("second"));
|
||||
assert!(out.contains("False none") || out.ends_with("</think>"));
|
||||
}
|
||||
let mut req = request(json!([{"role":"user","content":"hi"}]));
|
||||
req["tools"] = json!([
|
||||
{"type":"function","function":{"name":"first"}},
|
||||
{"type":"function","function":{"name":"second"}}
|
||||
]);
|
||||
req["tool_choice"] = json!({"type":"function","function":{"name":"second"}});
|
||||
req["reasoning"] = json!({"effort":"high"});
|
||||
let out = formatter.render(&req).unwrap();
|
||||
assert!(out.contains("second") && !out.contains("first"));
|
||||
assert!(out.contains("high") || out.contains("Reasoning Effort:"));
|
||||
req["tool_choice"] = json!("none");
|
||||
req["reasoning_effort"] = json!("none");
|
||||
req["reasoning"] = JsonValue::Null;
|
||||
let out = formatter.render(&req).unwrap();
|
||||
assert!(!out.contains("first") && !out.contains("second"));
|
||||
assert!(out.contains("False none") || out.ends_with("</think>"));
|
||||
}
|
||||
|
||||
/// Mirrors `protocol.py::normalize_reasoning_inputs`: effort decides both
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! SGLang's native DeepSeek serving semantics around Dynamo's V4 encoder.
|
||||
|
||||
use anyhow::{bail, ensure, Context, Result};
|
||||
use dynamo_renderer::deepseek::v4::{self, ReasoningEffort, ThinkingMode};
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use super::{adapter::ModelFiles, chat_formatter::ChatTemplateKwargs};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(super) enum V4Profile {
|
||||
#[default]
|
||||
Preview,
|
||||
Official,
|
||||
}
|
||||
|
||||
impl V4Profile {
|
||||
pub fn load(files: &ModelFiles, config: &Value) -> Result<Self> {
|
||||
if let Some(profile) = config
|
||||
.get("dsv4_reasoning_effort_profile")
|
||||
.filter(|v| !v.is_null())
|
||||
{
|
||||
return match profile.as_str() {
|
||||
Some("preview") => Ok(Self::Preview),
|
||||
Some("official") => Ok(Self::Official),
|
||||
_ => bail!("invalid dsv4_reasoning_effort_profile: {profile}"),
|
||||
};
|
||||
}
|
||||
// Inspect checkpoint source as data, never execute remote Python. Like
|
||||
// SGLang, an absent/unrecognized encoder falls back to the preview profile.
|
||||
let source = files.text("encoding/encoding_dsv4.py")?.unwrap_or_default();
|
||||
Ok(Self::detect(&source))
|
||||
}
|
||||
|
||||
/// `chat_encoding._detect_dsv4_reasoning_effort_profile`: official when the
|
||||
/// encoder declares a low default and low/high/max prompts, else preview.
|
||||
fn detect(source: &str) -> Self {
|
||||
if source.len() > 1 << 20 {
|
||||
return Self::Preview;
|
||||
}
|
||||
let low_default = source.lines().any(|line| {
|
||||
line.strip_prefix("DEFAULT_REASONING_EFFORT")
|
||||
.and_then(|rest| rest.split_once('='))
|
||||
.is_some_and(|(_, value)| matches!(value.trim().trim_matches(['"', '\'']), "low"))
|
||||
});
|
||||
let prompts = source
|
||||
.split_once("\nREASONING_EFFORT_PROMPTS")
|
||||
.and_then(|(_, rest)| rest.split_once('}'))
|
||||
.map_or("", |(body, _)| body);
|
||||
let has_keys = ["low", "high", "max"].iter().all(|key| {
|
||||
prompts.contains(&format!("\"{key}\"")) || prompts.contains(&format!("'{key}'"))
|
||||
});
|
||||
if low_default && has_keys {
|
||||
Self::Official
|
||||
} else {
|
||||
Self::Preview
|
||||
}
|
||||
}
|
||||
|
||||
pub fn render(
|
||||
self,
|
||||
request: &Value,
|
||||
mut messages: Vec<Value>,
|
||||
kwargs: &ChatTemplateKwargs,
|
||||
) -> Result<String> {
|
||||
ensure!(
|
||||
!messages.is_empty(),
|
||||
"DeepSeek requires messages after continuation extraction"
|
||||
);
|
||||
if messages[0]["role"] != "system" {
|
||||
messages.insert(0, json!({"role":"system", "content":""}));
|
||||
}
|
||||
// Unlike the HF template path, SGLang's native encoder takes *all*
|
||||
// request.tools, even for tool_choice=none or a named function.
|
||||
if let Some(tools) = request["tools"].as_array().filter(|t| !t.is_empty()) {
|
||||
messages[0]["tools"] = normalize_tools(tools)?.into();
|
||||
}
|
||||
// SGLang drops a later user's task when merging it into an existing
|
||||
// user/tool-result turn. Dynamo otherwise preserves that field.
|
||||
for index in 1..messages.len() {
|
||||
if messages[index]["role"] == "user"
|
||||
&& matches!(messages[index - 1]["role"].as_str(), Some("user" | "tool"))
|
||||
{
|
||||
messages[index].as_object_mut().unwrap().remove("task");
|
||||
}
|
||||
}
|
||||
let effort = match (
|
||||
self,
|
||||
request_effort(request).as_ref().and_then(Value::as_str),
|
||||
) {
|
||||
(Self::Official, Some("high")) | (Self::Preview, Some("max")) => {
|
||||
Some(ReasoningEffort::High)
|
||||
}
|
||||
(Self::Official, Some("max")) => Some(ReasoningEffort::Max),
|
||||
_ => None,
|
||||
};
|
||||
// Only thinking is a native-encoder kwarg. In particular, the engine
|
||||
// does not read kwargs.reasoning_effort or kwargs.drop_thinking.
|
||||
let thinking = kwargs
|
||||
.get("thinking")
|
||||
.is_some_and(|v| minijinja::Value::from_serialize(v).is_true());
|
||||
v4::encode_messages_with_options(
|
||||
&messages,
|
||||
if thinking {
|
||||
ThinkingMode::Thinking
|
||||
} else {
|
||||
ThinkingMode::Chat
|
||||
},
|
||||
true,
|
||||
true,
|
||||
effort,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// `serving_chat._convert_to_internal_request`: `chat_template_kwargs.reasoning_effort`
|
||||
/// replaces the validated request effort verbatim (thinking was already derived
|
||||
/// from the request fields); request-level values are coerced as pydantic does.
|
||||
pub(super) fn request_effort(request: &Value) -> Option<Value> {
|
||||
if let Some(effort) = request["chat_template_kwargs"]
|
||||
.get("reasoning_effort")
|
||||
.filter(|v| !v.is_null())
|
||||
{
|
||||
return Some(effort.clone());
|
||||
}
|
||||
let effort = [
|
||||
request["reasoning"].get("effort"),
|
||||
request["reasoning"].get("reasoning_effort"),
|
||||
request.get("reasoning_effort"),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find(|v| !v.is_null())?;
|
||||
Some(match effort {
|
||||
Value::String(s) => s
|
||||
.parse::<f64>()
|
||||
.map_or_else(|_| effort.clone(), Value::from),
|
||||
Value::Number(n) => n.as_f64().map_or_else(|| effort.clone(), Value::from),
|
||||
_ => effort.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Before continuation extraction, SGLang flattens V4 parts with spaces and
|
||||
/// parses assistant arguments as JSON objects. Dynamo expects those arguments
|
||||
/// serialized, but must not apply its permissive malformed-JSON fallback here.
|
||||
pub(super) fn normalize_messages(messages: &mut [Value]) -> Result<()> {
|
||||
for message in messages {
|
||||
if let Some(parts) = message["content"].as_array() {
|
||||
message["content"] = parts
|
||||
.iter()
|
||||
.filter(|p| matches!(p["type"].as_str(), Some("text" | "input_text")))
|
||||
.filter_map(|p| p["text"].as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.into();
|
||||
}
|
||||
if let Some(tools) = message["tools"].as_array() {
|
||||
if tools.is_empty() {
|
||||
message.as_object_mut().unwrap().remove("tools");
|
||||
} else {
|
||||
message["tools"] = normalize_tools(tools)?.into();
|
||||
}
|
||||
}
|
||||
if message["role"] == "assistant" {
|
||||
if message["tool_calls"].as_array().is_some_and(Vec::is_empty) {
|
||||
message.as_object_mut().unwrap().remove("tool_calls");
|
||||
}
|
||||
for call in message["tool_calls"].as_array_mut().into_iter().flatten() {
|
||||
let arguments = &mut call["function"]["arguments"];
|
||||
let parsed = match arguments.as_str() {
|
||||
Some(text) => serde_json::from_str::<Value>(text)
|
||||
.context("assistant tool arguments must be valid JSON")?,
|
||||
None => arguments.clone(),
|
||||
};
|
||||
ensure!(
|
||||
parsed.is_object(),
|
||||
"assistant tool arguments must be a JSON object"
|
||||
);
|
||||
*arguments = serde_json::to_string(&parsed)?.into();
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// protocol.py::Function.model_dump(), including declared field order and
|
||||
/// defaults. The native encoder serializes this dictionary verbatim.
|
||||
fn normalize_tools(tools: &[Value]) -> Result<Vec<Value>> {
|
||||
tools
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
let f = &tool["function"];
|
||||
let name = f["name"].as_str().context("tool function requires name")?;
|
||||
let mut function = Map::new();
|
||||
function.insert("description".into(), f["description"].clone());
|
||||
function.insert("name".into(), name.into());
|
||||
function.insert("parameters".into(), f["parameters"].clone());
|
||||
function.insert(
|
||||
"strict".into(),
|
||||
f.get("strict").cloned().unwrap_or(false.into()),
|
||||
);
|
||||
if let Some(defer) = f
|
||||
.get("defer_loading")
|
||||
.filter(|v| !v.is_null())
|
||||
.or_else(|| tool.get("defer_loading").filter(|v| !v.is_null()))
|
||||
{
|
||||
function.insert("defer_loading".into(), defer.clone());
|
||||
}
|
||||
Ok(json!({"type":"function", "function":function}))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn checkpoint_profile_and_override() {
|
||||
assert_eq!(
|
||||
V4Profile::detect("REASONING_EFFORT_MAX = 'text'"),
|
||||
V4Profile::Preview
|
||||
);
|
||||
assert_eq!(V4Profile::detect("DEFAULT_REASONING_EFFORT: str = 'low'\nREASONING_EFFORT_PROMPTS = {\n'low': '', 'high': 'H', 'max': 'M'\n}"), V4Profile::Official);
|
||||
assert_eq!(
|
||||
V4Profile::detect(
|
||||
"# DEFAULT_REASONING_EFFORT = 'low'\n# REASONING_EFFORT_PROMPTS = {}"
|
||||
),
|
||||
V4Profile::Preview
|
||||
);
|
||||
assert_eq!(
|
||||
V4Profile::detect("DEFAULT_REASONING_EFFORT = 'low'\nREASONING_EFFORT_PROMPTS = {}"),
|
||||
V4Profile::Preview
|
||||
);
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tokenizer.json");
|
||||
std::fs::write(&path, "{}").unwrap();
|
||||
let files = ModelFiles::open(path.to_str().unwrap());
|
||||
assert_eq!(
|
||||
V4Profile::load(&files, &json!({})).unwrap(),
|
||||
V4Profile::Preview
|
||||
);
|
||||
assert_eq!(
|
||||
V4Profile::load(&files, &json!({"dsv4_reasoning_effort_profile":"official"})).unwrap(),
|
||||
V4Profile::Official
|
||||
);
|
||||
assert!(V4Profile::load(&files, &json!({"dsv4_reasoning_effort_profile":"typo"})).is_err());
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
pub mod adapter;
|
||||
pub mod chat_formatter;
|
||||
mod deepseek;
|
||||
mod kimi;
|
||||
|
||||
use anyhow::Result;
|
||||
@@ -334,7 +335,7 @@ mod tests {
|
||||
assert_eq!(cfg["chat_template"], "X");
|
||||
}
|
||||
|
||||
/// Families the engine encodes in code skip a shipped template; V4.1 counts as V4.
|
||||
/// Families the engine encodes in code skip a shipped template.
|
||||
#[test]
|
||||
fn chat_formatter_load_preserves_native_precedence() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -356,8 +357,9 @@ mod tests {
|
||||
.render(&request)
|
||||
.unwrap()
|
||||
.contains("<|open|>message"));
|
||||
assert!(resolve("deepseek_v41").is_none());
|
||||
assert_eq!(
|
||||
resolve("deepseek_v41").unwrap().render(&request).unwrap(),
|
||||
resolve("deepseek_v4").unwrap().render(&request).unwrap(),
|
||||
"<｜begin▁of▁sentence｜><｜User｜>hi<｜Assistant｜></think>"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use sgl_router::tokenizer::{adapter, chat_formatter::ChatFormatter};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
fn check_fixture(fixture: &str, model_type: &str) {
|
||||
let fixture: Value = serde_json::from_str(fixture).unwrap();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tokenizer.json");
|
||||
std::fs::write(&path, "{}").unwrap();
|
||||
let hf_home = std::env::var_os("HF_HOME")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| dirs::home_dir().map(|p| p.join(".cache/huggingface")));
|
||||
let cached = hf_home.map(|p| {
|
||||
p.join("hub")
|
||||
.join(format!(
|
||||
"models--{}",
|
||||
fixture["model"].as_str().unwrap().replace('/', "--")
|
||||
))
|
||||
.join("snapshots")
|
||||
.join(fixture["revision"].as_str().unwrap())
|
||||
.join("tokenizer.json")
|
||||
});
|
||||
let tokenizer = cached
|
||||
.filter(|p| p.is_file())
|
||||
.map(|p| adapter::load(p.to_str().unwrap()).unwrap());
|
||||
if tokenizer.is_none() {
|
||||
eprintln!(
|
||||
"{}: checking rendered text; pinned tokenizer is not cached",
|
||||
fixture["model"]
|
||||
);
|
||||
}
|
||||
for case in fixture["cases"].as_array().unwrap() {
|
||||
std::fs::write(
|
||||
dir.path().join("config.json"),
|
||||
json!({"model_type":model_type, "dsv4_reasoning_effort_profile":case["profile"]})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap();
|
||||
let formatter = ChatFormatter::load("served-alias", path.to_str().unwrap())
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let text = formatter
|
||||
.render(&case["request"])
|
||||
.unwrap_or_else(|e| panic!("{}: {e:#}", case["name"]));
|
||||
assert_eq!(text, case["prompt"].as_str().unwrap(), "{}", case["name"]);
|
||||
if let Some(tokenizer) = &tokenizer {
|
||||
let ids = formatter.encode(tokenizer, &case["request"]).unwrap();
|
||||
let mut hash = Sha256::new();
|
||||
for id in &ids {
|
||||
hash.update(id.to_le_bytes());
|
||||
}
|
||||
assert_eq!(
|
||||
ids.len() as u64,
|
||||
case["token_count"].as_u64().unwrap(),
|
||||
"{}",
|
||||
case["name"]
|
||||
);
|
||||
assert_eq!(
|
||||
format!("{:x}", hash.finalize()),
|
||||
case["token_sha256"].as_str().unwrap(),
|
||||
"{}",
|
||||
case["name"]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_matches_sglang_serving() {
|
||||
check_fixture(
|
||||
include_str!("../../fixtures/deepseek/v4.json"),
|
||||
"deepseek_v4",
|
||||
);
|
||||
}
|
||||
@@ -4,3 +4,5 @@
|
||||
mod kimi;
|
||||
mod parity;
|
||||
mod render_parity;
|
||||
|
||||
mod deepseek;
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
{"model": "deepseek-ai/DeepSeek-V4-Flash", "revision": "60d8d70770c6776ff598c94bb586a859a38244f1", "cases": [
|
||||
{"name":"preview/user","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}]},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think>","token_count":5,"token_sha256":"95aeed952d0173ec8849b13faec66eb61ec4412cd9c41dae0c405257559365a6"},
|
||||
{"name":"preview/system","profile":"preview","request":{"model":"m","messages":[{"role":"system","content":"Be terse"},{"role":"user","content":"Hi"}]},"prompt":"<｜begin▁of▁sentence｜>Be terse<｜User｜>Hi<｜Assistant｜></think>","token_count":8,"token_sha256":"a577612d73073798010b2fca019f9f261242a452e826a7e1fc1e746f07e088b0"},
|
||||
{"name":"preview/multi_turn","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}]},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think>Answer<｜end▁of▁sentence｜><｜User｜>Again<｜Assistant｜></think>","token_count":11,"token_sha256":"5ab68a07b5a4b1fa1811d68fe156ece6b5284af15cac9b76933c28d9909e4ee2"},
|
||||
{"name":"preview/parts","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}]}]},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hello world<｜Assistant｜></think>","token_count":6,"token_sha256":"ca8d375f13a4fe4f95b6ec298e03d412f93f3c2d353aa0647223efe74065e51b"},
|
||||
{"name":"preview/continuation_parts","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}]}],"continue_final_message":true},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think>Hello world","token_count":7,"token_sha256":"f436ace77f4a14b4d21ac6e87347a1263253e87cbb23d5e7a4ffc8af1cb36f21"},
|
||||
{"name":"preview/continuation_bos","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"<｜begin▁of▁sentence｜>abc"}],"continue_final_message":true},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think><｜begin▁of▁sentence｜>abc","token_count":6,"token_sha256":"defef25e49dca2301b583fb67cabe122b8aa1bdd889727ea1e8b083be7149c14"},
|
||||
{"name":"preview/tools","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}]},"prompt":"<｜begin▁of▁sentence｜>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜tool_calls>\" block like the following:\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n...\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>Hi<｜Assistant｜></think>","token_count":279,"token_sha256":"b8bcd88b2fc3015ee81bd9be25dfa2025765e2590cc407b227b6b5e241fe3adb"},
|
||||
{"name":"preview/tools_none","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"tool_choice":"none"},"prompt":"<｜begin▁of▁sentence｜>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜tool_calls>\" block like the following:\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n...\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>Hi<｜Assistant｜></think>","token_count":279,"token_sha256":"b8bcd88b2fc3015ee81bd9be25dfa2025765e2590cc407b227b6b5e241fe3adb"},
|
||||
{"name":"preview/tools_named","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"tool_choice":{"type":"function","function":{"name":"bar"}}},"prompt":"<｜begin▁of▁sentence｜>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜tool_calls>\" block like the following:\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n...\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>Hi<｜Assistant｜></think>","token_count":279,"token_sha256":"b8bcd88b2fc3015ee81bd9be25dfa2025765e2590cc407b227b6b5e241fe3adb"},
|
||||
{"name":"preview/message_tools","profile":"preview","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}]},{"role":"user","content":"Hi"}]},"prompt":"<｜begin▁of▁sentence｜>SYS\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜tool_calls>\" block like the following:\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n...\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>Hi<｜Assistant｜></think>","token_count":281,"token_sha256":"d43bde4fa301d6ef32903e59288c6c35febe0e3e60df344a144b7e90f2234d4c"},
|
||||
{"name":"preview/empty_message_tools","profile":"preview","request":{"model":"m","messages":[{"role":"system","content":"SYS","tools":[]},{"role":"user","content":"Hi"}]},"prompt":"<｜begin▁of▁sentence｜>SYS<｜User｜>Hi<｜Assistant｜></think>","token_count":7,"token_sha256":"c8211376cce728f63cfce621a7a28ac296a8701a27e084a8e8aa57ced3e6c973"},
|
||||
{"name":"preview/tool_results","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":null,"tool_calls":[{"id":"a","type":"function","function":{"name":"foo","arguments":"{\"x\":\"雪\", \"y\":[1, true]}"}},{"id":"b","type":"function","function":{"name":"bar","arguments":{"z":2}}}],"reasoning_content":"Call both"},{"role":"tool","content":[{"type":"text","text":"Hello"},{"type":"text","text":"world"}],"tool_call_id":"b"},{"role":"tool","content":null,"tool_call_id":"a"}],"tools":[{"type":"function","function":{"name":"foo","parameters":{"type":"object","properties":{}}}},{"type":"function","function":{"name":"bar","description":"第二个","strict":true}}],"chat_template_kwargs":{"thinking":true}},"prompt":"<｜begin▁of▁sentence｜>\n\n## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜tool_calls>\" block like the following:\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"$TOOL_NAME\">\n<｜DSML｜parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜parameter>\n...\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"$TOOL_NAME2\">\n...\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.\n\nOtherwise, output directly after </think> with tool calls or final response.\n\n### Available Tool Schemas\n\n{\"description\": null, \"name\": \"foo\", \"parameters\": {\"type\": \"object\", \"properties\": {}}, \"strict\": false}\n{\"description\": \"第二个\", \"name\": \"bar\", \"parameters\": null, \"strict\": true}\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n<｜User｜>Hi<｜Assistant｜><think>Call both</think>\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"foo\">\n<｜DSML｜parameter name=\"x\" string=\"true\">雪</｜DSML｜parameter>\n<｜DSML｜parameter name=\"y\" string=\"false\">[1, true]</｜DSML｜parameter>\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"bar\">\n<｜DSML｜parameter name=\"z\" string=\"false\">2</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls><｜end▁of▁sentence｜><｜User｜><tool_result></tool_result>\n\n<tool_result>Hello world</tool_result><｜Assistant｜><think>","token_count":394,"token_sha256":"943afa4c1eaabc806f8e1b6b4a51aceb2bef811c308aa04033e7424ae9889e62"},
|
||||
{"name":"preview/kwargs_effort","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"chat_template_kwargs":{"thinking":true,"reasoning_effort":"max"}},"prompt":"<｜begin▁of▁sentence｜>Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n<｜User｜>Hi<｜Assistant｜><think>","token_count":84,"token_sha256":"daf853d041e8dde87f07e95a201f8f07459c88482fe53e46472605d63fbdbeac"},
|
||||
{"name":"preview/effort_conflict","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"reasoning_effort":"low"}},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜><think>","token_count":5,"token_sha256":"c6c0b4c94f78803396fe1b21a3613dd337930a46baf38d285233fe84a60ae7c7"},
|
||||
{"name":"preview/drop_thinking_ignored","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"assistant","content":"Answer","reasoning_content":"Prior reasoning"},{"role":"user","content":"Again"}],"chat_template_kwargs":{"thinking":true,"drop_thinking":false}},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think>Answer<｜end▁of▁sentence｜><｜User｜>Again<｜Assistant｜><think>","token_count":11,"token_sha256":"73695fa1d286fac76f65bccacff0cd8b2dbee97ca18c3cdc3939dc9048a9037a"},
|
||||
{"name":"preview/thinking_false","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high","chat_template_kwargs":{"thinking":false}},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think>","token_count":5,"token_sha256":"95aeed952d0173ec8849b13faec66eb61ec4412cd9c41dae0c405257559365a6"},
|
||||
{"name":"preview/thinking_none","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"none","chat_template_kwargs":{"thinking":true}},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜><think>","token_count":5,"token_sha256":"c6c0b4c94f78803396fe1b21a3613dd337930a46baf38d285233fe84a60ae7c7"},
|
||||
{"name":"preview/task_after_developer","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"developer","content":"Instruction"}],"task":"domain"},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜User｜>Instruction<｜domain｜>","token_count":6,"token_sha256":"0e73d27ea0537f419ab6c581e697be2217299186a5101bb84d193f1e5569a73b"},
|
||||
{"name":"preview/consecutive_task","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"},{"role":"user","content":"Again"}],"task":"domain"},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi\n\nAgain<｜Assistant｜></think>","token_count":7,"token_sha256":"d05e248d8a06f2c06df20b7a71af0d4dfae1cd91ce9f614719b5842674042d6b"},
|
||||
{"name":"preview/effort_high","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜><think>","token_count":5,"token_sha256":"c6c0b4c94f78803396fe1b21a3613dd337930a46baf38d285233fe84a60ae7c7"},
|
||||
{"name":"preview/effort_max","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"max"},"prompt":"<｜begin▁of▁sentence｜>Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n<｜User｜>Hi<｜Assistant｜><think>","token_count":84,"token_sha256":"daf853d041e8dde87f07e95a201f8f07459c88482fe53e46472605d63fbdbeac"},
|
||||
{"name":"preview/task_action","profile":"preview","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"task":"action"},"prompt":"<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜></think><｜action｜>","token_count":6,"token_sha256":"925bcaae248cc41efabd3e5d8f660f17513e43174f1d34aba51a1399aba76e8e"},
|
||||
{"name":"official/effort_high","profile":"official","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"high"},"prompt":"<｜begin▁of▁sentence｜>Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n<｜User｜>Hi<｜Assistant｜><think>","token_count":84,"token_sha256":"daf853d041e8dde87f07e95a201f8f07459c88482fe53e46472605d63fbdbeac"},
|
||||
{"name":"official/effort_max","profile":"official","request":{"model":"m","messages":[{"role":"user","content":"Hi"}],"reasoning_effort":"max"},"prompt":"<｜begin▁of▁sentence｜>Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\nYou MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\nDo not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n<｜User｜>Hi<｜Assistant｜><think>","token_count":97,"token_sha256":"82ee92542b8545282dcc47a141cb9262d6ab3e20463e0baf445557ca0a9b905c"}
|
||||
]}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Regenerate native DeepSeek fixtures with SGLang's actual serving pipeline.
|
||||
|
||||
Run from experimental/sgl-router in a SGLang Python environment:
|
||||
python tests/scripts/generate_deepseek_parity.py
|
||||
|
||||
The Rust tests always compare rendered text, even without cached model files.
|
||||
With the pinned HF snapshot cached they also compare exact token-ID digests.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
os.environ["SGLANG_DEFAULT_THINKING"] = "false"
|
||||
os.environ["SGLANG_DSV4_REASONING_EFFORT"] = ""
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] / "fixtures/deepseek"
|
||||
MODELS = {
|
||||
"v4": ("deepseek-ai/DeepSeek-V4-Flash", "60d8d70770c6776ff598c94bb586a859a38244f1"),
|
||||
}
|
||||
|
||||
|
||||
class RecordingTokenizer:
|
||||
def __init__(self, tokenizer):
|
||||
self.tokenizer = tokenizer
|
||||
self.texts = []
|
||||
|
||||
def encode(self, text):
|
||||
self.texts.append(text)
|
||||
return self.tokenizer.encode(text)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.tokenizer, name)
|
||||
|
||||
|
||||
def main():
|
||||
ROOT.mkdir(exist_ok=True)
|
||||
for family, (model, revision) in MODELS.items():
|
||||
path = snapshot_download(model, revision=revision, local_files_only=True)
|
||||
tok = RecordingTokenizer(
|
||||
AutoTokenizer.from_pretrained(path, local_files_only=True)
|
||||
)
|
||||
server = object.__new__(OpenAIServingChat)
|
||||
server.chat_encoding_spec = "ds" + family
|
||||
server.tokenizer_manager = SimpleNamespace(tokenizer=tok)
|
||||
server.template_manager = SimpleNamespace(
|
||||
jinja_template_content_format="string"
|
||||
)
|
||||
fixture_path = ROOT / (family + ".json")
|
||||
fixture = json.loads(fixture_path.read_text())
|
||||
for case in fixture["cases"]:
|
||||
server._dsv4_reasoning_effort_profile = case["profile"]
|
||||
tok.texts.clear()
|
||||
request = ChatCompletionRequest(**copy.deepcopy(case["request"]))
|
||||
# _convert_to_internal_request: kwargs effort replaces the request effort.
|
||||
if request.chat_template_kwargs:
|
||||
effort = request.chat_template_kwargs.pop("reasoning_effort", None)
|
||||
if effort is not None:
|
||||
request.reasoning_effort = effort
|
||||
ids = server._apply_jinja_template(
|
||||
request, tools=None, is_multimodal=False
|
||||
).prompt_ids
|
||||
case.update(
|
||||
prompt="".join(tok.texts),
|
||||
token_count=len(ids),
|
||||
token_sha256=hashlib.sha256(
|
||||
b"".join(i.to_bytes(4, "little") for i in ids)
|
||||
).hexdigest(),
|
||||
)
|
||||
metadata = {k: v for k, v in fixture.items() if k != "cases"}
|
||||
fixture_path.write_text(
|
||||
json.dumps(metadata)[:-1]
|
||||
+ ', "cases": [\n'
|
||||
+ ",\n".join(
|
||||
json.dumps(c, ensure_ascii=False, separators=(",", ":"))
|
||||
for c in fixture["cases"]
|
||||
)
|
||||
+ "\n]}\n"
|
||||
)
|
||||
print(f"wrote {family}: {len(fixture['cases'])} cases")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user