[Feat] openai: Render DeepSeek-V4/V4.1 chat prompts with deepseek-recipe (#45315)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Jackey Hua
2026-10-23 18:05:35 -07:00
committed by GitHub
co-authored by hnyls2002
parent 49a2fbf817
commit e22a530859
25 changed files with 1107 additions and 2494 deletions
@@ -757,12 +757,12 @@ SGLang supports various environment variables that can be used to configure its
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts <code>high</code> and <code>max</code>; the official profile accepts <code>low</code>, <code>high</code>, and <code>max</code>. The profile is detected from the bundled encoder. Override it with <code>--json-model-override-args '&#123;"dsv4_reasoning_effort_profile":"official"&#125;'</code>.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for DeepSeek-V4 when a request does not set it: a deepseek-recipe tier name. Unset means deepseek-recipe's default.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV41_REASONING_EFFORT</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek-V4.1 chat encoder when a request does not set it: one of <code>low</code>, <code>high</code>, <code>xhigh</code>, <code>max</code>, or an integer budget in [1, 100]. Unset means the encoder default <code>high</code>.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for DeepSeek-V4.1 when a request does not set it: a deepseek-recipe tier name or an integer budget in [1, 100]. Unset means deepseek-recipe's default.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
</tr>
<tr>
+1
View File
@@ -32,6 +32,7 @@ dependencies = [
"cuda-python>=13.0",
"datasets",
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
"distro",
"easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports
"einops",
+1
View File
@@ -23,6 +23,7 @@ dependencies = [
"build",
"compressed-tensors",
"datasets",
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
"easydict",
"einops",
"fastapi",
+1
View File
@@ -24,6 +24,7 @@ dependencies = [
"compressed-tensors",
"datasets",
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
"easydict",
"einops",
"fastapi",
+1
View File
@@ -28,6 +28,7 @@ runtime_base = [
"blobfile==3.0.0",
"build",
"datasets",
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
"easydict",
"einops",
"fastapi",
+1
View File
@@ -24,6 +24,7 @@ dependencies = [
"build",
"compressed-tensors",
"datasets",
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
"easydict",
"einops",
"fastapi",
@@ -7,102 +7,7 @@ it here instead of re-deriving it from model architectures themselves.
from __future__ import annotations
import ast
import logging
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv41
logger = logging.getLogger(__name__)
DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile"
_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py"
_MAX_DSV4_ENCODER_BYTES = 1 << 20
def _detect_dsv4_reasoning_effort_profile(
model_path: str, revision: Optional[str] = None
) -> Optional[str]:
encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER
try:
if not encoder_path.is_file():
from huggingface_hub import hf_hub_download
encoder_path = Path(
hf_hub_download(
model_path,
_DSV4_REASONING_EFFORT_ENCODER,
revision=revision,
)
)
if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES:
return None
tree = ast.parse(encoder_path.read_text(encoding="utf-8"))
except Exception as error:
logger.debug(
"Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s",
encoder_path,
error,
)
return None
assignments = {}
for node in tree.body:
if isinstance(node, ast.Assign):
targets = node.targets
value = node.value
elif isinstance(node, ast.AnnAssign):
targets = [node.target]
value = node.value
else:
continue
for target in targets:
if not isinstance(target, ast.Name):
continue
try:
assignments[target.id] = ast.literal_eval(value)
except (TypeError, ValueError):
continue
prompts = assignments.get("REASONING_EFFORT_PROMPTS")
if (
assignments.get("DEFAULT_REASONING_EFFORT") == "low"
and isinstance(prompts, dict)
and {"low", "high", "max"} <= prompts.keys()
):
return "official"
if "REASONING_EFFORT_MAX" in assignments:
return "preview"
return None
def _validate_dsv4_reasoning_effort_profile(profile: str) -> str:
if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES:
raise ValueError(
f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; "
f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}"
)
return profile
def resolve_dsv4_reasoning_effort_profile(
*,
model_path: str,
revision: Optional[str] = None,
override: Optional[str] = None,
) -> str:
if override is not None:
return _validate_dsv4_reasoning_effort_profile(override)
return (
_detect_dsv4_reasoning_effort_profile(
model_path=model_path,
revision=revision,
)
or "preview"
)
from typing import Any, Dict, List, Optional
def is_deepseek_v41_arch(*, arch: str, model_type: str) -> bool:
@@ -154,56 +59,6 @@ def resolve_chat_encoding_spec(
return None
def parse_dsv41_reasoning_effort(value: Any) -> Union[str, int, None]:
"""Map an API ``reasoning_effort`` onto what the V4.1 encoder accepts.
An int budget only reaches here through ``chat_template_kwargs``; None
means unsupported, and the caller applies its default.
"""
if isinstance(value, bool):
return None
if isinstance(value, int):
return value if 1 <= value <= 100 else None
if isinstance(value, float):
return max(1, round(value * 100)) if 0.0 <= value <= 0.99 else None
if value in encoding_dsv41.REASONING_EFFORT_MAPPINGS:
return value
return None
_OPENAI_FUNCTION_FIELD_ORDER = ("name", "description", "parameters")
def dsv41_tool_payload(tool: Any) -> Dict[str, Any]:
"""The tool dict the V4.1 encoder renders verbatim into the prompt.
Only fields the client sent, in the OpenAI field order; pydantic would
otherwise add defaults (strict=false) and reorder keys by declaration.
"""
payload = tool.model_dump(exclude_unset=True, exclude_none=True)
function = dict(payload.get("function") or {})
ordered = {
k: function.pop(k) for k in _OPENAI_FUNCTION_FIELD_ORDER if k in function
}
ordered.update(function)
payload["function"] = ordered
return payload
def default_dsv41_reasoning_effort_from_env(raw: Optional[str]) -> Union[str, int]:
"""Parse ``SGLANG_DSV41_REASONING_EFFORT``; raises so a bad value fails at boot."""
if raw is None or not raw.strip():
return encoding_dsv41.DEFAULT_REASONING_EFFORT
value: Any = int(raw) if raw.strip().isdigit() else raw.strip()
effort = parse_dsv41_reasoning_effort(value)
if effort is None:
raise ValueError(
f"Invalid SGLANG_DSV41_REASONING_EFFORT={raw!r}; expected one of "
f"{list(encoding_dsv41.REASONING_EFFORT_MAPPINGS)} or an integer in [1, 100]"
)
return effort
def spec_supports_inline_system(spec: Optional[str]) -> bool:
"""Native encoders verified to render mid-conversation system messages in place."""
# K3's tokenizer renders in Python, so there is no Jinja source to probe.
@@ -244,34 +99,30 @@ def encode_simple_chat(
Minimal encode for offline tools: no tools, no multimodal content, no
continue_final_message; the serving path keeps its full request-level
pipeline in ``serving_chat``. System-message handling matches
``serving_chat``: dsv4/dsv32 get an empty one prepended, dsv41 does not
(it renders a system token even for empty content).
pipeline in ``serving_chat``. dsv32 gets an empty system message prepended,
as ``serving_chat`` does; DeepSeek-V4 / V4.1 render through deepseek-recipe.
"""
if spec == "inkling":
from sglang.srt.parser.inkling_renderer import render_inkling_messages
return render_inkling_messages(messages)
if spec in ("dsv4", "dsv32", "dsv41"):
if spec != "dsv41" and messages and messages[0]["role"] != "system":
if spec in ("dsv4", "dsv41"):
from sglang.srt.parser.deepseek_recipe_renderer import RecipeEncoder
encoder = RecipeEncoder.from_hf_tokenizer(spec, tokenizer)
return encoder.encode(
{"messages": list(messages), "thinking": thinking_mode == "thinking"}
)
if spec == "dsv32":
from sglang.srt.entrypoints.openai import encoding_dsv32
if messages and messages[0]["role"] != "system":
messages = [{"role": "system", "content": ""}] + list(messages)
if spec == "dsv4":
from sglang.srt.entrypoints.openai import encoding_dsv4
real_input = encoding_dsv4.encode_messages(
messages, thinking_mode=thinking_mode
)
elif spec == "dsv41":
real_input = encoding_dsv41.encode_messages(
messages, thinking_mode=thinking_mode
)
else:
from sglang.srt.entrypoints.openai import encoding_dsv32
real_input = encoding_dsv32.encode_messages(
messages, thinking_mode=thinking_mode
)
real_input = encoding_dsv32.encode_messages(
messages, thinking_mode=thinking_mode
)
return tokenizer.encode(real_input)
if getattr(tokenizer, "chat_template", None) is None:
@@ -70,6 +70,10 @@ class ChatEncodingAdapter(Protocol):
"""Whether media still has to be collected from the OpenAI messages after
``encode_messages`` returns prompt ids."""
strict_tool_call_arguments: bool
"""Whether history tool-call ``arguments`` must parse as JSON before
``encode_messages`` sees them; False when the encoding serializes them itself."""
@property
def output(self) -> Optional[ChatEncodingOutputAdapter]:
"""None when the configured parsers leave output to the text parsers."""
@@ -81,19 +85,26 @@ class ChatEncodingAdapter(Protocol):
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
tools: Optional[list[dict]],
) -> list[int]: ...
thinking: bool,
image_data: list,
) -> list[int]:
"""``thinking`` is the handler's thinking decision for ``request``. An
encoding that collects media itself appends the prompt's images to
``image_data`` in placeholder order."""
...
class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
"""Everything a builder may read; a builder takes only what its encoding needs.
Consumers:
tokenizer: prompt encoding (Inkling).
tokenizer: prompt encoding (Inkling, DeepSeek-V4 / V4.1).
reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing.
hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id).
model_path, revision: checkpoint files (DeepSeek tokenizer.json and the
encoder file for effort-profile detection).
backend: selects the builder; "recipe" is an explicit opt-in.
model_path, revision: checkpoint files.
default_chat_template_kwargs: the workers' --default-chat-template-kwargs
(deepseek-recipe validates the default effort at startup).
backend: selects the builder; every spec has a "builtin" one.
"""
chat_encoding_spec: str
@@ -104,6 +115,7 @@ class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
hf_config: Any
model_path: str
revision: Optional[str]
default_chat_template_kwargs: Optional[dict[str, Any]] = None
backend: EncodingBackend = "builtin"
def require(self, field: str) -> Any:
@@ -127,11 +139,32 @@ def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter:
)
def _build_deepseek_recipe(
ctx: ChatEncodingAdapterContext,
) -> Optional[ChatEncodingAdapter]:
if ctx.tokenizer is None:
return None # --skip-tokenizer-init: the server encodes no chat prompts.
from sglang.srt.entrypoints.openai.serving_deepseek_recipe import (
DeepSeekRecipeChatEncodingAdapter,
)
return DeepSeekRecipeChatEncodingAdapter(
chat_encoding_spec=ctx.chat_encoding_spec,
tokenizer=ctx.tokenizer,
default_chat_template_kwargs=ctx.default_chat_template_kwargs,
)
_BUILDERS: dict[
str,
dict[EncodingBackend, Callable[[ChatEncodingAdapterContext], ChatEncodingAdapter]],
dict[
EncodingBackend,
Callable[[ChatEncodingAdapterContext], Optional[ChatEncodingAdapter]],
],
] = {
"inkling": {"builtin": _build_inkling},
"dsv4": {"builtin": _build_deepseek_recipe},
"dsv41": {"builtin": _build_deepseek_recipe},
}
@@ -1,884 +0,0 @@
# Adapted from the DeepSeek-V4 release reference implementation.
"""
DeepSeek-V4 Encoding
A self-contained implementation for encoding/decoding DeepSeek-V4 chat messages
with tool calling, thinking mode, and quick instruction task support.
"""
import copy
import json
import re
from typing import Any, Dict, List, Optional, Tuple, Union
# ============================================================
# Special Tokens
# ============================================================
bos_token: str = "<|begin▁of▁sentence|>"
eos_token: str = "<|end▁of▁sentence|>"
thinking_start_token: str = "<think>"
thinking_end_token: str = "</think>"
dsml_token: str = "|DSML|"
USER_SP_TOKEN = "<|User|>"
ASSISTANT_SP_TOKEN = "<|Assistant|>"
LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>"
# Task special tokens for internal classification tasks
DS_TASK_SP_TOKENS = {
"action": "<|action|>",
"query": "<|query|>",
"authority": "<|authority|>",
"domain": "<|domain|>",
"title": "<|title|>",
"read_url": "<|read_url|>",
}
VALID_TASKS = set(DS_TASK_SP_TOKENS.keys())
# ============================================================
# Templates
# ============================================================
system_msg_template: str = "{content}"
user_msg_template: str = "{content}"
latest_reminder_msg_template: str = "{content}"
assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token
assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}"
thinking_template: str = "{reasoning_content}"
response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"
tool_call_template: str = (
'<{dsml_token}invoke name="{name}">\n{arguments}\n</{dsml_token}invoke>'
)
tool_calls_template = (
"<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
)
tool_calls_block_name: str = "tool_calls"
tool_output_template: str = "<tool_result>{content}</tool_result>"
REASONING_EFFORT_PREVIEW_MAX = (
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
)
REASONING_EFFORT_OFFICIAL_MAX = (
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
)
REASONING_EFFORT_PROFILES = {
"preview": {
"high": "",
"max": REASONING_EFFORT_PREVIEW_MAX,
},
"official": {
"low": "",
"high": REASONING_EFFORT_PREVIEW_MAX,
"max": REASONING_EFFORT_OFFICIAL_MAX,
},
}
TOOLS_TEMPLATE = """## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
<{dsml_token}tool_calls>
<{dsml_token}invoke name="$TOOL_NAME">
<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}parameter>
...
</{dsml_token}invoke>
<{dsml_token}invoke name="$TOOL_NAME2">
...
</{dsml_token}invoke>
</{dsml_token}tool_calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
Otherwise, output directly after {thinking_end_token} with tool calls or final response.
### Available Tool Schemas
{tool_schemas}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
"""
# ============================================================
# Utility Functions
# ============================================================
def to_json(value: Any) -> str:
"""Serialize a value to JSON string."""
try:
return json.dumps(value, ensure_ascii=False)
except:
return json.dumps(value, ensure_ascii=True)
def tools_from_openai_format(tools):
"""Extract function definitions from OpenAI-format tool list."""
return [tool["function"] for tool in tools]
def tool_calls_from_openai_format(tool_calls):
"""Convert OpenAI-format tool calls to internal format."""
return [
{
"name": tool_call["function"]["name"],
"arguments": tool_call["function"]["arguments"],
}
for tool_call in tool_calls
]
def tool_calls_to_openai_format(tool_calls):
"""Convert internal tool calls to OpenAI format."""
return [
{
"type": "function",
"function": {
"name": tool_call["name"],
"arguments": tool_call["arguments"],
},
}
for tool_call in tool_calls
]
def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str:
"""
Encode tool call arguments into DSML parameter format.
Args:
tool_call: Dict with "name" and "arguments" keys.
Returns:
DSML-formatted parameter string.
"""
p_dsml_template = '<{dsml_token}parameter name="{key}" string="{is_str}">{value}</{dsml_token}parameter>'
P_dsml_strs = []
raw_arguments = tool_call["arguments"]
arguments = (
json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
)
if not isinstance(arguments, dict):
raise ValueError(
"Assistant tool call function.arguments must be a JSON object."
)
for k, v in arguments.items():
p_dsml_str = p_dsml_template.format(
dsml_token=dsml_token,
key=k,
is_str="true" if isinstance(v, str) else "false",
value=v if isinstance(v, str) else to_json(v),
)
P_dsml_strs.append(p_dsml_str)
return "\n".join(P_dsml_strs)
def decode_dsml_to_arguments(
tool_name: str, tool_args: Dict[str, Tuple[str, str]]
) -> Dict[str, str]:
"""
Decode DSML parameters back to a tool call dict.
Args:
tool_name: Name of the tool.
tool_args: Dict mapping param_name -> (value, is_string_flag).
Returns:
Dict with "name" and "arguments" (JSON string) keys.
"""
def _decode_value(key: str, value: str, string: str):
if string == "true":
value = to_json(value)
return f"{to_json(key)}: {value}"
tool_args_json = (
"{"
+ ", ".join(
[_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]
)
+ "}"
)
return dict(name=tool_name, arguments=tool_args_json)
def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str:
"""
Render tool schemas into the system prompt format.
Args:
tools: List of tool schema dicts (each with name, description, parameters).
Returns:
Formatted tools section string.
"""
tools_json = [to_json(t) for t in tools]
return TOOLS_TEMPLATE.format(
tool_schemas="\n".join(tools_json),
dsml_token=dsml_token,
thinking_start_token=thinking_start_token,
thinking_end_token=thinking_end_token,
)
def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
"""Find the index of the last user/developer message."""
last_user_index = -1
for idx in range(len(messages) - 1, -1, -1):
if messages[idx].get("role") in ["user", "developer"]:
last_user_index = idx
break
return last_user_index
def attach_task_to_last_user_message(messages: List[Dict[str, Any]], task: str) -> None:
"""Set `task` on the most recent user/developer message; raise if none exists."""
idx = find_last_user_index(messages)
if idx == -1:
raise ValueError(
"`task` requires at least one message with role='user' or 'developer'."
)
messages[idx]["task"] = task
# ============================================================
# Message Rendering
# ============================================================
def render_message(
index: int,
messages: List[Dict[str, Any]],
thinking_mode: str,
drop_thinking: bool = True,
reasoning_effort: Optional[str] = None,
reasoning_effort_profile: str = "preview",
) -> str:
"""
Render a single message at the given index into its encoded string form.
This is the core function that converts each message in the conversation
into the DeepSeek-V4 format.
Args:
index: Index of the message to render.
messages: Full list of messages in the conversation.
thinking_mode: Either "chat" or "thinking".
drop_thinking: Whether to drop reasoning content from earlier turns.
reasoning_effort: Optional reasoning effort level. The preview profile accepts
"high" and "max"; the official profile accepts "low", "high", and "max".
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
Returns:
Encoded string for this message.
"""
assert 0 <= index < len(messages)
assert thinking_mode in [
"chat",
"thinking",
], f"Invalid thinking_mode `{thinking_mode}`"
prompt = ""
msg = messages[index]
last_user_idx = find_last_user_index(messages)
role = msg.get("role")
content = msg.get("content")
tools = msg.get("tools")
response_format = msg.get("response_format")
tool_calls = msg.get("tool_calls")
reasoning_content = msg.get("reasoning_content")
wo_eos = msg.get("wo_eos", False)
if tools:
tools = tools_from_openai_format(tools)
if tool_calls:
tool_calls = tool_calls_from_openai_format(tool_calls)
if reasoning_effort_profile not in REASONING_EFFORT_PROFILES:
raise ValueError(
f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; "
f"expected one of {list(REASONING_EFFORT_PROFILES)}"
)
effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile]
if reasoning_effort is None:
reasoning_effort = "low" if reasoning_effort_profile == "official" else "high"
if reasoning_effort not in effort_prompts:
raise ValueError(
f"Invalid reasoning effort {reasoning_effort!r} for profile "
f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}"
)
if index == 0 and thinking_mode == "thinking":
prompt += effort_prompts[reasoning_effort]
if role == "system":
prompt += system_msg_template.format(content=content or "")
if tools:
prompt += "\n\n" + render_tools(tools)
if response_format:
prompt += "\n\n" + response_format_template.format(
schema=to_json(response_format)
)
elif role == "developer":
assert content, f"Invalid message for role `{role}`: {msg}"
content_developer = USER_SP_TOKEN
content_developer += content
if tools:
content_developer += "\n\n" + render_tools(tools)
if response_format:
content_developer += "\n\n" + response_format_template.format(
schema=to_json(response_format)
)
prompt += user_msg_template.format(content=content_developer)
elif role == "user":
prompt += USER_SP_TOKEN
# Handle content blocks (tool results mixed with text)
content_blocks = msg.get("content_blocks")
if content_blocks:
parts = []
for block in content_blocks:
block_type = block.get("type")
if block_type == "text":
parts.append(block.get("text", ""))
elif block_type == "tool_result":
tool_content = block.get("content", "")
if isinstance(tool_content, list):
text_parts = []
for b in tool_content:
if b.get("type") == "text":
text_parts.append(b.get("text", ""))
else:
text_parts.append(f"[Unsupported {b.get('type')}]")
tool_content = "\n\n".join(text_parts)
parts.append(tool_output_template.format(content=tool_content))
else:
parts.append(f"[Unsupported {block_type}]")
prompt += "\n\n".join(parts)
else:
prompt += content or ""
elif role == "latest_reminder":
prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(
content=content
)
elif role == "tool":
raise NotImplementedError(
"deepseek_v4 merges tool messages into user; please preprocess with merge_tool_messages()"
)
elif role == "assistant":
thinking_part = ""
tc_content = ""
if tool_calls:
tc_list = [
tool_call_template.format(
dsml_token=dsml_token,
name=tc.get("name"),
arguments=encode_arguments_to_dsml(tc),
)
for tc in tool_calls
]
tc_content += "\n\n" + tool_calls_template.format(
dsml_token=dsml_token,
tool_calls="\n".join(tc_list),
tc_block_name=tool_calls_block_name,
)
summary_content = content or ""
rc = reasoning_content or ""
# Check if previous message has a task - if so, this is a task output (no thinking)
prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None
if thinking_mode == "thinking" and not prev_has_task:
if not drop_thinking or index > last_user_idx:
thinking_part = (
thinking_template.format(reasoning_content=rc) + thinking_end_token
)
else:
thinking_part = ""
if wo_eos:
prompt += assistant_msg_wo_eos_template.format(
reasoning=thinking_part,
content=summary_content,
tool_calls=tc_content,
)
else:
prompt += assistant_msg_template.format(
reasoning=thinking_part,
content=summary_content,
tool_calls=tc_content,
)
else:
raise NotImplementedError(f"Unknown role: {role}")
# Append transition tokens based on what follows
if index + 1 < len(messages) and messages[index + 1].get("role") not in [
"assistant",
"latest_reminder",
]:
return prompt
task = messages[index].get("task")
if task is not None:
# Task special token for internal classification tasks
assert task in VALID_TASKS, (
f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
)
task_sp_token = DS_TASK_SP_TOKENS[task]
if task != "action":
# Non-action tasks: append task sp token directly after the message
prompt += task_sp_token
else:
# Action task: append Assistant + thinking token + action sp token
prompt += ASSISTANT_SP_TOKEN
prompt += (
thinking_end_token
if thinking_mode != "thinking"
else thinking_start_token
)
prompt += task_sp_token
elif messages[index].get("role") in ["user", "developer"]:
# Normal generation: append Assistant + thinking token
prompt += ASSISTANT_SP_TOKEN
if not drop_thinking and thinking_mode == "thinking":
prompt += thinking_start_token
elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx:
prompt += thinking_start_token
else:
prompt += thinking_end_token
return prompt
# ============================================================
# Preprocessing
# ============================================================
def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Merge tool messages into the preceding user message using content_blocks format.
DeepSeek-V4 does not have a standalone "tool" role; instead, tool results
are encoded as <tool_result> blocks within user messages.
This function converts a standard OpenAI-format conversation (with separate
"tool" role messages) into V4 format where tool results are merged into
user messages.
Args:
messages: List of message dicts in OpenAI format.
Returns:
Processed message list with tool messages merged into user messages.
"""
merged: List[Dict[str, Any]] = []
for msg in messages:
msg = copy.deepcopy(msg)
role = msg.get("role")
if role == "tool":
# Convert tool message to a user message with tool_result block
tool_block = {
"type": "tool_result",
"tool_use_id": msg.get("tool_call_id", ""),
"content": msg.get("content", ""),
}
# Merge into previous message if it's already a user (merged tool)
if (
merged
and merged[-1].get("role") == "user"
and "content_blocks" in merged[-1]
):
merged[-1]["content_blocks"].append(tool_block)
else:
merged.append(
{
"role": "user",
"content_blocks": [tool_block],
}
)
elif role == "user":
text_block = {"type": "text", "text": msg.get("content", "")}
if (
merged
and merged[-1].get("role") == "user"
and "content_blocks" in merged[-1]
and merged[-1].get("task") is None
):
merged[-1]["content_blocks"].append(text_block)
else:
new_msg = {
"role": "user",
"content": msg.get("content", ""),
"content_blocks": [text_block],
}
# Preserve extra fields (task, wo_eos, mask, etc.)
for key in ("task", "wo_eos", "mask"):
if key in msg:
new_msg[key] = msg[key]
merged.append(new_msg)
else:
merged.append(msg)
return merged
def sort_tool_results_by_call_order(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""
Sort tool_result blocks within user messages by the order of tool_calls
in the preceding assistant message.
Args:
messages: Preprocessed message list (after merge_tool_messages).
Returns:
Message list with sorted tool result blocks.
"""
last_tool_call_order: Dict[str, int] = {}
for msg in messages:
role = msg.get("role")
if role == "assistant" and msg.get("tool_calls"):
last_tool_call_order = {}
for idx, tc in enumerate(msg["tool_calls"]):
tc_id = tc.get("id") or tc.get("function", {}).get("id", "")
if tc_id:
last_tool_call_order[tc_id] = idx
elif role == "user" and msg.get("content_blocks"):
tool_blocks = [
b for b in msg["content_blocks"] if b.get("type") == "tool_result"
]
if len(tool_blocks) > 1 and last_tool_call_order:
sorted_blocks = sorted(
tool_blocks,
key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0),
)
sorted_idx = 0
new_blocks = []
for block in msg["content_blocks"]:
if block.get("type") == "tool_result":
new_blocks.append(sorted_blocks[sorted_idx])
sorted_idx += 1
else:
new_blocks.append(block)
msg["content_blocks"] = new_blocks
return messages
# ============================================================
# Main Encoding Function
# ============================================================
def encode_messages(
messages: List[Dict[str, Any]],
thinking_mode: str,
context: Optional[List[Dict[str, Any]]] = None,
drop_thinking: bool = True,
add_default_bos_token: bool = True,
reasoning_effort: Optional[str] = None,
reasoning_effort_profile: str = "preview",
) -> str:
"""
Encode a list of messages into the DeepSeek-V4 prompt format.
This is the main entry point for encoding conversations. It handles:
- BOS token insertion
- Thinking mode with optional reasoning content dropping
- Tool message merging into user messages
- Multi-turn conversation context
Args:
messages: List of message dicts to encode.
thinking_mode: Either "chat" or "thinking".
context: Optional preceding context messages (already encoded prefix).
drop_thinking: If True, drop reasoning_content from earlier assistant turns
(only keep reasoning for messages after the last user message).
add_default_bos_token: Whether to prepend BOS token at conversation start.
reasoning_effort: Optional reasoning effort level. The preview profile accepts
"high" and "max"; the official profile accepts "low", "high", and "max".
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
Returns:
The encoded prompt string.
"""
context = context if context else []
# Preprocess: merge tool messages and sort tool results
messages = merge_tool_messages(messages)
messages = sort_tool_results_by_call_order(context + messages)[len(context) :]
if context:
context = merge_tool_messages(context)
context = sort_tool_results_by_call_order(context)
full_messages = context + messages
prompt = bos_token if add_default_bos_token and len(context) == 0 else ""
# Resolve drop_thinking: if any message has tools defined, don't drop thinking
effective_drop_thinking = drop_thinking
if any(m.get("tools") for m in full_messages):
effective_drop_thinking = False
if thinking_mode == "thinking" and effective_drop_thinking:
full_messages = _drop_thinking_messages(full_messages)
# After dropping, recalculate how many messages to render
# (context may have shrunk too)
num_to_render = len(full_messages) - len(_drop_thinking_messages(context))
context_len = len(full_messages) - num_to_render
else:
num_to_render = len(messages)
context_len = len(context)
for idx in range(num_to_render):
prompt += render_message(
idx + context_len,
full_messages,
thinking_mode=thinking_mode,
drop_thinking=effective_drop_thinking,
reasoning_effort=reasoning_effort,
reasoning_effort_profile=reasoning_effort_profile,
)
return prompt
def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
Drop reasoning_content and non-essential messages before the last user message.
Behavior:
- Messages with role in ["user", "system", "tool", "latest_reminder"] are always kept.
- Messages at or after the last user index are always kept.
- Assistant messages before the last user get reasoning_content removed.
- Developer messages before the last user are dropped entirely.
"""
last_user_idx = find_last_user_index(messages)
result = []
keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"}
for idx, msg in enumerate(messages):
role = msg.get("role")
if role in keep_roles or idx >= last_user_idx:
result.append(msg)
elif role == "assistant":
msg = copy.copy(msg)
msg.pop("reasoning_content", None)
result.append(msg)
# developer and other roles before last_user_idx are dropped
return result
# ============================================================
# Parsing (Decoding model output)
# ============================================================
def _read_until_stop(
index: int, text: str, stop: List[str]
) -> Tuple[int, str, Optional[str]]:
"""
Read text from index until one of the stop strings is found.
Returns:
Tuple of (new_index, content_before_stop, matched_stop_string_or_None).
"""
min_pos = len(text)
matched_stop = None
for s in stop:
pos = text.find(s, index)
if pos != -1 and pos < min_pos:
min_pos = pos
matched_stop = s
if matched_stop:
content = text[index:min_pos]
return min_pos + len(matched_stop), content, matched_stop
else:
content = text[index:]
return len(text), content, None
def parse_tool_calls(
index: int, text: str
) -> Tuple[int, Optional[str], List[Dict[str, str]]]:
"""
Parse DSML tool calls from text starting at the given index.
Args:
index: Starting position in text.
text: The full text to parse.
Returns:
Tuple of (new_index, last_stop_token, list_of_tool_call_dicts).
Each tool call dict has "name" and "arguments" keys.
"""
tool_calls: List[Dict[str, Any]] = []
stop_token = None
tool_calls_end_token = f"</{dsml_token}{tool_calls_block_name}>"
while index < len(text):
index, _, stop_token = _read_until_stop(
index, text, [f"<{dsml_token}invoke", tool_calls_end_token]
)
if _ != ">\n":
raise ValueError(f"Tool call format error: expected '>\\n' but got '{_}'")
if stop_token == tool_calls_end_token:
break
if stop_token is None:
raise ValueError("Missing special token in tool calls")
index, tool_name_content, stop_token = _read_until_stop(
index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"]
)
p_tool_name = re.findall(
r'^\s*name="(.*?)">\n$', tool_name_content, flags=re.DOTALL
)
if len(p_tool_name) != 1:
raise ValueError(f"Tool name format error: '{tool_name_content}'")
tool_name = p_tool_name[0]
tool_args: Dict[str, Tuple[str, str]] = {}
while stop_token == f"<{dsml_token}parameter":
index, param_content, stop_token = _read_until_stop(
index, text, [f"/{dsml_token}parameter"]
)
param_kv = re.findall(
r'^ name="(.*?)" string="(true|false)">(.*?)<$',
param_content,
flags=re.DOTALL,
)
if len(param_kv) != 1:
raise ValueError(f"Parameter format error: '{param_content}'")
param_name, string, param_value = param_kv[0]
if param_name in tool_args:
raise ValueError(f"Duplicate parameter name: '{param_name}'")
tool_args[param_name] = (param_value, string)
index, content, stop_token = _read_until_stop(
index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"]
)
if content != ">\n":
raise ValueError(
f"Parameter format error: expected '>\\n' but got '{content}'"
)
tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args)
tool_calls.append(tool_call)
return index, stop_token, tool_calls
def parse_message_from_completion_text(text: str, thinking_mode: str) -> Dict[str, Any]:
"""
Parse a model completion text into a structured assistant message.
This function takes the raw text output from the model (a single assistant turn)
and extracts:
- reasoning_content (thinking block)
- content (summary/response)
- tool_calls (if any)
NOTE: This function is designed to parse only correctly formatted strings and
will raise ValueError for malformed output.
Args:
text: The raw completion text (including EOS token).
thinking_mode: Either "chat" or "thinking".
Returns:
Dict with keys: "role", "content", "reasoning_content", "tool_calls".
tool_calls are in OpenAI format.
"""
summary_content, reasoning_content, tool_calls = "", "", []
index, stop_token = 0, None
tool_calls_start_token = f"\n\n<{dsml_token}{tool_calls_block_name}"
is_thinking = thinking_mode == "thinking"
is_tool_calling = False
if is_thinking:
index, content_delta, stop_token = _read_until_stop(
index, text, [thinking_end_token, tool_calls_start_token]
)
reasoning_content = content_delta
assert stop_token == thinking_end_token, (
"Invalid thinking format: missing </think>"
)
index, content_delta, stop_token = _read_until_stop(
index, text, [eos_token, tool_calls_start_token]
)
summary_content = content_delta
if stop_token == tool_calls_start_token:
is_tool_calling = True
else:
assert stop_token == eos_token, "Invalid format: missing EOS token"
if is_tool_calling:
index, stop_token, tool_calls = parse_tool_calls(index, text)
index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token])
assert not tool_ends_text, "Unexpected content after tool calls"
assert len(text) == index and stop_token in [
eos_token,
None,
], "Unexpected content at end"
for sp_token in [
bos_token,
eos_token,
thinking_start_token,
thinking_end_token,
dsml_token,
]:
assert sp_token not in summary_content and sp_token not in reasoning_content, (
f"Unexpected special token '{sp_token}' in content"
)
return {
"role": "assistant",
"content": summary_content,
"reasoning_content": reasoning_content,
"tool_calls": tool_calls_to_openai_format(tool_calls),
}
@@ -1,719 +0,0 @@
# Adapted from the DeepSeek-V4.1 release reference implementation.
"""Encode DeepSeek-V4.1 chat messages.
Mid-conversation system messages trigger the assistant generation header.
"""
import copy
import json
from typing import Any, Dict, List, Optional, Tuple, Union
# ============================================================
# Special Tokens
# ============================================================
bos_token: str = "<|begin▁of▁sentence|>"
eos_token: str = "<|end▁of▁sentence|>"
thinking_start_token: str = "<think>"
thinking_end_token: str = "</think>"
dsml_token: str = "|DSML|"
USER_SP_TOKEN = "<|User|>"
ASSISTANT_SP_TOKEN = "<|Assistant|>"
SYSTEM_SP_TOKEN = "<|System|>"
LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>"
IMAGE_PLACEHOLDER = "<|deepseek_image|>"
# Task special tokens for internal classification tasks
DS_TASK_SP_TOKENS = {
"action": "<|action|>",
"query": "<|query|>",
"authority": "<|authority|>",
"domain": "<|domain|>",
"title": "<|title|>",
"read_url": "<|read_url|>",
}
VALID_TASKS = set(DS_TASK_SP_TOKENS.keys())
# ============================================================
# Templates
# ============================================================
system_msg_template: str = "{content}"
user_msg_template: str = "{content}"
latest_reminder_msg_template: str = "{content}"
assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token
assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}"
thinking_template: str = "{reasoning_content}"
response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"
tool_calls_block_name: str = " calls"
tool_call_tag_name: str = " invoke"
tool_parameter_tag_name: str = " parameter"
tool_call_template: str = '<{dsml_token}{tool_call_tag_name} name="{name}">\n{arguments}\n</{dsml_token}{tool_call_tag_name}>'
tool_calls_template = (
"<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
)
tool_output_template: str = "<tool_result>{content}</tool_result>"
REASONING_EFFORT_TEMPLATE = (
"Reasoning Effort: {budget} "
"(range 1-100, the higher the value, the more thorough the reasoning)\n\n"
)
REASONING_EFFORT_MAPPINGS: Dict[str, int] = {
"low": 50,
"high": 75,
"xhigh": 75,
"max": 100,
}
DEFAULT_REASONING_EFFORT = "high"
TOOLS_TEMPLATE = """## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following:
<{dsml_token}{tc_block_name}>
<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME">
<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}{tool_parameter_tag_name}>
...
</{dsml_token}{tool_call_tag_name}>
<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2">
...
</{dsml_token}{tool_call_tag_name}>
</{dsml_token}{tc_block_name}>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
Otherwise, output directly after {thinking_end_token} with tool calls or final response.
### Available Tool Schemas
{tool_schemas}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
"""
# ============================================================
# Utility Functions
# ============================================================
def to_json(value: Any) -> str:
try:
return json.dumps(value, ensure_ascii=False)
except:
return json.dumps(value, ensure_ascii=True)
def tools_from_openai_format(tools):
return [tool["function"] for tool in tools]
def tool_calls_from_openai_format(tool_calls):
return [
{
"name": tool_call["function"]["name"],
"arguments": tool_call["function"]["arguments"],
}
for tool_call in tool_calls
]
def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str:
p_dsml_template = (
'<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">'
"{value}</{dsml_token}{tool_parameter_tag_name}>"
)
P_dsml_strs = []
raw_arguments = tool_call["arguments"]
arguments = (
json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
)
if not isinstance(arguments, dict):
raise ValueError(
"Assistant tool call function.arguments must be a JSON object."
)
for k, v in arguments.items():
P_dsml_strs.append(
p_dsml_template.format(
dsml_token=dsml_token,
tool_parameter_tag_name=tool_parameter_tag_name,
key=k,
is_str="true" if isinstance(v, str) else "false",
value=v if isinstance(v, str) else to_json(v),
)
)
return "\n".join(P_dsml_strs)
def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str:
tools_json = [to_json(t) for t in tools]
return TOOLS_TEMPLATE.format(
tool_schemas="\n".join(tools_json),
dsml_token=dsml_token,
tc_block_name=tool_calls_block_name,
tool_call_tag_name=tool_call_tag_name,
tool_parameter_tag_name=tool_parameter_tag_name,
thinking_start_token=thinking_start_token,
thinking_end_token=thinking_end_token,
)
def render_reasoning_effort(
index: int,
thinking_mode: str,
effort: Union[str, int, None],
) -> str:
"""Render the numeric reasoning effort prefix (thinking mode, index 0 only)."""
if effort is None:
effort = DEFAULT_REASONING_EFFORT
if not (
(type(effort) is int and 1 <= effort <= 100)
or effort in REASONING_EFFORT_MAPPINGS
):
raise ValueError(
f"Invalid reasoning effort for deepseek_v41: {effort!r}, should be "
f"int within [1,100] or {list(REASONING_EFFORT_MAPPINGS)}"
)
if type(effort) is str:
effort = REASONING_EFFORT_MAPPINGS[effort]
if index == 0 and thinking_mode == "thinking":
return REASONING_EFFORT_TEMPLATE.format(budget=effort)
return ""
def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
"""Mid-conversation system messages also count as user messages here;
they trigger the assistant generation header.
User messages containing only tool results do not start a new turn.
"""
last_user_index = -1
for idx in range(len(messages) - 1, -1, -1):
role = messages[idx].get("role")
if role in ["user", "developer"] or (role == "system" and idx > 0):
content_blocks = messages[idx].get("content_blocks")
if content_blocks and not any(
block.get("type") in ("text", "image", "image_url")
for block in content_blocks
):
continue
last_user_index = idx
break
return last_user_index
def attach_task_to_last_user_message(messages: List[Dict[str, Any]], task: str) -> None:
"""Set `task` on the most recent user/developer message; raise if none exists."""
idx = find_last_user_index(messages)
if idx == -1:
raise ValueError(
"`task` requires at least one message with role='user' or 'developer'."
)
messages[idx]["task"] = task
# ============================================================
# Message Rendering
# ============================================================
def render_message(
index: int,
messages: List[Dict[str, Any]],
thinking_mode: str,
drop_thinking: bool = True,
reasoning_effort: Union[str, int, None] = None,
) -> str:
assert 0 <= index < len(messages)
assert thinking_mode in [
"chat",
"thinking",
], f"Invalid thinking_mode `{thinking_mode}`"
msg = messages[index]
last_user_idx = find_last_user_index(messages)
role = msg.get("role")
content = msg.get("content")
tools = msg.get("tools")
response_format = msg.get("response_format")
tool_calls = msg.get("tool_calls")
reasoning_content = msg.get("reasoning_content")
wo_eos = msg.get("wo_eos", False)
if tools:
tools = tools_from_openai_format(tools)
if tool_calls:
tool_calls = tool_calls_from_openai_format(tool_calls)
reasoning_effort_prompt = render_reasoning_effort(
index, thinking_mode, reasoning_effort
)
# Index 0 also emits the system token for the effort prompt, even before a user.
prompt = (
SYSTEM_SP_TOKEN
if index == 0 and (reasoning_effort_prompt or role == "system")
else ""
)
prompt += reasoning_effort_prompt
if role == "system":
if index > 0:
prompt += SYSTEM_SP_TOKEN
prompt += system_msg_template.format(content=content or "")
if tools:
prompt += "\n\n" + render_tools(tools)
if response_format:
prompt += "\n\n" + response_format_template.format(
schema=to_json(response_format)
)
elif role == "developer":
assert content, f"Invalid message for role `{role}`: {msg}"
content_developer = USER_SP_TOKEN
content_developer += content
if tools:
content_developer += "\n\n" + render_tools(tools)
if response_format:
content_developer += "\n\n" + response_format_template.format(
schema=to_json(response_format)
)
prompt += user_msg_template.format(content=content_developer)
elif role == "user":
prompt += USER_SP_TOKEN
# Handle content blocks (tool results mixed with text)
content_blocks = msg.get("content_blocks")
if content_blocks:
parts = []
for block in content_blocks:
block_type = block.get("type")
if block_type == "text":
parts.append(block.get("text", ""))
elif block_type == "tool_result":
tool_content = block.get("content", "")
if isinstance(tool_content, list):
text_parts = []
for b in tool_content:
if b.get("type") == "text":
text_parts.append(b.get("text", ""))
else:
text_parts.append(f"[Unsupported {b.get('type')}]")
tool_content = "\n\n".join(text_parts)
parts.append(tool_output_template.format(content=tool_content))
else:
parts.append(f"[Unsupported {block_type}]")
prompt += "\n\n".join(parts)
else:
prompt += content or ""
elif role == "latest_reminder":
prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(
content=content
)
elif role == "tool":
raise NotImplementedError(
"deepseek_v41 merges tool messages into user; please preprocess with merge_tool_messages()"
)
elif role == "assistant":
thinking_part = ""
tc_content = ""
if tool_calls:
tc_list = [
tool_call_template.format(
dsml_token=dsml_token,
tool_call_tag_name=tool_call_tag_name,
name=tc.get("name"),
arguments=encode_arguments_to_dsml(tc),
)
for tc in tool_calls
]
tc_content += "\n\n" + tool_calls_template.format(
dsml_token=dsml_token,
tool_calls="\n".join(tc_list),
tc_block_name=tool_calls_block_name,
)
summary_content = content or ""
rc = reasoning_content or ""
# Check if previous message has a task - if so, this is a task output (no thinking)
prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None
if thinking_mode == "thinking" and not prev_has_task:
if not drop_thinking or index > last_user_idx:
thinking_part = (
thinking_template.format(reasoning_content=rc) + thinking_end_token
)
else:
thinking_part = ""
if wo_eos:
prompt += assistant_msg_wo_eos_template.format(
reasoning=thinking_part,
content=summary_content,
tool_calls=tc_content,
)
else:
prompt += assistant_msg_template.format(
reasoning=thinking_part,
content=summary_content,
tool_calls=tc_content,
)
else:
raise NotImplementedError(f"Unknown role: {role}")
# Append transition tokens based on what follows
if index + 1 < len(messages) and messages[index + 1].get("role") not in [
"assistant",
"latest_reminder",
]:
return prompt
task = messages[index].get("task")
if task is not None:
# Task special token for internal classification tasks
assert task in VALID_TASKS, (
f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
)
task_sp_token = DS_TASK_SP_TOKENS[task]
if task != "action":
# Non-action tasks: append task sp token directly after the message
prompt += task_sp_token
else:
# Action task: append Assistant + thinking token + action sp token
prompt += ASSISTANT_SP_TOKEN
prompt += (
thinking_end_token
if thinking_mode != "thinking"
else thinking_start_token
)
prompt += task_sp_token
elif messages[index].get("role") in ["user", "developer"] or (
messages[index].get("role") == "system" and index > 0
):
# Normal generation: append Assistant + thinking token
prompt += ASSISTANT_SP_TOKEN
if not drop_thinking and thinking_mode == "thinking":
prompt += thinking_start_token
elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx:
prompt += thinking_start_token
else:
prompt += thinking_end_token
return prompt
# ============================================================
# Preprocessing
# ============================================================
def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Tool results are encoded within user messages;
DeepSeek-V4.1 has no standalone tool role.
"""
merged: List[Dict[str, Any]] = []
for msg in messages:
msg = copy.deepcopy(msg)
role = msg.get("role")
if role == "tool":
# Convert tool message to a user message with tool_result block
content = msg.get("content", "")
if msg.get("content_blocks") and not isinstance(msg.get("content"), str):
content = msg["content_blocks"]
tool_block = {
"type": "tool_result",
"tool_use_id": msg.get("tool_call_id", ""),
"content": content,
}
# Merge into previous message if it's already a user (merged tool)
if (
merged
and merged[-1].get("role") == "user"
and "content_blocks" in merged[-1]
):
merged[-1]["content_blocks"].append(tool_block)
else:
merged.append(
{
"role": "user",
"content_blocks": [tool_block],
}
)
elif role == "user":
content_blocks = msg.get("content_blocks")
if content_blocks is None:
content = msg.get("content", "")
content_blocks = (
content
if isinstance(content, list)
else [{"type": "text", "text": content}]
)
if (
merged
and merged[-1].get("role") == "user"
and "content_blocks" in merged[-1]
and merged[-1].get("task") is None
):
merged[-1]["content_blocks"].extend(content_blocks)
else:
# Keeps structured content and every message-level field.
new_msg = msg
new_msg["content_blocks"] = content_blocks
merged.append(new_msg)
else:
merged.append(msg)
return merged
def sort_tool_results_by_call_order(
messages: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
last_tool_call_order: Dict[str, int] = {}
for msg in messages:
role = msg.get("role")
if role == "assistant" and msg.get("tool_calls"):
last_tool_call_order = {}
for idx, tc in enumerate(msg["tool_calls"]):
tc_id = tc.get("id") or tc.get("function", {}).get("id", "")
if tc_id:
last_tool_call_order[tc_id] = idx
elif role == "user" and msg.get("content_blocks"):
tool_blocks = [
b for b in msg["content_blocks"] if b.get("type") == "tool_result"
]
if len(tool_blocks) > 1 and last_tool_call_order:
sorted_blocks = sorted(
tool_blocks,
key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0),
)
sorted_idx = 0
new_blocks = []
for block in msg["content_blocks"]:
if block.get("type") == "tool_result":
new_blocks.append(sorted_blocks[sorted_idx])
sorted_idx += 1
else:
new_blocks.append(block)
msg["content_blocks"] = new_blocks
return messages
# ============================================================
# Vision Message Preprocessing
# ============================================================
def _is_image_block(block: Dict[str, Any]) -> bool:
return isinstance(block, dict) and block.get("type") == "image_url"
def _extract_image(block: Dict[str, Any]) -> Dict[str, Any]:
image_url = block.get("image_url")
url = image_url if isinstance(image_url, str) else (image_url or {}).get("url", "")
if not url:
raise ValueError("Image block does not contain a valid source")
return {"url": url}
def _process_image_blocks(
blocks: List[Any], image_placeholder: str = IMAGE_PLACEHOLDER
) -> Tuple[List[Any], List[Dict[str, Any]]]:
new_blocks: List[Any] = []
images: List[Dict[str, Any]] = []
for block in blocks:
if not isinstance(block, dict):
new_blocks.append(block)
continue
if _is_image_block(block):
new_blocks.append({"type": "text", "text": image_placeholder})
images.append(_extract_image(block))
elif block.get("type") == "tool_result" and isinstance(
block.get("content"), list
):
block = copy.copy(block)
block["content"], nested_images = _process_image_blocks(
block["content"], image_placeholder
)
new_blocks.append(block)
images.extend(nested_images)
elif block.get("type") == "text":
text = block.get("text") or ""
if IMAGE_PLACEHOLDER in text:
raise ValueError(
f"Text block contains image placeholder '{IMAGE_PLACEHOLDER}': "
f"'{text[:100]}'. Images should be separate content blocks."
)
new_blocks.append(block)
else:
new_blocks.append(block)
return new_blocks, images
def _validate_no_image_sp_tokens(msg: Dict[str, Any]) -> None:
content = msg.get("content")
if isinstance(content, str) and IMAGE_PLACEHOLDER in content:
raise ValueError(
f"Message content contains image special token '{IMAGE_PLACEHOLDER}'. "
"Images should be provided as image content blocks."
)
reasoning_content = msg.get("reasoning_content")
if isinstance(reasoning_content, str) and IMAGE_PLACEHOLDER in reasoning_content:
raise ValueError(
f"reasoning_content contains image special token '{IMAGE_PLACEHOLDER}'"
)
def process_image_messages(
messages: List[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
processed: List[Dict[str, Any]] = []
images: List[Dict[str, Any]] = []
for msg in messages:
msg = copy.deepcopy(msg)
_validate_no_image_sp_tokens(msg)
if isinstance(msg.get("content"), list) and "content_blocks" not in msg:
msg["content_blocks"] = msg.pop("content")
if msg.get("content_blocks"):
msg["content_blocks"], message_images = _process_image_blocks(
msg["content_blocks"]
)
images.extend(message_images)
if not isinstance(msg.get("content"), str):
texts = [
block.get("text", "")
for block in msg["content_blocks"]
if isinstance(block, dict) and block.get("type") == "text"
]
msg["content"] = "\n\n".join(texts)
processed.append(msg)
return processed, images
# ============================================================
# Main Encoding Function
# ============================================================
def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
last_user_idx = find_last_user_index(messages)
result = []
keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"}
for idx, msg in enumerate(messages):
role = msg.get("role")
if role in keep_roles or idx >= last_user_idx:
result.append(msg)
elif role == "assistant":
msg = copy.copy(msg)
msg.pop("reasoning_content", None)
result.append(msg)
# developer and other roles before last_user_idx are dropped
return result
def _encode_messages_text(
messages: List[Dict[str, Any]],
thinking_mode: str,
context: Optional[List[Dict[str, Any]]] = None,
drop_thinking: bool = True,
add_default_bos_token: bool = True,
reasoning_effort: Union[str, int, None] = None,
) -> str:
"""Encode preprocessed (text-only) messages into the V4.1 prompt format."""
context = context if context else []
full_messages = context + messages
prompt = bos_token if add_default_bos_token and len(context) == 0 else ""
# Resolve drop_thinking: if any message has tools defined, don't drop thinking
effective_drop_thinking = drop_thinking
if any(m.get("tools") for m in full_messages):
effective_drop_thinking = False
if thinking_mode == "thinking" and effective_drop_thinking:
full_messages = _drop_thinking_messages(full_messages)
num_to_render = len(full_messages) - len(_drop_thinking_messages(context))
context_len = len(full_messages) - num_to_render
else:
num_to_render = len(messages)
context_len = len(context)
for idx in range(num_to_render):
prompt += render_message(
idx + context_len,
full_messages,
thinking_mode=thinking_mode,
drop_thinking=effective_drop_thinking,
reasoning_effort=reasoning_effort,
)
return prompt
def encode_messages(
messages: List[Dict[str, Any]],
thinking_mode: str,
context: Optional[List[Dict[str, Any]]] = None,
drop_thinking: bool = True,
add_default_bos_token: bool = True,
reasoning_effort: Union[str, int, None] = None,
return_multi_modal_data: bool = False,
) -> Any:
"""Encode a list of messages into the DeepSeek-V4.1 prompt format.
Returns the prompt string, or ``(prompt, {"images": [...]})`` when
``return_multi_modal_data`` is set; the image records are in prompt order.
"""
# Validate before merging can discard message-level textual fields.
for msg in (context or []) + messages:
_validate_no_image_sp_tokens(msg)
# Collect images only after tool results are in their final prompt order.
context = merge_tool_messages(context or [])
messages = merge_tool_messages(messages)
messages = sort_tool_results_by_call_order(context + messages)[len(context) :]
processed_context, _ = process_image_messages(context) if context else ([], [])
processed_messages, images = process_image_messages(messages)
prompt = _encode_messages_text(
processed_messages,
thinking_mode=thinking_mode,
context=processed_context if processed_context else None,
drop_thinking=drop_thinking,
add_default_bos_token=add_default_bos_token,
reasoning_effort=reasoning_effort,
)
if return_multi_modal_data:
return prompt, {"images": images}
return prompt
@@ -745,6 +745,8 @@ class ChatCompletionMessageGenericParam(BaseModel):
reasoning_content: Optional[str] = None
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
# DeepSeek API prefix completion: continue this final assistant message.
prefix: Optional[bool] = None
@field_validator("role", mode="before")
@classmethod
@@ -838,9 +840,10 @@ class ToolChoice(BaseModel):
# OpenAI-spec string tiers for reasoning effort (current Responses/Chat API):
# none/minimal/low/medium/high/xhigh/max. Used as-is by /v1/responses.
# none/minimal/low/medium/high/xhigh/max, plus DeepSeek's ultra. Used as-is by
# /v1/responses.
ReasoningEffortTier = Literal[
"none", "minimal", "low", "medium", "high", "xhigh", "max"
"none", "minimal", "low", "medium", "high", "xhigh", "ultra", "max"
]
# The typed /v1/responses stream events validate against OpenAI's narrower
# ``Reasoning.effort``; echoing a tier outside this set kills the stream.
@@ -934,7 +937,7 @@ class ChatCompletionRequest(PDRoutingFields):
reasoning_effort: ReasoningEffortType = Field(
default=None,
description="Constrains effort on reasoning for reasoning models. "
"Accepts string levels ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max') or a "
"Accepts string levels ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'ultra', 'max') or a "
"float in [0.0, 0.99] for fine-grained control. "
"'none' disables reasoning entirely, 'low' is the least effort, 'high' is the most effort. "
"Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning "
@@ -1063,6 +1066,7 @@ class ChatCompletionRequest(PDRoutingFields):
"medium",
"high",
"xhigh",
"ultra",
"max",
}:
values["reasoning_effort"] = effort
@@ -1778,7 +1782,7 @@ class ResponseReasoningParam(BaseModel):
default="medium",
description="Constrains effort on reasoning for reasoning models. "
"Accepts the OpenAI string tiers "
"('none','minimal','low','medium','high','xhigh','max').",
"('none','minimal','low','medium','high','xhigh','ultra','max').",
)
summary: Optional[Literal["auto", "concise", "detailed"]] = Field(
default=None,
@@ -9,7 +9,7 @@ from collections import OrderedDict
from collections.abc import AsyncGenerator
from enum import Enum
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional
from sglang.srt.runtime_context import get_model, get_serving
@@ -42,9 +42,7 @@ from jsonschema import Draft202012Validator, SchemaError
from sglang.srt.entrypoints.openai import (
chat_encoding,
chat_encoding_adapters,
encoding_dsv4,
encoding_dsv32,
encoding_dsv41,
)
from sglang.srt.entrypoints.openai.parallel_prompt_encode import (
parallel_prompt_encode,
@@ -127,7 +125,6 @@ from sglang.srt.parser.template_detection import detect_inline_system_support
from sglang.srt.sampling.sampling_params import (
set_request_reasoning_end_token_ids,
)
from sglang.srt.utils import ImageData
from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata
if TYPE_CHECKING:
@@ -348,26 +345,7 @@ class OpenAIServingChat(OpenAIServingBase):
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
self.supports_inline_system = self._resolve_inline_system_support()
self._dsv4_reasoning_effort_profile = (
chat_encoding.resolve_dsv4_reasoning_effort_profile(
model_path=self.tokenizer_manager.model_path,
revision=get_model().revision,
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
),
)
if self.chat_encoding_spec == "dsv4"
else None
)
self._encoding_adapter = self._build_chat_encoding_adapter()
self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = (
chat_encoding.default_dsv41_reasoning_effort_from_env(
envs.SGLANG_DSV41_REASONING_EFFORT.get()
)
if self.chat_encoding_spec == "dsv41"
else None
)
# Per-request response parser for custom decoding (set by _encode_messages)
self._response_parser: ResponseParserProtocol | None = None
@@ -521,6 +499,7 @@ class OpenAIServingChat(OpenAIServingBase):
hf_config=self.tokenizer_manager.model_config.hf_config,
model_path=self.tokenizer_manager.model_path,
revision=get_model().revision,
default_chat_template_kwargs=get_serving().default_chat_template_kwargs,
)
)
@@ -625,6 +604,8 @@ class OpenAIServingChat(OpenAIServingBase):
request: ChatCompletionRequest,
thinking_mode: ThinkingMode,
tools: list[dict] | None = None,
*,
image_data: list,
) -> list[int] | None:
"""Encode messages for custom chat_encoding_spec values.
@@ -632,7 +613,11 @@ class OpenAIServingChat(OpenAIServingBase):
"""
if self._encoding_adapter is not None:
return self._encoding_adapter.encode_messages(
messages=messages, request=request, tools=tools
messages=messages,
request=request,
tools=tools,
thinking=thinking_mode == ThinkingMode.THINKING,
image_data=image_data,
)
if self.chat_encoding_spec == "kimi_k3":
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
@@ -701,21 +686,6 @@ class OpenAIServingChat(OpenAIServingBase):
return prompt_ids
return None
def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]:
"""Request effort for the V4.1 encoder; unsupported values warn and fall back."""
effort = chat_encoding.parse_dsv41_reasoning_effort(value)
if effort is not None:
return effort
if value is not None and value != "none":
logger.warning(
"DeepSeek-V4.1 does not support reasoning_effort=%r; using the "
"default %r (low/high/xhigh/max, a float in [0, 0.99], or an "
"integer budget in [1, 100] via chat_template_kwargs are accepted).",
value,
self._dsv41_default_reasoning_effort,
)
return self._dsv41_default_reasoning_effort
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
"""Extract text from response."""
return ret_item["text"]
@@ -1591,9 +1561,14 @@ class OpenAIServingChat(OpenAIServingBase):
ThinkingMode.THINKING if thinking_requested else ThinkingMode.CHAT
)
messages = [msg.model_dump() for msg in request.messages]
strict_tool_call_arguments = (
self._encoding_adapter.strict_tool_call_arguments
if self._encoding_adapter is not None
else self.chat_encoding_spec != "kimi_k3"
)
for message in messages:
normalize_assistant_tool_call_arguments(
message, strict=self.chat_encoding_spec != "kimi_k3"
message, strict=strict_tool_call_arguments
)
prompt_ids = self._encode_messages(
@@ -1601,9 +1576,12 @@ class OpenAIServingChat(OpenAIServingBase):
request,
thinking_mode,
tools=tools,
image_data=image_data,
)
if prompt_ids is not None:
if image_data and not is_multimodal:
raise ValueError("image input is not supported for this model")
collects_openai_media = (
self._encoding_adapter.collects_openai_media
if self._encoding_adapter is not None
@@ -1623,33 +1601,30 @@ class OpenAIServingChat(OpenAIServingBase):
modalities,
)
elif self.chat_encoding_spec is not None:
# dsv4/dsv41/dsv32 encoding path
# dsv32 encoding path
messages = copy.deepcopy(messages)
is_dsv41 = self.chat_encoding_spec == "dsv41"
for msg in messages:
if msg.get("content") is None:
msg["content"] = ""
# The V4.1 encoder consumes OpenAI parts lists itself; dsv4/dsv32
# are text-only, so their parts-list content is flattened first.
if not is_dsv41:
for i, msg in enumerate(messages):
if isinstance(msg.get("content"), list):
messages[i] = process_content_for_template_format(
msg, "string", [], [], [], []
)
for msg in messages:
processed_msg = process_content_for_template_format(
msg,
template_content_format,
image_data,
video_data,
audio_data,
modalities,
use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32",
# The dsv32 encoder is text-only, so parts-list content is flattened first.
for i, msg in enumerate(messages):
if isinstance(msg.get("content"), list):
messages[i] = process_content_for_template_format(
msg, "string", [], [], [], []
)
msg.update(processed_msg)
for msg in messages:
processed_msg = process_content_for_template_format(
msg,
template_content_format,
image_data,
video_data,
audio_data,
modalities,
use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32",
)
msg.update(processed_msg)
# Handle continue_final_message: separate final assistant message
messages, assistant_prefix = self._handle_last_assistant_message(
@@ -1658,78 +1633,16 @@ class OpenAIServingChat(OpenAIServingBase):
request_tools = request.tools if tools else None
# An empty system message hosts the request tools; dsv41 renders a
# system token for it, so it only gets one when tools need the host.
if messages[0]["role"] != "system" and (request_tools or not is_dsv41):
# An empty system message hosts the request tools.
if messages[0]["role"] != "system":
messages.insert(0, {"role": "system", "content": ""})
if request_tools:
messages[0]["tools"] = [
(
chat_encoding.dsv41_tool_payload(tool)
if is_dsv41
else tool.model_dump()
)
for tool in request_tools
]
messages[0]["tools"] = [tool.model_dump() for tool in request_tools]
# Default encoding (dsv4/dsv41/dsv32)
if self.chat_encoding_spec == "dsv4":
effort_source = request.reasoning_effort
if effort_source is None:
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
if env_val:
effort_source = env_val
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
assert reasoning_effort_profile is not None
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
reasoning_effort_profile
]
v4_reasoning_effort = (
effort_source if effort_source in accepted_efforts else None
)
if request.task is not None:
encoding_dsv4.attach_task_to_last_user_message(
messages, request.task
)
real_input = encoding_dsv4.encode_messages(
messages,
thinking_mode=thinking_mode,
reasoning_effort=v4_reasoning_effort,
reasoning_effort_profile=reasoning_effort_profile,
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
elif is_dsv41:
if request.task is not None:
encoding_dsv41.attach_task_to_last_user_message(
messages, request.task
)
real_input, media = encoding_dsv41.encode_messages(
messages,
thinking_mode=thinking_mode,
reasoning_effort=self._resolve_dsv41_reasoning_effort(
request.reasoning_effort
),
return_multi_modal_data=True,
)
if media["images"]:
if not is_multimodal:
raise ValueError("image input is not supported for this model")
image_data.extend(
ImageData(url=image["url"]) for image in media["images"]
)
tokenizer = self.tokenizer_manager.tokenizer
real_input = real_input.replace(
encoding_dsv41.IMAGE_PLACEHOLDER,
tokenizer.convert_ids_to_tokens(
self.tokenizer_manager.image_token_id
),
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
else:
real_input = encoding_dsv32.encode_messages(
messages, thinking_mode=thinking_mode
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
real_input = encoding_dsv32.encode_messages(
messages, thinking_mode=thinking_mode
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
# Append assistant prefix if continue_final_message is enabled
if assistant_prefix:
@@ -0,0 +1,127 @@
"""DeepSeek-V4 / V4.1 serving glue for the ``dsv4`` / ``dsv41`` chat encoding
specs: prompt encoding through deepseek-recipe under SGLang's request semantics."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional, Union
from sglang.srt.environ import envs
from sglang.srt.parser.deepseek_recipe_renderer import (
RecipeEncoder,
parse_effort_budget,
)
from sglang.srt.utils import ImageData
if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
DEFAULT_EFFORT_ENVS = {
"dsv4": envs.SGLANG_DSV4_REASONING_EFFORT,
"dsv41": envs.SGLANG_DSV41_REASONING_EFFORT,
}
class DeepSeekRecipeChatEncodingAdapter:
# recipe sorts tool results by call order, so images come from its render.
collects_openai_media = False
output = None
strict_tool_call_arguments = False # recipe serializes object arguments itself.
def __init__(
self,
*,
chat_encoding_spec: str,
tokenizer: Any,
default_chat_template_kwargs: Optional[dict[str, Any]],
):
self._spec = chat_encoding_spec
self.encoder = RecipeEncoder.from_hf_tokenizer(chat_encoding_spec, tokenizer)
raw = (DEFAULT_EFFORT_ENVS[chat_encoding_spec].get() or "").strip()
self._default_effort = int(raw) if raw.isdigit() else raw or None
# Encoded at boot, so a bad server default fails startup, not requests.
defaults = default_chat_template_kwargs or {}
self._encode(
{"messages": [{"role": "user", "content": ""}], "thinking": True},
effort=defaults.get("reasoning_effort"),
)
def encode_messages(
self,
*,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
tools: Optional[list[dict]],
thinking: bool,
image_data: list,
) -> list[int]:
# The wire request: the handler's message dumps carry None for unset fields.
wire = request.model_dump(exclude_unset=True, by_alias=True, mode="json")
wire_messages = wire["messages"]
continue_final = request.continue_final_message
task_prefix = None
# SGLang's trailing-assistant rule, as `_handle_last_assistant_message`.
last = wire_messages[-1] if wire_messages else None
if (
last
and last["role"] == "assistant"
and isinstance(last.get("content"), str)
):
if not continue_final:
wire_messages[-1] = {"role": "user", "content": last["content"]}
elif request.task is not None:
# A task renders its own tail, so the prefix follows it.
task_prefix = wire_messages.pop()["content"]
continue_final = False
body = {
"messages": wire_messages,
"thinking": thinking,
"continue_final_message": continue_final,
"response_format": wire.get("response_format"),
}
if tools is not None:
# `model_dump` adds None for unset fields, which recipe renders as values.
body["tools"] = [
{k: v for k, v in tool.items() if v is not None}
| {
"function": {
k: v for k, v in tool["function"].items() if v is not None
}
}
for tool in tools
]
body["tool_choice"] = wire.get("tool_choice")
body = {key: value for key, value in body.items() if value is not None}
image_urls = []
prompt_ids = self._encode(
body,
effort=request.reasoning_effort,
task=request.task,
image_urls=image_urls,
)
image_data.extend(ImageData(url=url) for url in image_urls)
if task_prefix:
prompt_ids += self.encoder.encode_text(task_prefix)
return prompt_ids
def _encode(
self,
body: dict[str, Any],
*,
effort: Union[str, int, float, None],
task: Optional[str] = None,
image_urls: Optional[list[str]] = None,
) -> list[int]:
if effort is None or effort == "none":
effort = self._default_effort
budget = None
if effort is not None and not isinstance(effort, str):
if self._spec != "dsv41":
raise ValueError(
f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}"
)
budget = parse_effort_budget(effort)
elif effort is not None:
body["reasoning_effort"] = effort
return self.encoder.encode(
body, task=task, effort_budget=budget, image_urls=image_urls
)
@@ -35,6 +35,7 @@ if TYPE_CHECKING:
class InklingChatEncodingAdapter:
collects_openai_media = True
strict_tool_call_arguments = True
def __init__(
self,
@@ -73,7 +74,11 @@ class InklingChatEncodingAdapter:
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
tools: list[dict] | None,
thinking: bool,
image_data: list,
) -> list[int]:
# Inkling's thinking is effort-conditioned and the handler collects its
# media, so `thinking` and `image_data` are not read.
reasoning_effort = parse_reasoning_effort(request.reasoning_effort)
if reasoning_effort is None:
reasoning_effort = self._default_reasoning_effort
+4 -2
View File
@@ -1699,10 +1699,12 @@ class Envs:
SGLANG_DSV4_FP4_EXPERTS = EnvBool(True)
# Set True to dequantize the FP4 experts to FP8 at runtime
SGLANG_DSV4_FP4_DEQUANT = EnvBool(False)
# Flash-0731 also accepts "low"; the active profile is checkpoint-resolved.
# DeepSeek-V4 default when a request carries no reasoning_effort: a
# deepseek-recipe tier name; unset -> deepseek-recipe's default.
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
# DeepSeek-V4.1 default when a request carries no reasoning_effort: one of
# low/high/xhigh/max or an integer budget in [1, 100]; unset -> the encoder default.
# a deepseek-recipe tier name or an integer budget in [1, 100]; unset ->
# deepseek-recipe's default.
SGLANG_DSV41_REASONING_EFFORT = EnvStr(None)
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
@@ -0,0 +1,199 @@
"""DeepSeek-V4 / V4.1 prompts through deepseek-recipe, plus the SGLang extensions
recipe does not model (quick instruction tasks, V4.1 integer effort)."""
from __future__ import annotations
import copy
import functools
import json
from types import ModuleType
from typing import Any, Optional, Union
DEEPSEEK_RECIPE_INSTALL_HINT = (
"Serving DeepSeek-V4 / V4.1 requires the `deepseek-recipe` package (DeepSeek's "
"reference encoder and parser; wheels for Linux and macOS). Install it with "
"`pip install deepseek-recipe`."
)
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
IMAGE_TOKEN = "<|deepseek_image|>"
ASSISTANT_TOKEN = "<|Assistant|>"
THINK_START_TOKEN = "<think>"
THINK_END_TOKEN = "</think>"
TASK_TOKENS = {
"action": "<|action|>",
"query": "<|query|>",
"authority": "<|authority|>",
"domain": "<|domain|>",
"title": "<|title|>",
"read_url": "<|read_url|>",
}
# V4.1 renders an integer budget as this line; recipe's "high" tier is budget 75.
BUDGET_TIER, BUDGET_TIER_VALUE = "high", 75
BUDGET_LINE = "Reasoning Effort: {budget}"
@functools.cache
def load_deepseek_recipe() -> ModuleType:
try:
import deepseek_recipe
except ImportError as exc:
raise ImportError(DEEPSEEK_RECIPE_INSTALL_HINT) from exc
return deepseek_recipe
def parse_effort_budget(value: Union[int, float]) -> int:
if isinstance(value, bool):
raise ValueError("reasoning_effort must not be a boolean")
if isinstance(value, int) and 1 <= value <= 100:
return value
if isinstance(value, float) and 0.0 <= value <= 0.99:
return max(1, round(value * 100))
raise ValueError(
f"reasoning_effort={value!r} is outside the integer budget [1, 100] "
"and the float range [0, 0.99]"
)
def pre_normalize(body: dict[str, Any]) -> dict[str, Any]:
"""Rewrite an SGLang chat request body for recipe's Chat Completions adapter.
`body` carries only what the prompt depends on: messages, thinking and, when
set, tools, tool_choice, response_format, continue_final_message and a named
reasoning_effort.
"""
request = copy.deepcopy(body)
request["thinking"] = {"type": "enabled" if request["thinking"] else "disabled"}
# The host enforces structural_tag with its grammar.
response_format = request.get("response_format")
if response_format and response_format.get("type") == "structural_tag":
del request["response_format"]
choice = request.get("tool_choice")
if not request.get("tools") or choice == "none":
request.pop("tools", None)
request.pop("tool_choice", None)
elif choice == "required" or isinstance(choice, dict):
# SGLang enforces the choice with its grammar, which expects the model
# to open the calls block itself.
request["tool_choice"] = "auto"
messages = request["messages"]
for message in messages:
if message["role"] == "developer":
message["role"] = "system"
for call in message.get("tool_calls") or []:
function = call["function"]
if "arguments" in function and not isinstance(function["arguments"], str):
function["arguments"] = json.dumps(
function["arguments"], ensure_ascii=False, separators=(",", ":")
)
continue_final = request.pop("continue_final_message", False)
if continue_final and messages and messages[-1]["role"] == "assistant":
messages[-1]["prefix"] = True
return request
def _error_message(body: Any) -> str:
try:
return json.loads(body)["error"]["message"]
except (TypeError, ValueError, KeyError):
return str(body)
class RecipeEncoder:
"""`tokenizer` is the checkpoint's `tokenizers.Tokenizer`; recipe encodes prompts
with its own copy of it, and the SGLang extensions splice ids from the original."""
def __init__(self, chat_encoding_spec: str, tokenizer: Any):
recipe = load_deepseek_recipe()
self.model = MODEL_NAMES[chat_encoding_spec]
self.hf_tokenizer = tokenizer
special = recipe.SpecialTokens(image=IMAGE_TOKEN)
self._options = recipe.ConversionOptions().with_special_tokens(special)
encoding = (
recipe.DeepseekV4Encoding()
if chat_encoding_spec == "dsv4"
else recipe.DeepseekV41Encoding()
)
self._encoding = encoding.with_special_tokens(special).with_tokenizer(
recipe.Tokenizer.from_str(tokenizer.to_str())
)
@classmethod
def from_hf_tokenizer(
cls, chat_encoding_spec: str, tokenizer: Any
) -> RecipeEncoder:
backend = getattr(tokenizer, "backend_tokenizer", None)
if backend is None:
raise ValueError(
"deepseek-recipe needs the checkpoint's fast tokenizer (tokenizer.json)"
)
return cls(chat_encoding_spec, backend)
def encode(
self,
body: dict[str, Any],
*,
task: Optional[str] = None,
effort_budget: Optional[int] = None,
image_urls: Optional[list[str]] = None,
) -> list[int]:
"""Prompt ids for `body` (see `pre_normalize`); a request recipe rejects
raises ValueError. `image_urls` receives the images in placeholder order."""
recipe = load_deepseek_recipe()
normalized = pre_normalize(body)
normalized["model"] = self.model
if effort_budget is not None:
normalized["reasoning_effort"] = BUDGET_TIER
try:
converted = recipe.ChatCompletionRequest(normalized).convert(self._options)
except recipe.ConversionError as error:
raise ValueError(_error_message(error.body)) from error
prompt_ids = self._encoding.encode(converted.conversation)
if image_urls is not None:
rendered = self._encoding.render_conversation(converted.conversation)
image_urls.extend(
source.url or source.data_url for source in rendered.image_sources
)
if effort_budget is not None and body["thinking"]:
prompt_ids = self._replace_effort_budget(prompt_ids, effort_budget)
if task is not None:
prompt_ids = self._attach_task(prompt_ids, task, body["thinking"])
return prompt_ids
def encode_text(self, text: str) -> list[int]:
return self.hf_tokenizer.encode(text, add_special_tokens=False).ids
def _replace_effort_budget(self, prompt_ids: list[int], budget: int) -> list[int]:
# The effort line opens the prompt, so its first match is the line itself.
rendered = self.encode_text(BUDGET_LINE.format(budget=BUDGET_TIER_VALUE))
for start in range(len(prompt_ids) - len(rendered) + 1):
if prompt_ids[start : start + len(rendered)] == rendered:
end = start + len(rendered)
return (
prompt_ids[:start]
+ self.encode_text(BUDGET_LINE.format(budget=budget))
+ prompt_ids[end:]
)
raise ValueError("deepseek-recipe rendered no reasoning effort line")
def _attach_task(
self, prompt_ids: list[int], task: str, thinking: bool
) -> list[int]:
"""A quick instruction task on the last user turn, as DeepSeek's checkpoint
encoder renders it: `action` follows the generation header, other tasks
replace it."""
header = [
self.hf_tokenizer.token_to_id(ASSISTANT_TOKEN),
self.hf_tokenizer.token_to_id(
THINK_START_TOKEN if thinking else THINK_END_TOKEN
),
]
if prompt_ids[-len(header) :] != header:
raise ValueError(
"task requires the conversation to end with a user message"
)
task_id = self.hf_tokenizer.token_to_id(TASK_TOKENS[task])
if task == "action":
return prompt_ids + [task_id]
return prompt_ids[: -len(header)] + [task_id]
+11 -13
View File
@@ -4,14 +4,6 @@ from typing import Dict, List, Optional, Tuple, Type
from transformers import PreTrainedTokenizerBase
from sglang.srt.entrypoints.openai.encoding_dsv4 import dsml_token as dsv4_dsml_token
from sglang.srt.entrypoints.openai.encoding_dsv4 import eos_token as dsv4_eos_token
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
thinking_end_token as dsv4_thinking_end_token,
)
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
thinking_start_token as dsv4_thinking_start_token,
)
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
ResponsesRequest,
@@ -1525,6 +1517,12 @@ class _DeepSeekV3Detector(Qwen3Detector):
self.reasoning_default = "explicit_thinking"
DSV4_EOS_TOKEN = "<|end▁of▁sentence|>"
DSV4_THINK_START_TOKEN = "<think>"
DSV4_THINK_END_TOKEN = "</think>"
DSV4_DSML_TOKEN = "|DSML|"
class DeepSeekV4Detector(BaseReasoningFormatDetector):
def __init__(
self,
@@ -1535,13 +1533,13 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
force_nonempty_content: bool = False,
):
super().__init__(
dsv4_thinking_start_token,
dsv4_thinking_end_token,
think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token],
DSV4_THINK_START_TOKEN,
DSV4_THINK_END_TOKEN,
think_excluded_tokens=[DSV4_EOS_TOKEN, DSV4_DSML_TOKEN],
# The reference parser splits reasoning at `\n\n<|DSML|tool_calls`;
# anchoring on the full tag at a line start keeps a mid-sentence
# mention of the tag inside reasoning from ending the block.
tool_start_token=f"<{dsv4_dsml_token}tool_calls>",
tool_start_token=f"<{DSV4_DSML_TOKEN}tool_calls>",
tool_start_at_line_start=True,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
@@ -1559,7 +1557,7 @@ class DeepSeekV41ReasoningDetector(DeepSeekV4Detector):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.tool_start_token = f"<{dsv4_dsml_token} calls>"
self.tool_start_token = f"<{DSV4_DSML_TOKEN} calls>"
class _MimoDetector(Qwen3Detector):
@@ -37,7 +37,6 @@ from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/parity"
DSV4_PROFILE = chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
class RecordingTokenizer:
@@ -87,18 +86,6 @@ def serving_chat(model, path, revision, config, tokenizer, tool_call_parser):
tokenizer=tokenizer,
tool_call_parser=tool_call_parser,
)
server._dsv4_reasoning_effort_profile = (
chat_encoding.resolve_dsv4_reasoning_effort_profile(
model_path=path, revision=revision, override=config.get(DSV4_PROFILE)
)
if server.chat_encoding_spec == "dsv4"
else None
)
server._dsv41_default_reasoning_effort = (
chat_encoding.default_dsv41_reasoning_effort_from_env(None)
if server.chat_encoding_spec == "dsv41"
else None
)
server._tokenizer_auto_adds_specials = len(tokenizer.encode("")) > 0
server._prompt_text_round_trip_is_lossy = server._probe_prompt_text_round_trip()
server._chat_template_cache = OrderedDict()
@@ -130,13 +117,10 @@ def generate(fixture_path):
tok,
fixture.get("tool_call_parser"),
)
# What the processor reads from config.json; the resolved profile stands in
# for the checkpoint's encoder file so the Rust test runs offline.
# What the processor reads from config.json.
fixture["config"] = {
key: config[key] for key in ("model_type", "architectures") if key in config
}
if server._dsv4_reasoning_effort_profile:
fixture["config"][DSV4_PROFILE] = server._dsv4_reasoning_effort_profile
fixture["bos_token_id"] = tok.bos_token_id
# Jinja checkpoints render with their template; keep it so the test runs offline.
if server.chat_encoding_spec is None:
@@ -261,7 +261,7 @@ class TestChatCompletionRequest(unittest.TestCase):
from pydantic import ValidationError
messages = [{"role": "user", "content": "Hello"}]
for effort in ("xhigh", "max"):
for effort in ("xhigh", "ultra", "max"):
with self.subTest(effort=effort, request_form="top-level"):
request = ChatCompletionRequest(
model="test-model",
@@ -283,7 +283,7 @@ class TestChatCompletionRequest(unittest.TestCase):
ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning_effort="ultra",
reasoning_effort="extreme",
)
def test_chat_completion_reasoning_effort_is_strictly_validated(self):
@@ -11,23 +11,18 @@ from sglang.test.test_utils import CustomTestCase, enter_override, maybe_stub_sg
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
import asyncio
import copy
import gc
import json
import tempfile
import unittest
import uuid
from http import HTTPStatus
from pathlib import Path
from typing import Optional
from unittest.mock import Mock, patch
from fastapi import Request
from transformers.utils.chat_template_utils import _compile_jinja_template
from sglang.srt.entrypoints.openai.chat_encoding import (
resolve_dsv4_reasoning_effort_profile,
)
from sglang.srt.entrypoints.openai.chat_encoding import resolve_chat_encoding_spec
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
Function,
@@ -157,15 +152,6 @@ _TOOL_RESULT_REORDER_TEMPLATE = """
"""
def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
model_dir = tempfile.TemporaryDirectory()
test_case.addCleanup(model_dir.cleanup)
encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py"
encoder_path.parent.mkdir(parents=True)
encoder_path.write_text(source, encoding="utf-8")
return model_dir.name
class _MockTokenizerManager:
"""Minimal mock that satisfies OpenAIServingChat."""
@@ -1915,7 +1901,6 @@ class ServingChatTestCase(CustomTestCase):
def test_dsv_tool_choice_none_omits_tools_from_prompt(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat._dsv4_reasoning_effort_profile = "preview"
tool = {
"type": "function",
"function": {
@@ -1924,25 +1909,21 @@ class ServingChatTestCase(CustomTestCase):
},
}
for chat_encoding_spec in ("dsv4", "dsv32", "dsv41"):
for tool_choice, expects_tools in (("none", False), ("auto", True)):
with self.subTest(
chat_encoding_spec=chat_encoding_spec,
self.chat.chat_encoding_spec = "dsv32"
for tool_choice, expects_tools in (("none", False), ("auto", True)):
with self.subTest(tool_choice=tool_choice):
self.tm.tokenizer.encode.reset_mock()
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Solve 100 / 7."}],
tools=[tool],
tool_choice=tool_choice,
):
self.chat.chat_encoding_spec = chat_encoding_spec
self.tm.tokenizer.encode.reset_mock()
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Solve 100 / 7."}],
tools=[tool],
tool_choice=tool_choice,
)
)
self.chat._process_messages(request, is_multimodal=False)
self.chat._process_messages(request, is_multimodal=False)
prompt = self.tm.tokenizer.encode.call_args.args[0]
self.assertEqual("calculator" in prompt, expects_tools)
prompt = self.tm.tokenizer.encode.call_args.args[0]
self.assertEqual("calculator" in prompt, expects_tools)
def test_kimi_k3_encoder_receives_wire_request_fields(self):
self.template_manager.chat_template_name = None
@@ -2200,76 +2181,70 @@ class ServingChatTestCase(CustomTestCase):
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat._dsv4_reasoning_effort_profile = "preview"
for chat_encoding_spec in ("dsv4", "dsv32"):
with self.subTest(chat_encoding_spec=chat_encoding_spec):
self.chat.chat_encoding_spec = chat_encoding_spec
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
self.chat.chat_encoding_spec = "dsv32"
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '"Beijing"',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '"Beijing"',
},
}
],
)
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
],
)
with self.assertRaisesRegex(ValueError, "must be a JSON object"):
self.chat._process_messages(req, is_multimodal=False)
with self.assertRaisesRegex(ValueError, "must be a JSON object"):
self.chat._process_messages(req, is_multimodal=False)
def test_dsv_encoders_accept_object_tool_call_arguments_string(self):
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat._dsv4_reasoning_effort_profile = "preview"
for chat_encoding_spec in ("dsv4", "dsv32"):
with self.subTest(chat_encoding_spec=chat_encoding_spec):
self.chat.chat_encoding_spec = chat_encoding_spec
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
self.chat.chat_encoding_spec = "dsv32"
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Beijing"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Beijing"}',
},
}
],
)
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
],
)
self.chat._process_messages(req, is_multimodal=False)
self.chat._process_messages(req, is_multimodal=False)
def test_stop_str_isolation_between_requests(self):
"""Test that stop strings from one request don't affect subsequent requests.
@@ -3110,17 +3085,14 @@ class ServingChatTestCase(CustomTestCase):
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
# (release ships a stale V3 jinja we deliberately override).
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
mock_hf_config.to_dict.return_value = {
"dsv4_reasoning_effort_profile": "preview"
}
tm.model_path = "deepseek-ai/DeepSeek-V4-Flash"
tm.tokenizer.chat_template = "stale v3 jinja"
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
tm.tokenizer.chat_template = None
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
for chat_template in ("stale v3 jinja", None):
tm.tokenizer.chat_template = chat_template
self.assertEqual(
resolve_chat_encoding_spec(
hf_config=mock_hf_config, tokenizer=tm.tokenizer
),
"dsv4",
)
def test_kimi_k3_encoding_detection(self):
from sglang.srt.parser.template_manager import TemplateManager
@@ -3151,7 +3123,7 @@ class ServingChatTestCase(CustomTestCase):
self.chat.chat_encoding_spec = None
self.assertFalse(self.chat.supports_native_reasoning_history())
# ------------- dsv4 task + latest_reminder -------------
# ------------- dsv4 task -------------
def test_dsv4_task_field_schema(self):
"""Top-level `task` accepts the 6 DS task tokens and rejects others."""
for valid in ("action", "query", "authority", "domain", "title", "read_url"):
@@ -3175,407 +3147,6 @@ class ServingChatTestCase(CustomTestCase):
task="bogus",
)
def test_attach_task_to_last_user_message(self):
"""Helper attaches task to the nearest user/developer message."""
from sglang.srt.entrypoints.openai import encoding_dsv4
messages = [{"role": "user", "content": "Hi"}]
encoding_dsv4.attach_task_to_last_user_message(messages, "domain")
self.assertEqual(messages[0]["task"], "domain")
# Prefers the LAST user message across a multi-turn conversation.
messages = [
{"role": "user", "content": "first"},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": "second"},
]
encoding_dsv4.attach_task_to_last_user_message(messages, "query")
self.assertNotIn("task", messages[0])
self.assertEqual(messages[2]["task"], "query")
# `developer` role is treated like `user` (matches encoder semantics).
messages = [{"role": "developer", "content": "dev"}]
encoding_dsv4.attach_task_to_last_user_message(messages, "authority")
self.assertEqual(messages[0]["task"], "authority")
# No user/developer present -> raises.
with self.assertRaises(ValueError):
encoding_dsv4.attach_task_to_last_user_message(
[{"role": "system", "content": "s"}], "domain"
)
def test_dsv4_content_parts_list_normalized(self):
"""OpenAI list-of-parts content flattens to text before reaching the encoder."""
from sglang.srt.entrypoints.openai import encoding_dsv4
from sglang.srt.parser.jinja_template_utils import (
process_content_for_template_format,
)
req = ChatCompletionRequest(
model="x",
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "say hi"}],
}
],
)
messages = [m.model_dump() for m in req.messages]
# Mirror the boundary normalization _process_messages does for any
# non-None chat_encoding_spec.
for i, msg in enumerate(messages):
if isinstance(msg.get("content"), list):
messages[i] = process_content_for_template_format(
msg, "string", [], [], [], []
)
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
self.assertIn("<|User|>say hi", out)
# Multiple text parts concat with single space; non-text parts dropped.
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": "describe"},
{"type": "image_url", "image_url": {"url": "x"}},
],
}
]
for i, msg in enumerate(messages):
if isinstance(msg.get("content"), list):
messages[i] = process_content_for_template_format(
msg, "string", [], [], [], []
)
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
self.assertIn("<|User|>describe", out)
self.assertNotIn("image_url", out)
def test_dsv4_task_and_reminder_encode_end_to_end(self):
"""Task + latest_reminder plumb through to the dsv4 encoder correctly."""
from sglang.srt.entrypoints.openai import encoding_dsv4
# 1) task='domain' in chat mode -> `<|domain|>` appended, no Assistant
# prefix (this is a single-shot classification, not a chat turn).
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is SGLang?"}],
task="domain",
)
messages = [m.model_dump() for m in req.messages]
encoding_dsv4.attach_task_to_last_user_message(messages, req.task)
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
self.assertIn("<|domain|>", out)
self.assertTrue(out.rstrip().endswith("<|domain|>"))
self.assertNotIn("<|Assistant|>", out)
# 2) task='action' in thinking mode -> Assistant + <think> + <|action|>
# (action is the one task that still runs a reasoning pass).
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi"}],
task="action",
)
messages = [m.model_dump() for m in req.messages]
encoding_dsv4.attach_task_to_last_user_message(messages, req.task)
out = encoding_dsv4.encode_messages(messages, thinking_mode="thinking")
self.assertIn("<|Assistant|>", out)
self.assertIn("<think>", out)
self.assertTrue(out.rstrip().endswith("<|action|>"))
# 3) latest_reminder preceding user -> reminder renders before user,
# Assistant prefix still comes after user.
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "latest_reminder", "content": "Be terse."},
{"role": "user", "content": "Hello"},
],
)
messages = [m.model_dump() for m in req.messages]
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
self.assertIn("<|latest_reminder|>Be terse.", out)
self.assertIn("<|User|>Hello", out)
self.assertLess(
out.index("<|latest_reminder|>"),
out.index("<|User|>"),
)
self.assertIn("<|Assistant|>", out)
def test_dsv4_reasoning_effort_profiles(self):
from sglang.srt.entrypoints.openai import encoding_dsv4
messages = [
{"role": "system", "content": ""},
{"role": "user", "content": "Solve this."},
]
absolute_maximum = (
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
)
beyond_maximum = (
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
)
def encode(profile, effort):
return encoding_dsv4.encode_messages(
messages,
thinking_mode="thinking",
reasoning_effort=effort,
reasoning_effort_profile=profile,
)
preview_high = encode("preview", "high")
preview_max = encode("preview", "max")
official_low = encode("official", "low")
official_high = encode("official", "high")
official_max = encode("official", "max")
self.assertNotIn("Reasoning Effort:", preview_high)
self.assertTrue(
preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum)
)
self.assertNotIn("Reasoning Effort:", official_low)
self.assertEqual(
official_high,
encoding_dsv4.bos_token
+ absolute_maximum
+ official_low.removeprefix(encoding_dsv4.bos_token),
)
self.assertEqual(
official_max,
encoding_dsv4.bos_token
+ beyond_maximum
+ official_low.removeprefix(encoding_dsv4.bos_token),
)
self.assertEqual(encode("preview", None), preview_high)
self.assertEqual(encode("official", None), official_low)
self.assertEqual(len({official_low, official_high, official_max}), 3)
with self.assertRaises(ValueError):
encode("preview", "low")
def test_dsv4_reasoning_effort_profile_resolution(self):
resolve = resolve_dsv4_reasoning_effort_profile
preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER)
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
inconclusive_model_path = _create_dsv4_checkpoint(
self, 'UNRELATED_METADATA = "value"\n'
)
self.assertEqual(resolve(model_path=preview_model_path), "preview")
self.assertEqual(resolve(model_path=official_model_path), "official")
self.assertEqual(resolve(model_path=inconclusive_model_path), "preview")
self.assertEqual(
resolve(model_path="renamed/model", override="official"), "official"
)
self.assertEqual(
resolve(model_path="renamed/model", override="preview"), "preview"
)
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
resolve(model_path="renamed/model", override="auto")
def test_dsv4_reasoning_effort_profile_from_checkpoint(self):
from sglang.srt.parser.template_manager import TemplateManager
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
tm.model_config.hf_config.to_dict.return_value = {}
tm.model_config.hf_config.dspark_block_size = 5
tm.model_config.hf_config.dspark_markov_rank = 256
tm.model_path = official_model_path
tm.server_args.model_path = tm.model_path
serving_chat = OpenAIServingChat(tm, TemplateManager())
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hello"}],
reasoning_effort="max",
)
serving_chat._process_messages(request, is_multimodal=False)
prompt = tm.tokenizer.encode.call_args.args[0]
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
def test_dsv4_reasoning_effort_profile_override_from_model_config(self):
from sglang.srt.parser.template_manager import TemplateManager
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
tm.model_config.hf_config.to_dict.return_value = {
"dsv4_reasoning_effort_profile": "official"
}
serving_chat = OpenAIServingChat(tm, TemplateManager())
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hello"}],
reasoning_effort="max",
)
serving_chat._process_messages(request, is_multimodal=False)
prompt = tm.tokenizer.encode.call_args.args[0]
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
def test_dsv4_invalid_profile_override_fails_at_construction(self):
from sglang.srt.parser.template_manager import TemplateManager
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
tm.model_config.hf_config.to_dict.return_value = {
"dsv4_reasoning_effort_profile": "invalid"
}
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
OpenAIServingChat(tm, TemplateManager())
def test_dsv41_rejects_image_placeholder_in_text(self):
from sglang.srt.entrypoints.openai import encoding_dsv41
for role in ("user", "tool"):
with self.subTest(role=role):
with self.assertRaises(ValueError):
encoding_dsv41.encode_messages(
[
{
"role": role,
"content": f"hi {encoding_dsv41.IMAGE_PLACEHOLDER}",
}
],
thinking_mode="chat",
)
def test_dsv41_tool_result_images_follow_prompt_order(self):
"""Out-of-order tool results are sorted by call order; images must follow."""
from sglang.srt.entrypoints.openai import encoding_dsv41
image = encoding_dsv41.IMAGE_PLACEHOLDER
history = [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "history.png"}},
],
},
{
"role": "assistant",
"tool_calls": [
{
"id": name,
"type": "function",
"function": {"name": name, "arguments": "{}"},
}
for name in ("a", "b")
],
},
]
results = [
{
"type": "tool_result",
"tool_use_id": name,
"content": [
{"type": "text", "text": f"result {name}"},
{"type": "image_url", "image_url": {"url": f"{name}.png"}},
],
}
for name in ("b", "a")
]
for input_format in (
"content_blocks",
"tool_messages",
"tool_content_blocks",
"context",
):
with self.subTest(input_format=input_format):
if input_format in ("tool_messages", "tool_content_blocks"):
content_field = (
"content_blocks"
if input_format == "tool_content_blocks"
else "content"
)
messages = history + [
{
"role": "tool",
"tool_call_id": block["tool_use_id"],
content_field: block["content"],
}
for block in results
]
context = []
elif input_format == "context":
messages = [{"role": "user", "content_blocks": results}]
context = history
else:
messages = history + [{"role": "user", "content_blocks": results}]
context = []
original = copy.deepcopy((messages, context))
prompt, media = encoding_dsv41.encode_messages(
messages,
thinking_mode="chat",
context=context,
return_multi_modal_data=True,
)
self.assertTrue(
prompt.endswith(
f"<|User|><tool_result>result a\n\n{image}</tool_result>\n\n"
f"<tool_result>result b\n\n{image}</tool_result>"
"<|Assistant|></think>"
),
prompt,
)
expected_urls = (
["a.png", "b.png"] if context else ["history.png", "a.png", "b.png"]
)
self.assertEqual(
media,
{"images": [{"url": url} for url in expected_urls]},
)
self.assertEqual((messages, context), original)
def test_dsv41_drop_thinking_without_tools(self):
"""Tool results do not start a new turn, so current-turn reasoning is kept."""
from sglang.srt.entrypoints.openai import encoding_dsv41
messages = [
{"role": "user", "content": "q1"},
{"role": "assistant", "content": "a1", "reasoning_content": "r1"},
{"role": "user", "content": "q2"},
{
"role": "assistant",
"reasoning_content": "r2",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "c1", "content": "value"},
]
prompt = encoding_dsv41.encode_messages(
messages, thinking_mode="thinking", drop_thinking=True
)
self.assertIn(
"<|User|>q1<|Assistant|></think>a1<|end▁of▁sentence|>", prompt
)
self.assertNotIn("r1", prompt)
self.assertIn("<think>r2</think>", prompt)
self.assertTrue(
prompt.endswith("<tool_result>value</tool_result><|Assistant|><think>")
)
next_turn = encoding_dsv41.encode_messages(
messages + [{"role": "user", "content": "q3"}],
thinking_mode="thinking",
)
self.assertNotIn("r2", next_turn)
self.assertTrue(next_turn.endswith("q3<|Assistant|><think>"))
def test_streaming_abort_yields_error(self):
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
err_msg = "Aborted by scheduler"
@@ -5499,7 +5070,7 @@ class ServingChatTestCase(CustomTestCase):
# ------------- hook method tests -------------
def test_encode_messages_returns_none_by_default(self):
"""Default _encode_messages returns None (use standard encoding)."""
result = self.chat._encode_messages([], Mock(), False)
result = self.chat._encode_messages([], Mock(), False, image_data=[])
self.assertIsNone(result)
@@ -0,0 +1,298 @@
import unittest
from types import SimpleNamespace
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase, is_in_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash-1017"
REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170"
def _tokenizer_file():
"""The V4.1 tokenizer; only outside CI does an unreachable Hub skip the test."""
from huggingface_hub import hf_hub_download
try:
return hf_hub_download(MODEL, "tokenizer.json", revision=REVISION)
except Exception as error: # noqa: BLE001
if is_in_ci():
raise
raise unittest.SkipTest(f"the tokenizer is not available: {error}")
def _serving(test, tokenizer, *, is_multimodal=False, **server_args):
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
reset_context()
test.addCleanup(reset_context)
args = ServerArgs(
model_path="dummy",
reasoning_parser="deepseek-v41",
tool_call_parser="deepseekv41",
**server_args,
)
publish(args, role="tokenizer")
manager = SimpleNamespace(
model_path="dummy",
served_model_name="dummy",
server_args=args,
tokenizer=tokenizer,
model_config=SimpleNamespace(
is_multimodal=is_multimodal,
context_len=131072,
hf_config=SimpleNamespace(
model_type="deepseek_v41",
architectures=["DeepseekV41ForCausalLM"],
to_dict=lambda: {},
),
get_default_sampling_params=lambda: {},
),
config_value=lambda name: getattr(args, name),
)
template_manager = SimpleNamespace(
chat_template_name=None,
jinja_template_content_format=None,
reasoning_config=None,
force_reasoning=False,
python_chat_template=None,
jinja_template_may_reorder_tool_results=False,
)
return OpenAIServingChat(manager, template_manager)
class TestAdapterRegistry(CustomTestCase):
def _context(self, spec, tokenizer=None, default_kwargs=None):
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
return registry.ChatEncodingAdapterContext(
chat_encoding_spec=spec,
tokenizer=tokenizer,
reasoning_parser=None,
tool_call_parser=None,
chat_hooks=registry.ChatOutputHooks(
tool_call_parsing_active=lambda request: False,
history_tool_calls_count=lambda request: 0,
tool_call_id=lambda item, index: "",
),
hf_config=SimpleNamespace(
model_type="deepseek_v41", architectures=["DeepseekV41ForCausalLM"]
),
model_path="",
revision=None,
default_chat_template_kwargs=default_kwargs,
)
def _fast_tokenizer(self):
from tokenizers import Tokenizer
return SimpleNamespace(backend_tokenizer=Tokenizer.from_file(_tokenizer_file()))
def test_bad_default_effort_fails_startup(self):
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
tokenizer = self._fast_tokenizer()
for spec, effort in (("dsv41", "extreme"), ("dsv4", 60)):
with self.subTest(spec=spec, effort=effort), self.assertRaises(ValueError):
registry.build_chat_encoding_adapter(
self._context(
spec,
tokenizer=tokenizer,
default_kwargs={"reasoning_effort": effort},
)
)
def test_no_tokenizer_builds_no_adapter(self):
"""--skip-tokenizer-init servers start, with no chat prompts to encode."""
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
for spec in ("dsv4", "dsv41"):
with self.subTest(spec=spec):
self.assertIsNone(
registry.build_chat_encoding_adapter(self._context(spec))
)
def test_tool_dumps_render_as_the_client_sent_them(self):
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
adapter = registry.build_chat_encoding_adapter(
self._context("dsv41", tokenizer=self._fast_tokenizer())
)
# The handler's tool dumps carry None for unset fields.
tool = {
"type": "function",
"function": {
"name": "f",
"description": "d",
"parameters": {"type": "object", "properties": {}},
},
}
dumped = {
"type": "function",
"function": {**tool["function"], "strict": None},
"defer_loading": None,
}
request = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}])
self.assertEqual(
adapter.encode_messages(
messages=[],
request=request,
tools=[dumped],
thinking=False,
image_data=[],
),
adapter.encode_messages(
messages=[],
request=request,
tools=[tool],
thinking=False,
image_data=[],
),
)
class TestHandlerRequestSemantics(CustomTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
from transformers import PreTrainedTokenizerFast
cls.tokenizer = PreTrainedTokenizerFast(tokenizer_file=_tokenizer_file())
def _token(self, token):
return self.tokenizer.convert_tokens_to_ids(token)
def _prompt(self, chat, **fields):
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
request = ChatCompletionRequest(
messages=[{"role": "user", "content": "hi"}], **fields
)
return chat._process_messages(request, False)
def test_thinking_defaults_off(self):
chat = _serving(self, self.tokenizer)
default = self._prompt(chat)
self.assertEqual(default.prompt_ids[-1], self._token("</think>"))
self.assertFalse(default.require_reasoning)
thinking = self._prompt(chat, chat_template_kwargs={"thinking": True})
self.assertEqual(thinking.prompt_ids[-1], self._token("<think>"))
self.assertTrue(thinking.require_reasoning)
def test_request_effort_wins_over_the_server_default(self):
thinking = {"chat_template_kwargs": {"thinking": True}}
plain = _serving(self, self.tokenizer)
low = self._prompt(plain, reasoning_effort="low", **thinking).prompt_ids
self.assertNotEqual(
low, self._prompt(plain, reasoning_effort="max", **thinking).prompt_ids
)
chat = _serving(
self,
self.tokenizer,
default_chat_template_kwargs={"reasoning_effort": "max"},
)
self.assertEqual(
self._prompt(chat, reasoning_effort="low", **thinking).prompt_ids, low
)
def test_trailing_assistant_follows_sglang_rules(self):
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
chat = _serving(self, self.tokenizer)
def prompt(messages, **fields):
request = ChatCompletionRequest(messages=messages, **fields)
return chat._process_messages(request, False).prompt_ids
answer = {"role": "assistant", "content": "Hel"}
# Without continue_final_message the trailing turn becomes a user turn.
self.assertEqual(
prompt([{"role": "user", "content": "hi"}, answer]),
prompt(
[{"role": "user", "content": "hi"}, {"role": "user", "content": "Hel"}]
),
)
# A task renders its own tail; the continued prefix follows it.
self.assertEqual(
prompt(
[{"role": "user", "content": "hi"}, answer],
task="title",
continue_final_message=True,
),
prompt([{"role": "user", "content": "hi"}], task="title")
+ self.tokenizer.encode("Hel", add_special_tokens=False),
)
def test_tool_result_images_follow_prompt_order(self):
"""Out-of-order tool results render in call order; their images must follow."""
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
chat = _serving(self, self.tokenizer, is_multimodal=True)
names = ("a", "b")
urls = {name: f"https://example.com/{name}.png" for name in names}
calls = [
{
"id": name,
"type": "function",
"function": {"name": name, "arguments": "{}"},
}
for name in names
]
results = [
{
"role": "tool",
"tool_call_id": name,
"content": [
{"type": "text", "text": f"result {name}"},
{"type": "image_url", "image_url": {"url": urls[name]}},
],
}
for name in reversed(names)
]
request = ChatCompletionRequest(
messages=[
{"role": "user", "content": "both?"},
{"role": "assistant", "content": "", "tool_calls": calls},
*results,
],
tools=[
{
"type": "function",
"function": {"name": name, "parameters": {"type": "object"}},
}
for name in names
],
)
processed = chat._process_messages(request, True)
prompt = self.tokenizer.decode(processed.prompt_ids)
self.assertLess(prompt.index("result a"), prompt.index("result b"))
self.assertEqual(
[image.url for image in processed.image_data], [urls["a"], urls["b"]]
)
def test_request_reaches_the_engine_with_sglang_semantics(self):
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
chat = _serving(self, self.tokenizer)
request = ChatCompletionRequest(
messages=[{"role": "user", "content": "hi"}],
temperature=0.0,
chat_template_kwargs={"thinking": True, "reasoning_effort": 30},
)
adapted, _ = chat._convert_to_internal_request(request)
self.assertEqual(adapted.sampling_params["temperature"], 0.0)
self.assertIsNone(adapted.sampling_params["max_new_tokens"])
self.assertIn(
"Reasoning Effort: 30 (range 1-100",
self.tokenizer.decode(adapted.input_ids),
)
titled = self._prompt(chat, task="title").prompt_ids
self.assertEqual(titled[-1], self._token("<|title|>"))
if __name__ == "__main__":
unittest.main()
@@ -111,6 +111,7 @@ class InklingPromptTest(unittest.TestCase):
[message.model_dump() for message in request.messages],
request,
thinking_mode=None,
image_data=[],
)
def test_prompt_tail(self):
@@ -626,6 +627,8 @@ class InklingTokenOutputTest(CustomTestCase):
messages=[message.model_dump() for message in request.messages],
request=request,
tools=None,
thinking=False,
image_data=[],
)
key = reasoner.get_request_grammar_key(
("regex", "yes"), input_ids=prompt_ids, thinking_budget=None
@@ -1448,7 +1448,7 @@ class ChatEncodingAdapterDispatchTestCase(CustomTestCase):
)
with patch.dict(sys.modules, {"tml_renderers": None}):
for encoding in ("kimi_k3", "dsv4", "dsv41", "dsv32"):
for encoding in ("kimi_k3", "dsv32"):
with self.subTest(encoding=encoding):
self.assertIsNone(
chat_encoding_adapters.build_chat_encoding_adapter(
@@ -8,7 +8,6 @@ import xgrammar as xgr
from xgrammar.structural_tag import JSONSchemaFormat
from xgrammar.testing import _is_grammar_accept_string
from sglang.srt.entrypoints.openai import encoding_dsv41
from sglang.srt.entrypoints.openai.protocol import (
Function,
Tool,
@@ -17,6 +16,7 @@ from sglang.srt.entrypoints.openai.protocol import (
)
from sglang.srt.function_call.deepseekv41_detector import DeepSeekV41Detector
from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.parser.deepseek_recipe_renderer import load_deepseek_recipe
from sglang.srt.parser.reasoning_parser import ReasoningParser
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -25,6 +25,7 @@ register_cpu_ci(est_time=2, suite="base-a-test-cpu")
CHUNK_SIZES = [1, 2, 3, 5, 7, 11, 23, 1000]
DSML = "|DSML|"
EOS = "<|end▁of▁sentence|>"
def _tools():
@@ -59,6 +60,34 @@ def _tools():
]
def _render_assistant(message, *, thinking):
"""The assistant turn as deepseek-recipe renders it, minus the generation
header (`<think>` / `</think>`) that ends the prompt before it."""
recipe = load_deepseek_recipe()
calls = [
{**call, "id": f"call_{i}"} for i, call in enumerate(message["tool_calls"])
]
results = [
{"role": "tool", "tool_call_id": call["id"], "content": "ok"} for call in calls
]
body = {
"model": "DeepSeek-V4.1-Flash",
"messages": [
{"role": "user", "content": "question"},
{**message, "tool_calls": calls},
*results,
],
"thinking": {"type": "enabled" if thinking else "disabled"},
# Declared tools keep the turn's reasoning in the rendered history.
"tools": [tool.model_dump(exclude_none=True) for tool in _tools()],
}
converted = recipe.ChatCompletionRequest(body).convert(recipe.ConversionOptions())
prompt = recipe.DeepseekV41Encoding().render_conversation(converted.conversation)
turn = prompt.prompt.split("<|User|>question<|Assistant|>", 1)[1]
turn = turn[: turn.index(EOS) + len(EOS)]
return turn.removeprefix("<think>" if thinking else "</think>")
def _assemble(calls):
"""Streamed ToolCallItems -> [(name, parsed arguments)] per tool_index."""
by_index = {}
@@ -81,26 +110,22 @@ class TestDeepSeekV41RoundTrip(CustomTestCase):
def setUp(self):
self.tools = _tools()
self.completion = encoding_dsv41.render_message(
1,
[
{"role": "user", "content": "question"},
{
"role": "assistant",
"reasoning_content": "reason",
"content": "summary",
"tool_calls": [
{
"type": "function",
"function": {
"name": "lookup",
"arguments": json.dumps(self.ARGUMENTS),
},
}
],
},
],
thinking_mode="thinking",
self.completion = _render_assistant(
{
"role": "assistant",
"reasoning_content": "reason",
"content": "summary",
"tool_calls": [
{
"type": "function",
"function": {
"name": "lookup",
"arguments": json.dumps(self.ARGUMENTS),
},
}
],
},
thinking=True,
)
self.completion = "<think>" + self.completion
self.expected = [("lookup", self.ARGUMENTS)]
@@ -221,29 +246,21 @@ class TestDeepSeekV41ParameterGrammar(CustomTestCase):
@staticmethod
def _render(arguments, *, thinking=False, count=1, name="get_weather"):
return encoding_dsv41.render_message(
1,
[
{"role": "user", "content": "question"},
{
"role": "assistant",
"content": "",
"reasoning_content": "reason",
"wo_eos": True,
"tool_calls": [
{
"type": "function",
"function": {
"name": name,
"arguments": json.dumps(arguments),
},
}
]
* count,
},
],
thinking_mode="thinking" if thinking else "chat",
)
return _render_assistant(
{
"role": "assistant",
"content": "",
"reasoning_content": "reason",
"tool_calls": [
{
"type": "function",
"function": {"name": name, "arguments": json.dumps(arguments)},
}
]
* count,
},
thinking=thinking,
).removesuffix(EOS)
def _grammar(self, choice="required", thinking=False, parallel=True):
constraint = FunctionCallParser(
@@ -0,0 +1,206 @@
import unittest
from sglang.srt.parser import deepseek_recipe_renderer as renderer
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase, is_in_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash-1017"
REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170"
def _tokenizer_file():
"""The V4.1 tokenizer; only outside CI does an unreachable Hub skip the test."""
from huggingface_hub import hf_hub_download
try:
return hf_hub_download(MODEL, "tokenizer.json", revision=REVISION)
except Exception as error: # noqa: BLE001
if is_in_ci():
raise
raise unittest.SkipTest(f"the tokenizer is not available: {error}")
class TestPreNormalize(CustomTestCase):
def test_developer_becomes_system(self):
body = renderer.pre_normalize(
{
"messages": [
{"role": "developer", "content": "Be terse."},
{"role": "user", "content": "x"},
],
"thinking": False,
}
)
self.assertEqual(body["messages"][0]["role"], "system")
def test_tool_choice_renders_as_auto(self):
tools = [
{"type": "function", "function": {"name": "a"}},
{"type": "function", "function": {"name": "b"}},
]
named = renderer.pre_normalize(
{
"messages": [{"role": "user", "content": "x"}],
"thinking": False,
"tools": tools,
"tool_choice": {"type": "function", "function": {"name": "b"}},
}
)
self.assertEqual(named["tool_choice"], "auto")
self.assertEqual(len(named["tools"]), 2)
required = renderer.pre_normalize(
{
"messages": [{"role": "user", "content": "x"}],
"thinking": False,
"tools": tools,
"tool_choice": "required",
}
)
self.assertEqual(required["tool_choice"], "auto")
for tools_field, choice in ((tools, "none"), ([], "auto")):
off = renderer.pre_normalize(
{
"messages": [{"role": "user", "content": "x"}],
"thinking": False,
"tools": tools_field,
"tool_choice": choice,
}
)
self.assertNotIn("tools", off)
self.assertNotIn("tool_choice", off)
def test_history_translation(self):
body = renderer.pre_normalize(
{
"messages": [
{"role": "user", "content": "weather?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {
"name": "get_weather",
"arguments": {"location": "Paris", "days": 1.5},
},
}
],
},
{"role": "tool", "tool_call_id": "call_0", "content": "sunny"},
{"role": "assistant", "content": "It is"},
],
"thinking": False,
"continue_final_message": True,
"response_format": {"type": "structural_tag", "structures": []},
}
)
call = body["messages"][1]["tool_calls"][0]
self.assertEqual(
call["function"]["arguments"], '{"location":"Paris","days":1.5}'
)
self.assertTrue(body["messages"][3]["prefix"])
self.assertNotIn("continue_final_message", body)
self.assertNotIn("response_format", body)
class TestEffortBudget(CustomTestCase):
def test_integer_and_float_budgets(self):
self.assertEqual(renderer.parse_effort_budget(30), 30)
self.assertEqual(renderer.parse_effort_budget(0.5), 50)
self.assertEqual(renderer.parse_effort_budget(0.0), 1)
for value in (0, 101, 1.0, True):
with self.subTest(value=value), self.assertRaises(ValueError):
renderer.parse_effort_budget(value)
class TestRecipeEncoder(CustomTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
from tokenizers import Tokenizer
cls.tokenizer = Tokenizer.from_file(_tokenizer_file())
cls.encoder = renderer.RecipeEncoder("dsv41", cls.tokenizer)
def _ids(self, *tokens):
return [self.tokenizer.token_to_id(token) for token in tokens]
def test_encodes_with_the_served_vocabulary(self):
self.assertEqual(
self.encoder.encode(
{"messages": [{"role": "user", "content": ""}], "thinking": False}
),
self._ids(
"<|begin▁of▁sentence|>", "<|User|>", "<|Assistant|>", "</think>"
),
)
thinking = self.encoder.encode(
{"messages": [{"role": "user", "content": "hi"}], "thinking": True}
)
self.assertEqual(thinking[-2:], self._ids("<|Assistant|>", "<think>"))
def test_tasks_replace_or_follow_the_generation_header(self):
for thinking in (False, True):
body = {
"messages": [{"role": "user", "content": "hello"}],
"thinking": thinking,
}
plain = self.encoder.encode(body)
for task, token in renderer.TASK_TOKENS.items():
with self.subTest(thinking=thinking, task=task):
expected = (
plain + self._ids(token)
if task == "action"
else plain[:-2] + self._ids(token)
)
self.assertEqual(self.encoder.encode(body, task=task), expected)
continued = {
"messages": [
{"role": "user", "content": "count"},
{"role": "assistant", "content": "1, 2,"},
],
"thinking": False,
"continue_final_message": True,
}
with self.assertRaisesRegex(ValueError, "end with a user message"):
self.encoder.encode(continued, task="query")
def test_effort_budget_rewrites_the_effort_line(self):
body = {"messages": [{"role": "user", "content": "hi"}], "thinking": True}
high = self.tokenizer.decode(
self.encoder.encode({**body, "reasoning_effort": "high"}),
skip_special_tokens=False,
)
self.assertIn("Reasoning Effort: 75 (range 1-100", high)
for budget in (1, 30, 100):
with self.subTest(budget=budget):
expected = high.replace(
"Reasoning Effort: 75 ", f"Reasoning Effort: {budget} ", 1
)
self.assertEqual(
self.encoder.encode(body, effort_budget=budget),
self.tokenizer.encode(expected, add_special_tokens=False).ids,
)
# Chat mode renders no effort line, so the budget changes nothing.
chat = {"messages": [{"role": "user", "content": "hi"}], "thinking": False}
self.assertEqual(
self.encoder.encode(chat, effort_budget=30), self.encoder.encode(chat)
)
def test_api_validation_surfaces_as_value_error(self):
with self.assertRaises(ValueError):
self.encoder.encode(
{
"messages": [{"role": "user", "content": "hi"}],
"thinking": True,
"reasoning_effort": "x",
}
)
if __name__ == "__main__":
unittest.main()