[DSV4] Add official DSV4 reasoning effort support (#33140)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
Mohammad Miadh Angkad
2026-08-05 12:50:41 +08:00
committed by GitHub
co-authored by Xinyuan Tong David Orman
parent 198a3bc29b
commit 059269594c
6 changed files with 321 additions and 24 deletions
@@ -712,7 +712,7 @@ SGLang supports various environment variables that can be used to configure its
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it (accepts <code>max</code>, <code>high</code>; empty means unset).</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts <code>high</code> and <code>max</code>; the official profile accepts <code>low</code>, <code>high</code>, and <code>max</code>. The profile is detected from the bundled encoder. Override it with <code>--json-model-override-args '&#123;"dsv4_reasoning_effort_profile":"official"&#125;'</code>.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
</tr>
<tr>
@@ -7,8 +7,103 @@ it here instead of re-deriving it from model architectures themselves.
from __future__ import annotations
import ast
import logging
from pathlib import Path
from typing import Any, Dict, List, Optional
from sglang.srt.entrypoints.openai import encoding_dsv4
logger = logging.getLogger(__name__)
DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile"
_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py"
_MAX_DSV4_ENCODER_BYTES = 1 << 20
def _detect_dsv4_reasoning_effort_profile(
model_path: str, revision: Optional[str] = None
) -> Optional[str]:
encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER
try:
if not encoder_path.is_file():
from huggingface_hub import hf_hub_download
encoder_path = Path(
hf_hub_download(
model_path,
_DSV4_REASONING_EFFORT_ENCODER,
revision=revision,
)
)
if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES:
return None
tree = ast.parse(encoder_path.read_text(encoding="utf-8"))
except Exception as error:
logger.debug(
"Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s",
encoder_path,
error,
)
return None
assignments = {}
for node in tree.body:
if isinstance(node, ast.Assign):
targets = node.targets
value = node.value
elif isinstance(node, ast.AnnAssign):
targets = [node.target]
value = node.value
else:
continue
for target in targets:
if not isinstance(target, ast.Name):
continue
try:
assignments[target.id] = ast.literal_eval(value)
except (TypeError, ValueError):
continue
prompts = assignments.get("REASONING_EFFORT_PROMPTS")
if (
assignments.get("DEFAULT_REASONING_EFFORT") == "low"
and isinstance(prompts, dict)
and {"low", "high", "max"} <= prompts.keys()
):
return "official"
if "REASONING_EFFORT_MAX" in assignments:
return "preview"
return None
def _validate_dsv4_reasoning_effort_profile(profile: str) -> str:
if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES:
raise ValueError(
f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; "
f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}"
)
return profile
def resolve_dsv4_reasoning_effort_profile(
*,
model_path: str,
revision: Optional[str] = None,
override: Optional[str] = None,
) -> str:
if override is not None:
return _validate_dsv4_reasoning_effort_profile(override)
return (
_detect_dsv4_reasoning_effort_profile(
model_path=model_path,
revision=revision,
)
or "preview"
)
def resolve_chat_encoding_spec(
*,
@@ -60,12 +60,30 @@ tool_calls_block_name: str = "tool_calls"
tool_output_template: str = "<tool_result>{content}</tool_result>"
REASONING_EFFORT_MAX = (
REASONING_EFFORT_PREVIEW_MAX = (
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
)
REASONING_EFFORT_OFFICIAL_MAX = (
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
)
REASONING_EFFORT_PROFILES = {
"preview": {
"high": "",
"max": REASONING_EFFORT_PREVIEW_MAX,
},
"official": {
"low": "",
"high": REASONING_EFFORT_PREVIEW_MAX,
"max": REASONING_EFFORT_OFFICIAL_MAX,
},
}
TOOLS_TEMPLATE = """## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
@@ -250,6 +268,7 @@ def render_message(
thinking_mode: str,
drop_thinking: bool = True,
reasoning_effort: Optional[str] = None,
reasoning_effort_profile: str = "preview",
) -> str:
"""
Render a single message at the given index into its encoded string form.
@@ -262,7 +281,9 @@ def render_message(
messages: Full list of messages in the conversation.
thinking_mode: Either "chat" or "thinking".
drop_thinking: Whether to drop reasoning content from earlier turns.
reasoning_effort: Optional reasoning effort level ("max", "high", or None).
reasoning_effort: Optional reasoning effort level. The preview profile accepts
"high" and "max"; the official profile accepts "low", "high", and "max".
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
Returns:
Encoded string for this message.
@@ -290,14 +311,21 @@ def render_message(
if tool_calls:
tool_calls = tool_calls_from_openai_format(tool_calls)
# Reasoning effort prefix (only at index 0 in thinking mode with max effort)
assert reasoning_effort in [
"max",
None,
"high",
], f"Invalid reasoning effort: {reasoning_effort}"
if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max":
prompt += REASONING_EFFORT_MAX
if reasoning_effort_profile not in REASONING_EFFORT_PROFILES:
raise ValueError(
f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; "
f"expected one of {list(REASONING_EFFORT_PROFILES)}"
)
effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile]
if reasoning_effort is None:
reasoning_effort = "low" if reasoning_effort_profile == "official" else "high"
if reasoning_effort not in effort_prompts:
raise ValueError(
f"Invalid reasoning effort {reasoning_effort!r} for profile "
f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}"
)
if index == 0 and thinking_mode == "thinking":
prompt += effort_prompts[reasoning_effort]
if role == "system":
prompt += system_msg_template.format(content=content or "")
@@ -583,6 +611,7 @@ def encode_messages(
drop_thinking: bool = True,
add_default_bos_token: bool = True,
reasoning_effort: Optional[str] = None,
reasoning_effort_profile: str = "preview",
) -> str:
"""
Encode a list of messages into the DeepSeek-V4 prompt format.
@@ -600,7 +629,9 @@ def encode_messages(
drop_thinking: If True, drop reasoning_content from earlier assistant turns
(only keep reasoning for messages after the last user message).
add_default_bos_token: Whether to prepend BOS token at conversation start.
reasoning_effort: Optional reasoning effort level ("max", "high", or None).
reasoning_effort: Optional reasoning effort level. The preview profile accepts
"high" and "max"; the official profile accepts "low", "high", and "max".
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
Returns:
The encoded prompt string.
@@ -640,6 +671,7 @@ def encode_messages(
thinking_mode=thinking_mode,
drop_thinking=effective_drop_thinking,
reasoning_effort=reasoning_effort,
reasoning_effort_profile=reasoning_effort_profile,
)
return prompt
@@ -24,7 +24,7 @@ from fastapi import Request
from fastapi.responses import ORJSONResponse, StreamingResponse
from jsonschema import Draft202012Validator, SchemaError
from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv32
from sglang.srt.entrypoints.openai import chat_encoding, encoding_dsv4, encoding_dsv32
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionMessageGenericParam,
ChatCompletionRequest,
@@ -253,6 +253,17 @@ class OpenAIServingChat(OpenAIServingBase):
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
self._dsv4_reasoning_effort_profile = (
chat_encoding.resolve_dsv4_reasoning_effort_profile(
model_path=self.tokenizer_manager.model_path,
revision=self.tokenizer_manager.server_args.revision,
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
),
)
if self.chat_encoding_spec == "dsv4"
else None
)
# Resolve the env-configured Inkling effort default once: the env var is
# frozen for the server's lifetime, and a misconfigured value should
@@ -339,11 +350,7 @@ class OpenAIServingChat(OpenAIServingBase):
Override in subclass to add custom encoding specs.
"""
from sglang.srt.entrypoints.openai.chat_encoding import (
resolve_chat_encoding_spec,
)
return resolve_chat_encoding_spec(
return chat_encoding.resolve_chat_encoding_spec(
hf_config=self.tokenizer_manager.model_config.hf_config,
tokenizer=self.tokenizer_manager.tokenizer,
tool_call_parser=self.tool_call_parser,
@@ -1216,16 +1223,18 @@ class OpenAIServingChat(OpenAIServingBase):
# Default encoding (dsv4/dsv32)
if self.chat_encoding_spec == "dsv4":
# V4 encoder only accepts "max" / "high" / None.
# OpenAI protocol defaults to "medium" which V4 rejects; drop it.
# Fallback: if request didn't set it, try env SGLANG_DSV4_REASONING_EFFORT.
effort_source = request.reasoning_effort
if effort_source is None:
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
if env_val:
effort_source = env_val
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
assert reasoning_effort_profile is not None
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
reasoning_effort_profile
]
v4_reasoning_effort = (
effort_source if effort_source in ("max", "high") else None
effort_source if effort_source in accepted_efforts else None
)
if request.task is not None:
encoding_dsv4.attach_task_to_last_user_message(
@@ -1235,6 +1244,7 @@ class OpenAIServingChat(OpenAIServingBase):
messages,
thinking_mode=thinking_mode,
reasoning_effort=v4_reasoning_effort,
reasoning_effort_profile=reasoning_effort_profile,
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
else:
+1 -2
View File
@@ -1156,8 +1156,7 @@ class Envs:
# Copy rank-local MoE slices into independent CPU storage before H2D when
# they reference a larger mmap-backed checkpoint storage.
SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False)
# Default reasoning_effort for dsv4 chat encoder when request doesn't set it.
# Accepts "", "max", "high" (empty string means unset); other values filtered to None.
# Flash-0731 also accepts "low"; the active profile is checkpoint-resolved.
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
@@ -11,14 +11,19 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
import json
import tempfile
import unittest
import uuid
from http import HTTPStatus
from pathlib import Path
from typing import Optional
from unittest.mock import Mock, patch
from fastapi import Request
from sglang.srt.entrypoints.openai.chat_encoding import (
resolve_dsv4_reasoning_effort_profile,
)
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
MessageProcessingResult,
@@ -35,6 +40,22 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
_DSV4_PREVIEW_ENCODER = 'REASONING_EFFORT_MAX = "preview"\n'
_DSV4_OFFICIAL_ENCODER = (
"REASONING_EFFORT_PROMPTS: Dict[str, str] = "
'{"low": "", "high": "h", "max": "m"}\n'
'DEFAULT_REASONING_EFFORT = "low"\n'
)
def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
model_dir = tempfile.TemporaryDirectory()
test_case.addCleanup(model_dir.cleanup)
encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py"
encoder_path.parent.mkdir(parents=True)
encoder_path.write_text(source, encoding="utf-8")
return model_dir.name
class _MockTokenizerManager:
"""Minimal mock that satisfies OpenAIServingChat."""
@@ -42,18 +63,22 @@ class _MockTokenizerManager:
def __init__(self):
self.model_config = Mock(is_multimodal=False)
self.server_args = Mock(
model_path="deepseek-ai/DeepSeek-V4-Flash",
revision=None,
enable_cache_report=False,
tool_call_parser="hermes",
reasoning_parser=None,
stream_response_default_include_usage=False,
default_chat_template_kwargs=None,
)
self.model_path = self.server_args.model_path
# The manager tracks the served name itself; a weight update rewrites it.
self.served_model_name = "test-model"
# Mock hf_config for _resolve_chat_encoding_spec check
mock_hf_config = Mock()
mock_hf_config.architectures = ["LlamaForCausalLM"]
mock_hf_config.to_dict.return_value = {}
self.model_config.hf_config = mock_hf_config
self.chat_template_name: Optional[str] = "llama-3"
@@ -1023,6 +1048,7 @@ class ServingChatTestCase(unittest.TestCase):
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat._dsv4_reasoning_effort_profile = "preview"
for chat_encoding_spec in ("dsv4", "dsv32"):
with self.subTest(chat_encoding_spec=chat_encoding_spec):
@@ -1060,6 +1086,7 @@ class ServingChatTestCase(unittest.TestCase):
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat._dsv4_reasoning_effort_profile = "preview"
for chat_encoding_spec in ("dsv4", "dsv32"):
with self.subTest(chat_encoding_spec=chat_encoding_spec):
@@ -1536,6 +1563,7 @@ class ServingChatTestCase(unittest.TestCase):
mock_hf_config = Mock()
mock_hf_config.architectures = ["DeepseekV32ForCausalLM"]
mock_hf_config.to_dict.return_value = {}
tm.model_config.hf_config = mock_hf_config
# Case 1: No chat template + DeepSeek V3.2 arch -> should use dsv32 encoding
@@ -1557,6 +1585,10 @@ class ServingChatTestCase(unittest.TestCase):
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
# (release ships a stale V3 jinja we deliberately override).
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
mock_hf_config.to_dict.return_value = {
"dsv4_reasoning_effort_profile": "preview"
}
tm.model_path = "deepseek-ai/DeepSeek-V4-Flash"
tm.tokenizer.chat_template = "stale v3 jinja"
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
@@ -1728,6 +1760,135 @@ class ServingChatTestCase(unittest.TestCase):
)
self.assertIn("<|Assistant|>", out)
def test_dsv4_reasoning_effort_profiles(self):
from sglang.srt.entrypoints.openai import encoding_dsv4
messages = [
{"role": "system", "content": ""},
{"role": "user", "content": "Solve this."},
]
absolute_maximum = (
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
)
beyond_maximum = (
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
)
def encode(profile, effort):
return encoding_dsv4.encode_messages(
messages,
thinking_mode="thinking",
reasoning_effort=effort,
reasoning_effort_profile=profile,
)
preview_high = encode("preview", "high")
preview_max = encode("preview", "max")
official_low = encode("official", "low")
official_high = encode("official", "high")
official_max = encode("official", "max")
self.assertNotIn("Reasoning Effort:", preview_high)
self.assertTrue(
preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum)
)
self.assertNotIn("Reasoning Effort:", official_low)
self.assertEqual(
official_high,
encoding_dsv4.bos_token
+ absolute_maximum
+ official_low.removeprefix(encoding_dsv4.bos_token),
)
self.assertEqual(
official_max,
encoding_dsv4.bos_token
+ beyond_maximum
+ official_low.removeprefix(encoding_dsv4.bos_token),
)
self.assertEqual(encode("preview", None), preview_high)
self.assertEqual(encode("official", None), official_low)
self.assertEqual(len({official_low, official_high, official_max}), 3)
with self.assertRaises(ValueError):
encode("preview", "low")
def test_dsv4_reasoning_effort_profile_resolution(self):
resolve = resolve_dsv4_reasoning_effort_profile
preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER)
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
inconclusive_model_path = _create_dsv4_checkpoint(
self, 'UNRELATED_METADATA = "value"\n'
)
self.assertEqual(resolve(model_path=preview_model_path), "preview")
self.assertEqual(resolve(model_path=official_model_path), "official")
self.assertEqual(resolve(model_path=inconclusive_model_path), "preview")
self.assertEqual(
resolve(model_path="renamed/model", override="official"), "official"
)
self.assertEqual(
resolve(model_path="renamed/model", override="preview"), "preview"
)
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
resolve(model_path="renamed/model", override="auto")
def test_dsv4_reasoning_effort_profile_from_checkpoint(self):
from sglang.srt.parser.template_manager import TemplateManager
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
tm.model_config.hf_config.to_dict.return_value = {}
tm.model_config.hf_config.dspark_block_size = 5
tm.model_config.hf_config.dspark_markov_rank = 256
tm.model_path = official_model_path
tm.server_args.model_path = tm.model_path
serving_chat = OpenAIServingChat(tm, TemplateManager())
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hello"}],
reasoning_effort="max",
)
serving_chat._process_messages(request, is_multimodal=False)
prompt = tm.tokenizer.encode.call_args.args[0]
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
def test_dsv4_reasoning_effort_profile_override_from_model_config(self):
from sglang.srt.parser.template_manager import TemplateManager
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
tm.model_config.hf_config.to_dict.return_value = {
"dsv4_reasoning_effort_profile": "official"
}
serving_chat = OpenAIServingChat(tm, TemplateManager())
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hello"}],
reasoning_effort="max",
)
serving_chat._process_messages(request, is_multimodal=False)
prompt = tm.tokenizer.encode.call_args.args[0]
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
def test_dsv4_invalid_profile_override_fails_at_construction(self):
from sglang.srt.parser.template_manager import TemplateManager
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
tm.model_config.hf_config.to_dict.return_value = {
"dsv4_reasoning_effort_profile": "invalid"
}
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
OpenAIServingChat(tm, TemplateManager())
def test_streaming_abort_yields_error(self):
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
err_msg = "Aborted by scheduler"