[DSV4] Add official DSV4 reasoning effort support (#33140)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
co-authored by
Xinyuan Tong
David Orman
parent
198a3bc29b
commit
059269594c
@@ -712,7 +712,7 @@ SGLang supports various environment variables that can be used to configure its
|
|||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it (accepts <code>max</code>, <code>high</code>; empty means unset).</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts <code>high</code> and <code>max</code>; the official profile accepts <code>low</code>, <code>high</code>, and <code>max</code>. The profile is detected from the bundled encoder. Override it with <code>--json-model-override-args '{"dsv4_reasoning_effort_profile":"official"}'</code>.</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
|
|||||||
@@ -7,8 +7,103 @@ it here instead of re-deriving it from model architectures themselves.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from sglang.srt.entrypoints.openai import encoding_dsv4
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile"
|
||||||
|
_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py"
|
||||||
|
_MAX_DSV4_ENCODER_BYTES = 1 << 20
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_dsv4_reasoning_effort_profile(
|
||||||
|
model_path: str, revision: Optional[str] = None
|
||||||
|
) -> Optional[str]:
|
||||||
|
encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER
|
||||||
|
try:
|
||||||
|
if not encoder_path.is_file():
|
||||||
|
from huggingface_hub import hf_hub_download
|
||||||
|
|
||||||
|
encoder_path = Path(
|
||||||
|
hf_hub_download(
|
||||||
|
model_path,
|
||||||
|
_DSV4_REASONING_EFFORT_ENCODER,
|
||||||
|
revision=revision,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES:
|
||||||
|
return None
|
||||||
|
tree = ast.parse(encoder_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception as error:
|
||||||
|
logger.debug(
|
||||||
|
"Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s",
|
||||||
|
encoder_path,
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
assignments = {}
|
||||||
|
for node in tree.body:
|
||||||
|
if isinstance(node, ast.Assign):
|
||||||
|
targets = node.targets
|
||||||
|
value = node.value
|
||||||
|
elif isinstance(node, ast.AnnAssign):
|
||||||
|
targets = [node.target]
|
||||||
|
value = node.value
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for target in targets:
|
||||||
|
if not isinstance(target, ast.Name):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
assignments[target.id] = ast.literal_eval(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
prompts = assignments.get("REASONING_EFFORT_PROMPTS")
|
||||||
|
if (
|
||||||
|
assignments.get("DEFAULT_REASONING_EFFORT") == "low"
|
||||||
|
and isinstance(prompts, dict)
|
||||||
|
and {"low", "high", "max"} <= prompts.keys()
|
||||||
|
):
|
||||||
|
return "official"
|
||||||
|
if "REASONING_EFFORT_MAX" in assignments:
|
||||||
|
return "preview"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_dsv4_reasoning_effort_profile(profile: str) -> str:
|
||||||
|
if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; "
|
||||||
|
f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}"
|
||||||
|
)
|
||||||
|
return profile
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_dsv4_reasoning_effort_profile(
|
||||||
|
*,
|
||||||
|
model_path: str,
|
||||||
|
revision: Optional[str] = None,
|
||||||
|
override: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
if override is not None:
|
||||||
|
return _validate_dsv4_reasoning_effort_profile(override)
|
||||||
|
|
||||||
|
return (
|
||||||
|
_detect_dsv4_reasoning_effort_profile(
|
||||||
|
model_path=model_path,
|
||||||
|
revision=revision,
|
||||||
|
)
|
||||||
|
or "preview"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def resolve_chat_encoding_spec(
|
def resolve_chat_encoding_spec(
|
||||||
*,
|
*,
|
||||||
|
|||||||
@@ -60,12 +60,30 @@ tool_calls_block_name: str = "tool_calls"
|
|||||||
|
|
||||||
tool_output_template: str = "<tool_result>{content}</tool_result>"
|
tool_output_template: str = "<tool_result>{content}</tool_result>"
|
||||||
|
|
||||||
REASONING_EFFORT_MAX = (
|
REASONING_EFFORT_PREVIEW_MAX = (
|
||||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
||||||
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
||||||
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
REASONING_EFFORT_OFFICIAL_MAX = (
|
||||||
|
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
|
||||||
|
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
|
||||||
|
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
REASONING_EFFORT_PROFILES = {
|
||||||
|
"preview": {
|
||||||
|
"high": "",
|
||||||
|
"max": REASONING_EFFORT_PREVIEW_MAX,
|
||||||
|
},
|
||||||
|
"official": {
|
||||||
|
"low": "",
|
||||||
|
"high": REASONING_EFFORT_PREVIEW_MAX,
|
||||||
|
"max": REASONING_EFFORT_OFFICIAL_MAX,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
TOOLS_TEMPLATE = """## Tools
|
TOOLS_TEMPLATE = """## Tools
|
||||||
|
|
||||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
|
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
|
||||||
@@ -250,6 +268,7 @@ def render_message(
|
|||||||
thinking_mode: str,
|
thinking_mode: str,
|
||||||
drop_thinking: bool = True,
|
drop_thinking: bool = True,
|
||||||
reasoning_effort: Optional[str] = None,
|
reasoning_effort: Optional[str] = None,
|
||||||
|
reasoning_effort_profile: str = "preview",
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Render a single message at the given index into its encoded string form.
|
Render a single message at the given index into its encoded string form.
|
||||||
@@ -262,7 +281,9 @@ def render_message(
|
|||||||
messages: Full list of messages in the conversation.
|
messages: Full list of messages in the conversation.
|
||||||
thinking_mode: Either "chat" or "thinking".
|
thinking_mode: Either "chat" or "thinking".
|
||||||
drop_thinking: Whether to drop reasoning content from earlier turns.
|
drop_thinking: Whether to drop reasoning content from earlier turns.
|
||||||
reasoning_effort: Optional reasoning effort level ("max", "high", or None).
|
reasoning_effort: Optional reasoning effort level. The preview profile accepts
|
||||||
|
"high" and "max"; the official profile accepts "low", "high", and "max".
|
||||||
|
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Encoded string for this message.
|
Encoded string for this message.
|
||||||
@@ -290,14 +311,21 @@ def render_message(
|
|||||||
if tool_calls:
|
if tool_calls:
|
||||||
tool_calls = tool_calls_from_openai_format(tool_calls)
|
tool_calls = tool_calls_from_openai_format(tool_calls)
|
||||||
|
|
||||||
# Reasoning effort prefix (only at index 0 in thinking mode with max effort)
|
if reasoning_effort_profile not in REASONING_EFFORT_PROFILES:
|
||||||
assert reasoning_effort in [
|
raise ValueError(
|
||||||
"max",
|
f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; "
|
||||||
None,
|
f"expected one of {list(REASONING_EFFORT_PROFILES)}"
|
||||||
"high",
|
)
|
||||||
], f"Invalid reasoning effort: {reasoning_effort}"
|
effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile]
|
||||||
if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max":
|
if reasoning_effort is None:
|
||||||
prompt += REASONING_EFFORT_MAX
|
reasoning_effort = "low" if reasoning_effort_profile == "official" else "high"
|
||||||
|
if reasoning_effort not in effort_prompts:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid reasoning effort {reasoning_effort!r} for profile "
|
||||||
|
f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}"
|
||||||
|
)
|
||||||
|
if index == 0 and thinking_mode == "thinking":
|
||||||
|
prompt += effort_prompts[reasoning_effort]
|
||||||
|
|
||||||
if role == "system":
|
if role == "system":
|
||||||
prompt += system_msg_template.format(content=content or "")
|
prompt += system_msg_template.format(content=content or "")
|
||||||
@@ -583,6 +611,7 @@ def encode_messages(
|
|||||||
drop_thinking: bool = True,
|
drop_thinking: bool = True,
|
||||||
add_default_bos_token: bool = True,
|
add_default_bos_token: bool = True,
|
||||||
reasoning_effort: Optional[str] = None,
|
reasoning_effort: Optional[str] = None,
|
||||||
|
reasoning_effort_profile: str = "preview",
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Encode a list of messages into the DeepSeek-V4 prompt format.
|
Encode a list of messages into the DeepSeek-V4 prompt format.
|
||||||
@@ -600,7 +629,9 @@ def encode_messages(
|
|||||||
drop_thinking: If True, drop reasoning_content from earlier assistant turns
|
drop_thinking: If True, drop reasoning_content from earlier assistant turns
|
||||||
(only keep reasoning for messages after the last user message).
|
(only keep reasoning for messages after the last user message).
|
||||||
add_default_bos_token: Whether to prepend BOS token at conversation start.
|
add_default_bos_token: Whether to prepend BOS token at conversation start.
|
||||||
reasoning_effort: Optional reasoning effort level ("max", "high", or None).
|
reasoning_effort: Optional reasoning effort level. The preview profile accepts
|
||||||
|
"high" and "max"; the official profile accepts "low", "high", and "max".
|
||||||
|
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The encoded prompt string.
|
The encoded prompt string.
|
||||||
@@ -640,6 +671,7 @@ def encode_messages(
|
|||||||
thinking_mode=thinking_mode,
|
thinking_mode=thinking_mode,
|
||||||
drop_thinking=effective_drop_thinking,
|
drop_thinking=effective_drop_thinking,
|
||||||
reasoning_effort=reasoning_effort,
|
reasoning_effort=reasoning_effort,
|
||||||
|
reasoning_effort_profile=reasoning_effort_profile,
|
||||||
)
|
)
|
||||||
|
|
||||||
return prompt
|
return prompt
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ from fastapi import Request
|
|||||||
from fastapi.responses import ORJSONResponse, StreamingResponse
|
from fastapi.responses import ORJSONResponse, StreamingResponse
|
||||||
from jsonschema import Draft202012Validator, SchemaError
|
from jsonschema import Draft202012Validator, SchemaError
|
||||||
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv32
|
from sglang.srt.entrypoints.openai import chat_encoding, encoding_dsv4, encoding_dsv32
|
||||||
from sglang.srt.entrypoints.openai.protocol import (
|
from sglang.srt.entrypoints.openai.protocol import (
|
||||||
ChatCompletionMessageGenericParam,
|
ChatCompletionMessageGenericParam,
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
@@ -253,6 +253,17 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
|
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
|
||||||
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
||||||
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
||||||
|
self._dsv4_reasoning_effort_profile = (
|
||||||
|
chat_encoding.resolve_dsv4_reasoning_effort_profile(
|
||||||
|
model_path=self.tokenizer_manager.model_path,
|
||||||
|
revision=self.tokenizer_manager.server_args.revision,
|
||||||
|
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
|
||||||
|
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if self.chat_encoding_spec == "dsv4"
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
# Resolve the env-configured Inkling effort default once: the env var is
|
# Resolve the env-configured Inkling effort default once: the env var is
|
||||||
# frozen for the server's lifetime, and a misconfigured value should
|
# frozen for the server's lifetime, and a misconfigured value should
|
||||||
@@ -339,11 +350,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
|
|
||||||
Override in subclass to add custom encoding specs.
|
Override in subclass to add custom encoding specs.
|
||||||
"""
|
"""
|
||||||
from sglang.srt.entrypoints.openai.chat_encoding import (
|
return chat_encoding.resolve_chat_encoding_spec(
|
||||||
resolve_chat_encoding_spec,
|
|
||||||
)
|
|
||||||
|
|
||||||
return resolve_chat_encoding_spec(
|
|
||||||
hf_config=self.tokenizer_manager.model_config.hf_config,
|
hf_config=self.tokenizer_manager.model_config.hf_config,
|
||||||
tokenizer=self.tokenizer_manager.tokenizer,
|
tokenizer=self.tokenizer_manager.tokenizer,
|
||||||
tool_call_parser=self.tool_call_parser,
|
tool_call_parser=self.tool_call_parser,
|
||||||
@@ -1216,16 +1223,18 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
|
|
||||||
# Default encoding (dsv4/dsv32)
|
# Default encoding (dsv4/dsv32)
|
||||||
if self.chat_encoding_spec == "dsv4":
|
if self.chat_encoding_spec == "dsv4":
|
||||||
# V4 encoder only accepts "max" / "high" / None.
|
|
||||||
# OpenAI protocol defaults to "medium" which V4 rejects; drop it.
|
|
||||||
# Fallback: if request didn't set it, try env SGLANG_DSV4_REASONING_EFFORT.
|
|
||||||
effort_source = request.reasoning_effort
|
effort_source = request.reasoning_effort
|
||||||
if effort_source is None:
|
if effort_source is None:
|
||||||
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
|
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
|
||||||
if env_val:
|
if env_val:
|
||||||
effort_source = env_val
|
effort_source = env_val
|
||||||
|
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
|
||||||
|
assert reasoning_effort_profile is not None
|
||||||
|
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
|
||||||
|
reasoning_effort_profile
|
||||||
|
]
|
||||||
v4_reasoning_effort = (
|
v4_reasoning_effort = (
|
||||||
effort_source if effort_source in ("max", "high") else None
|
effort_source if effort_source in accepted_efforts else None
|
||||||
)
|
)
|
||||||
if request.task is not None:
|
if request.task is not None:
|
||||||
encoding_dsv4.attach_task_to_last_user_message(
|
encoding_dsv4.attach_task_to_last_user_message(
|
||||||
@@ -1235,6 +1244,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
messages,
|
messages,
|
||||||
thinking_mode=thinking_mode,
|
thinking_mode=thinking_mode,
|
||||||
reasoning_effort=v4_reasoning_effort,
|
reasoning_effort=v4_reasoning_effort,
|
||||||
|
reasoning_effort_profile=reasoning_effort_profile,
|
||||||
)
|
)
|
||||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -1156,8 +1156,7 @@ class Envs:
|
|||||||
# Copy rank-local MoE slices into independent CPU storage before H2D when
|
# Copy rank-local MoE slices into independent CPU storage before H2D when
|
||||||
# they reference a larger mmap-backed checkpoint storage.
|
# they reference a larger mmap-backed checkpoint storage.
|
||||||
SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False)
|
SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False)
|
||||||
# Default reasoning_effort for dsv4 chat encoder when request doesn't set it.
|
# Flash-0731 also accepts "low"; the active profile is checkpoint-resolved.
|
||||||
# Accepts "", "max", "high" (empty string means unset); other values filtered to None.
|
|
||||||
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
|
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
|
||||||
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
|
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
|
||||||
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
|
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
|
||||||
|
|||||||
@@ -11,14 +11,19 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel
|
|||||||
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
import uuid
|
import uuid
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
|
|
||||||
|
from sglang.srt.entrypoints.openai.chat_encoding import (
|
||||||
|
resolve_dsv4_reasoning_effort_profile,
|
||||||
|
)
|
||||||
from sglang.srt.entrypoints.openai.protocol import (
|
from sglang.srt.entrypoints.openai.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
MessageProcessingResult,
|
MessageProcessingResult,
|
||||||
@@ -35,6 +40,22 @@ from sglang.test.ci.ci_register import register_cpu_ci
|
|||||||
|
|
||||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||||
|
|
||||||
|
_DSV4_PREVIEW_ENCODER = 'REASONING_EFFORT_MAX = "preview"\n'
|
||||||
|
_DSV4_OFFICIAL_ENCODER = (
|
||||||
|
"REASONING_EFFORT_PROMPTS: Dict[str, str] = "
|
||||||
|
'{"low": "", "high": "h", "max": "m"}\n'
|
||||||
|
'DEFAULT_REASONING_EFFORT = "low"\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
|
||||||
|
model_dir = tempfile.TemporaryDirectory()
|
||||||
|
test_case.addCleanup(model_dir.cleanup)
|
||||||
|
encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py"
|
||||||
|
encoder_path.parent.mkdir(parents=True)
|
||||||
|
encoder_path.write_text(source, encoding="utf-8")
|
||||||
|
return model_dir.name
|
||||||
|
|
||||||
|
|
||||||
class _MockTokenizerManager:
|
class _MockTokenizerManager:
|
||||||
"""Minimal mock that satisfies OpenAIServingChat."""
|
"""Minimal mock that satisfies OpenAIServingChat."""
|
||||||
@@ -42,18 +63,22 @@ class _MockTokenizerManager:
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.model_config = Mock(is_multimodal=False)
|
self.model_config = Mock(is_multimodal=False)
|
||||||
self.server_args = Mock(
|
self.server_args = Mock(
|
||||||
|
model_path="deepseek-ai/DeepSeek-V4-Flash",
|
||||||
|
revision=None,
|
||||||
enable_cache_report=False,
|
enable_cache_report=False,
|
||||||
tool_call_parser="hermes",
|
tool_call_parser="hermes",
|
||||||
reasoning_parser=None,
|
reasoning_parser=None,
|
||||||
stream_response_default_include_usage=False,
|
stream_response_default_include_usage=False,
|
||||||
default_chat_template_kwargs=None,
|
default_chat_template_kwargs=None,
|
||||||
)
|
)
|
||||||
|
self.model_path = self.server_args.model_path
|
||||||
# The manager tracks the served name itself; a weight update rewrites it.
|
# The manager tracks the served name itself; a weight update rewrites it.
|
||||||
self.served_model_name = "test-model"
|
self.served_model_name = "test-model"
|
||||||
|
|
||||||
# Mock hf_config for _resolve_chat_encoding_spec check
|
# Mock hf_config for _resolve_chat_encoding_spec check
|
||||||
mock_hf_config = Mock()
|
mock_hf_config = Mock()
|
||||||
mock_hf_config.architectures = ["LlamaForCausalLM"]
|
mock_hf_config.architectures = ["LlamaForCausalLM"]
|
||||||
|
mock_hf_config.to_dict.return_value = {}
|
||||||
self.model_config.hf_config = mock_hf_config
|
self.model_config.hf_config = mock_hf_config
|
||||||
|
|
||||||
self.chat_template_name: Optional[str] = "llama-3"
|
self.chat_template_name: Optional[str] = "llama-3"
|
||||||
@@ -1023,6 +1048,7 @@ class ServingChatTestCase(unittest.TestCase):
|
|||||||
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
|
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
|
||||||
self.template_manager.chat_template_name = None
|
self.template_manager.chat_template_name = None
|
||||||
self.template_manager.jinja_template_content_format = "string"
|
self.template_manager.jinja_template_content_format = "string"
|
||||||
|
self.chat._dsv4_reasoning_effort_profile = "preview"
|
||||||
|
|
||||||
for chat_encoding_spec in ("dsv4", "dsv32"):
|
for chat_encoding_spec in ("dsv4", "dsv32"):
|
||||||
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
||||||
@@ -1060,6 +1086,7 @@ class ServingChatTestCase(unittest.TestCase):
|
|||||||
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
|
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
|
||||||
self.template_manager.chat_template_name = None
|
self.template_manager.chat_template_name = None
|
||||||
self.template_manager.jinja_template_content_format = "string"
|
self.template_manager.jinja_template_content_format = "string"
|
||||||
|
self.chat._dsv4_reasoning_effort_profile = "preview"
|
||||||
|
|
||||||
for chat_encoding_spec in ("dsv4", "dsv32"):
|
for chat_encoding_spec in ("dsv4", "dsv32"):
|
||||||
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
||||||
@@ -1536,6 +1563,7 @@ class ServingChatTestCase(unittest.TestCase):
|
|||||||
|
|
||||||
mock_hf_config = Mock()
|
mock_hf_config = Mock()
|
||||||
mock_hf_config.architectures = ["DeepseekV32ForCausalLM"]
|
mock_hf_config.architectures = ["DeepseekV32ForCausalLM"]
|
||||||
|
mock_hf_config.to_dict.return_value = {}
|
||||||
tm.model_config.hf_config = mock_hf_config
|
tm.model_config.hf_config = mock_hf_config
|
||||||
|
|
||||||
# Case 1: No chat template + DeepSeek V3.2 arch -> should use dsv32 encoding
|
# Case 1: No chat template + DeepSeek V3.2 arch -> should use dsv32 encoding
|
||||||
@@ -1557,6 +1585,10 @@ class ServingChatTestCase(unittest.TestCase):
|
|||||||
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
|
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
|
||||||
# (release ships a stale V3 jinja we deliberately override).
|
# (release ships a stale V3 jinja we deliberately override).
|
||||||
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||||
|
mock_hf_config.to_dict.return_value = {
|
||||||
|
"dsv4_reasoning_effort_profile": "preview"
|
||||||
|
}
|
||||||
|
tm.model_path = "deepseek-ai/DeepSeek-V4-Flash"
|
||||||
tm.tokenizer.chat_template = "stale v3 jinja"
|
tm.tokenizer.chat_template = "stale v3 jinja"
|
||||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
||||||
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
|
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
|
||||||
@@ -1728,6 +1760,135 @@ class ServingChatTestCase(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertIn("<|Assistant|>", out)
|
self.assertIn("<|Assistant|>", out)
|
||||||
|
|
||||||
|
def test_dsv4_reasoning_effort_profiles(self):
|
||||||
|
from sglang.srt.entrypoints.openai import encoding_dsv4
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": ""},
|
||||||
|
{"role": "user", "content": "Solve this."},
|
||||||
|
]
|
||||||
|
absolute_maximum = (
|
||||||
|
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
||||||
|
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
||||||
|
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
||||||
|
)
|
||||||
|
beyond_maximum = (
|
||||||
|
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
|
||||||
|
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
|
||||||
|
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
def encode(profile, effort):
|
||||||
|
return encoding_dsv4.encode_messages(
|
||||||
|
messages,
|
||||||
|
thinking_mode="thinking",
|
||||||
|
reasoning_effort=effort,
|
||||||
|
reasoning_effort_profile=profile,
|
||||||
|
)
|
||||||
|
|
||||||
|
preview_high = encode("preview", "high")
|
||||||
|
preview_max = encode("preview", "max")
|
||||||
|
official_low = encode("official", "low")
|
||||||
|
official_high = encode("official", "high")
|
||||||
|
official_max = encode("official", "max")
|
||||||
|
|
||||||
|
self.assertNotIn("Reasoning Effort:", preview_high)
|
||||||
|
self.assertTrue(
|
||||||
|
preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum)
|
||||||
|
)
|
||||||
|
self.assertNotIn("Reasoning Effort:", official_low)
|
||||||
|
self.assertEqual(
|
||||||
|
official_high,
|
||||||
|
encoding_dsv4.bos_token
|
||||||
|
+ absolute_maximum
|
||||||
|
+ official_low.removeprefix(encoding_dsv4.bos_token),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
official_max,
|
||||||
|
encoding_dsv4.bos_token
|
||||||
|
+ beyond_maximum
|
||||||
|
+ official_low.removeprefix(encoding_dsv4.bos_token),
|
||||||
|
)
|
||||||
|
self.assertEqual(encode("preview", None), preview_high)
|
||||||
|
self.assertEqual(encode("official", None), official_low)
|
||||||
|
self.assertEqual(len({official_low, official_high, official_max}), 3)
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
encode("preview", "low")
|
||||||
|
|
||||||
|
def test_dsv4_reasoning_effort_profile_resolution(self):
|
||||||
|
resolve = resolve_dsv4_reasoning_effort_profile
|
||||||
|
preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER)
|
||||||
|
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
|
||||||
|
inconclusive_model_path = _create_dsv4_checkpoint(
|
||||||
|
self, 'UNRELATED_METADATA = "value"\n'
|
||||||
|
)
|
||||||
|
self.assertEqual(resolve(model_path=preview_model_path), "preview")
|
||||||
|
self.assertEqual(resolve(model_path=official_model_path), "official")
|
||||||
|
self.assertEqual(resolve(model_path=inconclusive_model_path), "preview")
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
resolve(model_path="renamed/model", override="official"), "official"
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
resolve(model_path="renamed/model", override="preview"), "preview"
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
|
||||||
|
resolve(model_path="renamed/model", override="auto")
|
||||||
|
|
||||||
|
def test_dsv4_reasoning_effort_profile_from_checkpoint(self):
|
||||||
|
from sglang.srt.parser.template_manager import TemplateManager
|
||||||
|
|
||||||
|
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
|
||||||
|
tm = _MockTokenizerManager()
|
||||||
|
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||||
|
tm.model_config.hf_config.to_dict.return_value = {}
|
||||||
|
tm.model_config.hf_config.dspark_block_size = 5
|
||||||
|
tm.model_config.hf_config.dspark_markov_rank = 256
|
||||||
|
tm.model_path = official_model_path
|
||||||
|
tm.server_args.model_path = tm.model_path
|
||||||
|
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
||||||
|
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model="x",
|
||||||
|
messages=[{"role": "user", "content": "Hello"}],
|
||||||
|
reasoning_effort="max",
|
||||||
|
)
|
||||||
|
serving_chat._process_messages(request, is_multimodal=False)
|
||||||
|
prompt = tm.tokenizer.encode.call_args.args[0]
|
||||||
|
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
|
||||||
|
|
||||||
|
def test_dsv4_reasoning_effort_profile_override_from_model_config(self):
|
||||||
|
from sglang.srt.parser.template_manager import TemplateManager
|
||||||
|
|
||||||
|
tm = _MockTokenizerManager()
|
||||||
|
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||||
|
tm.model_config.hf_config.to_dict.return_value = {
|
||||||
|
"dsv4_reasoning_effort_profile": "official"
|
||||||
|
}
|
||||||
|
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model="x",
|
||||||
|
messages=[{"role": "user", "content": "Hello"}],
|
||||||
|
reasoning_effort="max",
|
||||||
|
)
|
||||||
|
|
||||||
|
serving_chat._process_messages(request, is_multimodal=False)
|
||||||
|
|
||||||
|
prompt = tm.tokenizer.encode.call_args.args[0]
|
||||||
|
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
|
||||||
|
|
||||||
|
def test_dsv4_invalid_profile_override_fails_at_construction(self):
|
||||||
|
from sglang.srt.parser.template_manager import TemplateManager
|
||||||
|
|
||||||
|
tm = _MockTokenizerManager()
|
||||||
|
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||||
|
tm.model_config.hf_config.to_dict.return_value = {
|
||||||
|
"dsv4_reasoning_effort_profile": "invalid"
|
||||||
|
}
|
||||||
|
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
|
||||||
|
OpenAIServingChat(tm, TemplateManager())
|
||||||
|
|
||||||
def test_streaming_abort_yields_error(self):
|
def test_streaming_abort_yields_error(self):
|
||||||
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
|
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
|
||||||
err_msg = "Aborted by scheduler"
|
err_msg = "Aborted by scheduler"
|
||||||
|
|||||||
Reference in New Issue
Block a user