[DSV4] Add official DSV4 reasoning effort support (#33140)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: David Orman <ormandj@corenode.com>
This commit is contained in:
co-authored by
Xinyuan Tong
David Orman
parent
198a3bc29b
commit
059269594c
@@ -712,7 +712,7 @@ SGLang supports various environment variables that can be used to configure its
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it (accepts <code>max</code>, <code>high</code>; empty means unset).</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts <code>high</code> and <code>max</code>; the official profile accepts <code>low</code>, <code>high</code>, and <code>max</code>. The profile is detected from the bundled encoder. Override it with <code>--json-model-override-args '{"dsv4_reasoning_effort_profile":"official"}'</code>.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
@@ -7,8 +7,103 @@ it here instead of re-deriving it from model architectures themselves.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile"
|
||||
_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py"
|
||||
_MAX_DSV4_ENCODER_BYTES = 1 << 20
|
||||
|
||||
|
||||
def _detect_dsv4_reasoning_effort_profile(
|
||||
model_path: str, revision: Optional[str] = None
|
||||
) -> Optional[str]:
|
||||
encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER
|
||||
try:
|
||||
if not encoder_path.is_file():
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
encoder_path = Path(
|
||||
hf_hub_download(
|
||||
model_path,
|
||||
_DSV4_REASONING_EFFORT_ENCODER,
|
||||
revision=revision,
|
||||
)
|
||||
)
|
||||
if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES:
|
||||
return None
|
||||
tree = ast.parse(encoder_path.read_text(encoding="utf-8"))
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
"Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s",
|
||||
encoder_path,
|
||||
error,
|
||||
)
|
||||
return None
|
||||
|
||||
assignments = {}
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign):
|
||||
targets = node.targets
|
||||
value = node.value
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
targets = [node.target]
|
||||
value = node.value
|
||||
else:
|
||||
continue
|
||||
|
||||
for target in targets:
|
||||
if not isinstance(target, ast.Name):
|
||||
continue
|
||||
try:
|
||||
assignments[target.id] = ast.literal_eval(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
prompts = assignments.get("REASONING_EFFORT_PROMPTS")
|
||||
if (
|
||||
assignments.get("DEFAULT_REASONING_EFFORT") == "low"
|
||||
and isinstance(prompts, dict)
|
||||
and {"low", "high", "max"} <= prompts.keys()
|
||||
):
|
||||
return "official"
|
||||
if "REASONING_EFFORT_MAX" in assignments:
|
||||
return "preview"
|
||||
return None
|
||||
|
||||
|
||||
def _validate_dsv4_reasoning_effort_profile(profile: str) -> str:
|
||||
if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES:
|
||||
raise ValueError(
|
||||
f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; "
|
||||
f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}"
|
||||
)
|
||||
return profile
|
||||
|
||||
|
||||
def resolve_dsv4_reasoning_effort_profile(
|
||||
*,
|
||||
model_path: str,
|
||||
revision: Optional[str] = None,
|
||||
override: Optional[str] = None,
|
||||
) -> str:
|
||||
if override is not None:
|
||||
return _validate_dsv4_reasoning_effort_profile(override)
|
||||
|
||||
return (
|
||||
_detect_dsv4_reasoning_effort_profile(
|
||||
model_path=model_path,
|
||||
revision=revision,
|
||||
)
|
||||
or "preview"
|
||||
)
|
||||
|
||||
|
||||
def resolve_chat_encoding_spec(
|
||||
*,
|
||||
|
||||
@@ -60,12 +60,30 @@ tool_calls_block_name: str = "tool_calls"
|
||||
|
||||
tool_output_template: str = "<tool_result>{content}</tool_result>"
|
||||
|
||||
REASONING_EFFORT_MAX = (
|
||||
REASONING_EFFORT_PREVIEW_MAX = (
|
||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
||||
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
||||
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
||||
)
|
||||
|
||||
REASONING_EFFORT_OFFICIAL_MAX = (
|
||||
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
|
||||
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
|
||||
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
|
||||
)
|
||||
|
||||
REASONING_EFFORT_PROFILES = {
|
||||
"preview": {
|
||||
"high": "",
|
||||
"max": REASONING_EFFORT_PREVIEW_MAX,
|
||||
},
|
||||
"official": {
|
||||
"low": "",
|
||||
"high": REASONING_EFFORT_PREVIEW_MAX,
|
||||
"max": REASONING_EFFORT_OFFICIAL_MAX,
|
||||
},
|
||||
}
|
||||
|
||||
TOOLS_TEMPLATE = """## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
|
||||
@@ -250,6 +268,7 @@ def render_message(
|
||||
thinking_mode: str,
|
||||
drop_thinking: bool = True,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
reasoning_effort_profile: str = "preview",
|
||||
) -> str:
|
||||
"""
|
||||
Render a single message at the given index into its encoded string form.
|
||||
@@ -262,7 +281,9 @@ def render_message(
|
||||
messages: Full list of messages in the conversation.
|
||||
thinking_mode: Either "chat" or "thinking".
|
||||
drop_thinking: Whether to drop reasoning content from earlier turns.
|
||||
reasoning_effort: Optional reasoning effort level ("max", "high", or None).
|
||||
reasoning_effort: Optional reasoning effort level. The preview profile accepts
|
||||
"high" and "max"; the official profile accepts "low", "high", and "max".
|
||||
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
|
||||
|
||||
Returns:
|
||||
Encoded string for this message.
|
||||
@@ -290,14 +311,21 @@ def render_message(
|
||||
if tool_calls:
|
||||
tool_calls = tool_calls_from_openai_format(tool_calls)
|
||||
|
||||
# Reasoning effort prefix (only at index 0 in thinking mode with max effort)
|
||||
assert reasoning_effort in [
|
||||
"max",
|
||||
None,
|
||||
"high",
|
||||
], f"Invalid reasoning effort: {reasoning_effort}"
|
||||
if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max":
|
||||
prompt += REASONING_EFFORT_MAX
|
||||
if reasoning_effort_profile not in REASONING_EFFORT_PROFILES:
|
||||
raise ValueError(
|
||||
f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; "
|
||||
f"expected one of {list(REASONING_EFFORT_PROFILES)}"
|
||||
)
|
||||
effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile]
|
||||
if reasoning_effort is None:
|
||||
reasoning_effort = "low" if reasoning_effort_profile == "official" else "high"
|
||||
if reasoning_effort not in effort_prompts:
|
||||
raise ValueError(
|
||||
f"Invalid reasoning effort {reasoning_effort!r} for profile "
|
||||
f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}"
|
||||
)
|
||||
if index == 0 and thinking_mode == "thinking":
|
||||
prompt += effort_prompts[reasoning_effort]
|
||||
|
||||
if role == "system":
|
||||
prompt += system_msg_template.format(content=content or "")
|
||||
@@ -583,6 +611,7 @@ def encode_messages(
|
||||
drop_thinking: bool = True,
|
||||
add_default_bos_token: bool = True,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
reasoning_effort_profile: str = "preview",
|
||||
) -> str:
|
||||
"""
|
||||
Encode a list of messages into the DeepSeek-V4 prompt format.
|
||||
@@ -600,7 +629,9 @@ def encode_messages(
|
||||
drop_thinking: If True, drop reasoning_content from earlier assistant turns
|
||||
(only keep reasoning for messages after the last user message).
|
||||
add_default_bos_token: Whether to prepend BOS token at conversation start.
|
||||
reasoning_effort: Optional reasoning effort level ("max", "high", or None).
|
||||
reasoning_effort: Optional reasoning effort level. The preview profile accepts
|
||||
"high" and "max"; the official profile accepts "low", "high", and "max".
|
||||
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
|
||||
|
||||
Returns:
|
||||
The encoded prompt string.
|
||||
@@ -640,6 +671,7 @@ def encode_messages(
|
||||
thinking_mode=thinking_mode,
|
||||
drop_thinking=effective_drop_thinking,
|
||||
reasoning_effort=reasoning_effort,
|
||||
reasoning_effort_profile=reasoning_effort_profile,
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
@@ -24,7 +24,7 @@ from fastapi import Request
|
||||
from fastapi.responses import ORJSONResponse, StreamingResponse
|
||||
from jsonschema import Draft202012Validator, SchemaError
|
||||
|
||||
from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv32
|
||||
from sglang.srt.entrypoints.openai import chat_encoding, encoding_dsv4, encoding_dsv32
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionMessageGenericParam,
|
||||
ChatCompletionRequest,
|
||||
@@ -253,6 +253,17 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
|
||||
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
||||
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
||||
self._dsv4_reasoning_effort_profile = (
|
||||
chat_encoding.resolve_dsv4_reasoning_effort_profile(
|
||||
model_path=self.tokenizer_manager.model_path,
|
||||
revision=self.tokenizer_manager.server_args.revision,
|
||||
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
|
||||
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
|
||||
),
|
||||
)
|
||||
if self.chat_encoding_spec == "dsv4"
|
||||
else None
|
||||
)
|
||||
|
||||
# Resolve the env-configured Inkling effort default once: the env var is
|
||||
# frozen for the server's lifetime, and a misconfigured value should
|
||||
@@ -339,11 +350,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
Override in subclass to add custom encoding specs.
|
||||
"""
|
||||
from sglang.srt.entrypoints.openai.chat_encoding import (
|
||||
resolve_chat_encoding_spec,
|
||||
)
|
||||
|
||||
return resolve_chat_encoding_spec(
|
||||
return chat_encoding.resolve_chat_encoding_spec(
|
||||
hf_config=self.tokenizer_manager.model_config.hf_config,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
@@ -1216,16 +1223,18 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
# Default encoding (dsv4/dsv32)
|
||||
if self.chat_encoding_spec == "dsv4":
|
||||
# V4 encoder only accepts "max" / "high" / None.
|
||||
# OpenAI protocol defaults to "medium" which V4 rejects; drop it.
|
||||
# Fallback: if request didn't set it, try env SGLANG_DSV4_REASONING_EFFORT.
|
||||
effort_source = request.reasoning_effort
|
||||
if effort_source is None:
|
||||
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
|
||||
if env_val:
|
||||
effort_source = env_val
|
||||
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
|
||||
assert reasoning_effort_profile is not None
|
||||
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
|
||||
reasoning_effort_profile
|
||||
]
|
||||
v4_reasoning_effort = (
|
||||
effort_source if effort_source in ("max", "high") else None
|
||||
effort_source if effort_source in accepted_efforts else None
|
||||
)
|
||||
if request.task is not None:
|
||||
encoding_dsv4.attach_task_to_last_user_message(
|
||||
@@ -1235,6 +1244,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
messages,
|
||||
thinking_mode=thinking_mode,
|
||||
reasoning_effort=v4_reasoning_effort,
|
||||
reasoning_effort_profile=reasoning_effort_profile,
|
||||
)
|
||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
||||
else:
|
||||
|
||||
@@ -1156,8 +1156,7 @@ class Envs:
|
||||
# Copy rank-local MoE slices into independent CPU storage before H2D when
|
||||
# they reference a larger mmap-backed checkpoint storage.
|
||||
SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False)
|
||||
# Default reasoning_effort for dsv4 chat encoder when request doesn't set it.
|
||||
# Accepts "", "max", "high" (empty string means unset); other values filtered to None.
|
||||
# Flash-0731 also accepts "low"; the active profile is checkpoint-resolved.
|
||||
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
|
||||
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
|
||||
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
|
||||
|
||||
@@ -11,14 +11,19 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from sglang.srt.entrypoints.openai.chat_encoding import (
|
||||
resolve_dsv4_reasoning_effort_profile,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
MessageProcessingResult,
|
||||
@@ -35,6 +40,22 @@ from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
_DSV4_PREVIEW_ENCODER = 'REASONING_EFFORT_MAX = "preview"\n'
|
||||
_DSV4_OFFICIAL_ENCODER = (
|
||||
"REASONING_EFFORT_PROMPTS: Dict[str, str] = "
|
||||
'{"low": "", "high": "h", "max": "m"}\n'
|
||||
'DEFAULT_REASONING_EFFORT = "low"\n'
|
||||
)
|
||||
|
||||
|
||||
def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
|
||||
model_dir = tempfile.TemporaryDirectory()
|
||||
test_case.addCleanup(model_dir.cleanup)
|
||||
encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py"
|
||||
encoder_path.parent.mkdir(parents=True)
|
||||
encoder_path.write_text(source, encoding="utf-8")
|
||||
return model_dir.name
|
||||
|
||||
|
||||
class _MockTokenizerManager:
|
||||
"""Minimal mock that satisfies OpenAIServingChat."""
|
||||
@@ -42,18 +63,22 @@ class _MockTokenizerManager:
|
||||
def __init__(self):
|
||||
self.model_config = Mock(is_multimodal=False)
|
||||
self.server_args = Mock(
|
||||
model_path="deepseek-ai/DeepSeek-V4-Flash",
|
||||
revision=None,
|
||||
enable_cache_report=False,
|
||||
tool_call_parser="hermes",
|
||||
reasoning_parser=None,
|
||||
stream_response_default_include_usage=False,
|
||||
default_chat_template_kwargs=None,
|
||||
)
|
||||
self.model_path = self.server_args.model_path
|
||||
# The manager tracks the served name itself; a weight update rewrites it.
|
||||
self.served_model_name = "test-model"
|
||||
|
||||
# Mock hf_config for _resolve_chat_encoding_spec check
|
||||
mock_hf_config = Mock()
|
||||
mock_hf_config.architectures = ["LlamaForCausalLM"]
|
||||
mock_hf_config.to_dict.return_value = {}
|
||||
self.model_config.hf_config = mock_hf_config
|
||||
|
||||
self.chat_template_name: Optional[str] = "llama-3"
|
||||
@@ -1023,6 +1048,7 @@ class ServingChatTestCase(unittest.TestCase):
|
||||
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
|
||||
self.template_manager.chat_template_name = None
|
||||
self.template_manager.jinja_template_content_format = "string"
|
||||
self.chat._dsv4_reasoning_effort_profile = "preview"
|
||||
|
||||
for chat_encoding_spec in ("dsv4", "dsv32"):
|
||||
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
||||
@@ -1060,6 +1086,7 @@ class ServingChatTestCase(unittest.TestCase):
|
||||
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
|
||||
self.template_manager.chat_template_name = None
|
||||
self.template_manager.jinja_template_content_format = "string"
|
||||
self.chat._dsv4_reasoning_effort_profile = "preview"
|
||||
|
||||
for chat_encoding_spec in ("dsv4", "dsv32"):
|
||||
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
||||
@@ -1536,6 +1563,7 @@ class ServingChatTestCase(unittest.TestCase):
|
||||
|
||||
mock_hf_config = Mock()
|
||||
mock_hf_config.architectures = ["DeepseekV32ForCausalLM"]
|
||||
mock_hf_config.to_dict.return_value = {}
|
||||
tm.model_config.hf_config = mock_hf_config
|
||||
|
||||
# Case 1: No chat template + DeepSeek V3.2 arch -> should use dsv32 encoding
|
||||
@@ -1557,6 +1585,10 @@ class ServingChatTestCase(unittest.TestCase):
|
||||
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
|
||||
# (release ships a stale V3 jinja we deliberately override).
|
||||
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||
mock_hf_config.to_dict.return_value = {
|
||||
"dsv4_reasoning_effort_profile": "preview"
|
||||
}
|
||||
tm.model_path = "deepseek-ai/DeepSeek-V4-Flash"
|
||||
tm.tokenizer.chat_template = "stale v3 jinja"
|
||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
||||
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
|
||||
@@ -1728,6 +1760,135 @@ class ServingChatTestCase(unittest.TestCase):
|
||||
)
|
||||
self.assertIn("<|Assistant|>", out)
|
||||
|
||||
def test_dsv4_reasoning_effort_profiles(self):
|
||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": ""},
|
||||
{"role": "user", "content": "Solve this."},
|
||||
]
|
||||
absolute_maximum = (
|
||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
||||
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
||||
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
||||
)
|
||||
beyond_maximum = (
|
||||
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
|
||||
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
|
||||
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
|
||||
)
|
||||
|
||||
def encode(profile, effort):
|
||||
return encoding_dsv4.encode_messages(
|
||||
messages,
|
||||
thinking_mode="thinking",
|
||||
reasoning_effort=effort,
|
||||
reasoning_effort_profile=profile,
|
||||
)
|
||||
|
||||
preview_high = encode("preview", "high")
|
||||
preview_max = encode("preview", "max")
|
||||
official_low = encode("official", "low")
|
||||
official_high = encode("official", "high")
|
||||
official_max = encode("official", "max")
|
||||
|
||||
self.assertNotIn("Reasoning Effort:", preview_high)
|
||||
self.assertTrue(
|
||||
preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum)
|
||||
)
|
||||
self.assertNotIn("Reasoning Effort:", official_low)
|
||||
self.assertEqual(
|
||||
official_high,
|
||||
encoding_dsv4.bos_token
|
||||
+ absolute_maximum
|
||||
+ official_low.removeprefix(encoding_dsv4.bos_token),
|
||||
)
|
||||
self.assertEqual(
|
||||
official_max,
|
||||
encoding_dsv4.bos_token
|
||||
+ beyond_maximum
|
||||
+ official_low.removeprefix(encoding_dsv4.bos_token),
|
||||
)
|
||||
self.assertEqual(encode("preview", None), preview_high)
|
||||
self.assertEqual(encode("official", None), official_low)
|
||||
self.assertEqual(len({official_low, official_high, official_max}), 3)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
encode("preview", "low")
|
||||
|
||||
def test_dsv4_reasoning_effort_profile_resolution(self):
|
||||
resolve = resolve_dsv4_reasoning_effort_profile
|
||||
preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER)
|
||||
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
|
||||
inconclusive_model_path = _create_dsv4_checkpoint(
|
||||
self, 'UNRELATED_METADATA = "value"\n'
|
||||
)
|
||||
self.assertEqual(resolve(model_path=preview_model_path), "preview")
|
||||
self.assertEqual(resolve(model_path=official_model_path), "official")
|
||||
self.assertEqual(resolve(model_path=inconclusive_model_path), "preview")
|
||||
|
||||
self.assertEqual(
|
||||
resolve(model_path="renamed/model", override="official"), "official"
|
||||
)
|
||||
self.assertEqual(
|
||||
resolve(model_path="renamed/model", override="preview"), "preview"
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
|
||||
resolve(model_path="renamed/model", override="auto")
|
||||
|
||||
def test_dsv4_reasoning_effort_profile_from_checkpoint(self):
|
||||
from sglang.srt.parser.template_manager import TemplateManager
|
||||
|
||||
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
|
||||
tm = _MockTokenizerManager()
|
||||
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||
tm.model_config.hf_config.to_dict.return_value = {}
|
||||
tm.model_config.hf_config.dspark_block_size = 5
|
||||
tm.model_config.hf_config.dspark_markov_rank = 256
|
||||
tm.model_path = official_model_path
|
||||
tm.server_args.model_path = tm.model_path
|
||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="x",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
reasoning_effort="max",
|
||||
)
|
||||
serving_chat._process_messages(request, is_multimodal=False)
|
||||
prompt = tm.tokenizer.encode.call_args.args[0]
|
||||
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
|
||||
|
||||
def test_dsv4_reasoning_effort_profile_override_from_model_config(self):
|
||||
from sglang.srt.parser.template_manager import TemplateManager
|
||||
|
||||
tm = _MockTokenizerManager()
|
||||
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||
tm.model_config.hf_config.to_dict.return_value = {
|
||||
"dsv4_reasoning_effort_profile": "official"
|
||||
}
|
||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
||||
request = ChatCompletionRequest(
|
||||
model="x",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
reasoning_effort="max",
|
||||
)
|
||||
|
||||
serving_chat._process_messages(request, is_multimodal=False)
|
||||
|
||||
prompt = tm.tokenizer.encode.call_args.args[0]
|
||||
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
|
||||
|
||||
def test_dsv4_invalid_profile_override_fails_at_construction(self):
|
||||
from sglang.srt.parser.template_manager import TemplateManager
|
||||
|
||||
tm = _MockTokenizerManager()
|
||||
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||
tm.model_config.hf_config.to_dict.return_value = {
|
||||
"dsv4_reasoning_effort_profile": "invalid"
|
||||
}
|
||||
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
|
||||
OpenAIServingChat(tm, TemplateManager())
|
||||
|
||||
def test_streaming_abort_yields_error(self):
|
||||
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
|
||||
err_msg = "Aborted by scheduler"
|
||||
|
||||
Reference in New Issue
Block a user