diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 619e5af52..116150444 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -712,7 +712,7 @@ SGLang supports various environment variables that can be used to configure its SGLANG_DSV4_REASONING_EFFORT - Default reasoning_effort for the DeepSeek V4 chat encoder when a request does not set it (accepts max, high; empty means unset). + Default reasoning_effort for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts high and max; the official profile accepts low, high, and max. The profile is detected from the bundled encoder. Override it with --json-model-override-args '{"dsv4_reasoning_effort_profile":"official"}'. "" diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding.py b/python/sglang/srt/entrypoints/openai/chat_encoding.py index 9cb7769a9..b6b136ca9 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding.py @@ -7,8 +7,103 @@ it here instead of re-deriving it from model architectures themselves. from __future__ import annotations +import ast +import logging +from pathlib import Path from typing import Any, Dict, List, Optional +from sglang.srt.entrypoints.openai import encoding_dsv4 + +logger = logging.getLogger(__name__) + +DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile" +_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py" +_MAX_DSV4_ENCODER_BYTES = 1 << 20 + + +def _detect_dsv4_reasoning_effort_profile( + model_path: str, revision: Optional[str] = None +) -> Optional[str]: + encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER + try: + if not encoder_path.is_file(): + from huggingface_hub import hf_hub_download + + encoder_path = Path( + hf_hub_download( + model_path, + _DSV4_REASONING_EFFORT_ENCODER, + revision=revision, + ) + ) + if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES: + return None + tree = ast.parse(encoder_path.read_text(encoding="utf-8")) + except Exception as error: + logger.debug( + "Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s", + encoder_path, + error, + ) + return None + + assignments = {} + for node in tree.body: + if isinstance(node, ast.Assign): + targets = node.targets + value = node.value + elif isinstance(node, ast.AnnAssign): + targets = [node.target] + value = node.value + else: + continue + + for target in targets: + if not isinstance(target, ast.Name): + continue + try: + assignments[target.id] = ast.literal_eval(value) + except (TypeError, ValueError): + continue + + prompts = assignments.get("REASONING_EFFORT_PROMPTS") + if ( + assignments.get("DEFAULT_REASONING_EFFORT") == "low" + and isinstance(prompts, dict) + and {"low", "high", "max"} <= prompts.keys() + ): + return "official" + if "REASONING_EFFORT_MAX" in assignments: + return "preview" + return None + + +def _validate_dsv4_reasoning_effort_profile(profile: str) -> str: + if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES: + raise ValueError( + f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; " + f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}" + ) + return profile + + +def resolve_dsv4_reasoning_effort_profile( + *, + model_path: str, + revision: Optional[str] = None, + override: Optional[str] = None, +) -> str: + if override is not None: + return _validate_dsv4_reasoning_effort_profile(override) + + return ( + _detect_dsv4_reasoning_effort_profile( + model_path=model_path, + revision=revision, + ) + or "preview" + ) + def resolve_chat_encoding_spec( *, diff --git a/python/sglang/srt/entrypoints/openai/encoding_dsv4.py b/python/sglang/srt/entrypoints/openai/encoding_dsv4.py index a19d1f4b1..d6477ba4e 100644 --- a/python/sglang/srt/entrypoints/openai/encoding_dsv4.py +++ b/python/sglang/srt/entrypoints/openai/encoding_dsv4.py @@ -60,12 +60,30 @@ tool_calls_block_name: str = "tool_calls" tool_output_template: str = "{content}" -REASONING_EFFORT_MAX = ( +REASONING_EFFORT_PREVIEW_MAX = ( "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" ) +REASONING_EFFORT_OFFICIAL_MAX = ( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" + "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" + "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" +) + +REASONING_EFFORT_PROFILES = { + "preview": { + "high": "", + "max": REASONING_EFFORT_PREVIEW_MAX, + }, + "official": { + "low": "", + "high": REASONING_EFFORT_PREVIEW_MAX, + "max": REASONING_EFFORT_OFFICIAL_MAX, + }, +} + TOOLS_TEMPLATE = """## Tools You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: @@ -250,6 +268,7 @@ def render_message( thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None, + reasoning_effort_profile: str = "preview", ) -> str: """ Render a single message at the given index into its encoded string form. @@ -262,7 +281,9 @@ def render_message( messages: Full list of messages in the conversation. thinking_mode: Either "chat" or "thinking". drop_thinking: Whether to drop reasoning content from earlier turns. - reasoning_effort: Optional reasoning effort level ("max", "high", or None). + reasoning_effort: Optional reasoning effort level. The preview profile accepts + "high" and "max"; the official profile accepts "low", "high", and "max". + reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official"). Returns: Encoded string for this message. @@ -290,14 +311,21 @@ def render_message( if tool_calls: tool_calls = tool_calls_from_openai_format(tool_calls) - # Reasoning effort prefix (only at index 0 in thinking mode with max effort) - assert reasoning_effort in [ - "max", - None, - "high", - ], f"Invalid reasoning effort: {reasoning_effort}" - if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max": - prompt += REASONING_EFFORT_MAX + if reasoning_effort_profile not in REASONING_EFFORT_PROFILES: + raise ValueError( + f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; " + f"expected one of {list(REASONING_EFFORT_PROFILES)}" + ) + effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile] + if reasoning_effort is None: + reasoning_effort = "low" if reasoning_effort_profile == "official" else "high" + if reasoning_effort not in effort_prompts: + raise ValueError( + f"Invalid reasoning effort {reasoning_effort!r} for profile " + f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}" + ) + if index == 0 and thinking_mode == "thinking": + prompt += effort_prompts[reasoning_effort] if role == "system": prompt += system_msg_template.format(content=content or "") @@ -583,6 +611,7 @@ def encode_messages( drop_thinking: bool = True, add_default_bos_token: bool = True, reasoning_effort: Optional[str] = None, + reasoning_effort_profile: str = "preview", ) -> str: """ Encode a list of messages into the DeepSeek-V4 prompt format. @@ -600,7 +629,9 @@ def encode_messages( drop_thinking: If True, drop reasoning_content from earlier assistant turns (only keep reasoning for messages after the last user message). add_default_bos_token: Whether to prepend BOS token at conversation start. - reasoning_effort: Optional reasoning effort level ("max", "high", or None). + reasoning_effort: Optional reasoning effort level. The preview profile accepts + "high" and "max"; the official profile accepts "low", "high", and "max". + reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official"). Returns: The encoded prompt string. @@ -640,6 +671,7 @@ def encode_messages( thinking_mode=thinking_mode, drop_thinking=effective_drop_thinking, reasoning_effort=reasoning_effort, + reasoning_effort_profile=reasoning_effort_profile, ) return prompt diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index ad1e56cb8..d122fe37f 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -24,7 +24,7 @@ from fastapi import Request from fastapi.responses import ORJSONResponse, StreamingResponse from jsonschema import Draft202012Validator, SchemaError -from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv32 +from sglang.srt.entrypoints.openai import chat_encoding, encoding_dsv4, encoding_dsv32 from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionMessageGenericParam, ChatCompletionRequest, @@ -253,6 +253,17 @@ class OpenAIServingChat(OpenAIServingBase): # Which Python-based chat encoder (if any) bypasses apply_chat_template. # Values: "dsv32", "dsv4", or custom values set by subclass. None for default. self.chat_encoding_spec = self._resolve_chat_encoding_spec() + self._dsv4_reasoning_effort_profile = ( + chat_encoding.resolve_dsv4_reasoning_effort_profile( + model_path=self.tokenizer_manager.model_path, + revision=self.tokenizer_manager.server_args.revision, + override=self.tokenizer_manager.model_config.hf_config.to_dict().get( + chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE + ), + ) + if self.chat_encoding_spec == "dsv4" + else None + ) # Resolve the env-configured Inkling effort default once: the env var is # frozen for the server's lifetime, and a misconfigured value should @@ -339,11 +350,7 @@ class OpenAIServingChat(OpenAIServingBase): Override in subclass to add custom encoding specs. """ - from sglang.srt.entrypoints.openai.chat_encoding import ( - resolve_chat_encoding_spec, - ) - - return resolve_chat_encoding_spec( + return chat_encoding.resolve_chat_encoding_spec( hf_config=self.tokenizer_manager.model_config.hf_config, tokenizer=self.tokenizer_manager.tokenizer, tool_call_parser=self.tool_call_parser, @@ -1216,16 +1223,18 @@ class OpenAIServingChat(OpenAIServingBase): # Default encoding (dsv4/dsv32) if self.chat_encoding_spec == "dsv4": - # V4 encoder only accepts "max" / "high" / None. - # OpenAI protocol defaults to "medium" which V4 rejects; drop it. - # Fallback: if request didn't set it, try env SGLANG_DSV4_REASONING_EFFORT. effort_source = request.reasoning_effort if effort_source is None: env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get() if env_val: effort_source = env_val + reasoning_effort_profile = self._dsv4_reasoning_effort_profile + assert reasoning_effort_profile is not None + accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[ + reasoning_effort_profile + ] v4_reasoning_effort = ( - effort_source if effort_source in ("max", "high") else None + effort_source if effort_source in accepted_efforts else None ) if request.task is not None: encoding_dsv4.attach_task_to_last_user_message( @@ -1235,6 +1244,7 @@ class OpenAIServingChat(OpenAIServingBase): messages, thinking_mode=thinking_mode, reasoning_effort=v4_reasoning_effort, + reasoning_effort_profile=reasoning_effort_profile, ) prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) else: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 820d0b362..15209060e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1156,8 +1156,7 @@ class Envs: # Copy rank-local MoE slices into independent CPU storage before H2D when # they reference a larger mmap-backed checkpoint storage. SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D = EnvBool(False) - # Default reasoning_effort for dsv4 chat encoder when request doesn't set it. - # Accepts "", "max", "high" (empty string means unset); other values filtered to None. + # Flash-0731 also accepts "low"; the active profile is checkpoint-resolved. SGLANG_DSV4_REASONING_EFFORT = EnvStr("") # Quantize the SWA fp8 KV cache from bf16-rounded values (matches # trainer-side QAT and the DSA-CP path) instead of fp32 registers. diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 90d58f91b..692f1384e 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -11,14 +11,19 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel import json +import tempfile import unittest import uuid from http import HTTPStatus +from pathlib import Path from typing import Optional from unittest.mock import Mock, patch from fastapi import Request +from sglang.srt.entrypoints.openai.chat_encoding import ( + resolve_dsv4_reasoning_effort_profile, +) from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, MessageProcessingResult, @@ -35,6 +40,22 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") +_DSV4_PREVIEW_ENCODER = 'REASONING_EFFORT_MAX = "preview"\n' +_DSV4_OFFICIAL_ENCODER = ( + "REASONING_EFFORT_PROMPTS: Dict[str, str] = " + '{"low": "", "high": "h", "max": "m"}\n' + 'DEFAULT_REASONING_EFFORT = "low"\n' +) + + +def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str: + model_dir = tempfile.TemporaryDirectory() + test_case.addCleanup(model_dir.cleanup) + encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py" + encoder_path.parent.mkdir(parents=True) + encoder_path.write_text(source, encoding="utf-8") + return model_dir.name + class _MockTokenizerManager: """Minimal mock that satisfies OpenAIServingChat.""" @@ -42,18 +63,22 @@ class _MockTokenizerManager: def __init__(self): self.model_config = Mock(is_multimodal=False) self.server_args = Mock( + model_path="deepseek-ai/DeepSeek-V4-Flash", + revision=None, enable_cache_report=False, tool_call_parser="hermes", reasoning_parser=None, stream_response_default_include_usage=False, default_chat_template_kwargs=None, ) + self.model_path = self.server_args.model_path # The manager tracks the served name itself; a weight update rewrites it. self.served_model_name = "test-model" # Mock hf_config for _resolve_chat_encoding_spec check mock_hf_config = Mock() mock_hf_config.architectures = ["LlamaForCausalLM"] + mock_hf_config.to_dict.return_value = {} self.model_config.hf_config = mock_hf_config self.chat_template_name: Optional[str] = "llama-3" @@ -1023,6 +1048,7 @@ class ServingChatTestCase(unittest.TestCase): """DeepSeek encoders should reject history tool call scalars as BadRequest.""" self.template_manager.chat_template_name = None self.template_manager.jinja_template_content_format = "string" + self.chat._dsv4_reasoning_effort_profile = "preview" for chat_encoding_spec in ("dsv4", "dsv32"): with self.subTest(chat_encoding_spec=chat_encoding_spec): @@ -1060,6 +1086,7 @@ class ServingChatTestCase(unittest.TestCase): """DeepSeek encoders accept object-shaped OpenAI JSON string arguments.""" self.template_manager.chat_template_name = None self.template_manager.jinja_template_content_format = "string" + self.chat._dsv4_reasoning_effort_profile = "preview" for chat_encoding_spec in ("dsv4", "dsv32"): with self.subTest(chat_encoding_spec=chat_encoding_spec): @@ -1536,6 +1563,7 @@ class ServingChatTestCase(unittest.TestCase): mock_hf_config = Mock() mock_hf_config.architectures = ["DeepseekV32ForCausalLM"] + mock_hf_config.to_dict.return_value = {} tm.model_config.hf_config = mock_hf_config # Case 1: No chat template + DeepSeek V3.2 arch -> should use dsv32 encoding @@ -1557,6 +1585,10 @@ class ServingChatTestCase(unittest.TestCase): # Case 4: DeepseekV4 arch -> always dsv4, even with chat_template # (release ships a stale V3 jinja we deliberately override). mock_hf_config.architectures = ["DeepseekV4ForCausalLM"] + mock_hf_config.to_dict.return_value = { + "dsv4_reasoning_effort_profile": "preview" + } + tm.model_path = "deepseek-ai/DeepSeek-V4-Flash" tm.tokenizer.chat_template = "stale v3 jinja" serving_chat = OpenAIServingChat(tm, TemplateManager()) self.assertEqual(serving_chat.chat_encoding_spec, "dsv4") @@ -1728,6 +1760,135 @@ class ServingChatTestCase(unittest.TestCase): ) self.assertIn("<｜Assistant｜>", out) + def test_dsv4_reasoning_effort_profiles(self): + from sglang.srt.entrypoints.openai import encoding_dsv4 + + messages = [ + {"role": "system", "content": ""}, + {"role": "user", "content": "Solve this."}, + ] + absolute_maximum = ( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" + "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" + "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" + ) + beyond_maximum = ( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" + "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" + "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" + ) + + def encode(profile, effort): + return encoding_dsv4.encode_messages( + messages, + thinking_mode="thinking", + reasoning_effort=effort, + reasoning_effort_profile=profile, + ) + + preview_high = encode("preview", "high") + preview_max = encode("preview", "max") + official_low = encode("official", "low") + official_high = encode("official", "high") + official_max = encode("official", "max") + + self.assertNotIn("Reasoning Effort:", preview_high) + self.assertTrue( + preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum) + ) + self.assertNotIn("Reasoning Effort:", official_low) + self.assertEqual( + official_high, + encoding_dsv4.bos_token + + absolute_maximum + + official_low.removeprefix(encoding_dsv4.bos_token), + ) + self.assertEqual( + official_max, + encoding_dsv4.bos_token + + beyond_maximum + + official_low.removeprefix(encoding_dsv4.bos_token), + ) + self.assertEqual(encode("preview", None), preview_high) + self.assertEqual(encode("official", None), official_low) + self.assertEqual(len({official_low, official_high, official_max}), 3) + + with self.assertRaises(ValueError): + encode("preview", "low") + + def test_dsv4_reasoning_effort_profile_resolution(self): + resolve = resolve_dsv4_reasoning_effort_profile + preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER) + official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER) + inconclusive_model_path = _create_dsv4_checkpoint( + self, 'UNRELATED_METADATA = "value"\n' + ) + self.assertEqual(resolve(model_path=preview_model_path), "preview") + self.assertEqual(resolve(model_path=official_model_path), "official") + self.assertEqual(resolve(model_path=inconclusive_model_path), "preview") + + self.assertEqual( + resolve(model_path="renamed/model", override="official"), "official" + ) + self.assertEqual( + resolve(model_path="renamed/model", override="preview"), "preview" + ) + with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"): + resolve(model_path="renamed/model", override="auto") + + def test_dsv4_reasoning_effort_profile_from_checkpoint(self): + from sglang.srt.parser.template_manager import TemplateManager + + official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER) + tm = _MockTokenizerManager() + tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"] + tm.model_config.hf_config.to_dict.return_value = {} + tm.model_config.hf_config.dspark_block_size = 5 + tm.model_config.hf_config.dspark_markov_rank = 256 + tm.model_path = official_model_path + tm.server_args.model_path = tm.model_path + serving_chat = OpenAIServingChat(tm, TemplateManager()) + + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort="max", + ) + serving_chat._process_messages(request, is_multimodal=False) + prompt = tm.tokenizer.encode.call_args.args[0] + self.assertIn("Reasoning Effort: Beyond maximum", prompt) + + def test_dsv4_reasoning_effort_profile_override_from_model_config(self): + from sglang.srt.parser.template_manager import TemplateManager + + tm = _MockTokenizerManager() + tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"] + tm.model_config.hf_config.to_dict.return_value = { + "dsv4_reasoning_effort_profile": "official" + } + serving_chat = OpenAIServingChat(tm, TemplateManager()) + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort="max", + ) + + serving_chat._process_messages(request, is_multimodal=False) + + prompt = tm.tokenizer.encode.call_args.args[0] + self.assertIn("Reasoning Effort: Beyond maximum", prompt) + + def test_dsv4_invalid_profile_override_fails_at_construction(self): + from sglang.srt.parser.template_manager import TemplateManager + + tm = _MockTokenizerManager() + tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"] + tm.model_config.hf_config.to_dict.return_value = { + "dsv4_reasoning_effort_profile": "invalid" + } + with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"): + OpenAIServingChat(tm, TemplateManager()) + def test_streaming_abort_yields_error(self): """Test that an abort finish reason during streaming correctly yields an error and stops.""" err_msg = "Aborted by scheduler"