minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[Bugfix] Map reasoning_effort=low to Nemotron-3 Super low_effort + warn on unsupported levels (#30463)

Browse Source 
Co-authored-by: EazyReal <8047065+EazyReal@users.noreply.github.com>
This commit is contained in:
Xinyuan Tong
2026-07-08 12:19:44 -07:00
committed by GitHub
co-authored by EazyReal
parent 8d2b66fd90
commit 45019b56ce
6 changed files with 140 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/protocol.py
+9 -12
View File
@@ -800,6 +800,7 @@ class ChatCompletionRequest(BaseModel):
@classmethod
def normalize_reasoning_inputs(cls, values: Dict):
r = values.get("reasoning")
thinking = None
if r is not None and isinstance(r, dict):
effort = r.get("effort") or r.get("reasoning_effort")
@@ -814,25 +815,21 @@ class ChatCompletionRequest(BaseModel):
if isinstance(enabled, str):
enabled = enabled.strip().lower() in {"1", "true", "yes", "y", "on"}
if enabled:
ctk = values.get("chat_template_kwargs")
if not isinstance(ctk, dict):
ctk = {}
# different models check different keys:
# - "thinking" for deepseek-v3, kimi_k2
# - "enable_thinking" for qwen3, glm45, nemotron_3, interns1, mimo
ctk.setdefault("thinking", True)
ctk.setdefault("enable_thinking", True)
values["chat_template_kwargs"] = ctk
thinking = True
if values.get("reasoning_effort") == "none":
effort = values.get("reasoning_effort")
if effort is not None:
thinking = effort != "none"
if thinking is not None:
ctk = values.get("chat_template_kwargs")
if not isinstance(ctk, dict):
ctk = {}
# different models check different keys:
# - "thinking" for deepseek-v3, kimi_k2
# - "enable_thinking" for qwen3, glm45, nemotron_3, interns1
ctk.setdefault("thinking", False)
ctk.setdefault("enable_thinking", False)
ctk.setdefault("thinking", thinking)
ctk.setdefault("enable_thinking", thinking)
values["chat_template_kwargs"] = ctk
return values
python/sglang/srt/entrypoints/openai/serving_chat.py
+12
View File
@@ -851,6 +851,18 @@ class OpenAIServingChat(OpenAIServingBase):
if request.chat_template_kwargs:
extra_template_kwargs.update(request.chat_template_kwargs)
rc = self.template_manager.reasoning_config
if rc is not None and rc.effort_kwarg is not None:
if request.reasoning_effort == "low":
extra_template_kwargs.setdefault(rc.effort_kwarg, True)
elif request.reasoning_effort in ("medium", "high", "max"):
logger.warning(
"Model '%s' supports only 'low' reasoning effort; "
"requested '%s' treated as default thinking",
self.tokenizer_manager.server_args.served_model_name,
request.reasoning_effort,
)
# Split apply_chat_template(tokenize=True) into render + encode so we
# can skip add_special_tokens=False on tokenizers that don't auto-add
# specials (Kimi-like, OpenAI-chat analogue of #25265). Chat
python/sglang/srt/managers/template_detection.py
+15 -2
View File
@@ -64,6 +64,7 @@ class ReasoningToggleConfig:
toggle_param: Optional[str] = None
default_enabled: Optional[bool] = None
special_case: Optional[str] = None
effort_kwarg: Optional[str] = None
@property
def always_on(self) -> bool:
@@ -104,6 +105,16 @@ REASONING_MODE_RULES = (
re.DOTALL,
),
),
DetectionRule(
name="nemotron_3_super_low_effort",
value=ReasoningToggleConfig(
toggle_param="enable_thinking",
default_enabled=True,
effort_kwarg="low_effort",
),
predicate=lambda ctx: ctx.has_text("low_effort")
and ctx.has_text("truncate_history_thinking"),
),
DetectionRule(
name="enable_thinking_default_true",
value=ReasoningToggleConfig(
@@ -193,8 +204,10 @@ def _is_kimi_k2(ctx):
def _is_nemotron_3(ctx):
return ctx.has_text("truncate_history_thinking") and ctx.reasoning_config == (
ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True)
return ctx.has_text("truncate_history_thinking") and (
ctx.reasoning_config is not None
and ctx.reasoning_config.toggle_param == "enable_thinking"
and ctx.reasoning_config.default_enabled is True
)
test/registered/unit/entrypoints/openai/test_protocol.py
+25
View File
@@ -210,6 +210,20 @@ class TestChatCompletionRequest(unittest.TestCase):
{"thinking": True, "enable_thinking": True},
)
def test_chat_completion_reasoning_effort_high_enables_thinking(self):
"""Top-level reasoning_effort='high' enables thinking."""
messages = [{"role": "user", "content": "Hello"}]
request = ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning_effort="high",
)
self.assertEqual(request.reasoning_effort, "high")
self.assertEqual(
request.chat_template_kwargs,
{"thinking": True, "enable_thinking": True},
)
def test_chat_completion_reasoning_effort_none(self):
"""Test reasoning_effort='none' disables thinking"""
messages = [{"role": "user", "content": "Hello"}]
@@ -234,6 +248,17 @@ class TestChatCompletionRequest(unittest.TestCase):
self.assertFalse(request.chat_template_kwargs.get("thinking"))
self.assertFalse(request.chat_template_kwargs.get("enable_thinking"))
def test_chat_completion_reasoning_effort_none_overrides_enabled(self):
messages = [{"role": "user", "content": "Hello"}]
request = ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning={"enabled": True, "effort": "none"},
)
self.assertEqual(request.reasoning_effort, "none")
self.assertFalse(request.chat_template_kwargs.get("thinking"))
self.assertFalse(request.chat_template_kwargs.get("enable_thinking"))
def test_chat_completion_reasoning_effort_max(self):
"""`max` is an sglang extension on chat completion's top-level
`reasoning_effort` only; the Responses-API-style nested
test/registered/unit/entrypoints/openai/test_serving_chat.py
+61
View File
@@ -2001,6 +2001,67 @@ class ServingChatTestCase(unittest.TestCase):
req.reasoning_effort = effort
self.assertEqual(chat._get_reasoning_from_request(req), expected)
def _setup_nemotron_super(self):
"""Drive _apply_jinja_template (chat_template_name=None) with a
Nemotron-3 Super reasoning_config carrying effort_kwarg."""
self.tm.server_args.reasoning_parser = "nemotron_3"
self.chat.reasoning_parser = "nemotron_3"
self.tm.server_args.served_model_name = (
"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
)
self.template_manager.chat_template_name = None
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="enable_thinking",
default_enabled=True,
effort_kwarg="low_effort",
)
self.chat.chat_encoding_spec = None
def _run_jinja_with_effort(self, effort):
req = ChatCompletionRequest(
model="nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16",
messages=[{"role": "user", "content": "hi"}],
reasoning_effort=effort,
)
with (
patch.object(self.chat, "_encode_messages", return_value=None),
patch.object(
self.chat,
"_handle_last_assistant_message",
return_value=([{"role": "user", "content": "hi"}], None),
),
):
self.chat._process_messages(req, False)
_, kwargs = self.tm.tokenizer.apply_chat_template.call_args
return kwargs
def test_nemotron_super_low_effort_mapped_to_kwarg(self):
self._setup_nemotron_super()
kwargs = self._run_jinja_with_effort("low")
self.assertTrue(kwargs.get("low_effort"))
def test_nemotron_super_high_effort_warns_without_kwarg(self):
self._setup_nemotron_super()
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="WARNING"
) as logs:
kwargs = self._run_jinja_with_effort("high")
self.assertNotIn("low_effort", kwargs)
self.assertTrue(any("only 'low' reasoning effort" in m for m in logs.output))
def test_nemotron_nano_no_effort_kwarg(self):
# Nano template has no low_effort, so effort_kwarg stays None and no
# warning is emitted even for non-low effort.
self.tm.server_args.reasoning_parser = "nemotron_3"
self.chat.reasoning_parser = "nemotron_3"
self.template_manager.chat_template_name = None
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
)
self.chat.chat_encoding_spec = None
kwargs = self._run_jinja_with_effort("high")
self.assertNotIn("low_effort", kwargs)
def test_non_stream_reasoning_response_preserves_payload_whitespace(self):
self.chat.reasoning_parser = "qwen3"
self.template_manager.force_reasoning = False
test/registered/unit/managers/test_template_manager.py
+18
View File
@@ -103,6 +103,24 @@ class TestTemplateManagerReasoningDetection(unittest.TestCase):
)
self.assertEqual(parser, "nemotron_3")
def test_nemotron_super_detects_low_effort_kwarg(self):
template = """
{% set enable_thinking = enable_thinking if enable_thinking is defined else True %}
{%- set low_effort = low_effort if low_effort is defined else False %}
{% set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
"""
_, config, parser = self._detect(template, [""])
self.assertEqual(
config,
ReasoningToggleConfig(
toggle_param="enable_thinking",
default_enabled=True,
effort_kwarg="low_effort",
),
)
self.assertEqual(parser, "nemotron_3")
def test_minimax_uses_template_signature_without_toggle_config(self):
template = """
{%- set toolcall_begin_token = '<minimax:tool_call>' -%}