From 45019b56ce0804f734f7361e182226c0e5445765 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Thu, 9 Jul 2026 03:19:44 +0800 Subject: [PATCH] [Bugfix] Map reasoning_effort=low to Nemotron-3 Super low_effort + warn on unsupported levels (#30463) Co-authored-by: EazyReal <8047065+EazyReal@users.noreply.github.com> --- .../sglang/srt/entrypoints/openai/protocol.py | 21 +++---- .../srt/entrypoints/openai/serving_chat.py | 12 ++++ .../sglang/srt/managers/template_detection.py | 17 +++++- .../unit/entrypoints/openai/test_protocol.py | 25 ++++++++ .../entrypoints/openai/test_serving_chat.py | 61 +++++++++++++++++++ .../unit/managers/test_template_manager.py | 18 ++++++ 6 files changed, 140 insertions(+), 14 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index e562dea8a..2677a0637 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -800,6 +800,7 @@ class ChatCompletionRequest(BaseModel): @classmethod def normalize_reasoning_inputs(cls, values: Dict): r = values.get("reasoning") + thinking = None if r is not None and isinstance(r, dict): effort = r.get("effort") or r.get("reasoning_effort") @@ -814,25 +815,21 @@ class ChatCompletionRequest(BaseModel): if isinstance(enabled, str): enabled = enabled.strip().lower() in {"1", "true", "yes", "y", "on"} if enabled: - ctk = values.get("chat_template_kwargs") - if not isinstance(ctk, dict): - ctk = {} - # different models check different keys: - # - "thinking" for deepseek-v3, kimi_k2 - # - "enable_thinking" for qwen3, glm45, nemotron_3, interns1, mimo - ctk.setdefault("thinking", True) - ctk.setdefault("enable_thinking", True) - values["chat_template_kwargs"] = ctk + thinking = True - if values.get("reasoning_effort") == "none": + effort = values.get("reasoning_effort") + if effort is not None: + thinking = effort != "none" + + if thinking is not None: ctk = values.get("chat_template_kwargs") if not isinstance(ctk, dict): ctk = {} # different models check different keys: # - "thinking" for deepseek-v3, kimi_k2 # - "enable_thinking" for qwen3, glm45, nemotron_3, interns1 - ctk.setdefault("thinking", False) - ctk.setdefault("enable_thinking", False) + ctk.setdefault("thinking", thinking) + ctk.setdefault("enable_thinking", thinking) values["chat_template_kwargs"] = ctk return values diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 37062a502..2a02ac408 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -851,6 +851,18 @@ class OpenAIServingChat(OpenAIServingBase): if request.chat_template_kwargs: extra_template_kwargs.update(request.chat_template_kwargs) + rc = self.template_manager.reasoning_config + if rc is not None and rc.effort_kwarg is not None: + if request.reasoning_effort == "low": + extra_template_kwargs.setdefault(rc.effort_kwarg, True) + elif request.reasoning_effort in ("medium", "high", "max"): + logger.warning( + "Model '%s' supports only 'low' reasoning effort; " + "requested '%s' treated as default thinking", + self.tokenizer_manager.server_args.served_model_name, + request.reasoning_effort, + ) + # Split apply_chat_template(tokenize=True) into render + encode so we # can skip add_special_tokens=False on tokenizers that don't auto-add # specials (Kimi-like, OpenAI-chat analogue of #25265). Chat diff --git a/python/sglang/srt/managers/template_detection.py b/python/sglang/srt/managers/template_detection.py index 74050fae1..f6ea7d05a 100644 --- a/python/sglang/srt/managers/template_detection.py +++ b/python/sglang/srt/managers/template_detection.py @@ -64,6 +64,7 @@ class ReasoningToggleConfig: toggle_param: Optional[str] = None default_enabled: Optional[bool] = None special_case: Optional[str] = None + effort_kwarg: Optional[str] = None @property def always_on(self) -> bool: @@ -104,6 +105,16 @@ REASONING_MODE_RULES = ( re.DOTALL, ), ), + DetectionRule( + name="nemotron_3_super_low_effort", + value=ReasoningToggleConfig( + toggle_param="enable_thinking", + default_enabled=True, + effort_kwarg="low_effort", + ), + predicate=lambda ctx: ctx.has_text("low_effort") + and ctx.has_text("truncate_history_thinking"), + ), DetectionRule( name="enable_thinking_default_true", value=ReasoningToggleConfig( @@ -193,8 +204,10 @@ def _is_kimi_k2(ctx): def _is_nemotron_3(ctx): - return ctx.has_text("truncate_history_thinking") and ctx.reasoning_config == ( - ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True) + return ctx.has_text("truncate_history_thinking") and ( + ctx.reasoning_config is not None + and ctx.reasoning_config.toggle_param == "enable_thinking" + and ctx.reasoning_config.default_enabled is True ) diff --git a/test/registered/unit/entrypoints/openai/test_protocol.py b/test/registered/unit/entrypoints/openai/test_protocol.py index 19f880455..ed1c20689 100644 --- a/test/registered/unit/entrypoints/openai/test_protocol.py +++ b/test/registered/unit/entrypoints/openai/test_protocol.py @@ -210,6 +210,20 @@ class TestChatCompletionRequest(unittest.TestCase): {"thinking": True, "enable_thinking": True}, ) + def test_chat_completion_reasoning_effort_high_enables_thinking(self): + """Top-level reasoning_effort='high' enables thinking.""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest( + model="test-model", + messages=messages, + reasoning_effort="high", + ) + self.assertEqual(request.reasoning_effort, "high") + self.assertEqual( + request.chat_template_kwargs, + {"thinking": True, "enable_thinking": True}, + ) + def test_chat_completion_reasoning_effort_none(self): """Test reasoning_effort='none' disables thinking""" messages = [{"role": "user", "content": "Hello"}] @@ -234,6 +248,17 @@ class TestChatCompletionRequest(unittest.TestCase): self.assertFalse(request.chat_template_kwargs.get("thinking")) self.assertFalse(request.chat_template_kwargs.get("enable_thinking")) + def test_chat_completion_reasoning_effort_none_overrides_enabled(self): + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest( + model="test-model", + messages=messages, + reasoning={"enabled": True, "effort": "none"}, + ) + self.assertEqual(request.reasoning_effort, "none") + self.assertFalse(request.chat_template_kwargs.get("thinking")) + self.assertFalse(request.chat_template_kwargs.get("enable_thinking")) + def test_chat_completion_reasoning_effort_max(self): """`max` is an sglang extension on chat completion's top-level `reasoning_effort` only; the Responses-API-style nested diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index a28b90832..bebe83ddf 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -2001,6 +2001,67 @@ class ServingChatTestCase(unittest.TestCase): req.reasoning_effort = effort self.assertEqual(chat._get_reasoning_from_request(req), expected) + def _setup_nemotron_super(self): + """Drive _apply_jinja_template (chat_template_name=None) with a + Nemotron-3 Super reasoning_config carrying effort_kwarg.""" + self.tm.server_args.reasoning_parser = "nemotron_3" + self.chat.reasoning_parser = "nemotron_3" + self.tm.server_args.served_model_name = ( + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" + ) + self.template_manager.chat_template_name = None + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="enable_thinking", + default_enabled=True, + effort_kwarg="low_effort", + ) + self.chat.chat_encoding_spec = None + + def _run_jinja_with_effort(self, effort): + req = ChatCompletionRequest( + model="nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort=effort, + ) + with ( + patch.object(self.chat, "_encode_messages", return_value=None), + patch.object( + self.chat, + "_handle_last_assistant_message", + return_value=([{"role": "user", "content": "hi"}], None), + ), + ): + self.chat._process_messages(req, False) + _, kwargs = self.tm.tokenizer.apply_chat_template.call_args + return kwargs + + def test_nemotron_super_low_effort_mapped_to_kwarg(self): + self._setup_nemotron_super() + kwargs = self._run_jinja_with_effort("low") + self.assertTrue(kwargs.get("low_effort")) + + def test_nemotron_super_high_effort_warns_without_kwarg(self): + self._setup_nemotron_super() + with self.assertLogs( + "sglang.srt.entrypoints.openai.serving_chat", level="WARNING" + ) as logs: + kwargs = self._run_jinja_with_effort("high") + self.assertNotIn("low_effort", kwargs) + self.assertTrue(any("only 'low' reasoning effort" in m for m in logs.output)) + + def test_nemotron_nano_no_effort_kwarg(self): + # Nano template has no low_effort, so effort_kwarg stays None and no + # warning is emitted even for non-low effort. + self.tm.server_args.reasoning_parser = "nemotron_3" + self.chat.reasoning_parser = "nemotron_3" + self.template_manager.chat_template_name = None + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="enable_thinking", default_enabled=True + ) + self.chat.chat_encoding_spec = None + kwargs = self._run_jinja_with_effort("high") + self.assertNotIn("low_effort", kwargs) + def test_non_stream_reasoning_response_preserves_payload_whitespace(self): self.chat.reasoning_parser = "qwen3" self.template_manager.force_reasoning = False diff --git a/test/registered/unit/managers/test_template_manager.py b/test/registered/unit/managers/test_template_manager.py index 4b386bdb2..2774d0d4b 100644 --- a/test/registered/unit/managers/test_template_manager.py +++ b/test/registered/unit/managers/test_template_manager.py @@ -103,6 +103,24 @@ class TestTemplateManagerReasoningDetection(unittest.TestCase): ) self.assertEqual(parser, "nemotron_3") + def test_nemotron_super_detects_low_effort_kwarg(self): + template = """ + {% set enable_thinking = enable_thinking if enable_thinking is defined else True %} + {%- set low_effort = low_effort if low_effort is defined else False %} + {% set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %} + """ + _, config, parser = self._detect(template, [""]) + + self.assertEqual( + config, + ReasoningToggleConfig( + toggle_param="enable_thinking", + default_enabled=True, + effort_kwarg="low_effort", + ), + ) + self.assertEqual(parser, "nemotron_3") + def test_minimax_uses_template_signature_without_toggle_config(self): template = """ {%- set toolcall_begin_token = '' -%}