From 0ee236ebdff36fe413d7ca2e7fc99875d5554c89 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Mon, 13 Jul 2026 15:50:27 -0700 Subject: [PATCH] more fixes for Nemotron 3 parser for tool call and force nonempty content (#30533) --- .../srt/entrypoints/openai/serving_chat.py | 17 ++- python/sglang/srt/parser/reasoning_parser.py | 110 +++++++++++++++--- .../unit/parser/test_reasoning_parser.py | 68 +++++++++++ 3 files changed, 175 insertions(+), 20 deletions(-) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 11baf6abf..3a678173f 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -374,7 +374,12 @@ class OpenAIServingChat(OpenAIServingBase): # Handle reasoning content if self.reasoning_parser and request.separate_reasoning: reasoning_text, delta = self._process_reasoning_stream( - index, delta, reasoning_parser_dict, content, request + index, + delta, + reasoning_parser_dict, + content, + request, + finish_reason_type, ) if reasoning_text: usage = None @@ -1635,6 +1640,7 @@ class OpenAIServingChat(OpenAIServingBase): reasoning_parser_dict: Dict[int, ReasoningParser], content: Dict[str, Any], request: ChatCompletionRequest, + finish_reason_type: Optional[str] = None, ) -> tuple[Optional[str], str]: """Process reasoning content in streaming response""" if index not in reasoning_parser_dict: @@ -1650,7 +1656,14 @@ class OpenAIServingChat(OpenAIServingBase): tokenizer=self.tokenizer_manager.tokenizer, ) reasoning_parser = reasoning_parser_dict[index] - return reasoning_parser.parse_stream_chunk(delta) + reasoning_text, normal_text = reasoning_parser.parse_stream_chunk(delta) + if finish_reason_type is not None and finish_reason_type != "abort": + end_reasoning_text, end_normal_text = reasoning_parser.parse_stream_end() + if end_reasoning_text: + reasoning_text = (reasoning_text or "") + end_reasoning_text + if end_normal_text: + normal_text = (normal_text or "") + end_normal_text + return reasoning_text, normal_text def _get_history_tool_calls_cnt(self, request: ChatCompletionRequest) -> int: """Counts the number of tool calls in the request's message history. diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 4c9f7fef0..a507f34f6 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -33,6 +33,7 @@ class BaseReasoningFormatDetector: previous_content: str = "", thinks_internally: bool = False, reasoning_default: str = "always", + force_nonempty_content: bool = False, ): self.think_start_token = think_start_token self.think_end_token = think_end_token @@ -48,6 +49,9 @@ class BaseReasoningFormatDetector: self.stripped_think_start = False self.think_start_self_label = "" + self._force_nonempty_content = force_nonempty_content + self._accumulated_reasoning = "" + self.continue_final_message = continue_final_message if self.continue_final_message: self.previous_content = previous_content @@ -61,11 +65,23 @@ class BaseReasoningFormatDetector: if self.think_end_token in self.previous_content: self._in_reasoning = False + def _maybe_apply_force_nonempty_content( + self, ret: StreamingParseResult + ) -> StreamingParseResult: + if self._force_nonempty_content and not ret.normal_text: + ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text + return ret + def detect_and_parse(self, text: str) -> StreamingParseResult: """ One-time parsing: Detects and parses reasoning sections in the provided text. Returns both reasoning content and normal text separately. """ + return self._maybe_apply_force_nonempty_content( + self._detect_and_parse_impl(text) + ) + + def _detect_and_parse_impl(self, text: str) -> StreamingParseResult: in_reasoning = self._in_reasoning or self.think_start_token in text if not in_reasoning: @@ -121,6 +137,15 @@ class BaseReasoningFormatDetector: If stream_reasoning is True: Streams reasoning content as it arrives """ + ret = self._parse_streaming_increment_impl(new_text) + if self._force_nonempty_content: + if self._in_reasoning: + self._accumulated_reasoning += ret.reasoning_text + else: + self._accumulated_reasoning = "" + return ret + + def _parse_streaming_increment_impl(self, new_text: str) -> StreamingParseResult: self._buffer += new_text current_text = self._buffer @@ -183,6 +208,26 @@ class BaseReasoningFormatDetector: return StreamingParseResult() + def finish(self) -> StreamingParseResult: + """ + Called once when the stream ends. If force_nonempty_content is set + and the stream ended mid-reasoning, reclassifies the accumulated + reasoning (plus any partial token still buffered) as normal text. + """ + if self._force_nonempty_content and self._in_reasoning: + # stream_reasoning=False never clears _buffer, so the opening think + # token (stripped only from the base class's local view) survives here. + buffer = self._buffer + think_start_text = self.think_start_token + self.think_start_self_label + if buffer.startswith(think_start_text): + buffer = buffer[len(think_start_text) :] + normal_text = self._accumulated_reasoning + buffer + self._accumulated_reasoning = "" + self._buffer = "" + if normal_text: + return StreamingParseResult(normal_text=normal_text) + return StreamingParseResult() + class DeepSeekR1Detector(BaseReasoningFormatDetector): """ @@ -211,6 +256,7 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector): force_reasoning: bool = True, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): # DeepSeek-R1 is assumed to be reasoning until `` token super().__init__( @@ -220,6 +266,7 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector): stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) # https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599 @@ -246,6 +293,7 @@ class Qwen3Detector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): think_excluded_tokens = [ "", @@ -266,6 +314,7 @@ class Qwen3Detector(BaseReasoningFormatDetector): previous_content=previous_content, thinks_internally=True, reasoning_default="enable_thinking", + force_nonempty_content=force_nonempty_content, ) @@ -284,6 +333,7 @@ class KimiDetector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): super().__init__( "◁think▷", @@ -292,6 +342,7 @@ class KimiDetector(BaseReasoningFormatDetector): stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) @@ -311,6 +362,7 @@ class KimiK2Detector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): think_excluded_tokens = [ "", @@ -334,6 +386,7 @@ class KimiK2Detector(BaseReasoningFormatDetector): continue_final_message=continue_final_message, previous_content=previous_content, reasoning_default="thinking", + force_nonempty_content=force_nonempty_content, ) @@ -350,7 +403,12 @@ class Glm45Detector(BaseReasoningFormatDetector): If True, streams reasoning content as it arrives. """ - def __init__(self, stream_reasoning: bool = True, force_reasoning: bool = False): + def __init__( + self, + stream_reasoning: bool = True, + force_reasoning: bool = False, + force_nonempty_content: bool = False, + ): think_excluded_tokens = [ "", "", @@ -367,6 +425,7 @@ class Glm45Detector(BaseReasoningFormatDetector): tool_start_token="", thinks_internally=True, reasoning_default="enable_thinking", + force_nonempty_content=force_nonempty_content, ) @@ -381,6 +440,7 @@ class GptOssDetector(BaseReasoningFormatDetector): force_reasoning: bool = True, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): super().__init__( "<|channel|>analysis<|message|>", @@ -389,6 +449,7 @@ class GptOssDetector(BaseReasoningFormatDetector): stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) self.parser = HarmonyParser() @@ -410,9 +471,11 @@ class GptOssDetector(BaseReasoningFormatDetector): normal_text = "".join(normal_parts) # Tool call events preserve raw text with structural markers - return StreamingParseResult( - normal_text=normal_text, - reasoning_text=reasoning_text, + return self._maybe_apply_force_nonempty_content( + StreamingParseResult( + normal_text=normal_text, + reasoning_text=reasoning_text, + ) ) def parse_streaming_increment(self, new_text: str) -> StreamingParseResult: @@ -447,6 +510,7 @@ class MiniMaxAppendThinkDetector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): # scheduler.py need `reasoning_parser.detector.think_end_token` super().__init__( @@ -456,6 +520,7 @@ class MiniMaxAppendThinkDetector(BaseReasoningFormatDetector): stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) self.is_first_chunk = False @@ -489,17 +554,12 @@ class Nemotron3Detector(BaseReasoningFormatDetector): "", force_reasoning=force_reasoning, stream_reasoning=stream_reasoning, + tool_start_token="", continue_final_message=continue_final_message, previous_content=previous_content, reasoning_default="enable_thinking", + force_nonempty_content=force_nonempty_content, ) - self._force_nonempty_content = force_nonempty_content - - def detect_and_parse(self, text: str) -> StreamingParseResult: - ret = super().detect_and_parse(text) - if self._force_nonempty_content and not ret.normal_text: - ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text - return ret class MiniMaxM3Detector(BaseReasoningFormatDetector): @@ -571,6 +631,7 @@ class MistralDetector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): super().__init__( "[THINK]", @@ -580,6 +641,7 @@ class MistralDetector(BaseReasoningFormatDetector): continue_final_message=continue_final_message, previous_content=previous_content, reasoning_default="mistral", + force_nonempty_content=force_nonempty_content, ) @@ -597,6 +659,7 @@ class HunyuanDetector(BaseReasoningFormatDetector): continue_final_message: bool = False, previous_content: str = "", tokenizer=None, + force_nonempty_content: bool = False, ): t = resolve_hunyuan_tokens(tokenizer) think_open = t["think"] @@ -611,6 +674,7 @@ class HunyuanDetector(BaseReasoningFormatDetector): tool_start_token=t["tool_calls"], continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) @@ -623,6 +687,7 @@ class Gemma4Detector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): super().__init__( "<|channel>", @@ -632,6 +697,7 @@ class Gemma4Detector(BaseReasoningFormatDetector): continue_final_message=continue_final_message, previous_content=previous_content, reasoning_default="explicit_enable_thinking", + force_nonempty_content=force_nonempty_content, ) self.think_start_self_label = "thought\n" @@ -684,9 +750,9 @@ class Apertus2509Detector(BaseReasoningFormatDetector): stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) self._force_reasoning = force_reasoning - self._force_nonempty_content = force_nonempty_content self._tool_start_token = "<|tools_prefix|>[" self._tool_end_token = "<|tools_suffix|>" self._reasoning_acc: str = "" @@ -707,9 +773,7 @@ class Apertus2509Detector(BaseReasoningFormatDetector): normal_text="".join(text_parts), reasoning_text="".join(reasoning_parts), ) - if self._force_nonempty_content and not ret.normal_text: - ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text - return ret + return self._maybe_apply_force_nonempty_content(ret) def detect_and_parse_block_sequence(self, text: str) -> list[tuple[str, str]]: """Return an ordered sequence of blocks: [("reasoning"|"text", content), ...]""" @@ -917,6 +981,7 @@ class CohereCommand4Detector(BaseReasoningFormatDetector): force_reasoning: bool = True, continue_final_message: bool = False, previous_content: str = "", + force_nonempty_content: bool = False, ): # The chat template puts <|START_THINKING|> in the assistant prefix # when reasoning is enabled, so the *generated* text usually starts @@ -930,6 +995,7 @@ class CohereCommand4Detector(BaseReasoningFormatDetector): stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, previous_content=previous_content, + force_nonempty_content=force_nonempty_content, ) # Streaming state machine. The model emits, in order: # 1. reasoning (between START_THINKING [in prefix] and END_THINKING) @@ -1008,9 +1074,11 @@ class CohereCommand4Detector(BaseReasoningFormatDetector): if reasoning.startswith(think_start_text): reasoning = reasoning[len(think_start_text) :] - return StreamingParseResult( - normal_text=self._strip_text_markers(rest), - reasoning_text=reasoning, + return self._maybe_apply_force_nonempty_content( + StreamingParseResult( + normal_text=self._strip_text_markers(rest), + reasoning_text=reasoning, + ) ) def parse_streaming_increment(self, new_text: str) -> StreamingParseResult: @@ -1230,3 +1298,9 @@ class ReasoningParser: """Streaming call: incremental parsing""" ret = self.detector.parse_streaming_increment(chunk_text) return ret.reasoning_text, ret.normal_text + + def parse_stream_end(self) -> Tuple[Optional[str], Optional[str]]: + """Streaming call: flush any detector-specific buffered state once + the stream ends.""" + ret = self.detector.finish() + return ret.reasoning_text, ret.normal_text diff --git a/test/registered/unit/parser/test_reasoning_parser.py b/test/registered/unit/parser/test_reasoning_parser.py index ef0d26b8a..44784e95f 100644 --- a/test/registered/unit/parser/test_reasoning_parser.py +++ b/test/registered/unit/parser/test_reasoning_parser.py @@ -3,6 +3,7 @@ import unittest from sglang.srt.parser.reasoning_parser import ( + Apertus2509Detector, BaseReasoningFormatDetector, DeepSeekR1Detector, Gemma4Detector, @@ -350,6 +351,73 @@ class TestNemotron3Detector(CustomTestCase): self.assertEqual(result.normal_text, "Truncated reasoning without end token") self.assertEqual(result.reasoning_text, "") + def test_streaming_truncated_reasoning_reclassified_on_finish(self): + """force_nonempty_content: truncated reasoning (no think_end) is flushed + as normal_text when the stream ends, so streaming content is non-empty.""" + detector = Nemotron3Detector(force_nonempty_content=True) + detector.parse_streaming_increment(detector.think_start_token) + detector.parse_streaming_increment("reasoning part one") + detector.parse_streaming_increment(" more reasoning") + end = detector.finish() + self.assertEqual(end.reasoning_text, "") + self.assertEqual(end.normal_text, "reasoning part one more reasoning") + + def test_streaming_tool_start_ends_reasoning_and_noops_finish(self): + """tool_start_token interrupts reasoning; finish() then no-ops because + _in_reasoning is already False.""" + detector = Nemotron3Detector(force_nonempty_content=True) + detector.parse_streaming_increment(detector.think_start_token) + detector.parse_streaming_increment("reasoning here") + result = detector.parse_streaming_increment( + detector.tool_start_token + "payload" + ) + self.assertEqual(result.reasoning_text, "") + self.assertEqual(result.normal_text, detector.tool_start_token + "payload") + self.assertFalse(detector._in_reasoning) + end = detector.finish() + self.assertEqual(end.normal_text, "") + + def test_streaming_truncated_no_stream_reasoning_strips_think_start(self): + """force_nonempty_content + stream_reasoning=False: the opening think + token must not leak into content when truncation is flushed on finish. + + Regression: with stream_reasoning=False the base parse_streaming_increment + never clears _buffer, so the stripped think_start survives in _buffer and + finish() would prepend it to the reclassified content.""" + detector = Nemotron3Detector( + force_nonempty_content=True, stream_reasoning=False + ) + detector.parse_streaming_increment(detector.think_start_token) + detector.parse_streaming_increment("hidden reasoning") + end = detector.finish() + self.assertEqual(end.reasoning_text, "") + self.assertEqual(end.normal_text, "hidden reasoning") + self.assertNotIn(detector.think_start_token, end.normal_text) + + +class TestApertus2509DetectorForceNonempty(CustomTestCase): + """force_nonempty_content swap on Apertus2509 (non-streaming, via base helper).""" + + def test_swap_when_only_reasoning(self): + detector = Apertus2509Detector(force_nonempty_content=True) + text = ( + detector.think_start_token + + "apertus reasoning only" + + detector.think_end_token + ) + result = detector.detect_and_parse(text) + self.assertEqual(result.normal_text, "apertus reasoning only") + self.assertEqual(result.reasoning_text, "") + + def test_no_swap_when_normal_exists(self): + detector = Apertus2509Detector(force_nonempty_content=True) + text = ( + detector.think_start_token + "reason" + detector.think_end_token + "answer" + ) + result = detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "reason") + self.assertEqual(result.normal_text, "answer") + class TestGemma4Detector(CustomTestCase): def setUp(self):