minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

more fixes for Nemotron 3 parser for tool call and force nonempty content (#30533)

Browse Source
This commit is contained in:
Brayden Zhong
2026-07-14 06:50:27 +08:00
committed by GitHub
parent 50ed4c011f
commit 0ee236ebdf
3 changed files with 175 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/serving_chat.py
+15 -2
View File
@@ -374,7 +374,12 @@ class OpenAIServingChat(OpenAIServingBase):
# Handle reasoning content
if self.reasoning_parser and request.separate_reasoning:
reasoning_text, delta = self._process_reasoning_stream(
index, delta, reasoning_parser_dict, content, request
index,
delta,
reasoning_parser_dict,
content,
request,
finish_reason_type,
)
if reasoning_text:
usage = None
@@ -1635,6 +1640,7 @@ class OpenAIServingChat(OpenAIServingBase):
reasoning_parser_dict: Dict[int, ReasoningParser],
content: Dict[str, Any],
request: ChatCompletionRequest,
finish_reason_type: Optional[str] = None,
) -> tuple[Optional[str], str]:
"""Process reasoning content in streaming response"""
if index not in reasoning_parser_dict:
@@ -1650,7 +1656,14 @@ class OpenAIServingChat(OpenAIServingBase):
tokenizer=self.tokenizer_manager.tokenizer,
)
reasoning_parser = reasoning_parser_dict[index]
return reasoning_parser.parse_stream_chunk(delta)
reasoning_text, normal_text = reasoning_parser.parse_stream_chunk(delta)
if finish_reason_type is not None and finish_reason_type != "abort":
end_reasoning_text, end_normal_text = reasoning_parser.parse_stream_end()
if end_reasoning_text:
reasoning_text = (reasoning_text or "") + end_reasoning_text
if end_normal_text:
normal_text = (normal_text or "") + end_normal_text
return reasoning_text, normal_text
def _get_history_tool_calls_cnt(self, request: ChatCompletionRequest) -> int:
"""Counts the number of tool calls in the request's message history.
python/sglang/srt/parser/reasoning_parser.py
+92 -18
View File
@@ -33,6 +33,7 @@ class BaseReasoningFormatDetector:
previous_content: str = "",
thinks_internally: bool = False,
reasoning_default: str = "always",
force_nonempty_content: bool = False,
):
self.think_start_token = think_start_token
self.think_end_token = think_end_token
@@ -48,6 +49,9 @@ class BaseReasoningFormatDetector:
self.stripped_think_start = False
self.think_start_self_label = ""
self._force_nonempty_content = force_nonempty_content
self._accumulated_reasoning = ""
self.continue_final_message = continue_final_message
if self.continue_final_message:
self.previous_content = previous_content
@@ -61,11 +65,23 @@ class BaseReasoningFormatDetector:
if self.think_end_token in self.previous_content:
self._in_reasoning = False
def _maybe_apply_force_nonempty_content(
self, ret: StreamingParseResult
) -> StreamingParseResult:
if self._force_nonempty_content and not ret.normal_text:
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
return ret
def detect_and_parse(self, text: str) -> StreamingParseResult:
"""
One-time parsing: Detects and parses reasoning sections in the provided text.
Returns both reasoning content and normal text separately.
"""
return self._maybe_apply_force_nonempty_content(
self._detect_and_parse_impl(text)
)
def _detect_and_parse_impl(self, text: str) -> StreamingParseResult:
in_reasoning = self._in_reasoning or self.think_start_token in text
if not in_reasoning:
@@ -121,6 +137,15 @@ class BaseReasoningFormatDetector:
If stream_reasoning is True:
Streams reasoning content as it arrives
"""
ret = self._parse_streaming_increment_impl(new_text)
if self._force_nonempty_content:
if self._in_reasoning:
self._accumulated_reasoning += ret.reasoning_text
else:
self._accumulated_reasoning = ""
return ret
def _parse_streaming_increment_impl(self, new_text: str) -> StreamingParseResult:
self._buffer += new_text
current_text = self._buffer
@@ -183,6 +208,26 @@ class BaseReasoningFormatDetector:
return StreamingParseResult()
def finish(self) -> StreamingParseResult:
"""
Called once when the stream ends. If force_nonempty_content is set
and the stream ended mid-reasoning, reclassifies the accumulated
reasoning (plus any partial token still buffered) as normal text.
"""
if self._force_nonempty_content and self._in_reasoning:
# stream_reasoning=False never clears _buffer, so the opening think
# token (stripped only from the base class's local view) survives here.
buffer = self._buffer
think_start_text = self.think_start_token + self.think_start_self_label
if buffer.startswith(think_start_text):
buffer = buffer[len(think_start_text) :]
normal_text = self._accumulated_reasoning + buffer
self._accumulated_reasoning = ""
self._buffer = ""
if normal_text:
return StreamingParseResult(normal_text=normal_text)
return StreamingParseResult()
class DeepSeekR1Detector(BaseReasoningFormatDetector):
"""
@@ -211,6 +256,7 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector):
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# DeepSeek-R1 is assumed to be reasoning until `</think>` token
super().__init__(
@@ -220,6 +266,7 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
# https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599
@@ -246,6 +293,7 @@ class Qwen3Detector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
think_excluded_tokens = [
"<tool_call>",
@@ -266,6 +314,7 @@ class Qwen3Detector(BaseReasoningFormatDetector):
previous_content=previous_content,
thinks_internally=True,
reasoning_default="enable_thinking",
force_nonempty_content=force_nonempty_content,
)
@@ -284,6 +333,7 @@ class KimiDetector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"◁think▷",
@@ -292,6 +342,7 @@ class KimiDetector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
@@ -311,6 +362,7 @@ class KimiK2Detector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
think_excluded_tokens = [
"<think>",
@@ -334,6 +386,7 @@ class KimiK2Detector(BaseReasoningFormatDetector):
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="thinking",
force_nonempty_content=force_nonempty_content,
)
@@ -350,7 +403,12 @@ class Glm45Detector(BaseReasoningFormatDetector):
If True, streams reasoning content as it arrives.
"""
def __init__(self, stream_reasoning: bool = True, force_reasoning: bool = False):
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
force_nonempty_content: bool = False,
):
think_excluded_tokens = [
"<tool_call>",
"</tool_call>",
@@ -367,6 +425,7 @@ class Glm45Detector(BaseReasoningFormatDetector):
tool_start_token="<tool_call>",
thinks_internally=True,
reasoning_default="enable_thinking",
force_nonempty_content=force_nonempty_content,
)
@@ -381,6 +440,7 @@ class GptOssDetector(BaseReasoningFormatDetector):
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<|channel|>analysis<|message|>",
@@ -389,6 +449,7 @@ class GptOssDetector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self.parser = HarmonyParser()
@@ -410,9 +471,11 @@ class GptOssDetector(BaseReasoningFormatDetector):
normal_text = "".join(normal_parts)
# Tool call events preserve raw text with structural markers
return StreamingParseResult(
normal_text=normal_text,
reasoning_text=reasoning_text,
return self._maybe_apply_force_nonempty_content(
StreamingParseResult(
normal_text=normal_text,
reasoning_text=reasoning_text,
)
)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
@@ -447,6 +510,7 @@ class MiniMaxAppendThinkDetector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# scheduler.py need `reasoning_parser.detector.think_end_token`
super().__init__(
@@ -456,6 +520,7 @@ class MiniMaxAppendThinkDetector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self.is_first_chunk = False
@@ -489,17 +554,12 @@ class Nemotron3Detector(BaseReasoningFormatDetector):
"</think>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token="<tool_call>",
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="enable_thinking",
force_nonempty_content=force_nonempty_content,
)
self._force_nonempty_content = force_nonempty_content
def detect_and_parse(self, text: str) -> StreamingParseResult:
ret = super().detect_and_parse(text)
if self._force_nonempty_content and not ret.normal_text:
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
return ret
class MiniMaxM3Detector(BaseReasoningFormatDetector):
@@ -571,6 +631,7 @@ class MistralDetector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"[THINK]",
@@ -580,6 +641,7 @@ class MistralDetector(BaseReasoningFormatDetector):
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="mistral",
force_nonempty_content=force_nonempty_content,
)
@@ -597,6 +659,7 @@ class HunyuanDetector(BaseReasoningFormatDetector):
continue_final_message: bool = False,
previous_content: str = "",
tokenizer=None,
force_nonempty_content: bool = False,
):
t = resolve_hunyuan_tokens(tokenizer)
think_open = t["think"]
@@ -611,6 +674,7 @@ class HunyuanDetector(BaseReasoningFormatDetector):
tool_start_token=t["tool_calls"],
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
@@ -623,6 +687,7 @@ class Gemma4Detector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<|channel>",
@@ -632,6 +697,7 @@ class Gemma4Detector(BaseReasoningFormatDetector):
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="explicit_enable_thinking",
force_nonempty_content=force_nonempty_content,
)
self.think_start_self_label = "thought\n"
@@ -684,9 +750,9 @@ class Apertus2509Detector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self._force_reasoning = force_reasoning
self._force_nonempty_content = force_nonempty_content
self._tool_start_token = "<|tools_prefix|>["
self._tool_end_token = "<|tools_suffix|>"
self._reasoning_acc: str = ""
@@ -707,9 +773,7 @@ class Apertus2509Detector(BaseReasoningFormatDetector):
normal_text="".join(text_parts),
reasoning_text="".join(reasoning_parts),
)
if self._force_nonempty_content and not ret.normal_text:
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
return ret
return self._maybe_apply_force_nonempty_content(ret)
def detect_and_parse_block_sequence(self, text: str) -> list[tuple[str, str]]:
"""Return an ordered sequence of blocks: [("reasoning"|"text", content), ...]"""
@@ -917,6 +981,7 @@ class CohereCommand4Detector(BaseReasoningFormatDetector):
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# The chat template puts <|START_THINKING|> in the assistant prefix
# when reasoning is enabled, so the *generated* text usually starts
@@ -930,6 +995,7 @@ class CohereCommand4Detector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
# Streaming state machine. The model emits, in order:
# 1. reasoning (between START_THINKING [in prefix] and END_THINKING)
@@ -1008,9 +1074,11 @@ class CohereCommand4Detector(BaseReasoningFormatDetector):
if reasoning.startswith(think_start_text):
reasoning = reasoning[len(think_start_text) :]
return StreamingParseResult(
normal_text=self._strip_text_markers(rest),
reasoning_text=reasoning,
return self._maybe_apply_force_nonempty_content(
StreamingParseResult(
normal_text=self._strip_text_markers(rest),
reasoning_text=reasoning,
)
)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
@@ -1230,3 +1298,9 @@ class ReasoningParser:
"""Streaming call: incremental parsing"""
ret = self.detector.parse_streaming_increment(chunk_text)
return ret.reasoning_text, ret.normal_text
def parse_stream_end(self) -> Tuple[Optional[str], Optional[str]]:
"""Streaming call: flush any detector-specific buffered state once
the stream ends."""
ret = self.detector.finish()
return ret.reasoning_text, ret.normal_text