minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Fix] Make DeepSeek-V4 reasoning and tool-call streaming parsing chunk-invariant (#34458)

Browse Source 
Co-authored-by: hao-cyber <89575785+hao-cyber@users.noreply.github.com>
Co-authored-by: Enrico Falco <enrico9034@gmail.com>
Co-authored-by: Svyatoslav <85786374+slivanovich@users.noreply.github.com>
Co-authored-by: Andreas Hassellof <andreas@ombori.com>
Co-authored-by: Leoyzen <leoyzen@gmail.com>
Co-authored-by: Chenglun Hu <chenglunhu@gmail.com>
Co-authored-by: robellliu-dev <robell.liu@huawei.com>
Co-authored-by: Gavin.Zhu <gavin.z@gmicloud.ai>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: tancheng33 <garrytancheng@gmail.com>
Co-authored-by: dineshx29 <dinesh.b.offl@gmail.com>
Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
This commit is contained in:
Liangsheng Yin
2026-08-11 20:03:28 -07:00
committed by GitHub
co-authored by hao-cyber Enrico Falco Svyatoslav Andreas Hassellof Leoyzen Chenglun Hu robellliu-dev Gavin.Zhu Xinyuan Tong tancheng33 dineshx29 Kangyan Zhou
parent 2be9773a21
commit 5899674504
4 changed files with 383 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/function_call/deepseekv32_detector.py
+35 -19
View File
@@ -2,6 +2,7 @@ import json
import logging
import re
from partial_json_parser.core.exceptions import MalformedJSON
from partial_json_parser.core.options import Allow
from sglang.srt.entrypoints.openai.protocol import Tool
@@ -179,7 +180,7 @@ class DeepSeekV32Detector(BaseFormatDetector):
parameters[param_name] = _partial_json_loads(
param_value, Allow.ALL
)[0]
except json.JSONDecodeError:
except (json.JSONDecodeError, MalformedJSON, ValueError):
parameters[param_name] = param_value.strip()
return json.dumps(parameters, ensure_ascii=False)
@@ -199,26 +200,25 @@ class DeepSeekV32Detector(BaseFormatDetector):
calls = []
try:
# Extract content between function_calls tags
function_calls_match = re.search(
self.function_calls_regex,
text,
re.DOTALL,
)
if not function_calls_match:
sections = re.findall(self.function_calls_regex, text, re.DOTALL)
if not sections:
return StreamingParseResult(normal_text=normal_text, calls=[])
function_calls_content = function_calls_match.group(1)
# Find all invoke blocks
for invoke_match in re.finditer(
self.invoke_regex, function_calls_content, re.DOTALL
):
func_name, invoke_content, _ = self._unpack_invoke_match(invoke_match)
func_args = self._parse_parameters_from_xml(invoke_content)
# construct match_result for parse_base_json
match_result = {"name": func_name, "parameters": json.loads(func_args)}
calls.extend(self.parse_base_json(match_result, tools))
for function_calls_content in sections:
for invoke_match in re.finditer(
self.invoke_regex, function_calls_content, re.DOTALL
):
func_name, invoke_content, _ = self._unpack_invoke_match(
invoke_match
)
func_args = self._parse_parameters_from_xml(invoke_content)
# construct match_result for parse_base_json
match_result = {
"name": func_name,
"parameters": json.loads(func_args),
}
calls.extend(self.parse_base_json(match_result, tools))
return StreamingParseResult(normal_text=normal_text, calls=calls)
except Exception as e:
@@ -259,6 +259,9 @@ class DeepSeekV32Detector(BaseFormatDetector):
return StreamingParseResult(normal_text=current_text)
all_calls: list[ToolCallItem] = []
# Only recovered for the first call: the DSML guard above never releases a
# buffer that still holds a marker, so later prose stays buffered.
preamble = ""
try:
# Loop to handle multiple consecutive invoke blocks
while True:
@@ -280,6 +283,12 @@ class DeepSeekV32Detector(BaseFormatDetector):
self.current_tool_id = 0
self.prev_tool_call_arr = []
self.streamed_args_for_tool = [""]
call_start = invoke_match.start()
bot_pos = current_text.rfind(self.bot_token, 0, call_start)
if bot_pos != -1:
call_start = bot_pos
# Same trailing-newline trim as detect_and_parse, so both agree.
preamble = current_text[:call_start].removesuffix("\n\n")
# Ensure arrays are large enough for current tool
while len(self.prev_tool_call_arr) <= self.current_tool_id:
@@ -355,10 +364,17 @@ class DeepSeekV32Detector(BaseFormatDetector):
break
# No more invoke blocks found
return StreamingParseResult(normal_text="", calls=all_calls)
return StreamingParseResult(normal_text=preamble, calls=all_calls)
except Exception as e:
logger.error(f"Error in parse_streaming_increment: {e}")
# Re-emit verbatim rather than swallowing the turn; the preamble is
# still inside current_text unless a completed call advanced past it.
# Calls are dropped on purpose: the failure can land between a tool's
# name and its arguments, and a half-formed call is worse than none.
self._buffer = ""
if not current_text.startswith(preamble):
current_text = preamble + current_text
return StreamingParseResult(normal_text=current_text)
def structure_info(self) -> _GetInfoFunc:
python/sglang/srt/parser/reasoning_parser.py
+42 -17
View File
@@ -205,6 +205,8 @@ class BaseReasoningFormatDetector:
# Strip `<think>` token if present
if not self.stripped_think_start and think_start_text in current_text:
current_text = current_text.replace(think_start_text, "", 1)
# Write back, or stream_reasoning=False carries the token into finish().
self._buffer = current_text
self.stripped_think_start = True
self._in_reasoning = True
@@ -224,7 +226,8 @@ class BaseReasoningFormatDetector:
# Continue with reasoning content
if self._in_reasoning:
# Check for tool_start_token interruption
# Check for tool_start_token interruption. Streaming cannot see a
# think_end_token that has not arrived yet; see the chunk_dependent test.
if self.tool_start_token and self.tool_start_token in current_text:
tool_idx = current_text.find(self.tool_start_token)
reasoning_text = current_text[:tool_idx]
@@ -236,9 +239,21 @@ class BaseReasoningFormatDetector:
normal_text=normal_text, reasoning_text=reasoning_text
)
if self.stream_reasoning:
# Stream the content immediately
self._buffer = ""
return StreamingParseResult(reasoning_text=current_text)
# Minus any trailing slice that could be a token split across chunks.
holdback_tokens = [self.think_end_token]
if self.tool_start_token:
holdback_tokens.append(self.tool_start_token)
if not self.stripped_think_start:
# force_reasoning never saw the opening token; it can still split.
holdback_tokens.append(think_start_text)
holdback = max(
self._ends_with_partial_token(current_text, token)
for token in holdback_tokens
)
self._buffer = current_text[len(current_text) - holdback :]
return StreamingParseResult(
reasoning_text=current_text[: len(current_text) - holdback]
)
else:
return StreamingParseResult()
@@ -255,15 +270,30 @@ class BaseReasoningFormatDetector:
return text[len(think_start_text) :]
return text
@staticmethod
def _ends_with_partial_token(buffer: str, token: str) -> int:
"""Length of the longest trailing slice of `buffer` that is a strict prefix
of `token`. Longest, so a token whose prefix repeats inside itself does not
get cut short and leak the rest of the marker."""
for i in range(min(len(buffer), len(token) - 1), 0, -1):
if token.startswith(buffer[-i:]):
return i
return 0
def finish(self) -> StreamingParseResult:
"""Flush reasoning buffered under stream_reasoning=False when the stream ends
before the end token (e.g. max_tokens cut it short), instead of dropping it.
"""Flush reasoning still buffered when the stream ends before the end token
(e.g. max_tokens cut it short), instead of dropping it: the whole block under
stream_reasoning=False, the held-back token suffix under stream_reasoning=True.
force_nonempty_content emits it as normal_text, else as reasoning_text."""
if not self._in_reasoning:
return StreamingParseResult()
# Same as the reasoning-side flush below: a held-back slice that never
# became a token is content.
leftover = self._buffer
self._buffer = ""
return StreamingParseResult(normal_text=leftover)
# stream_reasoning=False never clears _buffer, so the opening think token
# (stripped only from the base class's local view) survives here.
# Defensive: subclasses that fill _buffer themselves may not have stripped
# the opening think token that _parse_streaming_increment_impl removes.
buffer = self._strip_leading_think_start(self._buffer)
self._buffer = ""
@@ -274,7 +304,7 @@ class BaseReasoningFormatDetector:
return StreamingParseResult(normal_text=normal_text)
return StreamingParseResult()
if not self.stream_reasoning and buffer:
if buffer:
return StreamingParseResult(reasoning_text=buffer)
return StreamingParseResult()
@@ -1127,6 +1157,8 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
dsv4_thinking_start_token,
dsv4_thinking_end_token,
think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token],
# Leading "<" included: has_tool_call() matches on it.
tool_start_token=f"<{dsv4_dsml_token}",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
@@ -1185,13 +1217,6 @@ class Apertus2509Detector(BaseReasoningFormatDetector):
self._reasoning_acc: str = ""
self._in_inner_tool: bool = False
@staticmethod
def _ends_with_partial_token(buffer: str, token: str) -> int:
for i in range(1, min(len(buffer) + 1, len(token))):
if token.startswith(buffer[-i:]):
return i
return 0
def detect_and_parse(self, text: str) -> StreamingParseResult:
blocks = self.detect_and_parse_block_sequence(text)
reasoning_parts = [t for k, t in blocks if k == "reasoning"]