fix(function_call): buffer complete DeepSeek DSML invokes (#39632)
Co-authored-by: Xinyuan Tong <justintong0323@gmail.com> Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
co-authored by
Xinyuan Tong
Xinyuan Tong
parent
b77833c504
commit
19ee4b566b
@@ -94,6 +94,7 @@ from sglang.srt.function_call.json_array_parser import JsonArrayParser
|
||||
from sglang.srt.function_call.utils import (
|
||||
get_json_schema_constraint,
|
||||
normalize_json_schema_types,
|
||||
strip_structural_tag_excludes,
|
||||
)
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
from sglang.srt.parser.conversation import generate_chat_conv
|
||||
@@ -1351,6 +1352,20 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
parallel_tool_calls=request.parallel_tool_calls,
|
||||
thinking_mode=xgrammar_reasoning,
|
||||
)
|
||||
if (
|
||||
tool_call_constraint is not None
|
||||
and tool_call_constraint[0] == "structural_tag"
|
||||
and self._reasoning_detector is not None
|
||||
):
|
||||
# Same ownership rule: the reasoning parser handles think
|
||||
# tokens, so the grammar must not forbid them in free text.
|
||||
strip_structural_tag_excludes(
|
||||
tool_call_constraint[1],
|
||||
(
|
||||
self._reasoning_detector.think_start_token,
|
||||
self._reasoning_detector.think_end_token,
|
||||
),
|
||||
)
|
||||
required_parsed_natively = parser.detector.parses_required_natively()
|
||||
if self.chat_encoding_spec == "kimi_k3":
|
||||
tool_call_stop = parser.detector.eot_token
|
||||
|
||||
@@ -2,9 +2,6 @@ import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from partial_json_parser.core.exceptions import MalformedJSON
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import Tool
|
||||
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
|
||||
from sglang.srt.function_call.core_types import (
|
||||
@@ -13,11 +10,14 @@ from sglang.srt.function_call.core_types import (
|
||||
ToolCallItem,
|
||||
_GetInfoFunc,
|
||||
)
|
||||
from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reject_json_constant(value: str) -> None:
|
||||
raise ValueError(f"Invalid JSON constant: {value}")
|
||||
|
||||
|
||||
class DeepSeekV32Detector(BaseFormatDetector):
|
||||
"""
|
||||
Detector for DeepSeek V3.2 model function call format.
|
||||
@@ -89,9 +89,6 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
self.parameter_regex = (
|
||||
rf'<{parameter}\s+name="([^"]+)"\s+string="([^"]+)"\s*>(.*?)</{parameter}>'
|
||||
)
|
||||
self.partial_parameter_regex = (
|
||||
rf'<{parameter}\s+name="([^"]+)"\s+string="([^"]+)"\s*>(.*)$'
|
||||
)
|
||||
self.function_calls_regex = rf"<{block}>(.*?)</{block}>"
|
||||
# Long-form `<|DSML|invoke name="x">...</|DSML|invoke>` and the
|
||||
# self-closing `<|DSML|invoke name="x"/>` shape V4 emits for zero-arg
|
||||
@@ -101,25 +98,24 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
r"(?:(?P<self_close>/>)"
|
||||
rf"|>(?P<body>.*?)(?P<end>(?:</{invoke}>|$)))"
|
||||
)
|
||||
# Consumed right-to-left by rstrip (a character set, not a suffix), so the
|
||||
# invoke name is split to limit how much of a partial value gets eaten.
|
||||
self.prefix_parameter_end_call = [
|
||||
"</",
|
||||
self.dsml_token,
|
||||
self.parameter_tag_name,
|
||||
]
|
||||
self.prefix_invoke_end_call = [
|
||||
"</",
|
||||
self.dsml_token,
|
||||
self.invoke_tag_name[:-3],
|
||||
self.invoke_tag_name[-3:],
|
||||
]
|
||||
self.current_tool_id = -1
|
||||
|
||||
def has_tool_call(self, text: str) -> bool:
|
||||
"""Check if the text contains a deepseek v32 format tool call."""
|
||||
return self.bot_token in text or self.invoke_start_token in text
|
||||
|
||||
def _text_before_dsml(self, text: str) -> str:
|
||||
"""Prose preceding the first DSML tag, with the trailing blank line the
|
||||
chat template inserts before a tool call removed."""
|
||||
idx = text.find(self.dsml_token)
|
||||
if idx == -1:
|
||||
return text
|
||||
if idx >= 2 and text[idx - 2 : idx] == "</":
|
||||
idx -= 2
|
||||
elif idx >= 1 and text[idx - 1] == "<":
|
||||
idx -= 1
|
||||
return text[:idx].removesuffix("\n\n")
|
||||
|
||||
@staticmethod
|
||||
def _unpack_invoke_match(m: "re.Match[str]") -> tuple[str, str, bool]:
|
||||
"""Returns (name, body, is_complete) for an invoke_regex match.
|
||||
@@ -134,9 +130,7 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
return name, "", True
|
||||
return name, m.group("body"), bool(m.group("end"))
|
||||
|
||||
def _parse_parameters_from_xml(
|
||||
self, invoke_content: str, allow_partial: bool = False
|
||||
) -> str:
|
||||
def _parse_parameters_from_xml(self, invoke_content: str) -> str:
|
||||
"""
|
||||
Parse parameters from either XML-like format or JSON format to str.
|
||||
|
||||
@@ -147,13 +141,12 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
# First, try to parse as direct JSON (new format)
|
||||
invoke_content_stripped = invoke_content.strip()
|
||||
if invoke_content_stripped.startswith("{"):
|
||||
if allow_partial:
|
||||
# Remove incomplete invoke end call prefix in case they are captured by param
|
||||
for token in reversed(self.prefix_invoke_end_call):
|
||||
invoke_content_stripped = invoke_content_stripped.rstrip(token)
|
||||
return invoke_content_stripped
|
||||
elif invoke_content_stripped.endswith("}"):
|
||||
return invoke_content_stripped
|
||||
parsed = json.loads(
|
||||
invoke_content_stripped, parse_constant=_reject_json_constant
|
||||
)
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("DeepSeek tool arguments must be a JSON object")
|
||||
return invoke_content_stripped
|
||||
|
||||
# Fall back to XML parameter tag parsing (original format)
|
||||
parameters = {}
|
||||
@@ -162,13 +155,18 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
re.finditer(self.parameter_regex, invoke_content, re.DOTALL)
|
||||
)
|
||||
|
||||
# Stray prose around the parameter tags is ignored, as before. Leftover
|
||||
# text holding a DSML fragment is a parameter tag that did not match
|
||||
# (unclosed, self-closing, missing `string=`), and emitting the call
|
||||
# without that argument would be worse than emitting no call.
|
||||
last_match_end = 0
|
||||
leftover = []
|
||||
for match in param_matches:
|
||||
leftover.append(invoke_content[last_match_end : match.start()])
|
||||
param_name = match.group(1)
|
||||
param_type = match.group(2)
|
||||
param_value = match.group(3)
|
||||
last_match_end = match.end()
|
||||
|
||||
# Convert value based on type
|
||||
if param_type == "true": # string type
|
||||
parameters[param_name] = (
|
||||
@@ -179,39 +177,18 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
else:
|
||||
# Try to parse as JSON for other types
|
||||
try:
|
||||
parameters[param_name] = json.loads(param_value.strip())
|
||||
parameters[param_name] = json.loads(
|
||||
param_value.strip(), parse_constant=_reject_json_constant
|
||||
)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
parameters[param_name] = param_value.strip()
|
||||
leftover.append(invoke_content[last_match_end:])
|
||||
|
||||
# If allowed, try to parse a partial parameter at the end
|
||||
if allow_partial:
|
||||
remaining_content = invoke_content[last_match_end:]
|
||||
|
||||
# Remove incomplete parameter_end_call prefix in case they are captured by param
|
||||
for token in reversed(self.prefix_parameter_end_call):
|
||||
remaining_content = remaining_content.rstrip(token)
|
||||
|
||||
# Match start of a parameter tag + value (potentially incomplete)
|
||||
# Regex: <tag name="..." string="...">VALUE... (no end tag)
|
||||
partial_match = re.search(
|
||||
self.partial_parameter_regex, remaining_content, re.DOTALL
|
||||
)
|
||||
|
||||
if partial_match and (param_value := partial_match.group(3)):
|
||||
param_name = partial_match.group(1)
|
||||
if partial_match.group(2) == "true":
|
||||
parameters[param_name] = (
|
||||
param_value.strip()
|
||||
if self.strip_string_param_value
|
||||
else param_value
|
||||
)
|
||||
else:
|
||||
try:
|
||||
parameters[param_name] = _partial_json_loads(
|
||||
param_value, Allow.ALL
|
||||
)[0]
|
||||
except (json.JSONDecodeError, MalformedJSON, ValueError):
|
||||
parameters[param_name] = param_value.strip()
|
||||
leftover_text = "".join(leftover)
|
||||
if self.dsml_token in leftover_text or (
|
||||
not param_matches and leftover_text.strip()
|
||||
):
|
||||
raise ValueError("Malformed DeepSeek tool parameter")
|
||||
|
||||
return json.dumps(parameters, ensure_ascii=False)
|
||||
|
||||
@@ -224,25 +201,41 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
:return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls.
|
||||
"""
|
||||
idx = text.find(self.bot_token)
|
||||
normal_text = text[:idx].removesuffix("\n\n") if idx != -1 else text
|
||||
if self.bot_token not in text:
|
||||
if idx != -1:
|
||||
normal_text = text[:idx].removesuffix("\n\n")
|
||||
sections = re.findall(self.function_calls_regex, text, re.DOTALL)
|
||||
else:
|
||||
# The streaming path accepts bare invokes without the outer
|
||||
# wrapper; parse them here too instead of leaking raw DSML.
|
||||
start = text.find(self.invoke_start_token)
|
||||
if start == -1:
|
||||
return StreamingParseResult(normal_text=text, calls=[])
|
||||
normal_text = self._text_before_dsml(text)
|
||||
sections = [text[start:]]
|
||||
|
||||
if not sections:
|
||||
return StreamingParseResult(normal_text=normal_text, calls=[])
|
||||
|
||||
calls = []
|
||||
try:
|
||||
sections = re.findall(self.function_calls_regex, text, re.DOTALL)
|
||||
if not sections:
|
||||
return StreamingParseResult(normal_text=normal_text, calls=[])
|
||||
|
||||
# Find all invoke blocks
|
||||
for function_calls_content in sections:
|
||||
for invoke_match in re.finditer(
|
||||
self.invoke_regex, function_calls_content, re.DOTALL
|
||||
):
|
||||
func_name, invoke_content, _ = self._unpack_invoke_match(
|
||||
func_name, invoke_content, is_complete = self._unpack_invoke_match(
|
||||
invoke_match
|
||||
)
|
||||
func_args = self._parse_parameters_from_xml(invoke_content)
|
||||
if not is_complete:
|
||||
logger.warning(
|
||||
f"Dropping unclosed DeepSeek invoke '{func_name}'"
|
||||
)
|
||||
continue
|
||||
try:
|
||||
func_args = self._parse_parameters_from_xml(invoke_content)
|
||||
except ValueError as e:
|
||||
logger.warning(f"Dropping malformed DeepSeek invoke: {e}")
|
||||
continue
|
||||
# construct match_result for parse_base_json
|
||||
match_result = {
|
||||
"name": func_name,
|
||||
@@ -253,26 +246,27 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
return StreamingParseResult(normal_text=normal_text, calls=calls)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in detect_and_parse: {e}")
|
||||
# return the normal text if parsing fails
|
||||
return StreamingParseResult(normal_text=text)
|
||||
# Fail closed: keep the prose, never surface DSML as content.
|
||||
return StreamingParseResult(normal_text=normal_text, calls=calls)
|
||||
|
||||
def parse_streaming_increment(
|
||||
self, new_text: str, tools: list[Tool]
|
||||
) -> StreamingParseResult:
|
||||
"""
|
||||
Streaming incremental parsing tool calls for DeepSeekV32 format.
|
||||
Supports multiple consecutive invoke blocks and argument streaming.
|
||||
Supports multiple consecutive invoke blocks.
|
||||
"""
|
||||
self._buffer += new_text
|
||||
current_text = self._buffer
|
||||
|
||||
# Check if buffer contains any DSML markers or ends with potential tag prefix
|
||||
# This handles partial/streaming DSML content
|
||||
dsml_markers = ["|DSML|", "<|", "</|"]
|
||||
bar = self.dsml_token[0]
|
||||
dsml_markers = [self.dsml_token, f"<{bar}", f"</{bar}"]
|
||||
potentially_dsml = any(marker in current_text for marker in dsml_markers)
|
||||
|
||||
# Also check if text ends with start of a tag (to handle "<" arriving separately)
|
||||
dsml_prefixes = ["<", "<|", "</", "</|"]
|
||||
dsml_prefixes = ["<", f"<{bar}", "</", f"</{bar}"]
|
||||
ends_with_prefix = any(
|
||||
current_text.rstrip().endswith(prefix) for prefix in dsml_prefixes
|
||||
)
|
||||
@@ -282,7 +276,13 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
and not potentially_dsml
|
||||
and not ends_with_prefix
|
||||
):
|
||||
self._buffer = ""
|
||||
# Trailing whitespace may be the blank line ahead of a DSML block;
|
||||
# hold it so it is trimmed with the preamble or dropped with the block.
|
||||
stripped = current_text.rstrip()
|
||||
if not stripped:
|
||||
return StreamingParseResult()
|
||||
self._buffer = current_text[len(stripped) :]
|
||||
current_text = stripped
|
||||
for e_token in [self.eot_token, self.invoke_end_token]:
|
||||
if e_token in current_text:
|
||||
current_text = current_text.replace(e_token, "")
|
||||
@@ -307,12 +307,16 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
func_name, invoke_content, is_tool_end = self._unpack_invoke_match(
|
||||
invoke_match
|
||||
)
|
||||
if not is_tool_end:
|
||||
break
|
||||
|
||||
# Initialize state if this is the first tool call
|
||||
# Initialize state on the first complete invoke, malformed or
|
||||
# not: the preamble is released once and the trailing DSML is
|
||||
# withheld by finish() either way.
|
||||
if self.current_tool_id == -1:
|
||||
self.current_tool_id = 0
|
||||
self.prev_tool_call_arr = []
|
||||
self.streamed_args_for_tool = [""]
|
||||
self.streamed_args_for_tool = []
|
||||
call_start = invoke_match.start()
|
||||
bot_pos = current_text.rfind(self.bot_token, 0, call_start)
|
||||
if bot_pos != -1:
|
||||
@@ -320,55 +324,29 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
# Same trailing-newline trim as detect_and_parse, so both agree.
|
||||
preamble = current_text[:call_start].removesuffix("\n\n")
|
||||
|
||||
try:
|
||||
current_params = self._parse_parameters_from_xml(invoke_content)
|
||||
except ValueError as e:
|
||||
# Fail closed: drop this invoke, keep going for the next one.
|
||||
logger.warning(f"Dropping malformed DeepSeek invoke: {e}")
|
||||
self._buffer = current_text[invoke_match.end() :]
|
||||
current_text = self._buffer
|
||||
continue
|
||||
|
||||
# Ensure arrays are large enough for current tool
|
||||
while len(self.prev_tool_call_arr) <= self.current_tool_id:
|
||||
self.prev_tool_call_arr.append({})
|
||||
while len(self.streamed_args_for_tool) <= self.current_tool_id:
|
||||
self.streamed_args_for_tool.append("")
|
||||
|
||||
# 1. Send tool name if not sent yet
|
||||
if not self.current_tool_name_sent:
|
||||
all_calls.append(
|
||||
ToolCallItem(
|
||||
tool_index=self.current_tool_id,
|
||||
name=func_name,
|
||||
parameters="",
|
||||
)
|
||||
all_calls.append(
|
||||
ToolCallItem(
|
||||
tool_index=self.current_tool_id,
|
||||
name=func_name,
|
||||
parameters=current_params,
|
||||
)
|
||||
self.current_tool_name_sent = True
|
||||
|
||||
# 2. Parse current parameters (partial or complete)
|
||||
current_params = self._parse_parameters_from_xml(
|
||||
invoke_content, allow_partial=not is_tool_end
|
||||
)
|
||||
|
||||
# 3. Calculate and send incremental arguments
|
||||
sent_len = len(self.streamed_args_for_tool[self.current_tool_id])
|
||||
prev_params = self.prev_tool_call_arr[self.current_tool_id].get(
|
||||
"arguments"
|
||||
)
|
||||
|
||||
argument_diff = None
|
||||
|
||||
if is_tool_end:
|
||||
# If complete, send everything remaining
|
||||
argument_diff = current_params[sent_len:]
|
||||
elif prev_params is not None:
|
||||
# If partial, send stable prefix diff
|
||||
if current_params != prev_params:
|
||||
prefix = _find_common_prefix(current_params, prev_params)
|
||||
if len(prefix) > sent_len:
|
||||
argument_diff = prefix[sent_len:]
|
||||
|
||||
if argument_diff:
|
||||
all_calls.append(
|
||||
ToolCallItem(
|
||||
tool_index=self.current_tool_id,
|
||||
name=None,
|
||||
parameters=argument_diff,
|
||||
)
|
||||
)
|
||||
self.streamed_args_for_tool[self.current_tool_id] += argument_diff
|
||||
self.streamed_args_for_tool[self.current_tool_id] = current_params
|
||||
|
||||
# Update the stored arguments
|
||||
self.prev_tool_call_arr[self.current_tool_id] = {
|
||||
@@ -376,36 +354,33 @@ class DeepSeekV32Detector(BaseFormatDetector):
|
||||
"arguments": current_params,
|
||||
}
|
||||
|
||||
# Check if tool call is complete (has closing tag)
|
||||
if is_tool_end:
|
||||
# Remove the completed tool call from buffer
|
||||
self._buffer = current_text[invoke_match.end() :]
|
||||
current_text = self._buffer # Update for next iteration
|
||||
|
||||
# Move to next tool call
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
|
||||
# Continue loop to check for more invoke blocks
|
||||
continue
|
||||
else:
|
||||
# Tool call not complete yet, don't return anything
|
||||
# Wait for more chunks until we see </|DSML|invoke>
|
||||
break
|
||||
# Remove the completed tool call from buffer and check for another.
|
||||
self._buffer = current_text[invoke_match.end() :]
|
||||
current_text = self._buffer
|
||||
self.current_tool_id += 1
|
||||
|
||||
# No more invoke blocks found
|
||||
return StreamingParseResult(normal_text=preamble, calls=all_calls)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in parse_streaming_increment: {e}")
|
||||
# Re-emit verbatim rather than swallowing the turn; the preamble is
|
||||
# still inside current_text unless a completed call advanced past it.
|
||||
# Calls are dropped on purpose: the failure can land between a tool's
|
||||
# name and its arguments, and a half-formed call is worse than none.
|
||||
# Fail closed: drop the buffer, keep any prose ahead of the DSML,
|
||||
# and never surface DSML as content.
|
||||
self._buffer = ""
|
||||
if not current_text.startswith(preamble):
|
||||
current_text = preamble + current_text
|
||||
return StreamingParseResult(normal_text=current_text)
|
||||
if self.current_tool_id == -1:
|
||||
preamble = self._text_before_dsml(current_text)
|
||||
return StreamingParseResult(normal_text=preamble, calls=all_calls)
|
||||
|
||||
def finish(self, tools: list[Tool]) -> StreamingParseResult:
|
||||
if not self._buffer:
|
||||
return StreamingParseResult()
|
||||
|
||||
buffered = self._buffer
|
||||
self._buffer = ""
|
||||
if self.current_tool_id != -1:
|
||||
return StreamingParseResult()
|
||||
|
||||
return StreamingParseResult(normal_text=self._text_before_dsml(buffered))
|
||||
|
||||
def structure_info(self) -> _GetInfoFunc:
|
||||
return lambda name: StructureInfo(
|
||||
|
||||
@@ -3,7 +3,7 @@ import threading
|
||||
import warnings
|
||||
from json import JSONDecodeError, JSONDecoder
|
||||
from json.decoder import WHITESPACE
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
||||
from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union
|
||||
|
||||
import orjson
|
||||
import partial_json_parser
|
||||
@@ -476,3 +476,40 @@ def get_json_schema_constraint(
|
||||
return json_schema
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def strip_structural_tag_excludes(structural_tag: Any, tokens: Iterable[str]) -> None:
|
||||
"""Remove ``tokens`` from every free-text ``excludes`` list in a structural
|
||||
tag, in place.
|
||||
|
||||
xgrammar's builtin tags forbid the model's think tokens in free text. When
|
||||
SGLang's reasoning parser owns the reasoning section that exclusion only
|
||||
fights the model (e.g. an empty ``</think>`` emitted before prose).
|
||||
"""
|
||||
tokens = set(tokens)
|
||||
if not tokens:
|
||||
return
|
||||
|
||||
def visit(node: Any) -> None:
|
||||
if isinstance(node, list):
|
||||
for item in node:
|
||||
visit(item)
|
||||
return
|
||||
if isinstance(node, dict):
|
||||
excludes = node.get("excludes")
|
||||
if isinstance(excludes, list):
|
||||
node["excludes"] = [s for s in excludes if s not in tokens]
|
||||
for key in ("format", "content", "tags", "elements"):
|
||||
if key in node:
|
||||
visit(node[key])
|
||||
return
|
||||
if not hasattr(node, "__dict__"):
|
||||
return
|
||||
excludes = getattr(node, "excludes", None)
|
||||
if isinstance(excludes, list):
|
||||
node.excludes = [s for s in excludes if s not in tokens]
|
||||
for key in ("format", "content", "tags", "elements"):
|
||||
if hasattr(node, key):
|
||||
visit(getattr(node, key))
|
||||
|
||||
visit(structural_tag)
|
||||
|
||||
@@ -75,11 +75,15 @@ class BaseReasoningFormatDetector:
|
||||
thinks_internally: bool = False,
|
||||
reasoning_default: str = "always",
|
||||
force_nonempty_content: bool = False,
|
||||
tool_start_at_line_start: bool = False,
|
||||
):
|
||||
self.think_start_token = think_start_token
|
||||
self.think_end_token = think_end_token
|
||||
self.think_excluded_tokens = think_excluded_tokens
|
||||
self.tool_start_token = tool_start_token
|
||||
# Only a tool_start_token that begins a line interrupts reasoning, so
|
||||
# the model mentioning the tag mid-sentence does not end the block.
|
||||
self.tool_start_at_line_start = tool_start_at_line_start
|
||||
self.force_reasoning = force_reasoning
|
||||
self._in_reasoning = force_reasoning
|
||||
self.stream_reasoning = stream_reasoning
|
||||
@@ -87,6 +91,7 @@ class BaseReasoningFormatDetector:
|
||||
self.reasoning_default = reasoning_default
|
||||
|
||||
self._buffer = ""
|
||||
self._streamed_reasoning_tail = ""
|
||||
self.stripped_think_start = False
|
||||
self.think_start_self_label = ""
|
||||
|
||||
@@ -113,6 +118,23 @@ class BaseReasoningFormatDetector:
|
||||
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
|
||||
return ret
|
||||
|
||||
def _find_tool_start(self, text: str, preceded_by: str = "") -> int:
|
||||
"""Index of the first tool_start_token that interrupts reasoning, or -1.
|
||||
|
||||
`preceded_by` is the character just before `text`, for a streaming
|
||||
buffer whose leading newline was already emitted as reasoning."""
|
||||
if not self.tool_start_token:
|
||||
return -1
|
||||
idx = text.find(self.tool_start_token)
|
||||
while idx != -1:
|
||||
prev = text[idx - 1] if idx > 0 else preceded_by
|
||||
# Nothing precedes the token at the absolute start of generation;
|
||||
# that counts as line start too.
|
||||
if not self.tool_start_at_line_start or prev in ("", "\n"):
|
||||
return idx
|
||||
idx = text.find(self.tool_start_token, idx + 1)
|
||||
return -1
|
||||
|
||||
def detect_and_parse(self, text: str) -> StreamingParseResult:
|
||||
"""
|
||||
One-time parsing: Detects and parses reasoning sections in the provided text.
|
||||
@@ -139,13 +161,8 @@ class BaseReasoningFormatDetector:
|
||||
and self.think_end_token not in self.previous_content
|
||||
):
|
||||
# Check for tool_start_token interruption
|
||||
if (
|
||||
in_reasoning
|
||||
and self.tool_start_token is not None
|
||||
and self.tool_start_token in processed_text
|
||||
):
|
||||
# Find the first occurrence of tool_start_token and split there
|
||||
tool_idx = processed_text.find(self.tool_start_token)
|
||||
tool_idx = self._find_tool_start(processed_text) if in_reasoning else -1
|
||||
if tool_idx != -1:
|
||||
reasoning_text = processed_text[:tool_idx]
|
||||
# Preserve tool_start_token in normal text
|
||||
normal_text = processed_text[tool_idx:]
|
||||
@@ -153,7 +170,9 @@ class BaseReasoningFormatDetector:
|
||||
normal_text=normal_text, reasoning_text=reasoning_text
|
||||
)
|
||||
# Assume reasoning was truncated before end token
|
||||
return StreamingParseResult(reasoning_text=processed_text)
|
||||
return StreamingParseResult(
|
||||
reasoning_text=self._strip_partial_tool_start(processed_text)
|
||||
)
|
||||
|
||||
# Extract reasoning content
|
||||
if self.think_end_token in processed_text:
|
||||
@@ -228,8 +247,10 @@ class BaseReasoningFormatDetector:
|
||||
if self._in_reasoning:
|
||||
# Check for tool_start_token interruption. Streaming cannot see a
|
||||
# think_end_token that has not arrived yet; see the chunk_dependent test.
|
||||
if self.tool_start_token and self.tool_start_token in current_text:
|
||||
tool_idx = current_text.find(self.tool_start_token)
|
||||
tool_idx = self._find_tool_start(
|
||||
current_text, self._streamed_reasoning_tail
|
||||
)
|
||||
if tool_idx != -1:
|
||||
reasoning_text = current_text[:tool_idx]
|
||||
# Preserve tool_start_token in normal text
|
||||
normal_text = current_text[tool_idx:]
|
||||
@@ -251,9 +272,10 @@ class BaseReasoningFormatDetector:
|
||||
for token in holdback_tokens
|
||||
)
|
||||
self._buffer = current_text[len(current_text) - holdback :]
|
||||
return StreamingParseResult(
|
||||
reasoning_text=current_text[: len(current_text) - holdback]
|
||||
)
|
||||
reasoning_text = current_text[: len(current_text) - holdback]
|
||||
if reasoning_text:
|
||||
self._streamed_reasoning_tail = reasoning_text[-1]
|
||||
return StreamingParseResult(reasoning_text=reasoning_text)
|
||||
else:
|
||||
return StreamingParseResult()
|
||||
|
||||
@@ -280,6 +302,21 @@ class BaseReasoningFormatDetector:
|
||||
return i
|
||||
return 0
|
||||
|
||||
def _strip_partial_tool_start(self, text: str, preceded_by: str = "") -> str:
|
||||
"""Drop a trailing partial tool-start token that begins a line: generation
|
||||
ended mid-tag, and flushing the fragment as reasoning would surface raw
|
||||
markup to the client. `preceded_by` is as in `_find_tool_start`."""
|
||||
if not self.tool_start_at_line_start or not text:
|
||||
return text
|
||||
n = self._ends_with_partial_token(text, self.tool_start_token or "")
|
||||
if n == 0:
|
||||
return text
|
||||
start = len(text) - n
|
||||
prev = text[start - 1] if start > 0 else preceded_by
|
||||
if prev in ("", "\n"):
|
||||
return text[:start]
|
||||
return text
|
||||
|
||||
def finish(self) -> StreamingParseResult:
|
||||
"""Flush reasoning still buffered when the stream ends before the end token
|
||||
(e.g. max_tokens cut it short), instead of dropping it: the whole block under
|
||||
@@ -295,6 +332,7 @@ class BaseReasoningFormatDetector:
|
||||
# Defensive: subclasses that fill _buffer themselves may not have stripped
|
||||
# the opening think token that _parse_streaming_increment_impl removes.
|
||||
buffer = self._strip_leading_think_start(self._buffer)
|
||||
buffer = self._strip_partial_tool_start(buffer, self._streamed_reasoning_tail)
|
||||
self._buffer = ""
|
||||
|
||||
if self._force_nonempty_content:
|
||||
@@ -1446,8 +1484,11 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
||||
dsv4_thinking_start_token,
|
||||
dsv4_thinking_end_token,
|
||||
think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token],
|
||||
# Leading "<" included: has_tool_call() matches on it.
|
||||
tool_start_token=f"<{dsv4_dsml_token}",
|
||||
# The reference parser splits reasoning at `\n\n<|DSML|tool_calls`;
|
||||
# anchoring on the full tag at a line start keeps a mid-sentence
|
||||
# mention of the tag inside reasoning from ending the block.
|
||||
tool_start_token=f"<{dsv4_dsml_token}tool_calls>",
|
||||
tool_start_at_line_start=True,
|
||||
force_reasoning=force_reasoning,
|
||||
stream_reasoning=stream_reasoning,
|
||||
continue_final_message=continue_final_message,
|
||||
@@ -1458,6 +1499,15 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV41ReasoningDetector(DeepSeekV4Detector):
|
||||
"""V4.1 emits the calls block with a spaced tag name (`<|DSML| calls>`),
|
||||
so the unspaced V4 tool-start anchor never interrupts its reasoning."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.tool_start_token = f"<{dsv4_dsml_token} calls>"
|
||||
|
||||
|
||||
class _MimoDetector(Qwen3Detector):
|
||||
"""MIMO reuses Qwen3 tokens but requires explicit enable_thinking=True to enable."""
|
||||
|
||||
@@ -2166,7 +2216,7 @@ class ReasoningParser:
|
||||
"deepseek-r1": DeepSeekR1Detector,
|
||||
"deepseek-v3": _DeepSeekV3Detector,
|
||||
"deepseek-v4": DeepSeekV4Detector,
|
||||
"deepseek-v41": DeepSeekV4Detector,
|
||||
"deepseek-v41": DeepSeekV41ReasoningDetector,
|
||||
"dots": Qwen3Detector,
|
||||
"glm45": Glm45Detector,
|
||||
"ling3": Ling3Detector,
|
||||
|
||||
Reference in New Issue
Block a user