diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index e8b2ed61d7..b737c83acd 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -94,6 +94,7 @@ from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.function_call.utils import ( get_json_schema_constraint, normalize_json_schema_types, + strip_structural_tag_excludes, ) from sglang.srt.managers.io_struct import GenerateReqInput from sglang.srt.parser.conversation import generate_chat_conv @@ -1351,6 +1352,20 @@ class OpenAIServingChat(OpenAIServingBase): parallel_tool_calls=request.parallel_tool_calls, thinking_mode=xgrammar_reasoning, ) + if ( + tool_call_constraint is not None + and tool_call_constraint[0] == "structural_tag" + and self._reasoning_detector is not None + ): + # Same ownership rule: the reasoning parser handles think + # tokens, so the grammar must not forbid them in free text. + strip_structural_tag_excludes( + tool_call_constraint[1], + ( + self._reasoning_detector.think_start_token, + self._reasoning_detector.think_end_token, + ), + ) required_parsed_natively = parser.detector.parses_required_natively() if self.chat_encoding_spec == "kimi_k3": tool_call_stop = parser.detector.eot_token diff --git a/python/sglang/srt/function_call/deepseekv32_detector.py b/python/sglang/srt/function_call/deepseekv32_detector.py index 653ce7d503..57d3a57bbf 100644 --- a/python/sglang/srt/function_call/deepseekv32_detector.py +++ b/python/sglang/srt/function_call/deepseekv32_detector.py @@ -2,9 +2,6 @@ import json import logging import re -from partial_json_parser.core.exceptions import MalformedJSON -from partial_json_parser.core.options import Allow - from sglang.srt.entrypoints.openai.protocol import Tool from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.core_types import ( @@ -13,11 +10,14 @@ from sglang.srt.function_call.core_types import ( ToolCallItem, _GetInfoFunc, ) -from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads logger = logging.getLogger(__name__) +def _reject_json_constant(value: str) -> None: + raise ValueError(f"Invalid JSON constant: {value}") + + class DeepSeekV32Detector(BaseFormatDetector): """ Detector for DeepSeek V3.2 model function call format. @@ -89,9 +89,6 @@ class DeepSeekV32Detector(BaseFormatDetector): self.parameter_regex = ( rf'<{parameter}\s+name="([^"]+)"\s+string="([^"]+)"\s*>(.*?)' ) - self.partial_parameter_regex = ( - rf'<{parameter}\s+name="([^"]+)"\s+string="([^"]+)"\s*>(.*)$' - ) self.function_calls_regex = rf"<{block}>(.*?)" # Long-form `<|DSML|invoke name="x">...` and the # self-closing `<|DSML|invoke name="x"/>` shape V4 emits for zero-arg @@ -101,25 +98,24 @@ class DeepSeekV32Detector(BaseFormatDetector): r"(?:(?P/>)" rf"|>(?P.*?)(?P(?:|$)))" ) - # Consumed right-to-left by rstrip (a character set, not a suffix), so the - # invoke name is split to limit how much of a partial value gets eaten. - self.prefix_parameter_end_call = [ - " bool: """Check if the text contains a deepseek v32 format tool call.""" return self.bot_token in text or self.invoke_start_token in text + def _text_before_dsml(self, text: str) -> str: + """Prose preceding the first DSML tag, with the trailing blank line the + chat template inserts before a tool call removed.""" + idx = text.find(self.dsml_token) + if idx == -1: + return text + if idx >= 2 and text[idx - 2 : idx] == "= 1 and text[idx - 1] == "<": + idx -= 1 + return text[:idx].removesuffix("\n\n") + @staticmethod def _unpack_invoke_match(m: "re.Match[str]") -> tuple[str, str, bool]: """Returns (name, body, is_complete) for an invoke_regex match. @@ -134,9 +130,7 @@ class DeepSeekV32Detector(BaseFormatDetector): return name, "", True return name, m.group("body"), bool(m.group("end")) - def _parse_parameters_from_xml( - self, invoke_content: str, allow_partial: bool = False - ) -> str: + def _parse_parameters_from_xml(self, invoke_content: str) -> str: """ Parse parameters from either XML-like format or JSON format to str. @@ -147,13 +141,12 @@ class DeepSeekV32Detector(BaseFormatDetector): # First, try to parse as direct JSON (new format) invoke_content_stripped = invoke_content.strip() if invoke_content_stripped.startswith("{"): - if allow_partial: - # Remove incomplete invoke end call prefix in case they are captured by param - for token in reversed(self.prefix_invoke_end_call): - invoke_content_stripped = invoke_content_stripped.rstrip(token) - return invoke_content_stripped - elif invoke_content_stripped.endswith("}"): - return invoke_content_stripped + parsed = json.loads( + invoke_content_stripped, parse_constant=_reject_json_constant + ) + if not isinstance(parsed, dict): + raise ValueError("DeepSeek tool arguments must be a JSON object") + return invoke_content_stripped # Fall back to XML parameter tag parsing (original format) parameters = {} @@ -162,13 +155,18 @@ class DeepSeekV32Detector(BaseFormatDetector): re.finditer(self.parameter_regex, invoke_content, re.DOTALL) ) + # Stray prose around the parameter tags is ignored, as before. Leftover + # text holding a DSML fragment is a parameter tag that did not match + # (unclosed, self-closing, missing `string=`), and emitting the call + # without that argument would be worse than emitting no call. last_match_end = 0 + leftover = [] for match in param_matches: + leftover.append(invoke_content[last_match_end : match.start()]) param_name = match.group(1) param_type = match.group(2) param_value = match.group(3) last_match_end = match.end() - # Convert value based on type if param_type == "true": # string type parameters[param_name] = ( @@ -179,39 +177,18 @@ class DeepSeekV32Detector(BaseFormatDetector): else: # Try to parse as JSON for other types try: - parameters[param_name] = json.loads(param_value.strip()) + parameters[param_name] = json.loads( + param_value.strip(), parse_constant=_reject_json_constant + ) except (json.JSONDecodeError, ValueError): parameters[param_name] = param_value.strip() + leftover.append(invoke_content[last_match_end:]) - # If allowed, try to parse a partial parameter at the end - if allow_partial: - remaining_content = invoke_content[last_match_end:] - - # Remove incomplete parameter_end_call prefix in case they are captured by param - for token in reversed(self.prefix_parameter_end_call): - remaining_content = remaining_content.rstrip(token) - - # Match start of a parameter tag + value (potentially incomplete) - # Regex: VALUE... (no end tag) - partial_match = re.search( - self.partial_parameter_regex, remaining_content, re.DOTALL - ) - - if partial_match and (param_value := partial_match.group(3)): - param_name = partial_match.group(1) - if partial_match.group(2) == "true": - parameters[param_name] = ( - param_value.strip() - if self.strip_string_param_value - else param_value - ) - else: - try: - parameters[param_name] = _partial_json_loads( - param_value, Allow.ALL - )[0] - except (json.JSONDecodeError, MalformedJSON, ValueError): - parameters[param_name] = param_value.strip() + leftover_text = "".join(leftover) + if self.dsml_token in leftover_text or ( + not param_matches and leftover_text.strip() + ): + raise ValueError("Malformed DeepSeek tool parameter") return json.dumps(parameters, ensure_ascii=False) @@ -224,25 +201,41 @@ class DeepSeekV32Detector(BaseFormatDetector): :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. """ idx = text.find(self.bot_token) - normal_text = text[:idx].removesuffix("\n\n") if idx != -1 else text - if self.bot_token not in text: + if idx != -1: + normal_text = text[:idx].removesuffix("\n\n") + sections = re.findall(self.function_calls_regex, text, re.DOTALL) + else: + # The streaming path accepts bare invokes without the outer + # wrapper; parse them here too instead of leaking raw DSML. + start = text.find(self.invoke_start_token) + if start == -1: + return StreamingParseResult(normal_text=text, calls=[]) + normal_text = self._text_before_dsml(text) + sections = [text[start:]] + + if not sections: return StreamingParseResult(normal_text=normal_text, calls=[]) calls = [] try: - sections = re.findall(self.function_calls_regex, text, re.DOTALL) - if not sections: - return StreamingParseResult(normal_text=normal_text, calls=[]) - # Find all invoke blocks for function_calls_content in sections: for invoke_match in re.finditer( self.invoke_regex, function_calls_content, re.DOTALL ): - func_name, invoke_content, _ = self._unpack_invoke_match( + func_name, invoke_content, is_complete = self._unpack_invoke_match( invoke_match ) - func_args = self._parse_parameters_from_xml(invoke_content) + if not is_complete: + logger.warning( + f"Dropping unclosed DeepSeek invoke '{func_name}'" + ) + continue + try: + func_args = self._parse_parameters_from_xml(invoke_content) + except ValueError as e: + logger.warning(f"Dropping malformed DeepSeek invoke: {e}") + continue # construct match_result for parse_base_json match_result = { "name": func_name, @@ -253,26 +246,27 @@ class DeepSeekV32Detector(BaseFormatDetector): return StreamingParseResult(normal_text=normal_text, calls=calls) except Exception as e: logger.error(f"Error in detect_and_parse: {e}") - # return the normal text if parsing fails - return StreamingParseResult(normal_text=text) + # Fail closed: keep the prose, never surface DSML as content. + return StreamingParseResult(normal_text=normal_text, calls=calls) def parse_streaming_increment( self, new_text: str, tools: list[Tool] ) -> StreamingParseResult: """ Streaming incremental parsing tool calls for DeepSeekV32 format. - Supports multiple consecutive invoke blocks and argument streaming. + Supports multiple consecutive invoke blocks. """ self._buffer += new_text current_text = self._buffer # Check if buffer contains any DSML markers or ends with potential tag prefix # This handles partial/streaming DSML content - dsml_markers = ["|DSML|", "<|", " sent_len: - argument_diff = prefix[sent_len:] - - if argument_diff: - all_calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters=argument_diff, - ) - ) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff + self.streamed_args_for_tool[self.current_tool_id] = current_params # Update the stored arguments self.prev_tool_call_arr[self.current_tool_id] = { @@ -376,36 +354,33 @@ class DeepSeekV32Detector(BaseFormatDetector): "arguments": current_params, } - # Check if tool call is complete (has closing tag) - if is_tool_end: - # Remove the completed tool call from buffer - self._buffer = current_text[invoke_match.end() :] - current_text = self._buffer # Update for next iteration - - # Move to next tool call - self.current_tool_id += 1 - self.current_tool_name_sent = False - - # Continue loop to check for more invoke blocks - continue - else: - # Tool call not complete yet, don't return anything - # Wait for more chunks until we see - break + # Remove the completed tool call from buffer and check for another. + self._buffer = current_text[invoke_match.end() :] + current_text = self._buffer + self.current_tool_id += 1 # No more invoke blocks found return StreamingParseResult(normal_text=preamble, calls=all_calls) except Exception as e: logger.error(f"Error in parse_streaming_increment: {e}") - # Re-emit verbatim rather than swallowing the turn; the preamble is - # still inside current_text unless a completed call advanced past it. - # Calls are dropped on purpose: the failure can land between a tool's - # name and its arguments, and a half-formed call is worse than none. + # Fail closed: drop the buffer, keep any prose ahead of the DSML, + # and never surface DSML as content. self._buffer = "" - if not current_text.startswith(preamble): - current_text = preamble + current_text - return StreamingParseResult(normal_text=current_text) + if self.current_tool_id == -1: + preamble = self._text_before_dsml(current_text) + return StreamingParseResult(normal_text=preamble, calls=all_calls) + + def finish(self, tools: list[Tool]) -> StreamingParseResult: + if not self._buffer: + return StreamingParseResult() + + buffered = self._buffer + self._buffer = "" + if self.current_tool_id != -1: + return StreamingParseResult() + + return StreamingParseResult(normal_text=self._text_before_dsml(buffered)) def structure_info(self) -> _GetInfoFunc: return lambda name: StructureInfo( diff --git a/python/sglang/srt/function_call/utils.py b/python/sglang/srt/function_call/utils.py index d1a87a90c4..7e900d7fec 100644 --- a/python/sglang/srt/function_call/utils.py +++ b/python/sglang/srt/function_call/utils.py @@ -3,7 +3,7 @@ import threading import warnings from json import JSONDecodeError, JSONDecoder from json.decoder import WHITESPACE -from typing import Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union import orjson import partial_json_parser @@ -476,3 +476,40 @@ def get_json_schema_constraint( return json_schema return None + + +def strip_structural_tag_excludes(structural_tag: Any, tokens: Iterable[str]) -> None: + """Remove ``tokens`` from every free-text ``excludes`` list in a structural + tag, in place. + + xgrammar's builtin tags forbid the model's think tokens in free text. When + SGLang's reasoning parser owns the reasoning section that exclusion only + fights the model (e.g. an empty ```` emitted before prose). + """ + tokens = set(tokens) + if not tokens: + return + + def visit(node: Any) -> None: + if isinstance(node, list): + for item in node: + visit(item) + return + if isinstance(node, dict): + excludes = node.get("excludes") + if isinstance(excludes, list): + node["excludes"] = [s for s in excludes if s not in tokens] + for key in ("format", "content", "tags", "elements"): + if key in node: + visit(node[key]) + return + if not hasattr(node, "__dict__"): + return + excludes = getattr(node, "excludes", None) + if isinstance(excludes, list): + node.excludes = [s for s in excludes if s not in tokens] + for key in ("format", "content", "tags", "elements"): + if hasattr(node, key): + visit(getattr(node, key)) + + visit(structural_tag) diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index e4953935d0..d31a7a521c 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -75,11 +75,15 @@ class BaseReasoningFormatDetector: thinks_internally: bool = False, reasoning_default: str = "always", force_nonempty_content: bool = False, + tool_start_at_line_start: bool = False, ): self.think_start_token = think_start_token self.think_end_token = think_end_token self.think_excluded_tokens = think_excluded_tokens self.tool_start_token = tool_start_token + # Only a tool_start_token that begins a line interrupts reasoning, so + # the model mentioning the tag mid-sentence does not end the block. + self.tool_start_at_line_start = tool_start_at_line_start self.force_reasoning = force_reasoning self._in_reasoning = force_reasoning self.stream_reasoning = stream_reasoning @@ -87,6 +91,7 @@ class BaseReasoningFormatDetector: self.reasoning_default = reasoning_default self._buffer = "" + self._streamed_reasoning_tail = "" self.stripped_think_start = False self.think_start_self_label = "" @@ -113,6 +118,23 @@ class BaseReasoningFormatDetector: ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text return ret + def _find_tool_start(self, text: str, preceded_by: str = "") -> int: + """Index of the first tool_start_token that interrupts reasoning, or -1. + + `preceded_by` is the character just before `text`, for a streaming + buffer whose leading newline was already emitted as reasoning.""" + if not self.tool_start_token: + return -1 + idx = text.find(self.tool_start_token) + while idx != -1: + prev = text[idx - 1] if idx > 0 else preceded_by + # Nothing precedes the token at the absolute start of generation; + # that counts as line start too. + if not self.tool_start_at_line_start or prev in ("", "\n"): + return idx + idx = text.find(self.tool_start_token, idx + 1) + return -1 + def detect_and_parse(self, text: str) -> StreamingParseResult: """ One-time parsing: Detects and parses reasoning sections in the provided text. @@ -139,13 +161,8 @@ class BaseReasoningFormatDetector: and self.think_end_token not in self.previous_content ): # Check for tool_start_token interruption - if ( - in_reasoning - and self.tool_start_token is not None - and self.tool_start_token in processed_text - ): - # Find the first occurrence of tool_start_token and split there - tool_idx = processed_text.find(self.tool_start_token) + tool_idx = self._find_tool_start(processed_text) if in_reasoning else -1 + if tool_idx != -1: reasoning_text = processed_text[:tool_idx] # Preserve tool_start_token in normal text normal_text = processed_text[tool_idx:] @@ -153,7 +170,9 @@ class BaseReasoningFormatDetector: normal_text=normal_text, reasoning_text=reasoning_text ) # Assume reasoning was truncated before end token - return StreamingParseResult(reasoning_text=processed_text) + return StreamingParseResult( + reasoning_text=self._strip_partial_tool_start(processed_text) + ) # Extract reasoning content if self.think_end_token in processed_text: @@ -228,8 +247,10 @@ class BaseReasoningFormatDetector: if self._in_reasoning: # Check for tool_start_token interruption. Streaming cannot see a # think_end_token that has not arrived yet; see the chunk_dependent test. - if self.tool_start_token and self.tool_start_token in current_text: - tool_idx = current_text.find(self.tool_start_token) + tool_idx = self._find_tool_start( + current_text, self._streamed_reasoning_tail + ) + if tool_idx != -1: reasoning_text = current_text[:tool_idx] # Preserve tool_start_token in normal text normal_text = current_text[tool_idx:] @@ -251,9 +272,10 @@ class BaseReasoningFormatDetector: for token in holdback_tokens ) self._buffer = current_text[len(current_text) - holdback :] - return StreamingParseResult( - reasoning_text=current_text[: len(current_text) - holdback] - ) + reasoning_text = current_text[: len(current_text) - holdback] + if reasoning_text: + self._streamed_reasoning_tail = reasoning_text[-1] + return StreamingParseResult(reasoning_text=reasoning_text) else: return StreamingParseResult() @@ -280,6 +302,21 @@ class BaseReasoningFormatDetector: return i return 0 + def _strip_partial_tool_start(self, text: str, preceded_by: str = "") -> str: + """Drop a trailing partial tool-start token that begins a line: generation + ended mid-tag, and flushing the fragment as reasoning would surface raw + markup to the client. `preceded_by` is as in `_find_tool_start`.""" + if not self.tool_start_at_line_start or not text: + return text + n = self._ends_with_partial_token(text, self.tool_start_token or "") + if n == 0: + return text + start = len(text) - n + prev = text[start - 1] if start > 0 else preceded_by + if prev in ("", "\n"): + return text[:start] + return text + def finish(self) -> StreamingParseResult: """Flush reasoning still buffered when the stream ends before the end token (e.g. max_tokens cut it short), instead of dropping it: the whole block under @@ -295,6 +332,7 @@ class BaseReasoningFormatDetector: # Defensive: subclasses that fill _buffer themselves may not have stripped # the opening think token that _parse_streaming_increment_impl removes. buffer = self._strip_leading_think_start(self._buffer) + buffer = self._strip_partial_tool_start(buffer, self._streamed_reasoning_tail) self._buffer = "" if self._force_nonempty_content: @@ -1446,8 +1484,11 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector): dsv4_thinking_start_token, dsv4_thinking_end_token, think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token], - # Leading "<" included: has_tool_call() matches on it. - tool_start_token=f"<{dsv4_dsml_token}", + # The reference parser splits reasoning at `\n\n<|DSML|tool_calls`; + # anchoring on the full tag at a line start keeps a mid-sentence + # mention of the tag inside reasoning from ending the block. + tool_start_token=f"<{dsv4_dsml_token}tool_calls>", + tool_start_at_line_start=True, force_reasoning=force_reasoning, stream_reasoning=stream_reasoning, continue_final_message=continue_final_message, @@ -1458,6 +1499,15 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector): ) +class DeepSeekV41ReasoningDetector(DeepSeekV4Detector): + """V4.1 emits the calls block with a spaced tag name (`<|DSML| calls>`), + so the unspaced V4 tool-start anchor never interrupts its reasoning.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.tool_start_token = f"<{dsv4_dsml_token} calls>" + + class _MimoDetector(Qwen3Detector): """MIMO reuses Qwen3 tokens but requires explicit enable_thinking=True to enable.""" @@ -2166,7 +2216,7 @@ class ReasoningParser: "deepseek-r1": DeepSeekR1Detector, "deepseek-v3": _DeepSeekV3Detector, "deepseek-v4": DeepSeekV4Detector, - "deepseek-v41": DeepSeekV4Detector, + "deepseek-v41": DeepSeekV41ReasoningDetector, "dots": Qwen3Detector, "glm45": Glm45Detector, "ling3": Ling3Detector, diff --git a/test/registered/unit/function_call/test_deepseekv4_detector.py b/test/registered/unit/function_call/test_deepseekv4_detector.py index f6d0ceea5b..f0d18262d5 100644 --- a/test/registered/unit/function_call/test_deepseekv4_detector.py +++ b/test/registered/unit/function_call/test_deepseekv4_detector.py @@ -1,11 +1,12 @@ """Unit tests for DeepSeekV4Detector DSML streaming — no server, no model loading.""" +import unittest from unittest.mock import patch from sglang.srt.entrypoints.openai.protocol import Function, Tool from sglang.srt.function_call.deepseekv4_detector import DeepSeekV4Detector +from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase register_cpu_ci(1.0, "base-a-test-cpu") @@ -30,7 +31,7 @@ def _weather_call(city: str = "SF") -> str: return _wrapped(_invoke("get_weather", _param("city", "true", city))) -class TestDeepSeekV4Streaming(CustomTestCase): +class TestDeepSeekV4Streaming(unittest.TestCase): def setUp(self): self.tools = [ Tool( @@ -83,16 +84,310 @@ class TestDeepSeekV4Streaming(CustomTestCase): self.assertNotIn(DSML, normal) - def test_malformed_partial_json_falls_back_to_raw_value(self): - """A partial non-string parameter must not escape as MalformedJSON.""" + def test_streaming_waits_for_complete_invoke(self): detector = DeepSeekV4Detector() - result = detector.parse_streaming_increment( + partial = detector.parse_streaming_increment( f'<{DSML}tool_calls>\n<{DSML}invoke name="get_weather">\n' f'<{DSML}parameter name="city" string="false">{{"a"', self.tools, ) - self.assertEqual([c.name for c in result.calls if c.name], ["get_weather"]) + self.assertEqual(partial.calls, []) + + complete = detector.parse_streaming_increment( + f"", + self.tools, + ) + self.assertEqual(len(complete.calls), 1) + self.assertEqual(complete.calls[0].name, "get_weather") + self.assertEqual(complete.calls[0].parameters, '{"city": "{\\"a\\""}') + self.assertEqual( + detector.prev_tool_call_arr, + [{"name": "get_weather", "arguments": complete.calls[0].parameters}], + ) + self.assertEqual( + detector.streamed_args_for_tool, [complete.calls[0].parameters] + ) + + def test_streaming_matches_complete_parse_at_every_chunk_width(self): + tools = [ + Tool( + type="function", + function=Function( + name="create_tasks", + description="Create tasks", + parameters={ + "type": "object", + "properties": {"tasks": {"type": "array"}}, + }, + ), + ) + ] + values = ( + '[{"description":"first","priority":"medium"}]', + '[{"description":"unterminated,"priority":"medium"}]', + "[{]", + "[}", + ) + + for value in values: + text = _wrapped(_invoke("create_tasks", _param("tasks", "false", value))) + expected = DeepSeekV4Detector().detect_and_parse(text, tools).calls[0] + for width in range(1, len(text) + 1): + with self.subTest(value=value, width=width): + chunks = [ + text[index : index + width] + for index in range(0, len(text), width) + ] + normal, calls = self._feed_with_tools(chunks, tools) + + self.assertEqual(normal.strip(), "") + self.assertNotIn(DSML, normal) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].name, "create_tasks") + self.assertEqual(calls[0].parameters, expected.parameters) + + def test_complete_invoke_emits_name_and_arguments_together(self): + text = _weather_call() + + _, calls = self._feed([text[:-1], text[-1:]]) + + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].name, "get_weather") + self.assertEqual(calls[0].parameters, '{"city": "SF"}') + + def test_multiple_invokes_emit_one_complete_item_each(self): + text = _wrapped( + _invoke("get_weather", _param("city", "true", "SF")) + + "\n" + + _invoke("get_weather", _param("city", "true", "NY")) + ) + + _, calls = self._feed([text]) + + self.assertEqual( + [(call.tool_index, call.name, call.parameters) for call in calls], + [ + (0, "get_weather", '{"city": "SF"}'), + (1, "get_weather", '{"city": "NY"}'), + ], + ) + + def test_closed_invoke_rejects_incomplete_parameter(self): + text = _wrapped( + _invoke( + "get_weather", + f'<{DSML}parameter name="city" string="true">SF', + ) + ) + + normal, calls = self._feed([text]) + + self.assertEqual((normal, calls), ("", [])) + + def test_stray_prose_inside_invoke_is_ignored(self): + """Prose around the parameter tags cannot hide a lost argument, so the + call is still emitted, matching the one-shot parser and the previous + streaming behavior.""" + text = _wrapped( + _invoke( + "get_weather", + "thinking aloud\n" + _param("city", "true", "SF") + "\nnote to self", + ) + ) + + for width in (4, len(text)): + with self.subTest(width=width): + normal, calls = self._feed( + [text[i : i + width] for i in range(0, len(text), width)] + ) + self.assertEqual(normal, "") + self.assertEqual( + [(call.name, call.parameters) for call in calls], + [("get_weather", '{"city": "SF"}')], + ) + + result = DeepSeekV4Detector().detect_and_parse(text, self.tools) + self.assertEqual( + [(call.name, call.parameters) for call in result.calls], + [("get_weather", '{"city": "SF"}')], + ) + + def test_closed_invoke_rejects_unmatched_parameter_tag(self): + """Leftover text containing a DSML fragment is a parameter that failed + to match; the call is dropped rather than emitted without it.""" + bodies = { + "missing string attr": f'<{DSML}parameter name="city">SF', + "self-closing": f'<{DSML}parameter name="city" string="true"/>', + "unclosed after valid": _param("city", "true", "SF") + + f'<{DSML}parameter name="units" string="true">C', + } + for label, body in bodies.items(): + with self.subTest(label=label): + normal, calls = self._feed([_wrapped(_invoke("get_weather", body))]) + self.assertEqual((normal, calls), ("", [])) + + def test_closed_invoke_rejects_body_without_parameters(self): + text = _wrapped(_invoke("get_weather", "just prose, no parameters")) + + normal, calls = self._feed([text]) + + self.assertEqual((normal, calls), ("", [])) + + def test_closed_invoke_rejects_malformed_direct_json(self): + text = _wrapped(_invoke("get_weather", '{"city": }')) + + normal, calls = self._feed([text]) + + self.assertEqual((normal, calls), ("", [])) + + def test_closed_invoke_rejects_nonstandard_json_constant(self): + text = _wrapped(_invoke("get_weather", '{"city": NaN}')) + + normal, calls = self._feed([text]) + + self.assertEqual((normal, calls), ("", [])) + + def test_nonstandard_xml_constant_falls_back_to_string(self): + text = _wrapped(_invoke("get_weather", _param("city", "false", "NaN"))) + + _, calls = self._feed([text]) + + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].parameters, '{"city": "NaN"}') + + def test_malformed_first_invoke_does_not_create_flush_state(self): + detector = DeepSeekV4Detector() + + result = detector.parse_streaming_increment( + _wrapped(_invoke("get_weather", '{"city": }')), self.tools + ) + + self.assertEqual(result.calls, []) + self.assertEqual(detector.prev_tool_call_arr, []) + self.assertEqual(detector.streamed_args_for_tool, []) + + def test_malformed_later_invoke_preserves_completed_call(self): + text = _wrapped( + _invoke("get_weather", _param("city", "true", "SF")) + + "\n" + + _invoke("get_weather", '{"city": }') + ) + + normal, calls = self._feed([text]) + + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].name, "get_weather") + self.assertEqual(calls[0].parameters, '{"city": "SF"}') + self.assertEqual(normal, "") + + def test_malformed_first_invoke_keeps_prose_and_later_call(self): + """A malformed invoke is discarded on its own: the prose ahead of the + block survives, the next well-formed invoke still becomes call 0, and + no DSML reaches the client.""" + text = "Checking.\n\n" + _wrapped( + _invoke("get_weather", f'<{DSML}parameter name="city">SF') + + "\n" + + _invoke("get_weather", _param("city", "true", "NY")) + ) + + normal, calls = self._feed([text]) + + self.assertEqual(normal, "Checking.") + self.assertEqual( + [(call.tool_index, call.name, call.parameters) for call in calls], + [(0, "get_weather", '{"city": "NY"}')], + ) + + def test_malformed_invoke_fails_closed_at_every_chunk_width(self): + """Whatever the delta boundaries, a malformed invoke yields the prose, + no call, and no DSML, including the trailing tool_calls close.""" + text = "Checking.\n\n" + _wrapped( + _invoke("get_weather", f'<{DSML}parameter name="city" string="true">SF') + ) + + for width in range(1, len(text) + 1): + with self.subTest(width=width): + parser = FunctionCallParser(self.tools, "deepseekv4") + normal, calls = "", [] + for index in range(0, len(text), width): + chunk_normal, chunk_calls = parser.parse_stream_chunk( + text[index : index + width] + ) + normal += chunk_normal + calls.extend(chunk_calls) + end_normal, end_calls = parser.parse_stream_end() + normal += end_normal + calls.extend(end_calls) + + self.assertEqual(normal, "Checking.") + self.assertEqual(calls, []) + + def test_non_streaming_drops_only_the_malformed_invoke(self): + text = "Checking.\n\n" + _wrapped( + _invoke("get_weather", '{"city": }') + + "\n" + + _invoke("get_weather", _param("city", "true", "NY")) + ) + + result = DeepSeekV4Detector().detect_and_parse(text, self.tools) + + self.assertEqual(result.normal_text, "Checking.") + self.assertEqual( + [(call.name, call.parameters) for call in result.calls], + [("get_weather", '{"city": "NY"}')], + ) + + def test_non_streaming_drops_unclosed_invoke_in_closed_wrapper(self): + text = ( + f"<{DSML}tool_calls>\n" + f'<{DSML}invoke name="get_weather">\n{{"city": "SF"}}\n' + f"" + ) + + result = DeepSeekV4Detector().detect_and_parse(text, self.tools) + + self.assertEqual(result.calls, []) + self.assertNotIn(DSML, result.normal_text) + + def test_non_streaming_parses_bare_invoke_without_wrapper(self): + result = DeepSeekV4Detector().detect_and_parse( + "Checking.\n\n" + _invoke("get_weather", '{"city": "SF"}'), self.tools + ) + + self.assertEqual(result.normal_text, "Checking.") + self.assertEqual( + [(call.name, call.parameters) for call in result.calls], + [("get_weather", '{"city": "SF"}')], + ) + + def test_non_streaming_malformed_bare_invoke_fails_closed(self): + result = DeepSeekV4Detector().detect_and_parse( + _invoke("get_weather", '{"city": }'), self.tools + ) + + self.assertEqual(result.calls, []) + self.assertNotIn(DSML, result.normal_text) + + def test_stream_end_drops_incomplete_invoke(self): + parser = FunctionCallParser(self.tools, "deepseekv4") + normal, calls = parser.parse_stream_chunk( + "Checking.\n\n" + f'<{DSML}tool_calls><{DSML}invoke name="get_weather">' + f'<{DSML}parameter name="city" string="true">SF' + ) + + self.assertEqual((normal, calls), ("", [])) + self.assertEqual(parser.parse_stream_end(), ("Checking.", [])) + + def _feed_with_tools(self, chunks, tools): + detector = DeepSeekV4Detector() + normal, calls = "", [] + for chunk in chunks: + result = detector.parse_streaming_increment(chunk, tools) + normal += result.normal_text + calls.extend(result.calls) + return normal, calls def test_non_streaming_parses_every_tool_calls_section(self): """A turn with two tool_calls sections must yield both calls.""" @@ -102,9 +397,9 @@ class TestDeepSeekV4Streaming(CustomTestCase): self.assertEqual(len(result.calls), 2) - def test_parse_error_neither_swallows_nor_duplicates(self): - """An unexpected parse error must not empty the turn, and the dropped - buffer must not come back on the next delta.""" + def test_unexpected_parse_error_fails_closed(self): + """An unexpected parse error keeps the prose, drops the DSML and the + call, and the dropped buffer must not come back on the next delta.""" detector = DeepSeekV4Detector() with patch.object( @@ -112,18 +407,16 @@ class TestDeepSeekV4Streaming(CustomTestCase): "_parse_parameters_from_xml", side_effect=RuntimeError("boom"), ): - first = detector.parse_streaming_increment(_weather_call(), self.tools) + first = detector.parse_streaming_increment( + "Checking.\n\n" + _weather_call(), self.tools + ) self.assertEqual(detector._buffer, "") second = detector.parse_streaming_increment(" tail", self.tools) - self.assertIn("get_weather", first.normal_text) - self.assertNotIn("get_weather", second.normal_text) - # No half-formed call: the failure can land between a tool's name and its - # arguments, so an argument-less named call must not reach the client. + self.assertEqual(first.normal_text, "Checking.") self.assertEqual(first.calls, []) + self.assertEqual(second.normal_text, " tail") if __name__ == "__main__": - import unittest - unittest.main() diff --git a/test/registered/unit/function_call/test_function_call_parser.py b/test/registered/unit/function_call/test_function_call_parser.py index 359ae726db..93971258b9 100644 --- a/test/registered/unit/function_call/test_function_call_parser.py +++ b/test/registered/unit/function_call/test_function_call_parser.py @@ -1722,7 +1722,7 @@ class TestDeepSeekV32Detector(unittest.TestCase): call.parameters ) - self.assertGreater(num_tool_call_chunks, 8) + self.assertEqual(num_tool_call_chunks, 1) self.assertEqual(len(tool_calls_by_index), 1) self.assertEqual(tool_calls_by_index[0]["name"], "get_favorite_tourist_spot") @@ -1777,7 +1777,7 @@ class TestDeepSeekV32Detector(unittest.TestCase): call.parameters ) - self.assertGreater(num_tool_call_chunks, 8) + self.assertEqual(num_tool_call_chunks, 1) self.assertEqual(len(tool_calls_by_index), 1) self.assertEqual(tool_calls_by_index[0]["name"], "get_favorite_tourist_spot") @@ -2132,7 +2132,7 @@ class TestDeepSeekV4Detector(unittest.TestCase): call.parameters ) - self.assertGreater(num_tool_call_chunks, 8) + self.assertEqual(num_tool_call_chunks, 1) self.assertEqual(len(tool_calls_by_index), 1) self.assertEqual(tool_calls_by_index[0]["name"], "get_favorite_tourist_spot") diff --git a/test/registered/unit/function_call/test_structural_tag_excludes.py b/test/registered/unit/function_call/test_structural_tag_excludes.py new file mode 100644 index 0000000000..ca4d58ef9b --- /dev/null +++ b/test/registered/unit/function_call/test_structural_tag_excludes.py @@ -0,0 +1,99 @@ +"""Unit tests for strip_structural_tag_excludes — no server, no model loading.""" + +import unittest + +from sglang.srt.entrypoints.openai.protocol import Function, Tool +from sglang.srt.function_call.base_format_detector import get_model_structural_tag +from sglang.srt.function_call.deepseekv4_detector import DeepSeekV4Detector +from sglang.srt.function_call.utils import strip_structural_tag_excludes +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(1.0, "base-a-test-cpu") + +THINK_TOKENS = ("", "") + + +def _weather_tool() -> Tool: + return Tool( + type="function", + function=Function( + name="get_weather", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + ), + ) + + +def _collect_excludes(node, out): + if isinstance(node, list): + for item in node: + _collect_excludes(item, out) + elif isinstance(node, dict): + if "excludes" in node: + out.append(list(node["excludes"])) + for key in ("format", "content", "tags", "elements"): + if key in node: + _collect_excludes(node[key], out) + return out + + +class TestStripStructuralTagExcludes(unittest.TestCase): + def test_strips_only_think_tokens_from_every_free_text_section(self): + tag = { + "type": "structural_tag", + "format": { + "type": "sequence", + "elements": [ + { + "type": "triggered_tags", + "triggers": [""], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": ["", ""], + }, + "end": "", + } + ], + "excludes": ["", "", ""], + }, + {"type": "any_text", "excludes": ["", ""]}, + ], + }, + } + + strip_structural_tag_excludes(tag, THINK_TOKENS) + + self.assertEqual( + _collect_excludes(tag, []), + [[""], [""], []], + ) + + def test_no_tokens_is_a_no_op(self): + tag = {"type": "any_text", "excludes": [""]} + + strip_structural_tag_excludes(tag, ()) + + self.assertEqual(tag["excludes"], [""]) + + @unittest.skipIf(get_model_structural_tag is None, "xgrammar builtin tags absent") + def test_deepseek_v4_auto_tag_allows_think_tokens_after_strip(self): + tag = DeepSeekV4Detector().get_structural_tag([_weather_tool()], "auto") + before = _collect_excludes(tag.model_dump(by_alias=True), []) + self.assertTrue(any("" in ex for ex in before)) + + strip_structural_tag_excludes(tag, THINK_TOKENS) + + after = _collect_excludes(tag.model_dump(by_alias=True), []) + self.assertFalse(any(tok in ex for ex in after for tok in THINK_TOKENS)) + self.assertEqual(tag.format.triggers, ["<|DSML|tool_calls>"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/parser/test_reasoning_parser.py b/test/registered/unit/parser/test_reasoning_parser.py index acf2152d74..8e52598dc2 100644 --- a/test/registered/unit/parser/test_reasoning_parser.py +++ b/test/registered/unit/parser/test_reasoning_parser.py @@ -246,17 +246,73 @@ class TestDeepSeekV4Detector(CustomTestCase): self.assertTrue(detector.thinks_internally) def test_dsml_block_is_routed_out_of_reasoning(self): - """Without tool_start_token the DSML block stays in reasoning_content and - the tool call detector never sees it.""" + """A DSML block that starts a line inside reasoning is routed to + normal_text so the tool call detector sees it, even without .""" detector = ReasoningParser(model_type="deepseek-v4").detector - self.assertEqual(detector.tool_start_token, "<|DSML|") + self.assertEqual(detector.tool_start_token, "<|DSML|tool_calls>") + self.assertTrue(detector.tool_start_at_line_start) result = detector.parse_streaming_increment( - 'pick a tool<|DSML|tool_calls><|DSML|invoke name="s">' + 'pick a tool\n\n<|DSML|tool_calls><|DSML|invoke name="s">' ) - self.assertEqual(result.reasoning_text, "pick a tool") + self.assertEqual(result.reasoning_text, "pick a tool\n\n") self.assertTrue(result.normal_text.startswith("<|DSML|tool_calls>")) + def test_dsml_mention_mid_sentence_stays_in_reasoning(self): + """The system prompt shows the model the DSML tag, so it may talk about + it while thinking. A mid-sentence mention must not end the block.""" + detector = ReasoningParser(model_type="deepseek-v4").detector + + result = detector.parse_streaming_increment( + "I'll output a <|DSML|tool_calls> block with JSON.done" + ) + self.assertEqual( + result.reasoning_text, "I'll output a <|DSML|tool_calls> block with JSON." + ) + self.assertEqual(result.normal_text, "done") + + def test_truncated_calls_opener_is_not_flushed_into_reasoning(self): + """A generation cut off mid-calls-opener (e.g. max_tokens before `>`) + must not surface the partial tag in reasoning_content.""" + for model, partial in ( + ("deepseek-v4", "<|DSML|tool_calls"), + ("deepseek-v41", "<|DSML| calls"), + ): + with self.subTest(model=model): + text = "Choosing a tool\n\n" + partial + detector = ReasoningParser( + model_type=model, force_reasoning=True + ).detector + streamed = detector.parse_streaming_increment(text) + flushed = detector.finish() + reasoning = (streamed.reasoning_text or "") + ( + flushed.reasoning_text or "" + ) + normal = (streamed.normal_text or "") + (flushed.normal_text or "") + self.assertEqual(reasoning, "Choosing a tool\n\n") + self.assertEqual(normal, "") + + result = ReasoningParser( + model_type=model, force_reasoning=True + ).detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "Choosing a tool\n\n") + self.assertFalse(result.normal_text) + + def test_mid_sentence_partial_calls_opener_stays_in_reasoning(self): + """A partial calls opener that does not begin a line is a mention, + not a call, and must survive the EOS flush.""" + detector = ReasoningParser( + model_type="deepseek-v4", force_reasoning=True + ).detector + text = "mentioning <|DSML|tool" + + streamed = detector.parse_streaming_increment(text) + flushed = detector.finish() + + self.assertEqual( + (streamed.reasoning_text or "") + (flushed.reasoning_text or ""), text + ) + class TestInklingDetector(CustomTestCase): def test_streaming_routes_blocks_across_all_string_boundaries(self): @@ -1143,25 +1199,31 @@ class TestStreamingChunkSizeInvariance(CustomTestCase): (one_shot.reasoning_text, one_shot.normal_text), ("leadr", "tail") ) - def test_dsv4_reasoning_quoting_dsml_is_chunk_dependent(self): - """Accepted divergence: streaming ends the block at the DSML marker, while - one-shot waits to see whether a `` follows. Reachable because the - DSV4 system prompt shows that marker to the model.""" + def test_dsv4_reasoning_quoting_dsml_is_chunk_invariant(self): + """A mid-sentence mention of the DSML tag stays in reasoning at every + chunk width and in one-shot parsing. Reachable because the DSV4 system + prompt shows that marker to the model.""" text = f"format is <{self.DSML}tool_calls>answer" - by_output = {} - for chunk_size in self.CHUNK_SIZES: - by_output.setdefault( - self._feed(DeepSeekV4Detector(), text, chunk_size), [] - ).append(chunk_size) + self._assert_invariant( + DeepSeekV4Detector, text, (f"format is <{self.DSML}tool_calls>", "answer") + ) - self.assertEqual(len(by_output), 2, f"expected two variants, got {by_output}") - early_cut = ("format is ", f"<{self.DSML}tool_calls>answer") - whole_buffer = (f"format is <{self.DSML}tool_calls>", "answer") - self.assertIn(early_cut, by_output) - self.assertIn(whole_buffer, by_output) - - one_shot = DeepSeekV4Detector().detect_and_parse(text) - self.assertEqual((one_shot.reasoning_text, one_shot.normal_text), whole_buffer) + def test_dsv4_reasoning_mentioning_dsml_then_real_block(self): + """Mention inside reasoning, then a real block after : the + mention stays in reasoning and the block reaches normal_text intact.""" + tool_call = ( + f"<{self.DSML}tool_calls>\n" + f'<{self.DSML}invoke name="s">\n' + f"" + ) + text = ( + f"I'll use a <{self.DSML}tool_calls> block.\n\n{tool_call}" + ) + self._assert_invariant( + DeepSeekV4Detector, + text, + (f"I'll use a <{self.DSML}tool_calls> block.", f"\n\n{tool_call}"), + ) def test_dsv4_tool_block_after_think_end(self): tool_call = ( @@ -1176,8 +1238,9 @@ class TestStreamingChunkSizeInvariance(CustomTestCase): ) def test_dsv4_tool_block_without_think_end(self): - """DSML directly after reasoning must still be routed to normal_text so - the tool call detector can see it.""" + """A DSML block starting a line directly after reasoning must still be + routed to normal_text so the tool call detector can see it, including + when the preceding newline streamed out in an earlier chunk.""" tool_call = ( f"<{self.DSML}tool_calls>" f'<{self.DSML}invoke name="s">' @@ -1188,12 +1251,22 @@ class TestStreamingChunkSizeInvariance(CustomTestCase): self.assertEqual( self._feed( DeepSeekV4Detector(), - f"my reasoning{tool_call}", + f"my reasoning\n\n{tool_call}", chunk_size, ), - ("my reasoning", tool_call), + ("my reasoning\n\n", tool_call), ) + def test_dsv4_tool_block_mid_line_without_think_end_stays_in_reasoning(self): + """Accepted tradeoff: a block that neither follows nor starts a + line is indistinguishable from a mention, so it is left in reasoning.""" + tool_call = f"<{self.DSML}tool_calls>" + self._assert_invariant( + DeepSeekV4Detector, + f"my reasoning {tool_call}", + (f"my reasoning {tool_call}", ""), + ) + class TestGptOssDetector(CustomTestCase): """Test cases for GptOssDetector which delegates to HarmonyParser."""