feat: use structural tags to enable strict tool calling and reasoning for more models (#21722)

Signed-off-by: Yuchuan <yuchuan.7streams@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: Ubospica <ubospica@gmail.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>

python/pyproject.toml

@@ -76,7 +76,7 @@ dependencies = [
   "uvicorn",
   "uvloop",
   "watchfiles",
-  "xgrammar==0.1.32",
+  "xgrammar==0.2.0",
   "smg-grpc-servicer>=0.5.0",
   "kernels",
 ]

python/pyproject_cpu.toml

@@ -66,7 +66,7 @@ dependencies = [
   "triton==3.5.0",
   "uvicorn",
   "uvloop",
-  "xgrammar==0.1.32",
+  "xgrammar==0.2.0",
   "smg-grpc-servicer>=0.5.0",
 ]
 

python/pyproject_npu.toml

@@ -62,7 +62,7 @@ dependencies = [
   "transformers==5.6.0",
   "uvicorn",
   "uvloop",
-  "xgrammar==0.1.32",
+  "xgrammar==0.2.0",
   "smg-grpc-servicer>=0.5.0",
 ]
 

python/pyproject_other.toml

@@ -62,7 +62,7 @@ runtime_common = [
   "transformers==5.6.0",
   "uvicorn",
   "uvloop",
-  "xgrammar==0.1.32",
+  "xgrammar==0.2.0",
   "smg-grpc-servicer>=0.5.0",
 ]
 

python/pyproject_xpu.toml

@@ -66,6 +66,7 @@ dependencies = [
   "transformers==5.6.0",
   "uvicorn",
   "uvloop",
+  # "xgrammar==0.2.0", xgrammar depends on CUDA PyTorch and Triton only
   "smg-grpc-servicer>=0.5.0",
 ]
 

python/sglang/srt/entrypoints/openai/serving_chat.py

@@ -434,6 +434,13 @@ class OpenAIServingChat(OpenAIServingBase):
 
         self._patch_mistral_skip_special_tokens(request)
 
+        thinking_mode = self._get_reasoning_from_request(request)
+        # SGLang's ReasonerGrammarBackend owns the reasoning prefix
+        # when --reasoning-parser is configured, so builtin xgrammar
+        # tags must describe only the post-reasoning tool-call suffix.
+        xgrammar_reasoning = thinking_mode and (
+            self.tokenizer_manager.server_args.reasoning_parser is not None
+        )
         tool_call_constraint = None
 
         # Apply chat template and its stop strings
@@ -453,6 +460,7 @@ class OpenAIServingChat(OpenAIServingBase):
                 tool_call_constraint = parser.get_structure_constraint(
                     request.tool_choice,
                     parallel_tool_calls=request.parallel_tool_calls,
+                    thinking_mode=xgrammar_reasoning,
                 )
             # Fallback: use generic JSON schema for required/named tool choice
             # only when no parser-specific constraint was set

python/sglang/srt/function_call/base_format_detector.py

@@ -1,13 +1,19 @@
 import json
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Literal, Optional, Union
 
 import orjson
 from partial_json_parser.core.exceptions import MalformedJSON
 from partial_json_parser.core.options import Allow
 
-from sglang.srt.entrypoints.openai.protocol import Tool
+try:
+    from xgrammar import StructuralTag, get_model_structural_tag
+except ImportError:
+    StructuralTag = Any
+    get_model_structural_tag = None
+
+from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
 from sglang.srt.environ import envs
 from sglang.srt.function_call.core_types import (
     StreamingParseResult,
@@ -361,3 +367,45 @@ class BaseFormatDetector(ABC):
             A function that takes a tool name (str) and returns StructureInfo
         """
         raise NotImplementedError()
+
+    def get_structural_tag_name(self) -> Optional[str]:
+        """Return the XGrammar model name for native structural tags, if supported."""
+        return None
+
+    def get_structural_tag(
+        self,
+        tools: Union[List[Tool], None] = None,
+        tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
+        thinking_mode: bool = False,
+    ) -> Optional[StructuralTag]:
+        """
+        Return a model-native XGrammar structural tag when supported.
+
+        Args:
+            tools: List of available tools
+            tool_choice: The tool choice setting from the request
+            thinking_mode: Whether to include the model's reasoning prefix in
+                the returned structural tag. Pass False when SGLang's
+                ReasonerGrammarBackend will own the <think>...</think> prefix
+                (the typical case when --reasoning-parser is configured) so
+                only one layer constrains the reasoning section.
+
+        Returns:
+            StructuralTag if this detector supports model-native tags, otherwise None
+        """
+        structural_tag_name = self.get_structural_tag_name()
+        if not structural_tag_name or get_model_structural_tag is None:
+            return None
+
+        converted_tools = [tool.model_dump() for tool in tools or []]
+        converted_tool_choice = (
+            tool_choice.model_dump()
+            if isinstance(tool_choice, ToolChoice)
+            else tool_choice
+        )
+        return get_model_structural_tag(
+            model=structural_tag_name,
+            tools=converted_tools,
+            tool_choice=converted_tool_choice,
+            reasoning=thinking_mode,
+        )

python/sglang/srt/function_call/deepseekv32_detector.py

@@ -351,3 +351,6 @@ class DeepSeekV32Detector(BaseFormatDetector):
             end="</｜DSML｜invoke>",
             trigger="<｜DSML｜invoke",
         )
+
+    def get_structural_tag_name(self) -> str:
+        return "deepseek_v3_2"

python/sglang/srt/function_call/deepseekv4_detector.py

@@ -0,0 +1,67 @@
+import logging
+
+from sglang.srt.function_call.deepseekv32_detector import DeepSeekV32Detector
+
+logger = logging.getLogger(__name__)
+
+
+class DeepSeekV4Detector(DeepSeekV32Detector):
+    """
+    Detector for DeepSeek V4 model function call format.
+
+    The DeepSeek V4 format uses XML-like DSML tags to delimit function calls.
+    Supports two parameter formats:
+
+    Format 1 - XML Parameter Tags:
+    ```
+    <｜DSML｜tool_calls>
+        <｜DSML｜invoke name="function_name">
+        <｜DSML｜parameter name="param_name" string="true">value</｜DSML｜parameter>
+        ...
+    </｜DSML｜invoke>
+    </｜DSML｜tool_calls>
+    ```
+
+    Format 2 - Direct JSON:
+    ```
+    <｜DSML｜tool_calls>
+        <｜DSML｜invoke name="function_name">
+        {
+            "param_name": "value"
+        }
+    </｜DSML｜invoke>
+    </｜DSML｜tool_calls>
+    ```
+
+    Examples:
+    ```
+    <｜DSML｜tool_calls>
+        <｜DSML｜invoke name="get_favorite_tourist_spot">
+        <｜DSML｜parameter name="city" string="true">San Francisco</｜DSML｜parameter>
+    </｜DSML｜invoke>
+    </｜DSML｜tool_calls>
+
+    <｜DSML｜tool_calls>
+        <｜DSML｜invoke name="get_favorite_tourist_spot">
+        { "city": "San Francisco" }
+    </｜DSML｜invoke>
+    </｜DSML｜tool_calls>
+    ```
+
+    Key Components:
+    - Tool Calls Section: Wrapped between `<｜DSML｜tool_calls>` and `</｜DSML｜tool_calls>`
+    - Individual Tool Call: Wrapped between `<｜DSML｜invoke name="...">` and `</｜DSML｜invoke>`
+    - Parameters: Either XML tags or direct JSON format
+    - Supports multiple tool calls
+
+    Reference: DeepSeek V4 format specification
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.bot_token = "<｜DSML｜tool_calls>"
+        self.eot_token = "</｜DSML｜tool_calls>"
+        self.function_calls_regex = r"<｜DSML｜tool_calls>(.*?)</｜DSML｜tool_calls>"
+
+    def get_structural_tag_name(self) -> str:
+        return "deepseek_v4"

python/sglang/srt/function_call/function_call_parser.py

@@ -152,7 +152,7 @@ class FunctionCallParser:
 
         return final_normal_text, final_calls
 
-    def get_structure_tag(
+    def get_legacy_structural_tag(
         self, at_least_one: bool = False
     ) -> StructuralTagResponseFormat:
         """
@@ -208,6 +208,7 @@ class FunctionCallParser:
         self,
         tool_choice: Union[ToolChoice, Literal["auto", "required"]],
         parallel_tool_calls: bool = True,
+        thinking_mode: bool = False,
     ) -> Optional[ToolCallConstraint]:
         """
         Returns the appropriate structure constraint for tool calls based on the tool_choice.
@@ -220,27 +221,37 @@ class FunctionCallParser:
             A tuple of (constraint_type, constraint_value) to be added to sampling parameters,
             or None if no constraint applies.
         """
-        # NOTE: structural_tag only supports JSON-compatible content between the begin and end.
-        # It cannot parse or validate function call Pythonic or XML-ish syntax.
-        if self.detector.supports_structural_tag():
-            # For "required"/named: always use structural_tag to preserve the
-            # model's native tool call format. Schema is only included when
-            # strict=True, per OpenAI protocol semantics.
-            # For "auto": only constrain when strict is enabled.
-            is_required = tool_choice == "required" or isinstance(
-                tool_choice, ToolChoice
-            )
-            if is_required or (
-                tool_choice == "auto"
-                and (
-                    any(tool.function.strict for tool in self.tools)
-                    or self.tool_strict_level >= ToolStrictLevel.FUNCTION
+        is_required = tool_choice == "required" or isinstance(tool_choice, ToolChoice)
+        should_constrain_auto = tool_choice == "auto" and (
+            any(tool.function.strict for tool in self.tools)
+            or self.tool_strict_level >= ToolStrictLevel.FUNCTION
+        )
+
+        # Highest priority: model-native structural_tag when available.
+        try:
+            if is_required or should_constrain_auto:
+                structural_tag = self.detector.get_structural_tag(
+                    tools=self.tools,
+                    thinking_mode=thinking_mode,
+                    tool_choice=tool_choice,
                 )
-            ):
-                tag = self.get_structure_tag(at_least_one=is_required)
-                return ("structural_tag", tag)
-        elif tool_choice == "required" or isinstance(tool_choice, ToolChoice):
-            json_schema = get_json_schema_constraint(
-                self.tools, tool_choice, parallel_tool_calls=parallel_tool_calls
-            )
-            return ("json_schema", json_schema)
+                if structural_tag is not None:
+                    return ("structural_tag", structural_tag)
+
+                # Fallback to legacy structural tag if model-native tag is not supported.
+                if self.detector.supports_structural_tag():
+                    # For "required"/named: always use structural_tag to preserve the
+                    # model's native tool call format. Schema is only included when
+                    # strict=True, per OpenAI protocol semantics.
+                    # For "auto": only constrain when strict is enabled.
+                    tag = self.get_legacy_structural_tag(at_least_one=is_required)
+                    return ("structural_tag", tag)
+
+            if tool_choice == "required" or isinstance(tool_choice, ToolChoice):
+                json_schema = get_json_schema_constraint(
+                    self.tools, tool_choice, parallel_tool_calls=parallel_tool_calls
+                )
+                return ("json_schema", json_schema)
+        except Exception as e:
+            logger.error(f"Error getting structure constraint: {e}")
+            return None

python/sglang/srt/function_call/gpt_oss_detector.py

@@ -239,3 +239,6 @@ class GptOssDetector(BaseFormatDetector):
 
     def structure_info(self) -> _GetInfoFunc:
         raise NotImplementedError("structure_info not used with HarmonyParser")
+
+    def get_structural_tag_name(self) -> str:
+        return "harmony"

python/sglang/srt/function_call/kimik2_detector.py

@@ -253,3 +253,13 @@ class KimiK2Detector(BaseFormatDetector):
             )
 
         return get_info
+
+    # Kimi stays on the SGLang legacy structural tag path. xgrammar 0.2.0's
+    # get_kimi_structural_tag(tool_choice="auto") emits a bare
+    # <|tool_call_begin|>...<|tool_call_end|> grammar without the
+    # <|tool_calls_section_begin|>/<|tool_calls_section_end|> wrapper Kimi's
+    # chat template uses, and KimiK2Detector.has_tool_call() keys off the
+    # section marker — bare tool calls would be silently dropped. Inheriting
+    # the base get_structural_tag_name (returns None) keeps FunctionCallParser
+    # on the legacy path, whose structure_info bakes the section markers in.
+    # TODO: re-enable the builtin once https://github.com/mlc-ai/xgrammar/issues/622 is fixed.

python/sglang/srt/function_call/qwen3_coder_detector.py

@@ -468,7 +468,10 @@ class Qwen3CoderDetector(BaseFormatDetector):
         return StreamingParseResult(calls=calls, normal_text=normal_text)
 
     def supports_structural_tag(self) -> bool:
-        return False
+        return True
 
     def structure_info(self) -> _GetInfoFunc:
         raise NotImplementedError
+
+    def get_structural_tag_name(self) -> str:
+        return "qwen_3_coder"

python/sglang/srt/speculative/spec_utils.py

@@ -612,14 +612,14 @@ def traverse_tree(
         if accepted:
             if curr != 0:
                 # Accept the current token
-                grammar.accept_token(draft_tokens[curr])
+                grammar.accept_token(int(draft_tokens[curr]))
             if not grammar.is_terminated():
                 # Generate the bitmask for the current token
                 grammar.fill_vocab_mask(allocate_token_bitmask, curr)
                 if retrieve_next_token[curr] != -1:
                     # Visit the child node
                     dfs(
-                        retrieve_next_token[curr],
+                        int(retrieve_next_token[curr]),
                         retrieve_next_token,
                         retrieve_next_sibling,
                         curr,
@@ -632,7 +632,7 @@ def traverse_tree(
         if retrieve_next_sibling[curr] != -1:
             # Visit the sibling node
             dfs(
-                retrieve_next_sibling[curr],
+                int(retrieve_next_sibling[curr]),
                 retrieve_next_token,
                 retrieve_next_sibling,
                 parent_pos,