minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix tool call constrained decoding and parsing for models with native formats (#21593)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-04-10 20:37:23 -07:00
committed by GitHub
parent c2821dfbe9
commit 7c6db40540
9 changed files with 306 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/constrained/xgrammar_backend.py
+4 -1
View File
@@ -23,6 +23,7 @@ from xgrammar import (
CompiledGrammar,
GrammarCompiler,
GrammarMatcher,
StructuralTag,
StructuralTagItem,
TokenizerInfo,
allocate_token_bitmask,
@@ -295,9 +296,11 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
)
for structure in structural_tag["structures"]
]
ctx = self.grammar_compiler.compile_structural_tag(
new_tag = StructuralTag.from_legacy_structural_tag(
tags, structural_tag["triggers"]
)
new_tag.format.at_least_one = structural_tag.get("at_least_one", False)
ctx = self.grammar_compiler.compile_structural_tag(new_tag)
else:
format_dict = structural_tag.get("format")
if isinstance(format_dict, dict):
python/sglang/srt/entrypoints/openai/protocol.py
+1
View File
@@ -166,6 +166,7 @@ class LegacyStructuralTagResponseFormat(BaseModel):
type: Literal["structural_tag"]
structures: List[StructuresResponseFormat]
triggers: List[str]
at_least_one: bool = False
StructuralTagResponseFormat: TypeAlias = Union[
python/sglang/srt/entrypoints/openai/serving_chat.py
+67 -43
View File
@@ -361,9 +361,11 @@ class OpenAIServingChat(OpenAIServingBase):
request.tool_choice,
parallel_tool_calls=request.parallel_tool_calls,
)
# Handle JSON schema constraint directly for required or named tool choice
if request.tool_choice == "required" or isinstance(
request.tool_choice, ToolChoice
# Fallback: use generic JSON schema for required/named tool choice
# only when no parser-specific constraint was set
if tool_call_constraint is None and (
request.tool_choice == "required"
or isinstance(request.tool_choice, ToolChoice)
):
json_schema = get_json_schema_constraint(
request.tools,
@@ -1136,22 +1138,56 @@ class OpenAIServingChat(OpenAIServingBase):
) -> ToolCallProcessingResult:
"""Process tool calls in the response"""
# Handle required or named tool choice
if tool_choice == "required" or (
isinstance(tool_choice, ToolChoice) and tool_choice.type == "function"
):
# Set finish reason to tool_calls since we're processing tool calls
is_required = tool_choice == "required" or isinstance(tool_choice, ToolChoice)
# Try model-specific parser when output is in native format.
# For required/named: only use parser when structural_tag was used
# as constraint (mirrors the streaming path). For auto: always try.
if self.tool_call_parser:
parser = FunctionCallParser(tools, self.tool_call_parser)
should_try_parser = (
not is_required or parser.detector.supports_structural_tag()
)
if should_try_parser and parser.has_tool_call(text):
original_finish_type = finish_reason["type"]
if finish_reason["type"] == "stop":
finish_reason["type"] = "tool_calls"
finish_reason["matched"] = None
try:
text, call_info_list = parser.parse_non_stream(text)
tool_calls = []
for call_info in call_info_list:
tool_id = self._process_tool_call_id(
call_info, history_tool_calls_cnt
)
tool_calls.append(
ToolCall(
id=tool_id,
index=getattr(call_info, "tool_index", None),
function=FunctionResponse(
name=call_info.name,
arguments=call_info.parameters,
),
)
)
return ToolCallProcessingResult(tool_calls, text, finish_reason)
except Exception as e:
logger.error(f"Tool call parsing error: {e}")
finish_reason["type"] = original_finish_type
return ToolCallProcessingResult(None, text, finish_reason)
# json_schema constraint → JSON array output for required/named
if is_required:
original_finish_type = finish_reason["type"]
if finish_reason["type"] == "stop":
finish_reason["type"] = "tool_calls"
finish_reason["matched"] = None
try:
# For required tool choice, we expect a JSON array of tool calls
tool_call_data = orjson.loads(text)
tool_calls = []
for i, tool in enumerate(tool_call_data):
# Create a ToolCallItem from the JSON data
call_info = ToolCallItem(
tool_index=i, # Use the loop index as tool_index
tool_index=i,
name=tool["name"],
parameters=json.dumps(tool["parameters"], ensure_ascii=False),
)
@@ -1171,36 +1207,9 @@ class OpenAIServingChat(OpenAIServingBase):
)
)
return ToolCallProcessingResult(tool_calls, "", finish_reason)
except json.JSONDecodeError as e:
logger.error(f"Tool call parsing error: {e}")
return ToolCallProcessingResult(None, text, finish_reason)
# Use parser since output is not constrained by JSON schema
parser = FunctionCallParser(tools, self.tool_call_parser)
if parser.has_tool_call(text):
if finish_reason["type"] == "stop":
finish_reason["type"] = "tool_calls"
finish_reason["matched"] = None
try:
text, call_info_list = parser.parse_non_stream(text)
tool_calls = []
for call_info in call_info_list:
tool_id = self._process_tool_call_id(
call_info, history_tool_calls_cnt
)
tool_calls.append(
ToolCall(
id=tool_id,
index=getattr(call_info, "tool_index", None),
function=FunctionResponse(
name=call_info.name, arguments=call_info.parameters
),
)
)
return ToolCallProcessingResult(tool_calls, text, finish_reason)
except Exception as e:
logger.error(f"Tool call parsing error: {e}")
# Return error but don't fail the whole request
finish_reason["type"] = original_finish_type
return ToolCallProcessingResult(None, text, finish_reason)
return ToolCallProcessingResult(None, text, finish_reason)
@@ -1341,11 +1350,26 @@ class OpenAIServingChat(OpenAIServingBase):
):
"""Process tool calls in streaming response"""
if index not in parser_dict:
# Use JSON detector directly for required or named tool choice
if request.tool_choice == "required" or isinstance(
is_required = request.tool_choice == "required" or isinstance(
request.tool_choice, ToolChoice
):
parser_dict[index] = JsonArrayParser()
)
# For required/named tool choice: use JsonArrayParser when the
# constrained output is plain JSON (detector doesn't support
# structural_tag or no parser configured). Use FunctionCallParser
# only when the detector supports structural_tag and will produce
# native format output.
if is_required:
use_native_parser = False
if self.tool_call_parser:
probe = FunctionCallParser(
tools=request.tools,
tool_call_parser=self.tool_call_parser,
)
use_native_parser = probe.detector.supports_structural_tag()
if use_native_parser:
parser_dict[index] = probe
else:
parser_dict[index] = JsonArrayParser()
else:
parser_dict[index] = FunctionCallParser(
tools=request.tools,
python/sglang/srt/function_call/base_format_detector.py
+1 -1
View File
@@ -270,7 +270,7 @@ class BaseFormatDetector(ABC):
cur_arguments = current_tool_call.get("arguments")
res = StreamingParseResult()
if cur_arguments:
if cur_arguments is not None:
# Calculate how much of the arguments we've already streamed
sent = len(self.streamed_args_for_tool[self.current_tool_id])
cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
python/sglang/srt/function_call/deepseekv3_detector.py
+5 -3
View File
@@ -203,7 +203,9 @@ class DeepSeekV3Detector(BaseFormatDetector):
def structure_info(self) -> _GetInfoFunc:
return lambda name: StructureInfo(
begin=">" + name + "\n```json\n",
end="\n```<",
trigger=">" + name + "\n```json\n",
begin="<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
+ name
+ "\n```json\n",
end="\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>",
trigger="<｜tool▁calls▁begin｜>",
)
python/sglang/srt/function_call/function_call_parser.py
+35 -11
View File
@@ -3,6 +3,7 @@ from typing import Dict, List, Literal, Optional, Set, Tuple, Type, Union
from sglang.srt.entrypoints.openai.protocol import (
LegacyStructuralTagResponseFormat,
StructuralTagResponseFormat,
StructuresResponseFormat,
Tool,
ToolCallConstraint,
@@ -32,7 +33,10 @@ from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector
from sglang.srt.function_call.qwen25_detector import Qwen25Detector
from sglang.srt.function_call.step3_detector import Step3Detector
from sglang.srt.function_call.trinity_detector import TrinityDetector
from sglang.srt.function_call.utils import get_json_schema_constraint
from sglang.srt.function_call.utils import (
_get_tool_schema_defs,
get_json_schema_constraint,
)
logger = logging.getLogger(__name__)
@@ -146,12 +150,24 @@ class FunctionCallParser:
return final_normal_text, final_calls
def get_structure_tag(self) -> LegacyStructuralTagResponseFormat:
def get_structure_tag(
self, at_least_one: bool = False
) -> StructuralTagResponseFormat:
"""
Generate a structural tag response format for all available tools.
This creates the necessary structural tags that guide the model's output format.
Args:
at_least_one: If True, the grammar forces at least one tool call
(no free text allowed). Used for required/named tool_choice.
Raises:
ValueError: If tools have conflicting $defs schemas.
"""
# Validate $defs consistency before building structural tags
_get_tool_schema_defs(self.tools)
tool_structures: List[StructuresResponseFormat] = list()
tool_trigger_set: Set[str] = set()
@@ -183,6 +199,7 @@ class FunctionCallParser:
type="structural_tag",
structures=tool_structures,
triggers=list(tool_trigger_set),
at_least_one=at_least_one,
)
def get_structure_constraint(
@@ -203,16 +220,23 @@ class FunctionCallParser:
"""
# NOTE: structural_tag only supports JSON-compatible content between the begin and end.
# It cannot parse or validate function call Pythonic or XML-ish syntax.
if (
self.detector.supports_structural_tag()
and tool_choice == "auto"
and (
any(tool.function.strict for tool in self.tools)
or self.tool_strict_level >= ToolStrictLevel.FUNCTION
if self.detector.supports_structural_tag():
# For "required"/named: always use structural_tag to preserve the
# model's native tool call format. Schema is only included when
# strict=True, per OpenAI protocol semantics.
# For "auto": only constrain when strict is enabled.
is_required = tool_choice == "required" or isinstance(
tool_choice, ToolChoice
)
):
tag = self.get_structure_tag()
return ("structural_tag", tag)
if is_required or (
tool_choice == "auto"
and (
any(tool.function.strict for tool in self.tools)
or self.tool_strict_level >= ToolStrictLevel.FUNCTION
)
):
tag = self.get_structure_tag(at_least_one=is_required)
return ("structural_tag", tag)
elif tool_choice == "required" or isinstance(tool_choice, ToolChoice):
json_schema = get_json_schema_constraint(
self.tools, tool_choice, parallel_tool_calls=parallel_tool_calls