minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Upgrade xgrammar to 0.2.1 (#25676)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-05-29 11:40:07 +08:00
committed by GitHub
parent 272066566f
commit 79c844527c
13 changed files with 407 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/pyproject.toml
+1 -1
View File
@@ -80,7 +80,7 @@ dependencies = [
"uvicorn",
"uvloop",
"watchfiles",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
"kernels",
]
python/pyproject_cpu.toml
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"triton==3.5.0",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject_npu.toml
+1 -1
View File
@@ -63,7 +63,7 @@ dependencies = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject_other.toml
+1 -1
View File
@@ -62,7 +62,7 @@ runtime_common = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject_xpu.toml
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
# "xgrammar==0.2.0", xgrammar depends on CUDA PyTorch and Triton only
# "xgrammar==0.2.1", xgrammar depends on CUDA PyTorch and Triton only
"smg-grpc-servicer>=0.5.0",
]
python/sglang/srt/entrypoints/openai/serving_chat.py
+4 -2
View File
@@ -505,6 +505,8 @@ class OpenAIServingChat(OpenAIServingBase):
img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch(
request
)
require_reasoning = self._get_reasoning_from_request(request)
adapted_request = GenerateReqInput(
**prompt_kwargs,
image_data=processed_messages.image_data,
@@ -528,7 +530,7 @@ class OpenAIServingChat(OpenAIServingBase):
routed_experts_start_len=request.routed_experts_start_len,
rid=request.rid,
extra_key=self._compute_extra_key(request),
require_reasoning=self._get_reasoning_from_request(request),
require_reasoning=require_reasoning,
priority=request.priority,
routing_key=self.extract_routing_key(raw_request),
custom_labels=custom_labels,
@@ -556,7 +558,7 @@ class OpenAIServingChat(OpenAIServingBase):
# when --reasoning-parser is configured, so builtin xgrammar
# tags must describe only the post-reasoning tool-call suffix.
xgrammar_reasoning = thinking_mode and (
self.tokenizer_manager.server_args.reasoning_parser is not None
self.tokenizer_manager.server_args.reasoning_parser is None
)
tool_call_constraint = None
python/sglang/srt/function_call/deepseekv32_detector.py
+3 -115
View File
@@ -1,11 +1,10 @@
import json
import logging
import re
from typing import List, Literal, Optional, Union
from partial_json_parser.core.options import Allow
from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
from sglang.srt.entrypoints.openai.protocol import Tool
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
from sglang.srt.function_call.core_types import (
StreamingParseResult,
@@ -15,30 +14,8 @@ from sglang.srt.function_call.core_types import (
)
from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads
try:
from xgrammar import StructuralTag
from xgrammar.structural_tag import (
AnyTextFormat,
ConstStringFormat,
JSONSchemaFormat,
SequenceFormat,
TagFormat,
TagsWithSeparatorFormat,
TriggeredTagsFormat,
)
except ImportError:
StructuralTag = None # type: ignore
logger = logging.getLogger(__name__)
# Names mirror the DeepSeek-V3.2 official chat template tokens
# (see encoding_dsv32.TOOLS_SYSTEM_TEMPLATE).
_INVOKE_BEGIN_PREFIX = '<｜DSML｜invoke name="'
_INVOKE_BEGIN_SUFFIX = '">\n'
_THINK_TAG_END = "</think>"
_THINK_EXCLUDE_TOKENS = ["<think>", "</think>"]
_XML_STYLE = "deepseek_xml"
class DeepSeekV32Detector(BaseFormatDetector):
"""
@@ -391,94 +368,5 @@ class DeepSeekV32Detector(BaseFormatDetector):
trigger="<｜DSML｜invoke",
)
def get_structural_tag(
self,
tools: Union[List[Tool], None] = None,
tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
thinking_mode: bool = False,
) -> Optional["StructuralTag"]:
"""
Build an xgrammar StructuralTag locally for DeepSeek-V3.2.
Both layers — the outer `<｜DSML｜function_calls｜>...` wrapper and
the inner `<｜DSML｜invoke>...</｜DSML｜invoke>` blocks — are encoded
directly in the grammar with a single-newline join between
consecutive invokes, matching DeepSeek-V3.2's official chat
template. This avoids two layered defects that surfaced with the
prior `xgrammar.get_model_structural_tag("deepseek_v3_2")` path:
- the xgrammar builtin template (pre mlc-ai/xgrammar#638) forced
a double-newline join, which deterministically collapsed
parallel tool calls to one at greedy decoding.
- falling back to the legacy structural tag (built from
`structure_info()`) only constrains the inner invoke block;
the outer wrapper is off-grammar and the model can skip it
under `at_least_one=True`, leaving `detect_and_parse` with no
`<｜DSML｜function_calls>` marker to anchor on.
Returning a fully-formed StructuralTag from the detector keeps
both fixes local to sglang and decoupled from the xgrammar
release cadence.
"""
if not tools or StructuralTag is None:
return None
# `INVOKE_END` and the empty separator together yield a single `\n`
# between consecutive invokes — matching DeepSeek-V3.2's chat template
# `"\n".join(invoke_blocks)`.
function_calls_begin = self.bot_token + "\n"
invoke_end = self.invoke_end_token + "\n"
def _invoke_tag(tool: Tool) -> TagFormat:
return TagFormat(
begin=_INVOKE_BEGIN_PREFIX + tool.function.name + _INVOKE_BEGIN_SUFFIX,
content=JSONSchemaFormat(
json_schema=tool.function.parameters or {},
style=_XML_STYLE,
),
end=invoke_end,
)
if isinstance(tool_choice, ToolChoice):
target = next(
(t for t in tools if t.function.name == tool_choice.function.name),
None,
)
if target is None:
return None
invoke_tags = [_invoke_tag(target)]
is_required = True
else:
invoke_tags = [_invoke_tag(t) for t in tools]
is_required = tool_choice == "required"
inner_tool_calls = TagsWithSeparatorFormat(
tags=invoke_tags, separator="", at_least_one=True
)
if is_required:
suffix_tag = SequenceFormat(
elements=[
ConstStringFormat(value=function_calls_begin),
inner_tool_calls,
ConstStringFormat(value=self.eot_token),
]
)
else:
suffix_tag = TriggeredTagsFormat(
triggers=[self.bot_token],
tags=[
TagFormat(
begin=function_calls_begin,
content=inner_tool_calls,
end=self.eot_token,
)
],
excludes=_THINK_EXCLUDE_TOKENS,
)
if not thinking_mode:
return StructuralTag(format=suffix_tag)
prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=_THINK_TAG_END)
return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
def get_structural_tag_name(self) -> str:
return "deepseek_v3_2"
python/sglang/srt/function_call/kimik2_detector.py
+51 -12
View File
@@ -1,10 +1,14 @@
import json
import logging
import re
from typing import List
from typing import List, Literal, Optional, Union
from sglang.srt.entrypoints.openai.protocol import Tool
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
from sglang.srt.function_call.base_format_detector import (
BaseFormatDetector,
StructuralTag,
get_model_structural_tag,
)
from sglang.srt.function_call.core_types import (
StreamingParseResult,
StructureInfo,
@@ -23,6 +27,8 @@ _KIMI_K2_SPECIAL_TOKENS = [
"<|tool_call_argument_begin|>",
]
_KIMI_NON_STRICT_ARGUMENTS_SCHEMA = {"type": "object"}
def _strip_special_tokens(text: str) -> str:
"""Remove all Kimi-K2 tool-call special tokens from text."""
@@ -333,12 +339,45 @@ class KimiK2Detector(BaseFormatDetector):
return get_info
# Kimi stays on the SGLang legacy structural tag path. xgrammar 0.2.0's
# get_kimi_structural_tag(tool_choice="auto") emits a bare
# <|tool_call_begin|>...<|tool_call_end|> grammar without the
# <|tool_calls_section_begin|>/<|tool_calls_section_end|> wrapper Kimi's
# chat template uses, and KimiK2Detector.has_tool_call() keys off the
# section marker — bare tool calls would be silently dropped. Inheriting
# the base get_structural_tag_name (returns None) keeps FunctionCallParser
# on the legacy path, whose structure_info bakes the section markers in.
# TODO: re-enable the builtin once https://github.com/mlc-ai/xgrammar/issues/622 is fixed.
def get_structural_tag(
self,
tools: Union[List[Tool], None] = None,
tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
thinking_mode: bool = False,
) -> Optional[StructuralTag]:
if not (
tools and (tool_choice == "required" or isinstance(tool_choice, ToolChoice))
):
return super().get_structural_tag(
tools=tools, tool_choice=tool_choice, thinking_mode=thinking_mode
)
if get_model_structural_tag is None:
return None
converted_tools = []
for tool in tools:
converted_tool = tool.model_dump()
function = converted_tool["function"]
if not function.get("strict", False):
# Kimi's parser accepts only object-shaped tool arguments. XGrammar
# treats strict=False arguments as unconstrained JSON, which can
# generate strings/arrays/numbers that Kimi cannot parse. Keep
# non-strict semantics loose by constraining only the outer type.
function["strict"] = True
function["parameters"] = _KIMI_NON_STRICT_ARGUMENTS_SCHEMA
converted_tools.append(converted_tool)
converted_tool_choice = (
tool_choice.model_dump()
if isinstance(tool_choice, ToolChoice)
else tool_choice
)
return get_model_structural_tag(
model="kimi",
tools=converted_tools,
tool_choice=converted_tool_choice,
reasoning=thinking_mode,
)
def get_structural_tag_name(self) -> str:
return "kimi"