minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Upgrade xgrammar to 0.2.1 (#25676)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-05-29 11:40:07 +08:00
committed by GitHub
parent 272066566f
commit 79c844527c
13 changed files with 407 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/full-test-npu.yml
+4 -4
View File
@@ -135,7 +135,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -198,7 +198,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -261,7 +261,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -324,7 +324,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
.github/workflows/nightly-test-npu.yml
+5 -5
View File
@@ -131,7 +131,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -197,7 +197,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -264,7 +264,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -331,7 +331,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
@@ -398,7 +398,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 &
3rdparty/amd/wheel/sglang/pyproject.toml
Vendored
+1 -1
View File
@@ -62,7 +62,7 @@ runtime_common = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject.toml
+1 -1
View File
@@ -80,7 +80,7 @@ dependencies = [
"uvicorn",
"uvloop",
"watchfiles",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
"kernels",
]
python/pyproject_cpu.toml
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"triton==3.5.0",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject_npu.toml
+1 -1
View File
@@ -63,7 +63,7 @@ dependencies = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject_other.toml
+1 -1
View File
@@ -62,7 +62,7 @@ runtime_common = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
"xgrammar==0.2.0",
"xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0",
]
python/pyproject_xpu.toml
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"transformers==5.8.1",
"uvicorn",
"uvloop",
# "xgrammar==0.2.0", xgrammar depends on CUDA PyTorch and Triton only
# "xgrammar==0.2.1", xgrammar depends on CUDA PyTorch and Triton only
"smg-grpc-servicer>=0.5.0",
]
python/sglang/srt/entrypoints/openai/serving_chat.py
+4 -2
View File
@@ -505,6 +505,8 @@ class OpenAIServingChat(OpenAIServingBase):
img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch(
request
)
require_reasoning = self._get_reasoning_from_request(request)
adapted_request = GenerateReqInput(
**prompt_kwargs,
image_data=processed_messages.image_data,
@@ -528,7 +530,7 @@ class OpenAIServingChat(OpenAIServingBase):
routed_experts_start_len=request.routed_experts_start_len,
rid=request.rid,
extra_key=self._compute_extra_key(request),
require_reasoning=self._get_reasoning_from_request(request),
require_reasoning=require_reasoning,
priority=request.priority,
routing_key=self.extract_routing_key(raw_request),
custom_labels=custom_labels,
@@ -556,7 +558,7 @@ class OpenAIServingChat(OpenAIServingBase):
# when --reasoning-parser is configured, so builtin xgrammar
# tags must describe only the post-reasoning tool-call suffix.
xgrammar_reasoning = thinking_mode and (
self.tokenizer_manager.server_args.reasoning_parser is not None
self.tokenizer_manager.server_args.reasoning_parser is None
)
tool_call_constraint = None
python/sglang/srt/function_call/deepseekv32_detector.py
+3 -115
View File
@@ -1,11 +1,10 @@
import json
import logging
import re
from typing import List, Literal, Optional, Union
from partial_json_parser.core.options import Allow
from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
from sglang.srt.entrypoints.openai.protocol import Tool
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
from sglang.srt.function_call.core_types import (
StreamingParseResult,
@@ -15,30 +14,8 @@ from sglang.srt.function_call.core_types import (
)
from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads
try:
from xgrammar import StructuralTag
from xgrammar.structural_tag import (
AnyTextFormat,
ConstStringFormat,
JSONSchemaFormat,
SequenceFormat,
TagFormat,
TagsWithSeparatorFormat,
TriggeredTagsFormat,
)
except ImportError:
StructuralTag = None # type: ignore
logger = logging.getLogger(__name__)
# Names mirror the DeepSeek-V3.2 official chat template tokens
# (see encoding_dsv32.TOOLS_SYSTEM_TEMPLATE).
_INVOKE_BEGIN_PREFIX = '<｜DSML｜invoke name="'
_INVOKE_BEGIN_SUFFIX = '">\n'
_THINK_TAG_END = "</think>"
_THINK_EXCLUDE_TOKENS = ["<think>", "</think>"]
_XML_STYLE = "deepseek_xml"
class DeepSeekV32Detector(BaseFormatDetector):
"""
@@ -391,94 +368,5 @@ class DeepSeekV32Detector(BaseFormatDetector):
trigger="<｜DSML｜invoke",
)
def get_structural_tag(
self,
tools: Union[List[Tool], None] = None,
tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
thinking_mode: bool = False,
) -> Optional["StructuralTag"]:
"""
Build an xgrammar StructuralTag locally for DeepSeek-V3.2.
Both layers — the outer `<｜DSML｜function_calls｜>...` wrapper and
the inner `<｜DSML｜invoke>...</｜DSML｜invoke>` blocks — are encoded
directly in the grammar with a single-newline join between
consecutive invokes, matching DeepSeek-V3.2's official chat
template. This avoids two layered defects that surfaced with the
prior `xgrammar.get_model_structural_tag("deepseek_v3_2")` path:
- the xgrammar builtin template (pre mlc-ai/xgrammar#638) forced
a double-newline join, which deterministically collapsed
parallel tool calls to one at greedy decoding.
- falling back to the legacy structural tag (built from
`structure_info()`) only constrains the inner invoke block;
the outer wrapper is off-grammar and the model can skip it
under `at_least_one=True`, leaving `detect_and_parse` with no
`<｜DSML｜function_calls>` marker to anchor on.
Returning a fully-formed StructuralTag from the detector keeps
both fixes local to sglang and decoupled from the xgrammar
release cadence.
"""
if not tools or StructuralTag is None:
return None
# `INVOKE_END` and the empty separator together yield a single `\n`
# between consecutive invokes — matching DeepSeek-V3.2's chat template
# `"\n".join(invoke_blocks)`.
function_calls_begin = self.bot_token + "\n"
invoke_end = self.invoke_end_token + "\n"
def _invoke_tag(tool: Tool) -> TagFormat:
return TagFormat(
begin=_INVOKE_BEGIN_PREFIX + tool.function.name + _INVOKE_BEGIN_SUFFIX,
content=JSONSchemaFormat(
json_schema=tool.function.parameters or {},
style=_XML_STYLE,
),
end=invoke_end,
)
if isinstance(tool_choice, ToolChoice):
target = next(
(t for t in tools if t.function.name == tool_choice.function.name),
None,
)
if target is None:
return None
invoke_tags = [_invoke_tag(target)]
is_required = True
else:
invoke_tags = [_invoke_tag(t) for t in tools]
is_required = tool_choice == "required"
inner_tool_calls = TagsWithSeparatorFormat(
tags=invoke_tags, separator="", at_least_one=True
)
if is_required:
suffix_tag = SequenceFormat(
elements=[
ConstStringFormat(value=function_calls_begin),
inner_tool_calls,
ConstStringFormat(value=self.eot_token),
]
)
else:
suffix_tag = TriggeredTagsFormat(
triggers=[self.bot_token],
tags=[
TagFormat(
begin=function_calls_begin,
content=inner_tool_calls,
end=self.eot_token,
)
],
excludes=_THINK_EXCLUDE_TOKENS,
)
if not thinking_mode:
return StructuralTag(format=suffix_tag)
prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=_THINK_TAG_END)
return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
def get_structural_tag_name(self) -> str:
return "deepseek_v3_2"
python/sglang/srt/function_call/kimik2_detector.py
+51 -12
View File
@@ -1,10 +1,14 @@
import json
import logging
import re
from typing import List
from typing import List, Literal, Optional, Union
from sglang.srt.entrypoints.openai.protocol import Tool
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
from sglang.srt.function_call.base_format_detector import (
BaseFormatDetector,
StructuralTag,
get_model_structural_tag,
)
from sglang.srt.function_call.core_types import (
StreamingParseResult,
StructureInfo,
@@ -23,6 +27,8 @@ _KIMI_K2_SPECIAL_TOKENS = [
"<|tool_call_argument_begin|>",
]
_KIMI_NON_STRICT_ARGUMENTS_SCHEMA = {"type": "object"}
def _strip_special_tokens(text: str) -> str:
"""Remove all Kimi-K2 tool-call special tokens from text."""
@@ -333,12 +339,45 @@ class KimiK2Detector(BaseFormatDetector):
return get_info
# Kimi stays on the SGLang legacy structural tag path. xgrammar 0.2.0's
# get_kimi_structural_tag(tool_choice="auto") emits a bare
# <|tool_call_begin|>...<|tool_call_end|> grammar without the
# <|tool_calls_section_begin|>/<|tool_calls_section_end|> wrapper Kimi's
# chat template uses, and KimiK2Detector.has_tool_call() keys off the
# section marker — bare tool calls would be silently dropped. Inheriting
# the base get_structural_tag_name (returns None) keeps FunctionCallParser
# on the legacy path, whose structure_info bakes the section markers in.
# TODO: re-enable the builtin once https://github.com/mlc-ai/xgrammar/issues/622 is fixed.
def get_structural_tag(
self,
tools: Union[List[Tool], None] = None,
tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
thinking_mode: bool = False,
) -> Optional[StructuralTag]:
if not (
tools and (tool_choice == "required" or isinstance(tool_choice, ToolChoice))
):
return super().get_structural_tag(
tools=tools, tool_choice=tool_choice, thinking_mode=thinking_mode
)
if get_model_structural_tag is None:
return None
converted_tools = []
for tool in tools:
converted_tool = tool.model_dump()
function = converted_tool["function"]
if not function.get("strict", False):
# Kimi's parser accepts only object-shaped tool arguments. XGrammar
# treats strict=False arguments as unconstrained JSON, which can
# generate strings/arrays/numbers that Kimi cannot parse. Keep
# non-strict semantics loose by constraining only the outer type.
function["strict"] = True
function["parameters"] = _KIMI_NON_STRICT_ARGUMENTS_SCHEMA
converted_tools.append(converted_tool)
converted_tool_choice = (
tool_choice.model_dump()
if isinstance(tool_choice, ToolChoice)
else tool_choice
)
return get_model_structural_tag(
model="kimi",
tools=converted_tools,
tool_choice=converted_tool_choice,
reasoning=thinking_mode,
)
def get_structural_tag_name(self) -> str:
return "kimi"
test/registered/unit/entrypoints/openai/test_serving_chat.py
+284
View File
@@ -147,6 +147,241 @@ class ServingChatTestCase(unittest.TestCase):
self.assertFalse(adapted.stream)
self.assertEqual(processed, self.basic_req)
def test_kimi_tool_call_keeps_default_reasoning(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertTrue(adapted.require_reasoning)
def test_kimi_tool_call_keeps_explicit_reasoning(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": True},
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertTrue(adapted.require_reasoning)
def test_kimi_tool_call_respects_explicit_reasoning_disable(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": False},
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertFalse(adapted.require_reasoning)
def test_kimi_tool_call_keeps_template_default_thinking(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertNotIn("thinking", kwargs)
def test_kimi_tool_call_keeps_explicit_template_thinking(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": True},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertTrue(kwargs["thinking"])
def test_kimi_tool_call_keeps_explicit_template_thinking_false(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": False},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertFalse(kwargs["thinking"])
def test_jinja_uses_openai_tool_schema_first(self):
"""Ensure Jinja chat templates receive OpenAI-shaped tools by default."""
self.template_manager.chat_template_name = None
@@ -225,6 +460,55 @@ class ServingChatTestCase(unittest.TestCase):
second_tools, [tool.function.model_dump() for tool in req.tools]
)
def test_xgrammar_tag_omits_reasoning_when_parser_owns_it(self):
"""ReasonerGrammarBackend owns the thinking prefix when a parser is set."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "integer"},
"b": {"type": "integer"},
},
"required": ["a", "b"],
},
"strict": True,
},
}
],
tool_choice="required",
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as parser_cls:
parser = parser_cls.return_value
parser.get_structure_constraint.return_value = ("structural_tag", "tag")
self.chat._process_messages(req, is_multimodal=False)
parser.get_structure_constraint.assert_called_once()
self.assertFalse(
parser.get_structure_constraint.call_args.kwargs["thinking_mode"]
)
def test_stop_str_isolation_between_requests(self):
"""Test that stop strings from one request don't affect subsequent requests.
test/registered/unit/function_call/test_function_call_parser.py
+50 -28
View File
@@ -1643,12 +1643,17 @@ class TestDeepSeekV32Detector(unittest.TestCase):
def test_get_model_structural_tag(self):
import xgrammar as xgr
self.assertEqual(self.detector.get_structural_tag_name(), "deepseek_v3_2")
structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=True
)
self.assertIsInstance(structural_tag, xgr.StructuralTag)
grammar = xgr.Grammar.from_structural_tag(structural_tag)
self.assertIsInstance(grammar, xgr.Grammar)
serialized = structural_tag.model_dump_json()
self.assertIn("</｜DSML｜invoke>\\n", serialized)
self.assertNotIn("</｜DSML｜invoke>\\n\\n", serialized)
structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=False
@@ -2088,12 +2093,17 @@ class TestDeepSeekV4Detector(unittest.TestCase):
def test_get_model_structural_tag(self):
import xgrammar as xgr
self.assertEqual(self.detector.get_structural_tag_name(), "deepseek_v4")
structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=True
)
self.assertIsInstance(structural_tag, xgr.StructuralTag)
grammar = xgr.Grammar.from_structural_tag(structural_tag)
self.assertIsInstance(grammar, xgr.Grammar)
serialized = structural_tag.model_dump_json()
self.assertIn("</｜DSML｜invoke>\\n", serialized)
self.assertNotIn("</｜DSML｜invoke>\\n\\n", serialized)
structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=False
@@ -4686,46 +4696,54 @@ class TestGetStructureConstraint(unittest.TestCase):
return FunctionCallParser(self._make_tools(strict=strict), parser_name)
def _constraint_json(self, result):
return result[1].model_dump_json()
# --- structural_tag detectors (kimi_k2, deepseekv3, qwen25, etc.) ---
def test_kimi_required_strict_returns_structural_tag(self):
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("required")
self.assertIsNotNone(result)
self.assertEqual(result[0], "structural_tag")
self.assertTrue(result[1].at_least_one)
self.assertIsInstance(result[1], xgr.StructuralTag)
self.assertIn("<|tool_calls_section_begin|>", self._constraint_json(result))
def test_kimi_required_no_strict_returns_structural_tag(self):
"""required should use structural_tag even without strict, to preserve native format."""
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=False)
result = parser.get_structure_constraint("required")
self.assertIsNotNone(result)
self.assertEqual(result[0], "structural_tag")
self.assertTrue(result[1].at_least_one)
self.assertIsInstance(result[1], xgr.StructuralTag)
self.assertIn("<|tool_calls_section_begin|>", self._constraint_json(result))
def test_kimi_auto_strict_returns_structural_tag(self):
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("auto")
self.assertIsNotNone(result)
self.assertEqual(result[0], "structural_tag")
self.assertFalse(result[1].at_least_one)
self.assertIsInstance(result[1], xgr.StructuralTag)
serialized = self._constraint_json(result)
self.assertIn('"type":"triggered_tags"', serialized)
self.assertIn("<|tool_calls_section_begin|>", serialized)
def test_kimi_routes_through_legacy_with_section_markers(self):
"""xgrammar 0.2.0's get_kimi_structural_tag(tool_choice='auto') emits
a bare <|tool_call_begin|>...<|tool_call_end|> grammar without the
section wrapper Kimi's chat template uses, so the parser would drop
any generated tool calls. KimiK2Detector therefore stays on the
legacy path; pin that here so a future tweak doesn't silently
re-route Kimi through the broken builtin."""
from sglang.srt.entrypoints.openai.protocol import (
LegacyStructuralTagResponseFormat,
)
def test_kimi_routes_through_native_with_section_markers(self):
"""xgrammar 0.2.1's Kimi builtin keeps auto tool calls section-wrapped."""
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("auto")
self.assertIsInstance(result[1], LegacyStructuralTagResponseFormat)
self.assertIn("<|tool_calls_section_begin|>", result[1].structures[0].begin)
self.assertIn("<|tool_calls_section_end|>", result[1].structures[0].end)
self.assertIsInstance(result[1], xgr.StructuralTag)
serialized = self._constraint_json(result)
self.assertIn("<|tool_calls_section_begin|>", serialized)
self.assertIn("<|tool_calls_section_end|>", serialized)
def test_kimi_auto_no_strict_returns_none(self):
"""auto without strict should not constrain."""
@@ -4763,25 +4781,29 @@ class TestGetStructureConstraint(unittest.TestCase):
"""Verify structural_tag contains kimi-specific special tokens."""
parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("required")
tag = result[1]
self.assertTrue(len(tag.structures) > 0)
self.assertIn("<|tool_calls_section_begin|>", tag.structures[0].begin)
self.assertIn("<|tool_call_end|>", tag.structures[0].end)
serialized = self._constraint_json(result)
self.assertIn("<|tool_calls_section_begin|>", serialized)
self.assertIn("functions.get_weather:", serialized)
self.assertIn('"pattern":"\\\\d+"', serialized)
self.assertIn("<|tool_call_end|>", serialized)
self.assertIn("<|tool_calls_section_end|>", serialized)
def test_kimi_required_no_strict_uses_empty_schema(self):
"""Without strict, structural_tag should use empty schema per OpenAI
protocol: strict=False means no parameter schema enforcement."""
def test_kimi_required_no_strict_uses_loose_object_schema(self):
"""Kimi required calls keep non-strict arguments object-shaped but loose."""
parser = self._make_parser("kimi_k2", strict=False)
result = parser.get_structure_constraint("required")
self.assertEqual(result[1].structures[0].schema_, {})
serialized = self._constraint_json(result)
self.assertIn('"json_schema":{"type":"object"}', serialized)
self.assertNotIn('"additionalProperties":false', serialized)
self.assertNotIn('"properties"', serialized)
def test_kimi_required_strict_uses_tool_schema(self):
"""With strict, structural_tag should include the tool's parameter schema."""
"""With strict, native xgrammar should include the tool's parameter schema."""
parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("required")
schema = result[1].structures[0].schema_
self.assertIn("properties", schema)
self.assertIn("city", schema["properties"])
serialized = self._constraint_json(result)
self.assertIn('"properties"', serialized)
self.assertIn('"city"', serialized)
# --- reasoning-prefix ownership ---