minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Upgrade xgrammar to 0.2.1 (#25676)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-05-29 11:40:07 +08:00
committed by GitHub
parent 272066566f
commit 79c844527c
13 changed files with 407 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/full-test-npu.yml
+4 -4
View File
@@ -135,7 +135,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0 pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -198,7 +198,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0 pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -261,7 +261,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0 pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -324,7 +324,7 @@ jobs:
pip install sentence_transformers torchaudio==2.8.0 pip install sentence_transformers torchaudio==2.8.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
.github/workflows/nightly-test-npu.yml
+5 -5
View File
@@ -131,7 +131,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0 pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -197,7 +197,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0 pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -264,7 +264,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0 pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -331,7 +331,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0 pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
@@ -398,7 +398,7 @@ jobs:
pip install sentence_transformers torchaudio==2.10.0 pip install sentence_transformers torchaudio==2.10.0
pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap
pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1
pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv
git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
cd ./lmms-eval cd ./lmms-eval
nohup pip install . > lmmslog.txt 2>&1 & nohup pip install . > lmmslog.txt 2>&1 &
3rdparty/amd/wheel/sglang/pyproject.toml
Vendored
+1 -1
View File
@@ -62,7 +62,7 @@ runtime_common = [
"transformers==5.8.1", "transformers==5.8.1",
"uvicorn", "uvicorn",
"uvloop", "uvloop",
"xgrammar==0.2.0", "xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0", "smg-grpc-servicer>=0.5.0",
] ]
python/pyproject.toml
+1 -1
View File
@@ -80,7 +80,7 @@ dependencies = [
"uvicorn", "uvicorn",
"uvloop", "uvloop",
"watchfiles", "watchfiles",
"xgrammar==0.2.0", "xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0", "smg-grpc-servicer>=0.5.0",
"kernels", "kernels",
] ]
python/pyproject_cpu.toml
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"triton==3.5.0", "triton==3.5.0",
"uvicorn", "uvicorn",
"uvloop", "uvloop",
"xgrammar==0.2.0", "xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0", "smg-grpc-servicer>=0.5.0",
] ]
python/pyproject_npu.toml
+1 -1
View File
@@ -63,7 +63,7 @@ dependencies = [
"transformers==5.8.1", "transformers==5.8.1",
"uvicorn", "uvicorn",
"uvloop", "uvloop",
"xgrammar==0.2.0", "xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0", "smg-grpc-servicer>=0.5.0",
] ]
python/pyproject_other.toml
+1 -1
View File
@@ -62,7 +62,7 @@ runtime_common = [
"transformers==5.8.1", "transformers==5.8.1",
"uvicorn", "uvicorn",
"uvloop", "uvloop",
"xgrammar==0.2.0", "xgrammar==0.2.1",
"smg-grpc-servicer>=0.5.0", "smg-grpc-servicer>=0.5.0",
] ]
python/pyproject_xpu.toml
+1 -1
View File
@@ -66,7 +66,7 @@ dependencies = [
"transformers==5.8.1", "transformers==5.8.1",
"uvicorn", "uvicorn",
"uvloop", "uvloop",
# "xgrammar==0.2.0", xgrammar depends on CUDA PyTorch and Triton only # "xgrammar==0.2.1", xgrammar depends on CUDA PyTorch and Triton only
"smg-grpc-servicer>=0.5.0", "smg-grpc-servicer>=0.5.0",
] ]
python/sglang/srt/entrypoints/openai/serving_chat.py
+4 -2
View File
@@ -505,6 +505,8 @@ class OpenAIServingChat(OpenAIServingBase):
img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch( img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch(
request request
) )
require_reasoning = self._get_reasoning_from_request(request)
adapted_request = GenerateReqInput( adapted_request = GenerateReqInput(
**prompt_kwargs, **prompt_kwargs,
image_data=processed_messages.image_data, image_data=processed_messages.image_data,
@@ -528,7 +530,7 @@ class OpenAIServingChat(OpenAIServingBase):
routed_experts_start_len=request.routed_experts_start_len, routed_experts_start_len=request.routed_experts_start_len,
rid=request.rid, rid=request.rid,
extra_key=self._compute_extra_key(request), extra_key=self._compute_extra_key(request),
require_reasoning=self._get_reasoning_from_request(request), require_reasoning=require_reasoning,
priority=request.priority, priority=request.priority,
routing_key=self.extract_routing_key(raw_request), routing_key=self.extract_routing_key(raw_request),
custom_labels=custom_labels, custom_labels=custom_labels,
@@ -556,7 +558,7 @@ class OpenAIServingChat(OpenAIServingBase):
# when --reasoning-parser is configured, so builtin xgrammar # when --reasoning-parser is configured, so builtin xgrammar
# tags must describe only the post-reasoning tool-call suffix. # tags must describe only the post-reasoning tool-call suffix.
xgrammar_reasoning = thinking_mode and ( xgrammar_reasoning = thinking_mode and (
self.tokenizer_manager.server_args.reasoning_parser is not None self.tokenizer_manager.server_args.reasoning_parser is None
) )
tool_call_constraint = None tool_call_constraint = None
python/sglang/srt/function_call/deepseekv32_detector.py
+3 -115
View File
@@ -1,11 +1,10 @@
import json import json
import logging import logging
import re import re
from typing import List, Literal, Optional, Union
from partial_json_parser.core.options import Allow from partial_json_parser.core.options import Allow
from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice from sglang.srt.entrypoints.openai.protocol import Tool
from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.base_format_detector import BaseFormatDetector
from sglang.srt.function_call.core_types import ( from sglang.srt.function_call.core_types import (
StreamingParseResult, StreamingParseResult,
@@ -15,30 +14,8 @@ from sglang.srt.function_call.core_types import (
) )
from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads
try:
from xgrammar import StructuralTag
from xgrammar.structural_tag import (
AnyTextFormat,
ConstStringFormat,
JSONSchemaFormat,
SequenceFormat,
TagFormat,
TagsWithSeparatorFormat,
TriggeredTagsFormat,
)
except ImportError:
StructuralTag = None # type: ignore
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Names mirror the DeepSeek-V3.2 official chat template tokens
# (see encoding_dsv32.TOOLS_SYSTEM_TEMPLATE).
_INVOKE_BEGIN_PREFIX = '<｜DSML｜invoke name="'
_INVOKE_BEGIN_SUFFIX = '">\n'
_THINK_TAG_END = "</think>"
_THINK_EXCLUDE_TOKENS = ["<think>", "</think>"]
_XML_STYLE = "deepseek_xml"
class DeepSeekV32Detector(BaseFormatDetector): class DeepSeekV32Detector(BaseFormatDetector):
""" """
@@ -391,94 +368,5 @@ class DeepSeekV32Detector(BaseFormatDetector):
trigger="<｜DSML｜invoke", trigger="<｜DSML｜invoke",
) )
def get_structural_tag( def get_structural_tag_name(self) -> str:
self, return "deepseek_v3_2"
tools: Union[List[Tool], None] = None,
tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
thinking_mode: bool = False,
) -> Optional["StructuralTag"]:
"""
Build an xgrammar StructuralTag locally for DeepSeek-V3.2.
Both layers — the outer `<｜DSML｜function_calls｜>...` wrapper and
the inner `<｜DSML｜invoke>...</｜DSML｜invoke>` blocks — are encoded
directly in the grammar with a single-newline join between
consecutive invokes, matching DeepSeek-V3.2's official chat
template. This avoids two layered defects that surfaced with the
prior `xgrammar.get_model_structural_tag("deepseek_v3_2")` path:
- the xgrammar builtin template (pre mlc-ai/xgrammar#638) forced
a double-newline join, which deterministically collapsed
parallel tool calls to one at greedy decoding.
- falling back to the legacy structural tag (built from
`structure_info()`) only constrains the inner invoke block;
the outer wrapper is off-grammar and the model can skip it
under `at_least_one=True`, leaving `detect_and_parse` with no
`<｜DSML｜function_calls>` marker to anchor on.
Returning a fully-formed StructuralTag from the detector keeps
both fixes local to sglang and decoupled from the xgrammar
release cadence.
"""
if not tools or StructuralTag is None:
return None
# `INVOKE_END` and the empty separator together yield a single `\n`
# between consecutive invokes — matching DeepSeek-V3.2's chat template
# `"\n".join(invoke_blocks)`.
function_calls_begin = self.bot_token + "\n"
invoke_end = self.invoke_end_token + "\n"
def _invoke_tag(tool: Tool) -> TagFormat:
return TagFormat(
begin=_INVOKE_BEGIN_PREFIX + tool.function.name + _INVOKE_BEGIN_SUFFIX,
content=JSONSchemaFormat(
json_schema=tool.function.parameters or {},
style=_XML_STYLE,
),
end=invoke_end,
)
if isinstance(tool_choice, ToolChoice):
target = next(
(t for t in tools if t.function.name == tool_choice.function.name),
None,
)
if target is None:
return None
invoke_tags = [_invoke_tag(target)]
is_required = True
else:
invoke_tags = [_invoke_tag(t) for t in tools]
is_required = tool_choice == "required"
inner_tool_calls = TagsWithSeparatorFormat(
tags=invoke_tags, separator="", at_least_one=True
)
if is_required:
suffix_tag = SequenceFormat(
elements=[
ConstStringFormat(value=function_calls_begin),
inner_tool_calls,
ConstStringFormat(value=self.eot_token),
]
)
else:
suffix_tag = TriggeredTagsFormat(
triggers=[self.bot_token],
tags=[
TagFormat(
begin=function_calls_begin,
content=inner_tool_calls,
end=self.eot_token,
)
],
excludes=_THINK_EXCLUDE_TOKENS,
)
if not thinking_mode:
return StructuralTag(format=suffix_tag)
prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=_THINK_TAG_END)
return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
python/sglang/srt/function_call/kimik2_detector.py
+51 -12
View File
@@ -1,10 +1,14 @@
import json import json
import logging import logging
import re import re
from typing import List from typing import List, Literal, Optional, Union
from sglang.srt.entrypoints.openai.protocol import Tool from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice
from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.base_format_detector import (
BaseFormatDetector,
StructuralTag,
get_model_structural_tag,
)
from sglang.srt.function_call.core_types import ( from sglang.srt.function_call.core_types import (
StreamingParseResult, StreamingParseResult,
StructureInfo, StructureInfo,
@@ -23,6 +27,8 @@ _KIMI_K2_SPECIAL_TOKENS = [
"<|tool_call_argument_begin|>", "<|tool_call_argument_begin|>",
] ]
_KIMI_NON_STRICT_ARGUMENTS_SCHEMA = {"type": "object"}
def _strip_special_tokens(text: str) -> str: def _strip_special_tokens(text: str) -> str:
"""Remove all Kimi-K2 tool-call special tokens from text.""" """Remove all Kimi-K2 tool-call special tokens from text."""
@@ -333,12 +339,45 @@ class KimiK2Detector(BaseFormatDetector):
return get_info return get_info
# Kimi stays on the SGLang legacy structural tag path. xgrammar 0.2.0's def get_structural_tag(
# get_kimi_structural_tag(tool_choice="auto") emits a bare self,
# <|tool_call_begin|>...<|tool_call_end|> grammar without the tools: Union[List[Tool], None] = None,
# <|tool_calls_section_begin|>/<|tool_calls_section_end|> wrapper Kimi's tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
# chat template uses, and KimiK2Detector.has_tool_call() keys off the thinking_mode: bool = False,
# section marker — bare tool calls would be silently dropped. Inheriting ) -> Optional[StructuralTag]:
# the base get_structural_tag_name (returns None) keeps FunctionCallParser if not (
# on the legacy path, whose structure_info bakes the section markers in. tools and (tool_choice == "required" or isinstance(tool_choice, ToolChoice))
# TODO: re-enable the builtin once https://github.com/mlc-ai/xgrammar/issues/622 is fixed. ):
return super().get_structural_tag(
tools=tools, tool_choice=tool_choice, thinking_mode=thinking_mode
)
if get_model_structural_tag is None:
return None
converted_tools = []
for tool in tools:
converted_tool = tool.model_dump()
function = converted_tool["function"]
if not function.get("strict", False):
# Kimi's parser accepts only object-shaped tool arguments. XGrammar
# treats strict=False arguments as unconstrained JSON, which can
# generate strings/arrays/numbers that Kimi cannot parse. Keep
# non-strict semantics loose by constraining only the outer type.
function["strict"] = True
function["parameters"] = _KIMI_NON_STRICT_ARGUMENTS_SCHEMA
converted_tools.append(converted_tool)
converted_tool_choice = (
tool_choice.model_dump()
if isinstance(tool_choice, ToolChoice)
else tool_choice
)
return get_model_structural_tag(
model="kimi",
tools=converted_tools,
tool_choice=converted_tool_choice,
reasoning=thinking_mode,
)
def get_structural_tag_name(self) -> str:
return "kimi"
test/registered/unit/entrypoints/openai/test_serving_chat.py
+284
View File
@@ -147,6 +147,241 @@ class ServingChatTestCase(unittest.TestCase):
self.assertFalse(adapted.stream) self.assertFalse(adapted.stream)
self.assertEqual(processed, self.basic_req) self.assertEqual(processed, self.basic_req)
def test_kimi_tool_call_keeps_default_reasoning(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertTrue(adapted.require_reasoning)
def test_kimi_tool_call_keeps_explicit_reasoning(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": True},
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertTrue(adapted.require_reasoning)
def test_kimi_tool_call_respects_explicit_reasoning_disable(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": False},
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertFalse(adapted.require_reasoning)
def test_kimi_tool_call_keeps_template_default_thinking(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertNotIn("thinking", kwargs)
def test_kimi_tool_call_keeps_explicit_template_thinking(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": True},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertTrue(kwargs["thinking"])
def test_kimi_tool_call_keeps_explicit_template_thinking_false(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": False},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertFalse(kwargs["thinking"])
def test_jinja_uses_openai_tool_schema_first(self): def test_jinja_uses_openai_tool_schema_first(self):
"""Ensure Jinja chat templates receive OpenAI-shaped tools by default.""" """Ensure Jinja chat templates receive OpenAI-shaped tools by default."""
self.template_manager.chat_template_name = None self.template_manager.chat_template_name = None
@@ -225,6 +460,55 @@ class ServingChatTestCase(unittest.TestCase):
second_tools, [tool.function.model_dump() for tool in req.tools] second_tools, [tool.function.model_dump() for tool in req.tools]
) )
def test_xgrammar_tag_omits_reasoning_when_parser_owns_it(self):
"""ReasonerGrammarBackend owns the thinking prefix when a parser is set."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "integer"},
"b": {"type": "integer"},
},
"required": ["a", "b"],
},
"strict": True,
},
}
],
tool_choice="required",
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as parser_cls:
parser = parser_cls.return_value
parser.get_structure_constraint.return_value = ("structural_tag", "tag")
self.chat._process_messages(req, is_multimodal=False)
parser.get_structure_constraint.assert_called_once()
self.assertFalse(
parser.get_structure_constraint.call_args.kwargs["thinking_mode"]
)
def test_stop_str_isolation_between_requests(self): def test_stop_str_isolation_between_requests(self):
"""Test that stop strings from one request don't affect subsequent requests. """Test that stop strings from one request don't affect subsequent requests.
test/registered/unit/function_call/test_function_call_parser.py
+50 -28
View File
@@ -1643,12 +1643,17 @@ class TestDeepSeekV32Detector(unittest.TestCase):
def test_get_model_structural_tag(self): def test_get_model_structural_tag(self):
import xgrammar as xgr import xgrammar as xgr
self.assertEqual(self.detector.get_structural_tag_name(), "deepseek_v3_2")
structural_tag = self.detector.get_structural_tag( structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=True self.tools, thinking_mode=True
) )
self.assertIsInstance(structural_tag, xgr.StructuralTag) self.assertIsInstance(structural_tag, xgr.StructuralTag)
grammar = xgr.Grammar.from_structural_tag(structural_tag) grammar = xgr.Grammar.from_structural_tag(structural_tag)
self.assertIsInstance(grammar, xgr.Grammar) self.assertIsInstance(grammar, xgr.Grammar)
serialized = structural_tag.model_dump_json()
self.assertIn("</｜DSML｜invoke>\\n", serialized)
self.assertNotIn("</｜DSML｜invoke>\\n\\n", serialized)
structural_tag = self.detector.get_structural_tag( structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=False self.tools, thinking_mode=False
@@ -2088,12 +2093,17 @@ class TestDeepSeekV4Detector(unittest.TestCase):
def test_get_model_structural_tag(self): def test_get_model_structural_tag(self):
import xgrammar as xgr import xgrammar as xgr
self.assertEqual(self.detector.get_structural_tag_name(), "deepseek_v4")
structural_tag = self.detector.get_structural_tag( structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=True self.tools, thinking_mode=True
) )
self.assertIsInstance(structural_tag, xgr.StructuralTag) self.assertIsInstance(structural_tag, xgr.StructuralTag)
grammar = xgr.Grammar.from_structural_tag(structural_tag) grammar = xgr.Grammar.from_structural_tag(structural_tag)
self.assertIsInstance(grammar, xgr.Grammar) self.assertIsInstance(grammar, xgr.Grammar)
serialized = structural_tag.model_dump_json()
self.assertIn("</｜DSML｜invoke>\\n", serialized)
self.assertNotIn("</｜DSML｜invoke>\\n\\n", serialized)
structural_tag = self.detector.get_structural_tag( structural_tag = self.detector.get_structural_tag(
self.tools, thinking_mode=False self.tools, thinking_mode=False
@@ -4686,46 +4696,54 @@ class TestGetStructureConstraint(unittest.TestCase):
return FunctionCallParser(self._make_tools(strict=strict), parser_name) return FunctionCallParser(self._make_tools(strict=strict), parser_name)
def _constraint_json(self, result):
return result[1].model_dump_json()
# --- structural_tag detectors (kimi_k2, deepseekv3, qwen25, etc.) --- # --- structural_tag detectors (kimi_k2, deepseekv3, qwen25, etc.) ---
def test_kimi_required_strict_returns_structural_tag(self): def test_kimi_required_strict_returns_structural_tag(self):
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=True) parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("required") result = parser.get_structure_constraint("required")
self.assertIsNotNone(result) self.assertIsNotNone(result)
self.assertEqual(result[0], "structural_tag") self.assertEqual(result[0], "structural_tag")
self.assertTrue(result[1].at_least_one) self.assertIsInstance(result[1], xgr.StructuralTag)
self.assertIn("<|tool_calls_section_begin|>", self._constraint_json(result))
def test_kimi_required_no_strict_returns_structural_tag(self): def test_kimi_required_no_strict_returns_structural_tag(self):
"""required should use structural_tag even without strict, to preserve native format.""" """required should use structural_tag even without strict, to preserve native format."""
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=False) parser = self._make_parser("kimi_k2", strict=False)
result = parser.get_structure_constraint("required") result = parser.get_structure_constraint("required")
self.assertIsNotNone(result) self.assertIsNotNone(result)
self.assertEqual(result[0], "structural_tag") self.assertEqual(result[0], "structural_tag")
self.assertTrue(result[1].at_least_one) self.assertIsInstance(result[1], xgr.StructuralTag)
self.assertIn("<|tool_calls_section_begin|>", self._constraint_json(result))
def test_kimi_auto_strict_returns_structural_tag(self): def test_kimi_auto_strict_returns_structural_tag(self):
import xgrammar as xgr
parser = self._make_parser("kimi_k2", strict=True) parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("auto") result = parser.get_structure_constraint("auto")
self.assertIsNotNone(result) self.assertIsNotNone(result)
self.assertEqual(result[0], "structural_tag") self.assertEqual(result[0], "structural_tag")
self.assertFalse(result[1].at_least_one) self.assertIsInstance(result[1], xgr.StructuralTag)
serialized = self._constraint_json(result)
self.assertIn('"type":"triggered_tags"', serialized)
self.assertIn("<|tool_calls_section_begin|>", serialized)
def test_kimi_routes_through_legacy_with_section_markers(self): def test_kimi_routes_through_native_with_section_markers(self):
"""xgrammar 0.2.0's get_kimi_structural_tag(tool_choice='auto') emits """xgrammar 0.2.1's Kimi builtin keeps auto tool calls section-wrapped."""
a bare <|tool_call_begin|>...<|tool_call_end|> grammar without the import xgrammar as xgr
section wrapper Kimi's chat template uses, so the parser would drop
any generated tool calls. KimiK2Detector therefore stays on the
legacy path; pin that here so a future tweak doesn't silently
re-route Kimi through the broken builtin."""
from sglang.srt.entrypoints.openai.protocol import (
LegacyStructuralTagResponseFormat,
)
parser = self._make_parser("kimi_k2", strict=True) parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("auto") result = parser.get_structure_constraint("auto")
self.assertIsInstance(result[1], LegacyStructuralTagResponseFormat) self.assertIsInstance(result[1], xgr.StructuralTag)
self.assertIn("<|tool_calls_section_begin|>", result[1].structures[0].begin) serialized = self._constraint_json(result)
self.assertIn("<|tool_calls_section_end|>", result[1].structures[0].end) self.assertIn("<|tool_calls_section_begin|>", serialized)
self.assertIn("<|tool_calls_section_end|>", serialized)
def test_kimi_auto_no_strict_returns_none(self): def test_kimi_auto_no_strict_returns_none(self):
"""auto without strict should not constrain.""" """auto without strict should not constrain."""
@@ -4763,25 +4781,29 @@ class TestGetStructureConstraint(unittest.TestCase):
"""Verify structural_tag contains kimi-specific special tokens.""" """Verify structural_tag contains kimi-specific special tokens."""
parser = self._make_parser("kimi_k2", strict=True) parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("required") result = parser.get_structure_constraint("required")
tag = result[1] serialized = self._constraint_json(result)
self.assertTrue(len(tag.structures) > 0) self.assertIn("<|tool_calls_section_begin|>", serialized)
self.assertIn("<|tool_calls_section_begin|>", tag.structures[0].begin) self.assertIn("functions.get_weather:", serialized)
self.assertIn("<|tool_call_end|>", tag.structures[0].end) self.assertIn('"pattern":"\\\\d+"', serialized)
self.assertIn("<|tool_call_end|>", serialized)
self.assertIn("<|tool_calls_section_end|>", serialized)
def test_kimi_required_no_strict_uses_empty_schema(self): def test_kimi_required_no_strict_uses_loose_object_schema(self):
"""Without strict, structural_tag should use empty schema per OpenAI """Kimi required calls keep non-strict arguments object-shaped but loose."""
protocol: strict=False means no parameter schema enforcement."""
parser = self._make_parser("kimi_k2", strict=False) parser = self._make_parser("kimi_k2", strict=False)
result = parser.get_structure_constraint("required") result = parser.get_structure_constraint("required")
self.assertEqual(result[1].structures[0].schema_, {}) serialized = self._constraint_json(result)
self.assertIn('"json_schema":{"type":"object"}', serialized)
self.assertNotIn('"additionalProperties":false', serialized)
self.assertNotIn('"properties"', serialized)
def test_kimi_required_strict_uses_tool_schema(self): def test_kimi_required_strict_uses_tool_schema(self):
"""With strict, structural_tag should include the tool's parameter schema.""" """With strict, native xgrammar should include the tool's parameter schema."""
parser = self._make_parser("kimi_k2", strict=True) parser = self._make_parser("kimi_k2", strict=True)
result = parser.get_structure_constraint("required") result = parser.get_structure_constraint("required")
schema = result[1].structures[0].schema_ serialized = self._constraint_json(result)
self.assertIn("properties", schema) self.assertIn('"properties"', serialized)
self.assertIn("city", schema["properties"]) self.assertIn('"city"', serialized)
# --- reasoning-prefix ownership --- # --- reasoning-prefix ownership ---