diff --git a/.github/workflows/full-test-npu.yml b/.github/workflows/full-test-npu.yml index 1feb3f504..e57664a3d 100644 --- a/.github/workflows/full-test-npu.yml +++ b/.github/workflows/full-test-npu.yml @@ -135,7 +135,7 @@ jobs: pip install sentence_transformers torchaudio==2.8.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -198,7 +198,7 @@ jobs: pip install sentence_transformers torchaudio==2.8.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -261,7 +261,7 @@ jobs: pip install sentence_transformers torchaudio==2.8.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -324,7 +324,7 @@ jobs: pip install sentence_transformers torchaudio==2.8.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.1.25 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index 9b4253a76..0ca0e9ae7 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -131,7 +131,7 @@ jobs: pip install sentence_transformers torchaudio==2.10.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -197,7 +197,7 @@ jobs: pip install sentence_transformers torchaudio==2.10.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -264,7 +264,7 @@ jobs: pip install sentence_transformers torchaudio==2.10.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -331,7 +331,7 @@ jobs: pip install sentence_transformers torchaudio==2.10.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & @@ -398,7 +398,7 @@ jobs: pip install sentence_transformers torchaudio==2.10.0 pip install protobuf==6.31.1 zss pre-commit wandb>=0.16.0 tenacity==8.3.0 loguru openpyxl latex2sympy2 zstandard transformers-stream-generator tqdm-multiprocess pycocoevalcap pip install yt-dlp sentencepiece==0.1.99 nltk av ftfy sqlitedict==2.1.0 sacrebleu>=1.5.0 pytablewriter black==24.1.0 isort==5.13.2 peft>=0.2.0 accelerate>=0.29.1 - pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.0 numpy==1.26.4 dotenv + pip install jsonlines httpx==0.25.0 evaluate>=0.4.0 datasets==2.16.1 numexpr xgrammar==0.2.1 numpy==1.26.4 dotenv git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git cd ./lmms-eval nohup pip install . > lmmslog.txt 2>&1 & diff --git a/3rdparty/amd/wheel/sglang/pyproject.toml b/3rdparty/amd/wheel/sglang/pyproject.toml index 582a03c80..a32fef182 100644 --- a/3rdparty/amd/wheel/sglang/pyproject.toml +++ b/3rdparty/amd/wheel/sglang/pyproject.toml @@ -62,7 +62,7 @@ runtime_common = [ "transformers==5.8.1", "uvicorn", "uvloop", - "xgrammar==0.2.0", + "xgrammar==0.2.1", "smg-grpc-servicer>=0.5.0", ] diff --git a/python/pyproject.toml b/python/pyproject.toml index de3849335..793dd8bf8 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -80,7 +80,7 @@ dependencies = [ "uvicorn", "uvloop", "watchfiles", - "xgrammar==0.2.0", + "xgrammar==0.2.1", "smg-grpc-servicer>=0.5.0", "kernels", ] diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 722172054..9cb4e11f7 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -66,7 +66,7 @@ dependencies = [ "triton==3.5.0", "uvicorn", "uvloop", - "xgrammar==0.2.0", + "xgrammar==0.2.1", "smg-grpc-servicer>=0.5.0", ] diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 4de298651..79c846b08 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -63,7 +63,7 @@ dependencies = [ "transformers==5.8.1", "uvicorn", "uvloop", - "xgrammar==0.2.0", + "xgrammar==0.2.1", "smg-grpc-servicer>=0.5.0", ] diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 86ccfe910..b30d80f62 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -62,7 +62,7 @@ runtime_common = [ "transformers==5.8.1", "uvicorn", "uvloop", - "xgrammar==0.2.0", + "xgrammar==0.2.1", "smg-grpc-servicer>=0.5.0", ] diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index a8ee08d9e..837b390d3 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -66,7 +66,7 @@ dependencies = [ "transformers==5.8.1", "uvicorn", "uvloop", - # "xgrammar==0.2.0", xgrammar depends on CUDA PyTorch and Triton only + # "xgrammar==0.2.1", xgrammar depends on CUDA PyTorch and Triton only "smg-grpc-servicer>=0.5.0", ] diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index d55c3a6ad..cc02ebd6a 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -505,6 +505,8 @@ class OpenAIServingChat(OpenAIServingBase): img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch( request ) + require_reasoning = self._get_reasoning_from_request(request) + adapted_request = GenerateReqInput( **prompt_kwargs, image_data=processed_messages.image_data, @@ -528,7 +530,7 @@ class OpenAIServingChat(OpenAIServingBase): routed_experts_start_len=request.routed_experts_start_len, rid=request.rid, extra_key=self._compute_extra_key(request), - require_reasoning=self._get_reasoning_from_request(request), + require_reasoning=require_reasoning, priority=request.priority, routing_key=self.extract_routing_key(raw_request), custom_labels=custom_labels, @@ -556,7 +558,7 @@ class OpenAIServingChat(OpenAIServingBase): # when --reasoning-parser is configured, so builtin xgrammar # tags must describe only the post-reasoning tool-call suffix. xgrammar_reasoning = thinking_mode and ( - self.tokenizer_manager.server_args.reasoning_parser is not None + self.tokenizer_manager.server_args.reasoning_parser is None ) tool_call_constraint = None diff --git a/python/sglang/srt/function_call/deepseekv32_detector.py b/python/sglang/srt/function_call/deepseekv32_detector.py index 88b13d75d..a4c1b9dab 100644 --- a/python/sglang/srt/function_call/deepseekv32_detector.py +++ b/python/sglang/srt/function_call/deepseekv32_detector.py @@ -1,11 +1,10 @@ import json import logging import re -from typing import List, Literal, Optional, Union from partial_json_parser.core.options import Allow -from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice +from sglang.srt.entrypoints.openai.protocol import Tool from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.core_types import ( StreamingParseResult, @@ -15,30 +14,8 @@ from sglang.srt.function_call.core_types import ( ) from sglang.srt.function_call.utils import _find_common_prefix, _partial_json_loads -try: - from xgrammar import StructuralTag - from xgrammar.structural_tag import ( - AnyTextFormat, - ConstStringFormat, - JSONSchemaFormat, - SequenceFormat, - TagFormat, - TagsWithSeparatorFormat, - TriggeredTagsFormat, - ) -except ImportError: - StructuralTag = None # type: ignore - logger = logging.getLogger(__name__) -# Names mirror the DeepSeek-V3.2 official chat template tokens -# (see encoding_dsv32.TOOLS_SYSTEM_TEMPLATE). -_INVOKE_BEGIN_PREFIX = '<｜DSML｜invoke name="' -_INVOKE_BEGIN_SUFFIX = '">\n' -_THINK_TAG_END = "" -_THINK_EXCLUDE_TOKENS = ["", ""] -_XML_STYLE = "deepseek_xml" - class DeepSeekV32Detector(BaseFormatDetector): """ @@ -391,94 +368,5 @@ class DeepSeekV32Detector(BaseFormatDetector): trigger="<｜DSML｜invoke", ) - def get_structural_tag( - self, - tools: Union[List[Tool], None] = None, - tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto", - thinking_mode: bool = False, - ) -> Optional["StructuralTag"]: - """ - Build an xgrammar StructuralTag locally for DeepSeek-V3.2. - - Both layers — the outer `<｜DSML｜function_calls｜>...` wrapper and - the inner `<｜DSML｜invoke>...` blocks — are encoded - directly in the grammar with a single-newline join between - consecutive invokes, matching DeepSeek-V3.2's official chat - template. This avoids two layered defects that surfaced with the - prior `xgrammar.get_model_structural_tag("deepseek_v3_2")` path: - - - the xgrammar builtin template (pre mlc-ai/xgrammar#638) forced - a double-newline join, which deterministically collapsed - parallel tool calls to one at greedy decoding. - - falling back to the legacy structural tag (built from - `structure_info()`) only constrains the inner invoke block; - the outer wrapper is off-grammar and the model can skip it - under `at_least_one=True`, leaving `detect_and_parse` with no - `<｜DSML｜function_calls>` marker to anchor on. - - Returning a fully-formed StructuralTag from the detector keeps - both fixes local to sglang and decoupled from the xgrammar - release cadence. - """ - if not tools or StructuralTag is None: - return None - - # `INVOKE_END` and the empty separator together yield a single `\n` - # between consecutive invokes — matching DeepSeek-V3.2's chat template - # `"\n".join(invoke_blocks)`. - function_calls_begin = self.bot_token + "\n" - invoke_end = self.invoke_end_token + "\n" - - def _invoke_tag(tool: Tool) -> TagFormat: - return TagFormat( - begin=_INVOKE_BEGIN_PREFIX + tool.function.name + _INVOKE_BEGIN_SUFFIX, - content=JSONSchemaFormat( - json_schema=tool.function.parameters or {}, - style=_XML_STYLE, - ), - end=invoke_end, - ) - - if isinstance(tool_choice, ToolChoice): - target = next( - (t for t in tools if t.function.name == tool_choice.function.name), - None, - ) - if target is None: - return None - invoke_tags = [_invoke_tag(target)] - is_required = True - else: - invoke_tags = [_invoke_tag(t) for t in tools] - is_required = tool_choice == "required" - - inner_tool_calls = TagsWithSeparatorFormat( - tags=invoke_tags, separator="", at_least_one=True - ) - - if is_required: - suffix_tag = SequenceFormat( - elements=[ - ConstStringFormat(value=function_calls_begin), - inner_tool_calls, - ConstStringFormat(value=self.eot_token), - ] - ) - else: - suffix_tag = TriggeredTagsFormat( - triggers=[self.bot_token], - tags=[ - TagFormat( - begin=function_calls_begin, - content=inner_tool_calls, - end=self.eot_token, - ) - ], - excludes=_THINK_EXCLUDE_TOKENS, - ) - - if not thinking_mode: - return StructuralTag(format=suffix_tag) - - prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=_THINK_TAG_END) - return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) + def get_structural_tag_name(self) -> str: + return "deepseek_v3_2" diff --git a/python/sglang/srt/function_call/kimik2_detector.py b/python/sglang/srt/function_call/kimik2_detector.py index da2c76fd0..2345817de 100644 --- a/python/sglang/srt/function_call/kimik2_detector.py +++ b/python/sglang/srt/function_call/kimik2_detector.py @@ -1,10 +1,14 @@ import json import logging import re -from typing import List +from typing import List, Literal, Optional, Union -from sglang.srt.entrypoints.openai.protocol import Tool -from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.entrypoints.openai.protocol import Tool, ToolChoice +from sglang.srt.function_call.base_format_detector import ( + BaseFormatDetector, + StructuralTag, + get_model_structural_tag, +) from sglang.srt.function_call.core_types import ( StreamingParseResult, StructureInfo, @@ -23,6 +27,8 @@ _KIMI_K2_SPECIAL_TOKENS = [ "<|tool_call_argument_begin|>", ] +_KIMI_NON_STRICT_ARGUMENTS_SCHEMA = {"type": "object"} + def _strip_special_tokens(text: str) -> str: """Remove all Kimi-K2 tool-call special tokens from text.""" @@ -333,12 +339,45 @@ class KimiK2Detector(BaseFormatDetector): return get_info - # Kimi stays on the SGLang legacy structural tag path. xgrammar 0.2.0's - # get_kimi_structural_tag(tool_choice="auto") emits a bare - # <|tool_call_begin|>...<|tool_call_end|> grammar without the - # <|tool_calls_section_begin|>/<|tool_calls_section_end|> wrapper Kimi's - # chat template uses, and KimiK2Detector.has_tool_call() keys off the - # section marker — bare tool calls would be silently dropped. Inheriting - # the base get_structural_tag_name (returns None) keeps FunctionCallParser - # on the legacy path, whose structure_info bakes the section markers in. - # TODO: re-enable the builtin once https://github.com/mlc-ai/xgrammar/issues/622 is fixed. + def get_structural_tag( + self, + tools: Union[List[Tool], None] = None, + tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto", + thinking_mode: bool = False, + ) -> Optional[StructuralTag]: + if not ( + tools and (tool_choice == "required" or isinstance(tool_choice, ToolChoice)) + ): + return super().get_structural_tag( + tools=tools, tool_choice=tool_choice, thinking_mode=thinking_mode + ) + if get_model_structural_tag is None: + return None + + converted_tools = [] + for tool in tools: + converted_tool = tool.model_dump() + function = converted_tool["function"] + if not function.get("strict", False): + # Kimi's parser accepts only object-shaped tool arguments. XGrammar + # treats strict=False arguments as unconstrained JSON, which can + # generate strings/arrays/numbers that Kimi cannot parse. Keep + # non-strict semantics loose by constraining only the outer type. + function["strict"] = True + function["parameters"] = _KIMI_NON_STRICT_ARGUMENTS_SCHEMA + converted_tools.append(converted_tool) + + converted_tool_choice = ( + tool_choice.model_dump() + if isinstance(tool_choice, ToolChoice) + else tool_choice + ) + return get_model_structural_tag( + model="kimi", + tools=converted_tools, + tool_choice=converted_tool_choice, + reasoning=thinking_mode, + ) + + def get_structural_tag_name(self) -> str: + return "kimi" diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index c12284d14..101402aa5 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -147,6 +147,241 @@ class ServingChatTestCase(unittest.TestCase): self.assertFalse(adapted.stream) self.assertEqual(processed, self.basic_req) + def test_kimi_tool_call_keeps_default_reasoning(self): + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + }, + }, + } + ], + tool_choice="required", + ) + + with patch.object(self.chat, "_process_messages") as proc_mock: + proc_mock.return_value = MessageProcessingResult( + "", + [1, 2, 3], + None, + None, + [], + [], + None, + ) + + adapted, _ = self.chat._convert_to_internal_request(req) + + self.assertTrue(adapted.require_reasoning) + + def test_kimi_tool_call_keeps_explicit_reasoning(self): + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + }, + }, + } + ], + tool_choice="required", + chat_template_kwargs={"thinking": True}, + ) + + with patch.object(self.chat, "_process_messages") as proc_mock: + proc_mock.return_value = MessageProcessingResult( + "", + [1, 2, 3], + None, + None, + [], + [], + None, + ) + + adapted, _ = self.chat._convert_to_internal_request(req) + + self.assertTrue(adapted.require_reasoning) + + def test_kimi_tool_call_respects_explicit_reasoning_disable(self): + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + }, + }, + } + ], + tool_choice="required", + chat_template_kwargs={"thinking": False}, + ) + + with patch.object(self.chat, "_process_messages") as proc_mock: + proc_mock.return_value = MessageProcessingResult( + "", + [1, 2, 3], + None, + None, + [], + [], + None, + ) + + adapted, _ = self.chat._convert_to_internal_request(req) + + self.assertFalse(adapted.require_reasoning) + + def test_kimi_tool_call_keeps_template_default_thinking(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + }, + }, + } + ], + tool_choice="required", + ) + + self.chat._process_messages(req, is_multimodal=False) + + kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs + self.assertNotIn("thinking", kwargs) + + def test_kimi_tool_call_keeps_explicit_template_thinking(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + }, + }, + } + ], + tool_choice="required", + chat_template_kwargs={"thinking": True}, + ) + + self.chat._process_messages(req, is_multimodal=False) + + kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs + self.assertTrue(kwargs["thinking"]) + + def test_kimi_tool_call_keeps_explicit_template_thinking_false(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"a": {"type": "integer"}}, + }, + }, + } + ], + tool_choice="required", + chat_template_kwargs={"thinking": False}, + ) + + self.chat._process_messages(req, is_multimodal=False) + + kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs + self.assertFalse(kwargs["thinking"]) + def test_jinja_uses_openai_tool_schema_first(self): """Ensure Jinja chat templates receive OpenAI-shaped tools by default.""" self.template_manager.chat_template_name = None @@ -225,6 +460,55 @@ class ServingChatTestCase(unittest.TestCase): second_tools, [tool.function.model_dump() for tool in req.tools] ) + def test_xgrammar_tag_omits_reasoning_when_parser_owns_it(self): + """ReasonerGrammarBackend owns the thinking prefix when a parser is set.""" + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.template_manager.reasoning_config = ReasoningToggleConfig( + toggle_param="thinking", default_enabled=True + ) + self.tm.server_args.reasoning_parser = "kimi_k2" + self.tm.server_args.tool_call_parser = "kimi_k2" + self.chat.reasoning_parser = "kimi_k2" + self.chat.tool_call_parser = "kimi_k2" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + tools=[ + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": { + "a": {"type": "integer"}, + "b": {"type": "integer"}, + }, + "required": ["a", "b"], + }, + "strict": True, + }, + } + ], + tool_choice="required", + ) + + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as parser_cls: + parser = parser_cls.return_value + parser.get_structure_constraint.return_value = ("structural_tag", "tag") + + self.chat._process_messages(req, is_multimodal=False) + + parser.get_structure_constraint.assert_called_once() + self.assertFalse( + parser.get_structure_constraint.call_args.kwargs["thinking_mode"] + ) + def test_stop_str_isolation_between_requests(self): """Test that stop strings from one request don't affect subsequent requests. diff --git a/test/registered/unit/function_call/test_function_call_parser.py b/test/registered/unit/function_call/test_function_call_parser.py index 188b928e9..300f64f26 100644 --- a/test/registered/unit/function_call/test_function_call_parser.py +++ b/test/registered/unit/function_call/test_function_call_parser.py @@ -1643,12 +1643,17 @@ class TestDeepSeekV32Detector(unittest.TestCase): def test_get_model_structural_tag(self): import xgrammar as xgr + self.assertEqual(self.detector.get_structural_tag_name(), "deepseek_v3_2") + structural_tag = self.detector.get_structural_tag( self.tools, thinking_mode=True ) self.assertIsInstance(structural_tag, xgr.StructuralTag) grammar = xgr.Grammar.from_structural_tag(structural_tag) self.assertIsInstance(grammar, xgr.Grammar) + serialized = structural_tag.model_dump_json() + self.assertIn("\\n", serialized) + self.assertNotIn("\\n\\n", serialized) structural_tag = self.detector.get_structural_tag( self.tools, thinking_mode=False @@ -2088,12 +2093,17 @@ class TestDeepSeekV4Detector(unittest.TestCase): def test_get_model_structural_tag(self): import xgrammar as xgr + self.assertEqual(self.detector.get_structural_tag_name(), "deepseek_v4") + structural_tag = self.detector.get_structural_tag( self.tools, thinking_mode=True ) self.assertIsInstance(structural_tag, xgr.StructuralTag) grammar = xgr.Grammar.from_structural_tag(structural_tag) self.assertIsInstance(grammar, xgr.Grammar) + serialized = structural_tag.model_dump_json() + self.assertIn("\\n", serialized) + self.assertNotIn("\\n\\n", serialized) structural_tag = self.detector.get_structural_tag( self.tools, thinking_mode=False @@ -4686,46 +4696,54 @@ class TestGetStructureConstraint(unittest.TestCase): return FunctionCallParser(self._make_tools(strict=strict), parser_name) + def _constraint_json(self, result): + return result[1].model_dump_json() + # --- structural_tag detectors (kimi_k2, deepseekv3, qwen25, etc.) --- def test_kimi_required_strict_returns_structural_tag(self): + import xgrammar as xgr + parser = self._make_parser("kimi_k2", strict=True) result = parser.get_structure_constraint("required") self.assertIsNotNone(result) self.assertEqual(result[0], "structural_tag") - self.assertTrue(result[1].at_least_one) + self.assertIsInstance(result[1], xgr.StructuralTag) + self.assertIn("<|tool_calls_section_begin|>", self._constraint_json(result)) def test_kimi_required_no_strict_returns_structural_tag(self): """required should use structural_tag even without strict, to preserve native format.""" + import xgrammar as xgr + parser = self._make_parser("kimi_k2", strict=False) result = parser.get_structure_constraint("required") self.assertIsNotNone(result) self.assertEqual(result[0], "structural_tag") - self.assertTrue(result[1].at_least_one) + self.assertIsInstance(result[1], xgr.StructuralTag) + self.assertIn("<|tool_calls_section_begin|>", self._constraint_json(result)) def test_kimi_auto_strict_returns_structural_tag(self): + import xgrammar as xgr + parser = self._make_parser("kimi_k2", strict=True) result = parser.get_structure_constraint("auto") self.assertIsNotNone(result) self.assertEqual(result[0], "structural_tag") - self.assertFalse(result[1].at_least_one) + self.assertIsInstance(result[1], xgr.StructuralTag) + serialized = self._constraint_json(result) + self.assertIn('"type":"triggered_tags"', serialized) + self.assertIn("<|tool_calls_section_begin|>", serialized) - def test_kimi_routes_through_legacy_with_section_markers(self): - """xgrammar 0.2.0's get_kimi_structural_tag(tool_choice='auto') emits - a bare <|tool_call_begin|>...<|tool_call_end|> grammar without the - section wrapper Kimi's chat template uses, so the parser would drop - any generated tool calls. KimiK2Detector therefore stays on the - legacy path; pin that here so a future tweak doesn't silently - re-route Kimi through the broken builtin.""" - from sglang.srt.entrypoints.openai.protocol import ( - LegacyStructuralTagResponseFormat, - ) + def test_kimi_routes_through_native_with_section_markers(self): + """xgrammar 0.2.1's Kimi builtin keeps auto tool calls section-wrapped.""" + import xgrammar as xgr parser = self._make_parser("kimi_k2", strict=True) result = parser.get_structure_constraint("auto") - self.assertIsInstance(result[1], LegacyStructuralTagResponseFormat) - self.assertIn("<|tool_calls_section_begin|>", result[1].structures[0].begin) - self.assertIn("<|tool_calls_section_end|>", result[1].structures[0].end) + self.assertIsInstance(result[1], xgr.StructuralTag) + serialized = self._constraint_json(result) + self.assertIn("<|tool_calls_section_begin|>", serialized) + self.assertIn("<|tool_calls_section_end|>", serialized) def test_kimi_auto_no_strict_returns_none(self): """auto without strict should not constrain.""" @@ -4763,25 +4781,29 @@ class TestGetStructureConstraint(unittest.TestCase): """Verify structural_tag contains kimi-specific special tokens.""" parser = self._make_parser("kimi_k2", strict=True) result = parser.get_structure_constraint("required") - tag = result[1] - self.assertTrue(len(tag.structures) > 0) - self.assertIn("<|tool_calls_section_begin|>", tag.structures[0].begin) - self.assertIn("<|tool_call_end|>", tag.structures[0].end) + serialized = self._constraint_json(result) + self.assertIn("<|tool_calls_section_begin|>", serialized) + self.assertIn("functions.get_weather:", serialized) + self.assertIn('"pattern":"\\\\d+"', serialized) + self.assertIn("<|tool_call_end|>", serialized) + self.assertIn("<|tool_calls_section_end|>", serialized) - def test_kimi_required_no_strict_uses_empty_schema(self): - """Without strict, structural_tag should use empty schema per OpenAI - protocol: strict=False means no parameter schema enforcement.""" + def test_kimi_required_no_strict_uses_loose_object_schema(self): + """Kimi required calls keep non-strict arguments object-shaped but loose.""" parser = self._make_parser("kimi_k2", strict=False) result = parser.get_structure_constraint("required") - self.assertEqual(result[1].structures[0].schema_, {}) + serialized = self._constraint_json(result) + self.assertIn('"json_schema":{"type":"object"}', serialized) + self.assertNotIn('"additionalProperties":false', serialized) + self.assertNotIn('"properties"', serialized) def test_kimi_required_strict_uses_tool_schema(self): - """With strict, structural_tag should include the tool's parameter schema.""" + """With strict, native xgrammar should include the tool's parameter schema.""" parser = self._make_parser("kimi_k2", strict=True) result = parser.get_structure_constraint("required") - schema = result[1].structures[0].schema_ - self.assertIn("properties", schema) - self.assertIn("city", schema["properties"]) + serialized = self._constraint_json(result) + self.assertIn('"properties"', serialized) + self.assertIn('"city"', serialized) # --- reasoning-prefix ownership ---