Expand parser auto detection coverage (#28449)
This commit is contained in:
@@ -19,6 +19,7 @@ parser from chat templates and tokenizer vocabularies.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Optional, Tuple
|
from typing import Callable, Optional, Tuple
|
||||||
@@ -203,6 +204,12 @@ def _is_glm45(ctx):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_glm47(ctx):
|
||||||
|
return _is_glm45(ctx) and ctx.has_pattern(
|
||||||
|
r"\{\{[-\s]*['\"]<tool_call>['\"]\s*\+\s*tc\.name"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_xml_kv_tool_call(ctx):
|
def _is_xml_kv_tool_call(ctx):
|
||||||
# Structural signature for the GLM-4.5 / GLM-4.6 style tool-call format
|
# Structural signature for the GLM-4.5 / GLM-4.6 style tool-call format
|
||||||
# (`<tool_call>name<arg_key>k</arg_key>\n<arg_value>v</arg_value>...</tool_call>`).
|
# (`<tool_call>name<arg_key>k</arg_key>\n<arg_value>v</arg_value>...</tool_call>`).
|
||||||
@@ -213,6 +220,41 @@ def _is_xml_kv_tool_call(ctx):
|
|||||||
return ctx.has_vocab("<arg_key>") and ctx.has_vocab("<arg_value>")
|
return ctx.has_vocab("<arg_key>") and ctx.has_vocab("<arg_value>")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_deepseek_v31(ctx):
|
||||||
|
return ctx.has_text("<|tool▁calls▁begin|>") and ctx.has_text("<|tool▁sep|>")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_deepseek_v32(ctx):
|
||||||
|
return ctx.has_text("<|DSML|function_calls>")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_deepseek_v4(ctx):
|
||||||
|
return ctx.has_text("<|DSML|tool_calls>")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_hunyuan(ctx):
|
||||||
|
return (
|
||||||
|
(ctx.has_text("<tool_calls>") or ctx.has_vocab("<tool_calls>"))
|
||||||
|
and (ctx.has_text("<tool_sep>") or ctx.has_vocab("<tool_sep>"))
|
||||||
|
) or (ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking"))
|
||||||
|
|
||||||
|
|
||||||
|
def _is_poolside_v1(ctx):
|
||||||
|
has_poolside_tool_format = (
|
||||||
|
ctx.has_text("unescaped XML-like object")
|
||||||
|
and ctx.has_text("<tool_call>function-name")
|
||||||
|
and ctx.has_text("<arg_key>")
|
||||||
|
and ctx.has_text("<arg_value>")
|
||||||
|
)
|
||||||
|
return has_poolside_tool_format or (
|
||||||
|
ctx.reasoning_config
|
||||||
|
== ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=False)
|
||||||
|
and not _is_hunyuan(ctx)
|
||||||
|
and (ctx.has_text("<arg_key>") or ctx.has_vocab("<arg_key>"))
|
||||||
|
and (ctx.has_text("<arg_value>") or ctx.has_vocab("<arg_value>"))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_mimo(ctx):
|
def _is_mimo(ctx):
|
||||||
return ctx.reasoning_config == ReasoningToggleConfig(
|
return ctx.reasoning_config == ReasoningToggleConfig(
|
||||||
toggle_param="enable_thinking", default_enabled=False
|
toggle_param="enable_thinking", default_enabled=False
|
||||||
@@ -229,6 +271,30 @@ def _is_minicpm5(ctx):
|
|||||||
return ctx.has_pattern(r"<function\s+name=") and ctx.has_pattern(r"<param\s+name=")
|
return ctx.has_pattern(r"<function\s+name=") and ctx.has_pattern(r"<param\s+name=")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_lfm2(ctx):
|
||||||
|
return (
|
||||||
|
ctx.has_text("<|tool_call_start|>") or ctx.has_vocab("<|tool_call_start|>")
|
||||||
|
) and (ctx.has_text("<|tool_call_end|>") or ctx.has_vocab("<|tool_call_end|>"))
|
||||||
|
|
||||||
|
|
||||||
|
def _is_step3p5(ctx):
|
||||||
|
return ctx.has_pattern(r"Step-?3(?:\.|p)?[57]", re.IGNORECASE) or (
|
||||||
|
ctx.has_text("reasoning_effort")
|
||||||
|
and ctx.has_text("Reasoning: ")
|
||||||
|
and _is_qwen3_coder(ctx)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_step3(ctx):
|
||||||
|
return ctx.has_text("<steptml:invoke") or (
|
||||||
|
ctx.has_text("<|tool_calls_begin|>") and ctx.has_text("<|tool_sep|>")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_qwen3_coder(ctx):
|
||||||
|
return ctx.has_text("<function=") and ctx.has_text("<parameter=")
|
||||||
|
|
||||||
|
|
||||||
def _is_qwen3(ctx):
|
def _is_qwen3(ctx):
|
||||||
return ctx.reasoning_config == ReasoningToggleConfig(
|
return ctx.reasoning_config == ReasoningToggleConfig(
|
||||||
toggle_param="enable_thinking", default_enabled=True
|
toggle_param="enable_thinking", default_enabled=True
|
||||||
@@ -246,7 +312,7 @@ def _is_deepseek_r1(ctx):
|
|||||||
|
|
||||||
|
|
||||||
def _is_deepseek_r1_think_tags(ctx):
|
def _is_deepseek_r1_think_tags(ctx):
|
||||||
return ctx.has_text("<think>") or ctx.has_text("</think>")
|
return not _is_lfm2(ctx) and (ctx.has_text("<think>") or ctx.has_text("</think>"))
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -263,9 +329,14 @@ REASONING_PARSER_RULES = (
|
|||||||
DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
|
DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
|
||||||
DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3),
|
DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3),
|
||||||
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
|
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
|
||||||
|
DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
|
||||||
|
DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
|
||||||
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
|
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
|
||||||
DetectionRule(name="minimax", value="minimax", predicate=_is_minimax),
|
DetectionRule(name="minimax", value="minimax", predicate=_is_minimax),
|
||||||
|
DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
|
||||||
|
DetectionRule(name="step3", value="step3", predicate=_is_step3),
|
||||||
DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3),
|
DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3),
|
||||||
|
DetectionRule(name="deepseek_v4", value="deepseek-v4", predicate=_is_deepseek_v4),
|
||||||
DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3),
|
DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3),
|
||||||
DetectionRule(
|
DetectionRule(
|
||||||
name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1
|
name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1
|
||||||
@@ -289,12 +360,22 @@ TOOL_CALL_PARSER_RULES = (
|
|||||||
DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax),
|
DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax),
|
||||||
DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
|
DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
|
||||||
DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
|
DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
|
||||||
|
DetectionRule(name="deepseek_v4", value="deepseekv4", predicate=_is_deepseek_v4),
|
||||||
|
DetectionRule(name="deepseek_v32", value="deepseekv32", predicate=_is_deepseek_v32),
|
||||||
|
DetectionRule(name="deepseek_v31", value="deepseekv31", predicate=_is_deepseek_v31),
|
||||||
|
DetectionRule(name="lfm2", value="lfm2", predicate=_is_lfm2),
|
||||||
|
DetectionRule(name="glm47", value="glm47", predicate=_is_glm47),
|
||||||
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
|
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
|
||||||
DetectionRule(name="minicpm5", value="minicpm5", predicate=_is_minicpm5),
|
DetectionRule(name="minicpm5", value="minicpm5", predicate=_is_minicpm5),
|
||||||
|
DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
|
||||||
|
DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
|
||||||
|
DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
|
||||||
|
DetectionRule(name="step3", value="step3", predicate=_is_step3),
|
||||||
DetectionRule(
|
DetectionRule(
|
||||||
name="xml_kv_tool_call", value="glm45", predicate=_is_xml_kv_tool_call
|
name="xml_kv_tool_call", value="glm45", predicate=_is_xml_kv_tool_call
|
||||||
),
|
),
|
||||||
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
|
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
|
||||||
|
DetectionRule(name="qwen3_coder", value="qwen3_coder", predicate=_is_qwen3_coder),
|
||||||
DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3),
|
DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3),
|
||||||
DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3),
|
DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3),
|
||||||
DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1),
|
DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1),
|
||||||
@@ -424,6 +505,56 @@ def _resolve_auto_parser(
|
|||||||
setattr(server_args, attr, None)
|
setattr(server_args, attr, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[str]:
|
||||||
|
if not chat_template_arg or not isinstance(chat_template_arg, str):
|
||||||
|
return None
|
||||||
|
if not chat_template_arg.endswith(".jinja") or not os.path.exists(
|
||||||
|
chat_template_arg
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
with open(chat_template_arg, encoding="utf-8") as f:
|
||||||
|
return f.read().replace("\\n", "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _disable_auto_parser(server_args, attr: str, label: str) -> None:
|
||||||
|
logger.warning(
|
||||||
|
f"--{attr.replace('_', '-')}=auto specified but could not detect "
|
||||||
|
f"{label} from chat template. Disabling {label}."
|
||||||
|
)
|
||||||
|
setattr(server_args, attr, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_architecture_auto_parsers(server_args) -> None:
|
||||||
|
from sglang.srt.utils.hf_transformers_utils import get_config
|
||||||
|
|
||||||
|
config = get_config(
|
||||||
|
server_args.model_path,
|
||||||
|
trust_remote_code=server_args.trust_remote_code,
|
||||||
|
revision=getattr(server_args, "revision", None),
|
||||||
|
model_config_parser=getattr(server_args, "model_config_parser", "auto"),
|
||||||
|
)
|
||||||
|
architectures = getattr(config, "architectures", None) or []
|
||||||
|
arch = architectures[0] if architectures else ""
|
||||||
|
|
||||||
|
if "DeepseekV4" in arch:
|
||||||
|
reasoning_parser, tool_call_parser = "deepseek-v4", "deepseekv4"
|
||||||
|
elif "DeepseekV3" in arch:
|
||||||
|
reasoning_parser, tool_call_parser = "deepseek-v3", "deepseekv32"
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
|
||||||
|
for attr, detected in (
|
||||||
|
("reasoning_parser", reasoning_parser),
|
||||||
|
("tool_call_parser", tool_call_parser),
|
||||||
|
):
|
||||||
|
if getattr(server_args, attr) == "auto":
|
||||||
|
setattr(server_args, attr, detected)
|
||||||
|
logger.info(
|
||||||
|
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' "
|
||||||
|
f"from model architecture '{arch}'"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def resolve_auto_parsers(server_args) -> None:
|
def resolve_auto_parsers(server_args) -> None:
|
||||||
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
|
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
|
||||||
|
|
||||||
@@ -438,33 +569,58 @@ def resolve_auto_parsers(server_args) -> None:
|
|||||||
|
|
||||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||||
|
|
||||||
|
chat_template_arg = getattr(server_args, "chat_template", None)
|
||||||
|
try:
|
||||||
|
explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to load explicit Jinja chat template: %s", e)
|
||||||
|
explicit_jinja_template = None
|
||||||
|
has_explicit_template_without_detection = (
|
||||||
|
chat_template_arg is not None and explicit_jinja_template is None
|
||||||
|
)
|
||||||
|
|
||||||
|
tokenizer = None
|
||||||
try:
|
try:
|
||||||
tokenizer = get_tokenizer(
|
tokenizer = get_tokenizer(
|
||||||
server_args.model_path,
|
server_args.model_path,
|
||||||
trust_remote_code=server_args.trust_remote_code,
|
trust_remote_code=server_args.trust_remote_code,
|
||||||
)
|
)
|
||||||
template = getattr(tokenizer, "chat_template", None)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
|
logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
|
||||||
if needs_reasoning:
|
|
||||||
logger.warning(
|
template = explicit_jinja_template
|
||||||
"--reasoning-parser=auto specified but could not detect "
|
if template is None and tokenizer is not None:
|
||||||
"reasoning parser from chat template. Disabling reasoning parser."
|
template = getattr(tokenizer, "chat_template", None)
|
||||||
)
|
|
||||||
server_args.reasoning_parser = None
|
|
||||||
if needs_tool_call:
|
|
||||||
logger.warning(
|
|
||||||
"--tool-call-parser=auto specified but could not detect "
|
|
||||||
"tool-call parser from chat template. Disabling tool-call parser."
|
|
||||||
)
|
|
||||||
server_args.tool_call_parser = None
|
|
||||||
return
|
|
||||||
|
|
||||||
force_reasoning, reasoning_config = detect_reasoning_pattern(template)
|
force_reasoning, reasoning_config = detect_reasoning_pattern(template)
|
||||||
ctx = build_detection_context(
|
ctx = build_detection_context(
|
||||||
template, tokenizer, reasoning_config, force_reasoning
|
template, tokenizer, reasoning_config, force_reasoning
|
||||||
)
|
)
|
||||||
if ctx is None:
|
if ctx is None:
|
||||||
|
if has_explicit_template_without_detection:
|
||||||
|
logger.warning(
|
||||||
|
"--chat-template=%s is explicit but is not a readable Jinja template, so "
|
||||||
|
"parser auto-detection from chat template is not available.",
|
||||||
|
chat_template_arg,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
_resolve_architecture_auto_parsers(server_args)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to load model config for architecture-based auto-detection: %s",
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
if needs_reasoning:
|
||||||
|
if server_args.reasoning_parser == "auto":
|
||||||
|
_disable_auto_parser(
|
||||||
|
server_args, "reasoning_parser", "reasoning parser"
|
||||||
|
)
|
||||||
|
if needs_tool_call:
|
||||||
|
if server_args.tool_call_parser == "auto":
|
||||||
|
_disable_auto_parser(
|
||||||
|
server_args, "tool_call_parser", "tool-call parser"
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
if needs_reasoning:
|
if needs_reasoning:
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from types import SimpleNamespace
|
from types import ModuleType, SimpleNamespace
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from sglang.srt.managers.template_detection import (
|
from sglang.srt.managers.template_detection import (
|
||||||
|
REASONING_PARSER_RULES,
|
||||||
TOOL_CALL_PARSER_RULES,
|
TOOL_CALL_PARSER_RULES,
|
||||||
ReasoningToggleConfig,
|
ReasoningToggleConfig,
|
||||||
detect_reasoning_parser,
|
detect_reasoning_parser,
|
||||||
@@ -15,13 +19,22 @@ register_cpu_ci(2.0, "base-a-test-cpu")
|
|||||||
|
|
||||||
|
|
||||||
class _DummyTokenizer:
|
class _DummyTokenizer:
|
||||||
def __init__(self, vocab):
|
def __init__(self, vocab, chat_template=None):
|
||||||
self._vocab = vocab
|
self._vocab = vocab
|
||||||
|
self.chat_template = chat_template
|
||||||
|
|
||||||
def get_vocab(self):
|
def get_vocab(self):
|
||||||
return {token: i for i, token in enumerate(self._vocab)}
|
return {token: i for i, token in enumerate(self._vocab)}
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_hf_transformers_utils(get_tokenizer, get_config=None):
|
||||||
|
module = ModuleType("sglang.srt.utils.hf_transformers_utils")
|
||||||
|
module.get_tokenizer = get_tokenizer
|
||||||
|
if get_config is not None:
|
||||||
|
module.get_config = get_config
|
||||||
|
return patch.dict(sys.modules, {module.__name__: module})
|
||||||
|
|
||||||
|
|
||||||
class TestTemplateManagerReasoningDetection(unittest.TestCase):
|
class TestTemplateManagerReasoningDetection(unittest.TestCase):
|
||||||
|
|
||||||
def _detect(self, template, vocab):
|
def _detect(self, template, vocab):
|
||||||
@@ -136,6 +149,90 @@ class TestTemplateDetectionRuleMatrix(unittest.TestCase):
|
|||||||
"deepseek-v3",
|
"deepseek-v3",
|
||||||
"thinking",
|
"thinking",
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"deepseek_v4_dsml_tool_calls",
|
||||||
|
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}\n"
|
||||||
|
'<|DSML|tool_calls><|DSML|invoke name="tool"></|DSML|invoke>',
|
||||||
|
[],
|
||||||
|
"deepseek-v4",
|
||||||
|
"thinking",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"hunyuan_interleaved_thinking",
|
||||||
|
"{% set reasoning_effort = reasoning_effort | default('no_think', true) %}\n"
|
||||||
|
"{% set interleaved_thinking = interleaved_thinking | default(false, true) %}\n"
|
||||||
|
"<think>reasoning</think><tool_calls><tool_call>name<tool_sep>",
|
||||||
|
["<tool_calls>", "<tool_sep>", "<arg_key>", "<arg_value>"],
|
||||||
|
"hunyuan",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"poolside_v1_enable_thinking_false",
|
||||||
|
"{% if not enable_thinking is defined %}{% set enable_thinking = false %}{% endif %}\n"
|
||||||
|
"<tool_call>{{ name }}\n<arg_key>{{ key }}</arg_key><arg_value>{{ value }}</arg_value>",
|
||||||
|
["<tool_call>", "<arg_key>", "<arg_value>"],
|
||||||
|
"poolside_v1",
|
||||||
|
"enable_thinking",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"poolside_v1_actual_template_shape",
|
||||||
|
"{% set enable_thinking = enable_thinking | default(false) %}\n"
|
||||||
|
"Wrap your thinking in '<think>', '</think>' tags, followed by a function call.\n"
|
||||||
|
"For each function call, return an unescaped XML-like object with function name "
|
||||||
|
"and arguments within '<tool_call>' and '</tool_call>' tags, like here:\n"
|
||||||
|
"<tool_call>function-name\n"
|
||||||
|
"<arg_key>argument-key</arg_key>\n"
|
||||||
|
"<arg_value>value-of-argument-key</arg_value>\n"
|
||||||
|
"</tool_call>",
|
||||||
|
["<tool_call>"],
|
||||||
|
"poolside_v1",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"lfm2_not_deepseek_r1_from_history_cleanup",
|
||||||
|
"{% set keep_past_thinking = keep_past_thinking | default(false) %}\n"
|
||||||
|
"{% if not keep_past_thinking and '</think>' in content %}"
|
||||||
|
"{{ content.split('</think>')[-1] }}{% endif %}\n"
|
||||||
|
'<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>',
|
||||||
|
["<|tool_call_start|>", "<|tool_call_end|>"],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"qwen3_coder_actual_template_shape_has_no_reasoning",
|
||||||
|
"<tools>\n"
|
||||||
|
"<tool_call><function=get_weather>"
|
||||||
|
"<parameter=city>Paris</parameter></function></tool_call>",
|
||||||
|
["<tool_call>"],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"step3p5_actual_template_shape",
|
||||||
|
"{% if reasoning_effort is defined %}Reasoning: {{ reasoning_effort }}{% endif %}\n"
|
||||||
|
"<tool_call><function=get_weather>"
|
||||||
|
"<parameter=city>Paris</parameter></function></tool_call>\n"
|
||||||
|
"{% if '<think>' in content %}{{ content.split('</think>')[-1] }}{% endif %}",
|
||||||
|
["<tool_call>", "<tool_calls>"],
|
||||||
|
"step3p5",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"step3p5_think_tags",
|
||||||
|
"Step3.5-Flash\n<function=tool><parameter=arg>value</parameter></function>\n<think>",
|
||||||
|
[],
|
||||||
|
"step3p5",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"step3_steptml",
|
||||||
|
"<|tool_calls_begin|><|tool_call_begin|>function<|tool_sep|>"
|
||||||
|
'<steptml:invoke name="tool"><steptml:parameter name="arg">value</steptml:parameter>'
|
||||||
|
"</steptml:invoke><|tool_call_end|><think>",
|
||||||
|
[],
|
||||||
|
"step3",
|
||||||
|
None,
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"qwen3_enable_thinking_true",
|
"qwen3_enable_thinking_true",
|
||||||
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n",
|
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n",
|
||||||
@@ -267,6 +364,27 @@ class TestToolCallParserDetection(unittest.TestCase):
|
|||||||
[],
|
[],
|
||||||
"deepseekv3",
|
"deepseekv3",
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"deepseekv31",
|
||||||
|
(
|
||||||
|
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}\n"
|
||||||
|
"<|tool▁calls▁begin|><|tool▁call▁begin|>tool<|tool▁sep|>{}<|tool▁call▁end|>"
|
||||||
|
),
|
||||||
|
[],
|
||||||
|
"deepseekv31",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"deepseekv32",
|
||||||
|
'<|DSML|function_calls><|DSML|invoke name="tool"></|DSML|invoke></|DSML|function_calls>',
|
||||||
|
[],
|
||||||
|
"deepseekv32",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"deepseekv4",
|
||||||
|
'<|DSML|tool_calls><|DSML|invoke name="tool"></|DSML|invoke></|DSML|tool_calls>',
|
||||||
|
[],
|
||||||
|
"deepseekv4",
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"kimi_k2",
|
"kimi_k2",
|
||||||
"{% set thinking = thinking if thinking is defined else true %}\n<think>",
|
"{% set thinking = thinking if thinking is defined else true %}\n<think>",
|
||||||
@@ -284,6 +402,98 @@ class TestToolCallParserDetection(unittest.TestCase):
|
|||||||
["<function", "<param"],
|
["<function", "<param"],
|
||||||
"minicpm5",
|
"minicpm5",
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
"lfm2",
|
||||||
|
'<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>',
|
||||||
|
["<|tool_call_start|>", "<|tool_call_end|>"],
|
||||||
|
"lfm2",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"hunyuan",
|
||||||
|
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Paris</arg_value></tool_call></tool_calls>",
|
||||||
|
["<tool_calls>", "<tool_sep>", "<arg_key>", "<arg_value>"],
|
||||||
|
"hunyuan",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"poolside_v1",
|
||||||
|
"{% if not enable_thinking is defined %}{% set enable_thinking = false %}{% endif %}\n"
|
||||||
|
"<tool_call>get_weather\n<arg_key>city</arg_key><arg_value>Paris</arg_value></tool_call>",
|
||||||
|
["<tool_call>", "<arg_key>", "<arg_value>"],
|
||||||
|
"poolside_v1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"poolside_v1_actual_template_shape",
|
||||||
|
"{% set enable_thinking = enable_thinking | default(false) %}\n"
|
||||||
|
"return an unescaped XML-like object with function name and arguments "
|
||||||
|
"within '<tool_call>' and '</tool_call>' tags\n"
|
||||||
|
"<tool_call>function-name\n"
|
||||||
|
"<arg_key>argument-key</arg_key>\n"
|
||||||
|
"<arg_value>value-of-argument-key</arg_value>\n"
|
||||||
|
"</tool_call>",
|
||||||
|
["<tool_call>"],
|
||||||
|
"poolside_v1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"qwen3_coder",
|
||||||
|
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n"
|
||||||
|
"<tool_call><function=get_weather><parameter=city>Paris</parameter></function></tool_call>",
|
||||||
|
["<tool_call>"],
|
||||||
|
"qwen3_coder",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"step3p5",
|
||||||
|
"Step3.5-Flash\n<tool_call><function=get_weather><parameter=city>Paris</parameter></function></tool_call>",
|
||||||
|
["<tool_call>"],
|
||||||
|
"step3p5",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"step3p5_actual_template_shape",
|
||||||
|
"{% if reasoning_effort is defined %}Reasoning: {{ reasoning_effort }}{% endif %}\n"
|
||||||
|
"<tool_call><function=get_weather>"
|
||||||
|
"<parameter=city>Paris</parameter></function></tool_call>",
|
||||||
|
["<tool_call>", "<tool_calls>"],
|
||||||
|
"step3p5",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"step3",
|
||||||
|
"<|tool_calls_begin|><|tool_call_begin|>function<|tool_sep|>"
|
||||||
|
'<steptml:invoke name="get_weather"><steptml:parameter name="city">Paris</steptml:parameter>'
|
||||||
|
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>",
|
||||||
|
[],
|
||||||
|
"step3",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"glm47_compact_tool_call",
|
||||||
|
(
|
||||||
|
"[gMASK]<sop>\n"
|
||||||
|
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n"
|
||||||
|
"{% for tc in m.tool_calls %}\n"
|
||||||
|
"{{- '<tool_call>' + tc.name -}}\n"
|
||||||
|
"{% set _args = tc.arguments %}"
|
||||||
|
"{% for k, v in _args.items() %}"
|
||||||
|
"<arg_key>{{ k }}</arg_key><arg_value>{{ v }}</arg_value>"
|
||||||
|
"{% endfor %}</tool_call>\n"
|
||||||
|
"{% endfor %}"
|
||||||
|
),
|
||||||
|
["<tool_call>", "<arg_key>", "<arg_value>", "<|endoftext|>"],
|
||||||
|
"glm47",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"glm45_newline_tool_call",
|
||||||
|
(
|
||||||
|
"[gMASK]<sop>\n"
|
||||||
|
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n"
|
||||||
|
"{% for tc in m.tool_calls %}\n"
|
||||||
|
"{{ '\\n<tool_call>' + tc.name }}\n"
|
||||||
|
"{% set _args = tc.arguments %}\n"
|
||||||
|
"{% for k, v in _args.items() %}\n"
|
||||||
|
"<arg_key>{{ k }}</arg_key>\n<arg_value>{{ v }}</arg_value>\n"
|
||||||
|
"{% endfor %}\n</tool_call>\n"
|
||||||
|
"{% endfor %}"
|
||||||
|
),
|
||||||
|
["<tool_call>", "<arg_key>", "<arg_value>", "<|endoftext|>"],
|
||||||
|
"glm45",
|
||||||
|
),
|
||||||
(
|
(
|
||||||
"xml_kv_tool_call_via_vocab",
|
"xml_kv_tool_call_via_vocab",
|
||||||
"{% set reasoning_effort = reasoning_effort | default('high', true) %}\n<think>",
|
"{% set reasoning_effort = reasoning_effort | default('high', true) %}\n<think>",
|
||||||
@@ -304,8 +514,33 @@ class TestToolCallParserDetection(unittest.TestCase):
|
|||||||
# xml_kv_tool_call fallback. Both currently map to "glm45", so the
|
# xml_kv_tool_call fallback. Both currently map to "glm45", so the
|
||||||
# value-based test above can't catch a swap — assert positions directly.
|
# value-based test above can't catch a swap — assert positions directly.
|
||||||
rule_index = {rule.name: i for i, rule in enumerate(TOOL_CALL_PARSER_RULES)}
|
rule_index = {rule.name: i for i, rule in enumerate(TOOL_CALL_PARSER_RULES)}
|
||||||
|
self.assertLess(rule_index["glm47"], rule_index["glm45"])
|
||||||
self.assertLess(rule_index["glm45"], rule_index["xml_kv_tool_call"])
|
self.assertLess(rule_index["glm45"], rule_index["xml_kv_tool_call"])
|
||||||
|
|
||||||
|
def test_specific_rules_precede_broad_fallbacks(self):
|
||||||
|
reasoning_index = {
|
||||||
|
rule.name: i for i, rule in enumerate(REASONING_PARSER_RULES)
|
||||||
|
}
|
||||||
|
self.assertLess(reasoning_index["deepseek_v4"], reasoning_index["deepseek_v3"])
|
||||||
|
self.assertLess(
|
||||||
|
reasoning_index["hunyuan"], reasoning_index["deepseek_r1_think_tags"]
|
||||||
|
)
|
||||||
|
self.assertLess(reasoning_index["poolside_v1"], reasoning_index["mimo"])
|
||||||
|
self.assertLess(
|
||||||
|
reasoning_index["step3p5"], reasoning_index["deepseek_r1_think_tags"]
|
||||||
|
)
|
||||||
|
self.assertLess(
|
||||||
|
reasoning_index["step3"], reasoning_index["deepseek_r1_think_tags"]
|
||||||
|
)
|
||||||
|
|
||||||
|
tool_index = {rule.name: i for i, rule in enumerate(TOOL_CALL_PARSER_RULES)}
|
||||||
|
self.assertLess(tool_index["deepseek_v31"], tool_index["deepseek_v3"])
|
||||||
|
self.assertLess(tool_index["hunyuan"], tool_index["xml_kv_tool_call"])
|
||||||
|
self.assertLess(tool_index["poolside_v1"], tool_index["xml_kv_tool_call"])
|
||||||
|
self.assertLess(tool_index["step3p5"], tool_index["qwen3_coder"])
|
||||||
|
self.assertLess(tool_index["step3"], tool_index["deepseek_v3"])
|
||||||
|
self.assertLess(tool_index["qwen3_coder"], tool_index["qwen"])
|
||||||
|
|
||||||
def test_xml_kv_requires_both_arg_tokens(self):
|
def test_xml_kv_requires_both_arg_tokens(self):
|
||||||
template = "Hello {{ user }}"
|
template = "Hello {{ user }}"
|
||||||
force, config = detect_reasoning_pattern(template)
|
force, config = detect_reasoning_pattern(template)
|
||||||
@@ -345,31 +580,48 @@ class TestToolCallParserDetection(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class TestResolveAutoParsers(unittest.TestCase):
|
class TestResolveAutoParsers(unittest.TestCase):
|
||||||
"""Tests for resolve_auto_parsers() using real model tokenizers."""
|
"""Tests for resolve_auto_parsers()."""
|
||||||
|
|
||||||
def _make_server_args(self, reasoning_parser=None, tool_call_parser=None):
|
qwen3_template = "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}"
|
||||||
|
|
||||||
|
def _make_server_args(
|
||||||
|
self, reasoning_parser=None, tool_call_parser=None, chat_template=None
|
||||||
|
):
|
||||||
return SimpleNamespace(
|
return SimpleNamespace(
|
||||||
reasoning_parser=reasoning_parser,
|
reasoning_parser=reasoning_parser,
|
||||||
tool_call_parser=tool_call_parser,
|
tool_call_parser=tool_call_parser,
|
||||||
model_path="Qwen/Qwen3-0.6B",
|
model_path="Qwen/Qwen3-0.6B",
|
||||||
trust_remote_code=False,
|
trust_remote_code=False,
|
||||||
|
chat_template=chat_template,
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_resolves_both_parsers_with_real_model(self):
|
def test_resolves_both_parsers_with_tokenizer_template(self):
|
||||||
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
|
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
|
||||||
|
tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template)
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
|
||||||
resolve_auto_parsers(args)
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
self.assertEqual(args.reasoning_parser, "qwen3")
|
self.assertEqual(args.reasoning_parser, "qwen3")
|
||||||
self.assertEqual(args.tool_call_parser, "qwen")
|
self.assertEqual(args.tool_call_parser, "qwen")
|
||||||
|
|
||||||
def test_resolves_reasoning_parser_only(self):
|
def test_resolves_reasoning_parser_only(self):
|
||||||
args = self._make_server_args(reasoning_parser="auto", tool_call_parser=None)
|
args = self._make_server_args(reasoning_parser="auto", tool_call_parser=None)
|
||||||
|
tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template)
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
|
||||||
resolve_auto_parsers(args)
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
self.assertEqual(args.reasoning_parser, "qwen3")
|
self.assertEqual(args.reasoning_parser, "qwen3")
|
||||||
self.assertIsNone(args.tool_call_parser)
|
self.assertIsNone(args.tool_call_parser)
|
||||||
|
|
||||||
def test_resolves_tool_call_parser_only(self):
|
def test_resolves_tool_call_parser_only(self):
|
||||||
args = self._make_server_args(reasoning_parser="qwen3", tool_call_parser="auto")
|
args = self._make_server_args(reasoning_parser="qwen3", tool_call_parser="auto")
|
||||||
|
tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template)
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
|
||||||
resolve_auto_parsers(args)
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
self.assertEqual(args.reasoning_parser, "qwen3")
|
self.assertEqual(args.reasoning_parser, "qwen3")
|
||||||
self.assertEqual(args.tool_call_parser, "qwen")
|
self.assertEqual(args.tool_call_parser, "qwen")
|
||||||
|
|
||||||
@@ -386,10 +638,102 @@ class TestResolveAutoParsers(unittest.TestCase):
|
|||||||
model_path="nonexistent/model-does-not-exist-xyz",
|
model_path="nonexistent/model-does-not-exist-xyz",
|
||||||
trust_remote_code=False,
|
trust_remote_code=False,
|
||||||
)
|
)
|
||||||
|
with _patch_hf_transformers_utils(
|
||||||
|
Mock(side_effect=RuntimeError("tokenizer unavailable")),
|
||||||
|
Mock(side_effect=RuntimeError("config unavailable")),
|
||||||
|
):
|
||||||
resolve_auto_parsers(args)
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
self.assertIsNone(args.reasoning_parser)
|
self.assertIsNone(args.reasoning_parser)
|
||||||
self.assertIsNone(args.tool_call_parser)
|
self.assertIsNone(args.tool_call_parser)
|
||||||
|
|
||||||
|
def test_none_chat_template_disables_both_parsers(self):
|
||||||
|
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
|
||||||
|
tokenizer = _DummyTokenizer([])
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
|
||||||
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
|
self.assertIsNone(args.reasoning_parser)
|
||||||
|
self.assertIsNone(args.tool_call_parser)
|
||||||
|
|
||||||
|
def test_deepseek_v32_arch_without_chat_template_uses_custom_encoder(self):
|
||||||
|
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
|
||||||
|
tokenizer = _DummyTokenizer([])
|
||||||
|
config = SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(
|
||||||
|
Mock(return_value=tokenizer), Mock(return_value=config)
|
||||||
|
):
|
||||||
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
|
self.assertEqual(args.reasoning_parser, "deepseek-v3")
|
||||||
|
self.assertEqual(args.tool_call_parser, "deepseekv32")
|
||||||
|
|
||||||
|
def test_deepseek_v4_arch_without_chat_template_uses_custom_encoder(self):
|
||||||
|
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
|
||||||
|
tokenizer = _DummyTokenizer([])
|
||||||
|
config = SimpleNamespace(architectures=["DeepseekV4ForCausalLM"])
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(
|
||||||
|
Mock(return_value=tokenizer), Mock(return_value=config)
|
||||||
|
):
|
||||||
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
|
self.assertEqual(args.reasoning_parser, "deepseek-v4")
|
||||||
|
self.assertEqual(args.tool_call_parser, "deepseekv4")
|
||||||
|
|
||||||
|
def test_deepseek_arch_fallback_runs_when_tokenizer_load_fails(self):
|
||||||
|
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
|
||||||
|
config = SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(
|
||||||
|
Mock(side_effect=RuntimeError("tokenizer unavailable")),
|
||||||
|
Mock(return_value=config),
|
||||||
|
):
|
||||||
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
|
self.assertEqual(args.reasoning_parser, "deepseek-v3")
|
||||||
|
self.assertEqual(args.tool_call_parser, "deepseekv32")
|
||||||
|
|
||||||
|
def test_explicit_non_jinja_template_skips_architecture_fallback(self):
|
||||||
|
args = self._make_server_args(
|
||||||
|
reasoning_parser="auto",
|
||||||
|
tool_call_parser="auto",
|
||||||
|
chat_template="chatml",
|
||||||
|
)
|
||||||
|
args.model_path = "deepseek-ai/DeepSeek-V3.2"
|
||||||
|
tokenizer = _DummyTokenizer([])
|
||||||
|
get_config = Mock()
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(Mock(return_value=tokenizer), get_config):
|
||||||
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
|
get_config.assert_not_called()
|
||||||
|
self.assertIsNone(args.reasoning_parser)
|
||||||
|
self.assertIsNone(args.tool_call_parser)
|
||||||
|
|
||||||
|
def test_explicit_jinja_template_takes_precedence(self):
|
||||||
|
tokenizer = _DummyTokenizer([], chat_template=None)
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile("w", suffix=".jinja") as f:
|
||||||
|
f.write(
|
||||||
|
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}\n"
|
||||||
|
'<|DSML|function_calls><|DSML|invoke name="tool"></|DSML|invoke>'
|
||||||
|
)
|
||||||
|
f.flush()
|
||||||
|
args = self._make_server_args(
|
||||||
|
reasoning_parser="auto",
|
||||||
|
tool_call_parser="auto",
|
||||||
|
chat_template=f.name,
|
||||||
|
)
|
||||||
|
|
||||||
|
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
|
||||||
|
resolve_auto_parsers(args)
|
||||||
|
|
||||||
|
self.assertEqual(args.reasoning_parser, "deepseek-v3")
|
||||||
|
self.assertEqual(args.tool_call_parser, "deepseekv32")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user