Expand parser auto detection coverage (#28449)

This commit is contained in:
Xinyuan Tong
2026-06-23 12:26:37 -07:00
committed by GitHub
parent e0dc8b7137
commit 0c6e8e9477
2 changed files with 524 additions and 24 deletions
+171 -15
View File
@@ -19,6 +19,7 @@ parser from chat templates and tokenizer vocabularies.
""" """
import logging import logging
import os
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from typing import Callable, Optional, Tuple from typing import Callable, Optional, Tuple
@@ -203,6 +204,12 @@ def _is_glm45(ctx):
) )
def _is_glm47(ctx):
return _is_glm45(ctx) and ctx.has_pattern(
r"\{\{[-\s]*['\"]<tool_call>['\"]\s*\+\s*tc\.name"
)
def _is_xml_kv_tool_call(ctx): def _is_xml_kv_tool_call(ctx):
# Structural signature for the GLM-4.5 / GLM-4.6 style tool-call format # Structural signature for the GLM-4.5 / GLM-4.6 style tool-call format
# (`<tool_call>name<arg_key>k</arg_key>\n<arg_value>v</arg_value>...</tool_call>`). # (`<tool_call>name<arg_key>k</arg_key>\n<arg_value>v</arg_value>...</tool_call>`).
@@ -213,6 +220,41 @@ def _is_xml_kv_tool_call(ctx):
return ctx.has_vocab("<arg_key>") and ctx.has_vocab("<arg_value>") return ctx.has_vocab("<arg_key>") and ctx.has_vocab("<arg_value>")
def _is_deepseek_v31(ctx):
return ctx.has_text("<|tool▁calls▁begin|>") and ctx.has_text("<|tool▁sep|>")
def _is_deepseek_v32(ctx):
return ctx.has_text("<|DSML|function_calls>")
def _is_deepseek_v4(ctx):
return ctx.has_text("<|DSML|tool_calls>")
def _is_hunyuan(ctx):
return (
(ctx.has_text("<tool_calls>") or ctx.has_vocab("<tool_calls>"))
and (ctx.has_text("<tool_sep>") or ctx.has_vocab("<tool_sep>"))
) or (ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking"))
def _is_poolside_v1(ctx):
has_poolside_tool_format = (
ctx.has_text("unescaped XML-like object")
and ctx.has_text("<tool_call>function-name")
and ctx.has_text("<arg_key>")
and ctx.has_text("<arg_value>")
)
return has_poolside_tool_format or (
ctx.reasoning_config
== ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=False)
and not _is_hunyuan(ctx)
and (ctx.has_text("<arg_key>") or ctx.has_vocab("<arg_key>"))
and (ctx.has_text("<arg_value>") or ctx.has_vocab("<arg_value>"))
)
def _is_mimo(ctx): def _is_mimo(ctx):
return ctx.reasoning_config == ReasoningToggleConfig( return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=False toggle_param="enable_thinking", default_enabled=False
@@ -229,6 +271,30 @@ def _is_minicpm5(ctx):
return ctx.has_pattern(r"<function\s+name=") and ctx.has_pattern(r"<param\s+name=") return ctx.has_pattern(r"<function\s+name=") and ctx.has_pattern(r"<param\s+name=")
def _is_lfm2(ctx):
return (
ctx.has_text("<|tool_call_start|>") or ctx.has_vocab("<|tool_call_start|>")
) and (ctx.has_text("<|tool_call_end|>") or ctx.has_vocab("<|tool_call_end|>"))
def _is_step3p5(ctx):
return ctx.has_pattern(r"Step-?3(?:\.|p)?[57]", re.IGNORECASE) or (
ctx.has_text("reasoning_effort")
and ctx.has_text("Reasoning: ")
and _is_qwen3_coder(ctx)
)
def _is_step3(ctx):
return ctx.has_text("<steptml:invoke") or (
ctx.has_text("<|tool_calls_begin|>") and ctx.has_text("<|tool_sep|>")
)
def _is_qwen3_coder(ctx):
return ctx.has_text("<function=") and ctx.has_text("<parameter=")
def _is_qwen3(ctx): def _is_qwen3(ctx):
return ctx.reasoning_config == ReasoningToggleConfig( return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True toggle_param="enable_thinking", default_enabled=True
@@ -246,7 +312,7 @@ def _is_deepseek_r1(ctx):
def _is_deepseek_r1_think_tags(ctx): def _is_deepseek_r1_think_tags(ctx):
return ctx.has_text("<think>") or ctx.has_text("</think>") return not _is_lfm2(ctx) and (ctx.has_text("<think>") or ctx.has_text("</think>"))
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -263,9 +329,14 @@ REASONING_PARSER_RULES = (
DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2), DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3), DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3),
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45), DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo), DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
DetectionRule(name="minimax", value="minimax", predicate=_is_minimax), DetectionRule(name="minimax", value="minimax", predicate=_is_minimax),
DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
DetectionRule(name="step3", value="step3", predicate=_is_step3),
DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3), DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3),
DetectionRule(name="deepseek_v4", value="deepseek-v4", predicate=_is_deepseek_v4),
DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3), DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3),
DetectionRule( DetectionRule(
name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1 name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1
@@ -289,12 +360,22 @@ TOOL_CALL_PARSER_RULES = (
DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax), DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax),
DetectionRule(name="interns1", value="interns1", predicate=_is_interns1), DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
DetectionRule(name="mistral", value="mistral", predicate=_is_mistral), DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
DetectionRule(name="deepseek_v4", value="deepseekv4", predicate=_is_deepseek_v4),
DetectionRule(name="deepseek_v32", value="deepseekv32", predicate=_is_deepseek_v32),
DetectionRule(name="deepseek_v31", value="deepseekv31", predicate=_is_deepseek_v31),
DetectionRule(name="lfm2", value="lfm2", predicate=_is_lfm2),
DetectionRule(name="glm47", value="glm47", predicate=_is_glm47),
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45), DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
DetectionRule(name="minicpm5", value="minicpm5", predicate=_is_minicpm5), DetectionRule(name="minicpm5", value="minicpm5", predicate=_is_minicpm5),
DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
DetectionRule(name="step3", value="step3", predicate=_is_step3),
DetectionRule( DetectionRule(
name="xml_kv_tool_call", value="glm45", predicate=_is_xml_kv_tool_call name="xml_kv_tool_call", value="glm45", predicate=_is_xml_kv_tool_call
), ),
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo), DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
DetectionRule(name="qwen3_coder", value="qwen3_coder", predicate=_is_qwen3_coder),
DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3), DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3),
DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3), DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3),
DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1), DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1),
@@ -424,6 +505,56 @@ def _resolve_auto_parser(
setattr(server_args, attr, None) setattr(server_args, attr, None)
def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[str]:
if not chat_template_arg or not isinstance(chat_template_arg, str):
return None
if not chat_template_arg.endswith(".jinja") or not os.path.exists(
chat_template_arg
):
return None
with open(chat_template_arg, encoding="utf-8") as f:
return f.read().replace("\\n", "\n")
def _disable_auto_parser(server_args, attr: str, label: str) -> None:
logger.warning(
f"--{attr.replace('_', '-')}=auto specified but could not detect "
f"{label} from chat template. Disabling {label}."
)
setattr(server_args, attr, None)
def _resolve_architecture_auto_parsers(server_args) -> None:
from sglang.srt.utils.hf_transformers_utils import get_config
config = get_config(
server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
revision=getattr(server_args, "revision", None),
model_config_parser=getattr(server_args, "model_config_parser", "auto"),
)
architectures = getattr(config, "architectures", None) or []
arch = architectures[0] if architectures else ""
if "DeepseekV4" in arch:
reasoning_parser, tool_call_parser = "deepseek-v4", "deepseekv4"
elif "DeepseekV3" in arch:
reasoning_parser, tool_call_parser = "deepseek-v3", "deepseekv32"
else:
return
for attr, detected in (
("reasoning_parser", reasoning_parser),
("tool_call_parser", tool_call_parser),
):
if getattr(server_args, attr) == "auto":
setattr(server_args, attr, detected)
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' "
f"from model architecture '{arch}'"
)
def resolve_auto_parsers(server_args) -> None: def resolve_auto_parsers(server_args) -> None:
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler. """Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
@@ -438,33 +569,58 @@ def resolve_auto_parsers(server_args) -> None:
from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.srt.utils.hf_transformers_utils import get_tokenizer
chat_template_arg = getattr(server_args, "chat_template", None)
try:
explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
except Exception as e:
logger.warning("Failed to load explicit Jinja chat template: %s", e)
explicit_jinja_template = None
has_explicit_template_without_detection = (
chat_template_arg is not None and explicit_jinja_template is None
)
tokenizer = None
try: try:
tokenizer = get_tokenizer( tokenizer = get_tokenizer(
server_args.model_path, server_args.model_path,
trust_remote_code=server_args.trust_remote_code, trust_remote_code=server_args.trust_remote_code,
) )
template = getattr(tokenizer, "chat_template", None)
except Exception as e: except Exception as e:
logger.warning(f"Failed to load tokenizer for auto-detection: {e}") logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
if needs_reasoning:
logger.warning( template = explicit_jinja_template
"--reasoning-parser=auto specified but could not detect " if template is None and tokenizer is not None:
"reasoning parser from chat template. Disabling reasoning parser." template = getattr(tokenizer, "chat_template", None)
)
server_args.reasoning_parser = None
if needs_tool_call:
logger.warning(
"--tool-call-parser=auto specified but could not detect "
"tool-call parser from chat template. Disabling tool-call parser."
)
server_args.tool_call_parser = None
return
force_reasoning, reasoning_config = detect_reasoning_pattern(template) force_reasoning, reasoning_config = detect_reasoning_pattern(template)
ctx = build_detection_context( ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning template, tokenizer, reasoning_config, force_reasoning
) )
if ctx is None: if ctx is None:
if has_explicit_template_without_detection:
logger.warning(
"--chat-template=%s is explicit but is not a readable Jinja template, so "
"parser auto-detection from chat template is not available.",
chat_template_arg,
)
else:
try:
_resolve_architecture_auto_parsers(server_args)
except Exception as e:
logger.warning(
"Failed to load model config for architecture-based auto-detection: %s",
e,
)
if needs_reasoning:
if server_args.reasoning_parser == "auto":
_disable_auto_parser(
server_args, "reasoning_parser", "reasoning parser"
)
if needs_tool_call:
if server_args.tool_call_parser == "auto":
_disable_auto_parser(
server_args, "tool_call_parser", "tool-call parser"
)
return return
if needs_reasoning: if needs_reasoning:
@@ -1,7 +1,11 @@
import sys
import tempfile
import unittest import unittest
from types import SimpleNamespace from types import ModuleType, SimpleNamespace
from unittest.mock import Mock, patch
from sglang.srt.managers.template_detection import ( from sglang.srt.managers.template_detection import (
REASONING_PARSER_RULES,
TOOL_CALL_PARSER_RULES, TOOL_CALL_PARSER_RULES,
ReasoningToggleConfig, ReasoningToggleConfig,
detect_reasoning_parser, detect_reasoning_parser,
@@ -15,13 +19,22 @@ register_cpu_ci(2.0, "base-a-test-cpu")
class _DummyTokenizer: class _DummyTokenizer:
def __init__(self, vocab): def __init__(self, vocab, chat_template=None):
self._vocab = vocab self._vocab = vocab
self.chat_template = chat_template
def get_vocab(self): def get_vocab(self):
return {token: i for i, token in enumerate(self._vocab)} return {token: i for i, token in enumerate(self._vocab)}
def _patch_hf_transformers_utils(get_tokenizer, get_config=None):
module = ModuleType("sglang.srt.utils.hf_transformers_utils")
module.get_tokenizer = get_tokenizer
if get_config is not None:
module.get_config = get_config
return patch.dict(sys.modules, {module.__name__: module})
class TestTemplateManagerReasoningDetection(unittest.TestCase): class TestTemplateManagerReasoningDetection(unittest.TestCase):
def _detect(self, template, vocab): def _detect(self, template, vocab):
@@ -136,6 +149,90 @@ class TestTemplateDetectionRuleMatrix(unittest.TestCase):
"deepseek-v3", "deepseek-v3",
"thinking", "thinking",
), ),
(
"deepseek_v4_dsml_tool_calls",
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}\n"
'<|DSML|tool_calls><|DSML|invoke name="tool"></|DSML|invoke>',
[],
"deepseek-v4",
"thinking",
),
(
"hunyuan_interleaved_thinking",
"{% set reasoning_effort = reasoning_effort | default('no_think', true) %}\n"
"{% set interleaved_thinking = interleaved_thinking | default(false, true) %}\n"
"<think>reasoning</think><tool_calls><tool_call>name<tool_sep>",
["<tool_calls>", "<tool_sep>", "<arg_key>", "<arg_value>"],
"hunyuan",
None,
),
(
"poolside_v1_enable_thinking_false",
"{% if not enable_thinking is defined %}{% set enable_thinking = false %}{% endif %}\n"
"<tool_call>{{ name }}\n<arg_key>{{ key }}</arg_key><arg_value>{{ value }}</arg_value>",
["<tool_call>", "<arg_key>", "<arg_value>"],
"poolside_v1",
"enable_thinking",
),
(
"poolside_v1_actual_template_shape",
"{% set enable_thinking = enable_thinking | default(false) %}\n"
"Wrap your thinking in '<think>', '</think>' tags, followed by a function call.\n"
"For each function call, return an unescaped XML-like object with function name "
"and arguments within '<tool_call>' and '</tool_call>' tags, like here:\n"
"<tool_call>function-name\n"
"<arg_key>argument-key</arg_key>\n"
"<arg_value>value-of-argument-key</arg_value>\n"
"</tool_call>",
["<tool_call>"],
"poolside_v1",
None,
),
(
"lfm2_not_deepseek_r1_from_history_cleanup",
"{% set keep_past_thinking = keep_past_thinking | default(false) %}\n"
"{% if not keep_past_thinking and '</think>' in content %}"
"{{ content.split('</think>')[-1] }}{% endif %}\n"
'<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>',
["<|tool_call_start|>", "<|tool_call_end|>"],
None,
None,
),
(
"qwen3_coder_actual_template_shape_has_no_reasoning",
"<tools>\n"
"<tool_call><function=get_weather>"
"<parameter=city>Paris</parameter></function></tool_call>",
["<tool_call>"],
None,
None,
),
(
"step3p5_actual_template_shape",
"{% if reasoning_effort is defined %}Reasoning: {{ reasoning_effort }}{% endif %}\n"
"<tool_call><function=get_weather>"
"<parameter=city>Paris</parameter></function></tool_call>\n"
"{% if '<think>' in content %}{{ content.split('</think>')[-1] }}{% endif %}",
["<tool_call>", "<tool_calls>"],
"step3p5",
None,
),
(
"step3p5_think_tags",
"Step3.5-Flash\n<function=tool><parameter=arg>value</parameter></function>\n<think>",
[],
"step3p5",
None,
),
(
"step3_steptml",
"<|tool_calls_begin|><|tool_call_begin|>function<|tool_sep|>"
'<steptml:invoke name="tool"><steptml:parameter name="arg">value</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><think>",
[],
"step3",
None,
),
( (
"qwen3_enable_thinking_true", "qwen3_enable_thinking_true",
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n", "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n",
@@ -267,6 +364,27 @@ class TestToolCallParserDetection(unittest.TestCase):
[], [],
"deepseekv3", "deepseekv3",
), ),
(
"deepseekv31",
(
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}\n"
"<|tool▁calls▁begin|><|tool▁call▁begin|>tool<|tool▁sep|>{}<|tool▁call▁end|>"
),
[],
"deepseekv31",
),
(
"deepseekv32",
'<|DSML|function_calls><|DSML|invoke name="tool"></|DSML|invoke></|DSML|function_calls>',
[],
"deepseekv32",
),
(
"deepseekv4",
'<|DSML|tool_calls><|DSML|invoke name="tool"></|DSML|invoke></|DSML|tool_calls>',
[],
"deepseekv4",
),
( (
"kimi_k2", "kimi_k2",
"{% set thinking = thinking if thinking is defined else true %}\n<think>", "{% set thinking = thinking if thinking is defined else true %}\n<think>",
@@ -284,6 +402,98 @@ class TestToolCallParserDetection(unittest.TestCase):
["<function", "<param"], ["<function", "<param"],
"minicpm5", "minicpm5",
), ),
(
"lfm2",
'<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>',
["<|tool_call_start|>", "<|tool_call_end|>"],
"lfm2",
),
(
"hunyuan",
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Paris</arg_value></tool_call></tool_calls>",
["<tool_calls>", "<tool_sep>", "<arg_key>", "<arg_value>"],
"hunyuan",
),
(
"poolside_v1",
"{% if not enable_thinking is defined %}{% set enable_thinking = false %}{% endif %}\n"
"<tool_call>get_weather\n<arg_key>city</arg_key><arg_value>Paris</arg_value></tool_call>",
["<tool_call>", "<arg_key>", "<arg_value>"],
"poolside_v1",
),
(
"poolside_v1_actual_template_shape",
"{% set enable_thinking = enable_thinking | default(false) %}\n"
"return an unescaped XML-like object with function name and arguments "
"within '<tool_call>' and '</tool_call>' tags\n"
"<tool_call>function-name\n"
"<arg_key>argument-key</arg_key>\n"
"<arg_value>value-of-argument-key</arg_value>\n"
"</tool_call>",
["<tool_call>"],
"poolside_v1",
),
(
"qwen3_coder",
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n"
"<tool_call><function=get_weather><parameter=city>Paris</parameter></function></tool_call>",
["<tool_call>"],
"qwen3_coder",
),
(
"step3p5",
"Step3.5-Flash\n<tool_call><function=get_weather><parameter=city>Paris</parameter></function></tool_call>",
["<tool_call>"],
"step3p5",
),
(
"step3p5_actual_template_shape",
"{% if reasoning_effort is defined %}Reasoning: {{ reasoning_effort }}{% endif %}\n"
"<tool_call><function=get_weather>"
"<parameter=city>Paris</parameter></function></tool_call>",
["<tool_call>", "<tool_calls>"],
"step3p5",
),
(
"step3",
"<|tool_calls_begin|><|tool_call_begin|>function<|tool_sep|>"
'<steptml:invoke name="get_weather"><steptml:parameter name="city">Paris</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>",
[],
"step3",
),
(
"glm47_compact_tool_call",
(
"[gMASK]<sop>\n"
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n"
"{% for tc in m.tool_calls %}\n"
"{{- '<tool_call>' + tc.name -}}\n"
"{% set _args = tc.arguments %}"
"{% for k, v in _args.items() %}"
"<arg_key>{{ k }}</arg_key><arg_value>{{ v }}</arg_value>"
"{% endfor %}</tool_call>\n"
"{% endfor %}"
),
["<tool_call>", "<arg_key>", "<arg_value>", "<|endoftext|>"],
"glm47",
),
(
"glm45_newline_tool_call",
(
"[gMASK]<sop>\n"
"{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n"
"{% for tc in m.tool_calls %}\n"
"{{ '\\n<tool_call>' + tc.name }}\n"
"{% set _args = tc.arguments %}\n"
"{% for k, v in _args.items() %}\n"
"<arg_key>{{ k }}</arg_key>\n<arg_value>{{ v }}</arg_value>\n"
"{% endfor %}\n</tool_call>\n"
"{% endfor %}"
),
["<tool_call>", "<arg_key>", "<arg_value>", "<|endoftext|>"],
"glm45",
),
( (
"xml_kv_tool_call_via_vocab", "xml_kv_tool_call_via_vocab",
"{% set reasoning_effort = reasoning_effort | default('high', true) %}\n<think>", "{% set reasoning_effort = reasoning_effort | default('high', true) %}\n<think>",
@@ -304,8 +514,33 @@ class TestToolCallParserDetection(unittest.TestCase):
# xml_kv_tool_call fallback. Both currently map to "glm45", so the # xml_kv_tool_call fallback. Both currently map to "glm45", so the
# value-based test above can't catch a swap — assert positions directly. # value-based test above can't catch a swap — assert positions directly.
rule_index = {rule.name: i for i, rule in enumerate(TOOL_CALL_PARSER_RULES)} rule_index = {rule.name: i for i, rule in enumerate(TOOL_CALL_PARSER_RULES)}
self.assertLess(rule_index["glm47"], rule_index["glm45"])
self.assertLess(rule_index["glm45"], rule_index["xml_kv_tool_call"]) self.assertLess(rule_index["glm45"], rule_index["xml_kv_tool_call"])
def test_specific_rules_precede_broad_fallbacks(self):
reasoning_index = {
rule.name: i for i, rule in enumerate(REASONING_PARSER_RULES)
}
self.assertLess(reasoning_index["deepseek_v4"], reasoning_index["deepseek_v3"])
self.assertLess(
reasoning_index["hunyuan"], reasoning_index["deepseek_r1_think_tags"]
)
self.assertLess(reasoning_index["poolside_v1"], reasoning_index["mimo"])
self.assertLess(
reasoning_index["step3p5"], reasoning_index["deepseek_r1_think_tags"]
)
self.assertLess(
reasoning_index["step3"], reasoning_index["deepseek_r1_think_tags"]
)
tool_index = {rule.name: i for i, rule in enumerate(TOOL_CALL_PARSER_RULES)}
self.assertLess(tool_index["deepseek_v31"], tool_index["deepseek_v3"])
self.assertLess(tool_index["hunyuan"], tool_index["xml_kv_tool_call"])
self.assertLess(tool_index["poolside_v1"], tool_index["xml_kv_tool_call"])
self.assertLess(tool_index["step3p5"], tool_index["qwen3_coder"])
self.assertLess(tool_index["step3"], tool_index["deepseek_v3"])
self.assertLess(tool_index["qwen3_coder"], tool_index["qwen"])
def test_xml_kv_requires_both_arg_tokens(self): def test_xml_kv_requires_both_arg_tokens(self):
template = "Hello {{ user }}" template = "Hello {{ user }}"
force, config = detect_reasoning_pattern(template) force, config = detect_reasoning_pattern(template)
@@ -345,31 +580,48 @@ class TestToolCallParserDetection(unittest.TestCase):
class TestResolveAutoParsers(unittest.TestCase): class TestResolveAutoParsers(unittest.TestCase):
"""Tests for resolve_auto_parsers() using real model tokenizers.""" """Tests for resolve_auto_parsers()."""
def _make_server_args(self, reasoning_parser=None, tool_call_parser=None): qwen3_template = "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}"
def _make_server_args(
self, reasoning_parser=None, tool_call_parser=None, chat_template=None
):
return SimpleNamespace( return SimpleNamespace(
reasoning_parser=reasoning_parser, reasoning_parser=reasoning_parser,
tool_call_parser=tool_call_parser, tool_call_parser=tool_call_parser,
model_path="Qwen/Qwen3-0.6B", model_path="Qwen/Qwen3-0.6B",
trust_remote_code=False, trust_remote_code=False,
chat_template=chat_template,
) )
def test_resolves_both_parsers_with_real_model(self): def test_resolves_both_parsers_with_tokenizer_template(self):
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto") args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template)
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
resolve_auto_parsers(args) resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "qwen3") self.assertEqual(args.reasoning_parser, "qwen3")
self.assertEqual(args.tool_call_parser, "qwen") self.assertEqual(args.tool_call_parser, "qwen")
def test_resolves_reasoning_parser_only(self): def test_resolves_reasoning_parser_only(self):
args = self._make_server_args(reasoning_parser="auto", tool_call_parser=None) args = self._make_server_args(reasoning_parser="auto", tool_call_parser=None)
tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template)
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
resolve_auto_parsers(args) resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "qwen3") self.assertEqual(args.reasoning_parser, "qwen3")
self.assertIsNone(args.tool_call_parser) self.assertIsNone(args.tool_call_parser)
def test_resolves_tool_call_parser_only(self): def test_resolves_tool_call_parser_only(self):
args = self._make_server_args(reasoning_parser="qwen3", tool_call_parser="auto") args = self._make_server_args(reasoning_parser="qwen3", tool_call_parser="auto")
tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template)
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
resolve_auto_parsers(args) resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "qwen3") self.assertEqual(args.reasoning_parser, "qwen3")
self.assertEqual(args.tool_call_parser, "qwen") self.assertEqual(args.tool_call_parser, "qwen")
@@ -386,10 +638,102 @@ class TestResolveAutoParsers(unittest.TestCase):
model_path="nonexistent/model-does-not-exist-xyz", model_path="nonexistent/model-does-not-exist-xyz",
trust_remote_code=False, trust_remote_code=False,
) )
with _patch_hf_transformers_utils(
Mock(side_effect=RuntimeError("tokenizer unavailable")),
Mock(side_effect=RuntimeError("config unavailable")),
):
resolve_auto_parsers(args) resolve_auto_parsers(args)
self.assertIsNone(args.reasoning_parser) self.assertIsNone(args.reasoning_parser)
self.assertIsNone(args.tool_call_parser) self.assertIsNone(args.tool_call_parser)
def test_none_chat_template_disables_both_parsers(self):
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
tokenizer = _DummyTokenizer([])
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
resolve_auto_parsers(args)
self.assertIsNone(args.reasoning_parser)
self.assertIsNone(args.tool_call_parser)
def test_deepseek_v32_arch_without_chat_template_uses_custom_encoder(self):
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
tokenizer = _DummyTokenizer([])
config = SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
with _patch_hf_transformers_utils(
Mock(return_value=tokenizer), Mock(return_value=config)
):
resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "deepseek-v3")
self.assertEqual(args.tool_call_parser, "deepseekv32")
def test_deepseek_v4_arch_without_chat_template_uses_custom_encoder(self):
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
tokenizer = _DummyTokenizer([])
config = SimpleNamespace(architectures=["DeepseekV4ForCausalLM"])
with _patch_hf_transformers_utils(
Mock(return_value=tokenizer), Mock(return_value=config)
):
resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "deepseek-v4")
self.assertEqual(args.tool_call_parser, "deepseekv4")
def test_deepseek_arch_fallback_runs_when_tokenizer_load_fails(self):
args = self._make_server_args(reasoning_parser="auto", tool_call_parser="auto")
config = SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
with _patch_hf_transformers_utils(
Mock(side_effect=RuntimeError("tokenizer unavailable")),
Mock(return_value=config),
):
resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "deepseek-v3")
self.assertEqual(args.tool_call_parser, "deepseekv32")
def test_explicit_non_jinja_template_skips_architecture_fallback(self):
args = self._make_server_args(
reasoning_parser="auto",
tool_call_parser="auto",
chat_template="chatml",
)
args.model_path = "deepseek-ai/DeepSeek-V3.2"
tokenizer = _DummyTokenizer([])
get_config = Mock()
with _patch_hf_transformers_utils(Mock(return_value=tokenizer), get_config):
resolve_auto_parsers(args)
get_config.assert_not_called()
self.assertIsNone(args.reasoning_parser)
self.assertIsNone(args.tool_call_parser)
def test_explicit_jinja_template_takes_precedence(self):
tokenizer = _DummyTokenizer([], chat_template=None)
with tempfile.NamedTemporaryFile("w", suffix=".jinja") as f:
f.write(
"{% if not thinking is defined %}{% set thinking = false %}{% endif %}\n"
'<|DSML|function_calls><|DSML|invoke name="tool"></|DSML|invoke>'
)
f.flush()
args = self._make_server_args(
reasoning_parser="auto",
tool_call_parser="auto",
chat_template=f.name,
)
with _patch_hf_transformers_utils(Mock(return_value=tokenizer)):
resolve_auto_parsers(args)
self.assertEqual(args.reasoning_parser, "deepseek-v3")
self.assertEqual(args.tool_call_parser, "deepseekv32")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()