minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Expand parser auto detection coverage (#28449)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-06-23 12:26:37 -07:00
committed by GitHub
parent e0dc8b7137
commit 0c6e8e9477
2 changed files with 524 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/template_detection.py
+171 -15
View File
@@ -19,6 +19,7 @@ parser from chat templates and tokenizer vocabularies.
"""
import logging
import os
import re
from dataclasses import dataclass
from typing import Callable, Optional, Tuple
@@ -203,6 +204,12 @@ def _is_glm45(ctx):
)
def _is_glm47(ctx):
return _is_glm45(ctx) and ctx.has_pattern(
r"\{\{[-\s]*['\"]<tool_call>['\"]\s*\+\s*tc\.name"
)
def _is_xml_kv_tool_call(ctx):
# Structural signature for the GLM-4.5 / GLM-4.6 style tool-call format
# (`<tool_call>name<arg_key>k</arg_key>\n<arg_value>v</arg_value>...</tool_call>`).
@@ -213,6 +220,41 @@ def _is_xml_kv_tool_call(ctx):
return ctx.has_vocab("<arg_key>") and ctx.has_vocab("<arg_value>")
def _is_deepseek_v31(ctx):
return ctx.has_text("<｜tool▁calls▁begin｜>") and ctx.has_text("<｜tool▁sep｜>")
def _is_deepseek_v32(ctx):
return ctx.has_text("<｜DSML｜function_calls>")
def _is_deepseek_v4(ctx):
return ctx.has_text("<｜DSML｜tool_calls>")
def _is_hunyuan(ctx):
return (
(ctx.has_text("<tool_calls>") or ctx.has_vocab("<tool_calls>"))
and (ctx.has_text("<tool_sep>") or ctx.has_vocab("<tool_sep>"))
) or (ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking"))
def _is_poolside_v1(ctx):
has_poolside_tool_format = (
ctx.has_text("unescaped XML-like object")
and ctx.has_text("<tool_call>function-name")
and ctx.has_text("<arg_key>")
and ctx.has_text("<arg_value>")
)
return has_poolside_tool_format or (
ctx.reasoning_config
== ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=False)
and not _is_hunyuan(ctx)
and (ctx.has_text("<arg_key>") or ctx.has_vocab("<arg_key>"))
and (ctx.has_text("<arg_value>") or ctx.has_vocab("<arg_value>"))
)
def _is_mimo(ctx):
return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=False
@@ -229,6 +271,30 @@ def _is_minicpm5(ctx):
return ctx.has_pattern(r"<function\s+name=") and ctx.has_pattern(r"<param\s+name=")
def _is_lfm2(ctx):
return (
ctx.has_text("<|tool_call_start|>") or ctx.has_vocab("<|tool_call_start|>")
) and (ctx.has_text("<|tool_call_end|>") or ctx.has_vocab("<|tool_call_end|>"))
def _is_step3p5(ctx):
return ctx.has_pattern(r"Step-?3(?:\.|p)?[57]", re.IGNORECASE) or (
ctx.has_text("reasoning_effort")
and ctx.has_text("Reasoning: ")
and _is_qwen3_coder(ctx)
)
def _is_step3(ctx):
return ctx.has_text("<steptml:invoke") or (
ctx.has_text("<｜tool_calls_begin｜>") and ctx.has_text("<｜tool_sep｜>")
)
def _is_qwen3_coder(ctx):
return ctx.has_text("<function=") and ctx.has_text("<parameter=")
def _is_qwen3(ctx):
return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
@@ -246,7 +312,7 @@ def _is_deepseek_r1(ctx):
def _is_deepseek_r1_think_tags(ctx):
return ctx.has_text("<think>") or ctx.has_text("</think>")
return not _is_lfm2(ctx) and (ctx.has_text("<think>") or ctx.has_text("</think>"))
# ---------------------------------------------------------------------------
@@ -263,9 +329,14 @@ REASONING_PARSER_RULES = (
DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3),
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
DetectionRule(name="minimax", value="minimax", predicate=_is_minimax),
DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
DetectionRule(name="step3", value="step3", predicate=_is_step3),
DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3),
DetectionRule(name="deepseek_v4", value="deepseek-v4", predicate=_is_deepseek_v4),
DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3),
DetectionRule(
name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1
@@ -289,12 +360,22 @@ TOOL_CALL_PARSER_RULES = (
DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax),
DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
DetectionRule(name="deepseek_v4", value="deepseekv4", predicate=_is_deepseek_v4),
DetectionRule(name="deepseek_v32", value="deepseekv32", predicate=_is_deepseek_v32),
DetectionRule(name="deepseek_v31", value="deepseekv31", predicate=_is_deepseek_v31),
DetectionRule(name="lfm2", value="lfm2", predicate=_is_lfm2),
DetectionRule(name="glm47", value="glm47", predicate=_is_glm47),
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
DetectionRule(name="minicpm5", value="minicpm5", predicate=_is_minicpm5),
DetectionRule(name="hunyuan", value="hunyuan", predicate=_is_hunyuan),
DetectionRule(name="poolside_v1", value="poolside_v1", predicate=_is_poolside_v1),
DetectionRule(name="step3p5", value="step3p5", predicate=_is_step3p5),
DetectionRule(name="step3", value="step3", predicate=_is_step3),
DetectionRule(
name="xml_kv_tool_call", value="glm45", predicate=_is_xml_kv_tool_call
),
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
DetectionRule(name="qwen3_coder", value="qwen3_coder", predicate=_is_qwen3_coder),
DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3),
DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3),
DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1),
@@ -424,6 +505,56 @@ def _resolve_auto_parser(
setattr(server_args, attr, None)
def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[str]:
if not chat_template_arg or not isinstance(chat_template_arg, str):
return None
if not chat_template_arg.endswith(".jinja") or not os.path.exists(
chat_template_arg
):
return None
with open(chat_template_arg, encoding="utf-8") as f:
return f.read().replace("\\n", "\n")
def _disable_auto_parser(server_args, attr: str, label: str) -> None:
logger.warning(
f"--{attr.replace('_', '-')}=auto specified but could not detect "
f"{label} from chat template. Disabling {label}."
)
setattr(server_args, attr, None)
def _resolve_architecture_auto_parsers(server_args) -> None:
from sglang.srt.utils.hf_transformers_utils import get_config
config = get_config(
server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
revision=getattr(server_args, "revision", None),
model_config_parser=getattr(server_args, "model_config_parser", "auto"),
)
architectures = getattr(config, "architectures", None) or []
arch = architectures[0] if architectures else ""
if "DeepseekV4" in arch:
reasoning_parser, tool_call_parser = "deepseek-v4", "deepseekv4"
elif "DeepseekV3" in arch:
reasoning_parser, tool_call_parser = "deepseek-v3", "deepseekv32"
else:
return
for attr, detected in (
("reasoning_parser", reasoning_parser),
("tool_call_parser", tool_call_parser),
):
if getattr(server_args, attr) == "auto":
setattr(server_args, attr, detected)
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' "
f"from model architecture '{arch}'"
)
def resolve_auto_parsers(server_args) -> None:
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
@@ -438,33 +569,58 @@ def resolve_auto_parsers(server_args) -> None:
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
chat_template_arg = getattr(server_args, "chat_template", None)
try:
explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
except Exception as e:
logger.warning("Failed to load explicit Jinja chat template: %s", e)
explicit_jinja_template = None
has_explicit_template_without_detection = (
chat_template_arg is not None and explicit_jinja_template is None
)
tokenizer = None
try:
tokenizer = get_tokenizer(
server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
)
template = getattr(tokenizer, "chat_template", None)
except Exception as e:
logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
if needs_reasoning:
logger.warning(
"--reasoning-parser=auto specified but could not detect "
"reasoning parser from chat template. Disabling reasoning parser."
)
server_args.reasoning_parser = None
if needs_tool_call:
logger.warning(
"--tool-call-parser=auto specified but could not detect "
"tool-call parser from chat template. Disabling tool-call parser."
)
server_args.tool_call_parser = None
return
template = explicit_jinja_template
if template is None and tokenizer is not None:
template = getattr(tokenizer, "chat_template", None)
force_reasoning, reasoning_config = detect_reasoning_pattern(template)
ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning
)
if ctx is None:
if has_explicit_template_without_detection:
logger.warning(
"--chat-template=%s is explicit but is not a readable Jinja template, so "
"parser auto-detection from chat template is not available.",
chat_template_arg,
)
else:
try:
_resolve_architecture_auto_parsers(server_args)
except Exception as e:
logger.warning(
"Failed to load model config for architecture-based auto-detection: %s",
e,
)
if needs_reasoning:
if server_args.reasoning_parser == "auto":
_disable_auto_parser(
server_args, "reasoning_parser", "reasoning parser"
)
if needs_tool_call:
if server_args.tool_call_parser == "auto":
_disable_auto_parser(
server_args, "tool_call_parser", "tool-call parser"
)
return
if needs_reasoning: