minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

feat(reasoning): auto-detect reasoning/tool-call parser from chat template (#23952)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-05-07 14:19:16 -07:00
committed by GitHub
parent d2c1034163
commit d8f9d32a05
9 changed files with 1182 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/engine.py
+34
View File
@@ -82,6 +82,7 @@ from sglang.srt.managers.io_struct import (
)
from sglang.srt.managers.multi_tokenizer_mixin import MultiTokenizerRouter
from sglang.srt.managers.scheduler import run_scheduler_process
from sglang.srt.managers.template_detection import resolve_auto_parsers
from sglang.srt.managers.template_manager import TemplateManager
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info
@@ -140,6 +141,33 @@ def init_tokenizer_manager(
completion_template=server_args.completion_template,
)
# Resolve any remaining auto parsers using template manager's detection results
for attr, suggested, label in (
(
"reasoning_parser",
template_manager.suggested_reasoning_parser,
"reasoning parser",
),
(
"tool_call_parser",
template_manager.suggested_tool_call_parser,
"tool-call parser",
),
):
if getattr(server_args, attr) != "auto":
continue
if suggested is not None:
setattr(server_args, attr, suggested)
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{suggested}' from chat template"
)
else:
logger.warning(
f"--{attr.replace('_', '-')}=auto specified but could not detect "
f"{label} from chat template. Disabling {label}."
)
setattr(server_args, attr, None)
return tokenizer_manager, template_manager
@@ -695,6 +723,12 @@ class Engine(EngineScoreMixin, EngineBase):
host=server_args.host, port=bootstrap_port
)
if (
server_args.reasoning_parser == "auto"
or server_args.tool_call_parser == "auto"
):
resolve_auto_parsers(server_args)
# Launch scheduler processes
scheduler_init_result, scheduler_procs = cls._launch_scheduler_processes(
server_args, port_args, run_scheduler_process_func
python/sglang/srt/entrypoints/openai/serving_chat.py
+73 -38
View File
@@ -193,6 +193,19 @@ class OpenAIServingChat(OpenAIServingBase):
self.template_manager = template_manager
self.tool_call_parser = self.tokenizer_manager.server_args.tool_call_parser
self.reasoning_parser = self.tokenizer_manager.server_args.reasoning_parser
self._reasoning_detector = None
if self.reasoning_parser:
try:
rp = ReasoningParser(
model_type=self.reasoning_parser, stream_reasoning=True
)
self._reasoning_detector = rp.detector
except ValueError as e:
logger.warning(
"Failed to initialize reasoning detector for parser '%s': %s",
self.reasoning_parser,
e,
)
# Get default sampling parameters from model's generation config
self.default_sampling_params = (
@@ -683,10 +696,11 @@ class OpenAIServingChat(OpenAIServingBase):
prompt = prompt[: -len(conv.sep2)]
else:
prompt = conv.get_prompt()
if self._get_reasoning_from_request(
request
) and self.reasoning_parser not in ["qwen3", "qwen3-thinking", "glm4"]:
# qwen3 and glm4 think internally without a leading <think> token
if self._get_reasoning_from_request(request) and (
self._reasoning_detector is None
or not self._reasoning_detector.thinks_internally
):
# Models with thinks_internally=True think without a leading <think> token
prompt += "<think>" # Note(Xinyuan): hard code thinking token
image_data = conv.image_data if conv.image_data else None
@@ -1395,54 +1409,75 @@ class OpenAIServingChat(OpenAIServingBase):
request.skip_special_tokens = False
def _get_reasoning_from_request(self, request: ChatCompletionRequest) -> bool:
"""Judge whether the request needs reasoning for hybrid reasoning models
"""Determine whether reasoning mode should be enabled for this request.
NOTE: This is predefined based on model's chat template
"""
if not self.reasoning_parser:
return False
if self.reasoning_parser == "deepseek-v3":
# Models that require explicit enable thinking (thinking=True)
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get("thinking") is True
)
if self.reasoning_parser == "gemma4":
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get("enable_thinking") is True
)
if self.reasoning_parser in ["kimi_k2"]:
# Models that thinking by default, and can be disabled by setting thinking=False
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get("thinking") is not False
)
if self.reasoning_parser in ["qwen3", "glm45", "nemotron_3", "interns1"]:
# Models that thinking by default, and can be disabled by setting enable_thinking=False
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get("enable_thinking") is not False
)
if self.reasoning_parser in ["mimo"]:
# Models that require explicit enable thinking (enable_thinking=True)
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get("enable_thinking") is True
)
if self.reasoning_parser == "hunyuan":
# Hy3-preview template emits no <think> when reasoning_effort is
# "no_think" / "none" / unset; forcing reasoning would route all
# output into reasoning_content.
return request.reasoning_effort not in (None, "none", "no_think")
if self.reasoning_parser == "mistral":
# Mistral only reasons when reasoning_effort is explicitly set
# to a non-"none" value (typically "high").
config = self.template_manager.reasoning_config
if config is None:
# Fallback to parser-level defaults when template toggle config
# cannot be inferred (e.g., parser-only <think> templates).
mode = (
self._reasoning_detector.reasoning_default
if self._reasoning_detector is not None
else None
)
if mode is None:
return False
if mode == "always":
return True
if mode == "mistral":
return (
request.reasoning_effort is not None
and request.reasoning_effort != "none"
)
if mode in ("thinking", "enable_thinking"):
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get(mode) is not False
)
if mode in ("explicit_thinking", "explicit_enable_thinking"):
toggle = mode.replace("explicit_", "")
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get(toggle) is True
)
logger.warning(
"Unknown reasoning_default mode '%s', defaulting to reasoning disabled",
mode,
)
return False
if config.special_case == "always":
return True
if config.special_case == "mistral":
return (
request.reasoning_effort is not None
and request.reasoning_effort != "none"
)
return True # default
if config.toggle_param is None or config.default_enabled is None:
return False
if config.default_enabled:
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get(config.toggle_param) is not False
)
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get(config.toggle_param) is True
)
async def _process_tool_call_stream(
self,
python/sglang/srt/managers/template_detection.py
+471
View File
@@ -0,0 +1,471 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Template detection utilities for auto-detecting reasoning and tool-call parsers.
Provides rule-based detection of reasoning mode, reasoning parser, and tool-call
parser from chat templates and tokenizer vocabularies.
"""
import logging
import re
from dataclasses import dataclass
from typing import Callable, Optional, Tuple
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class TemplateDetectionContext:
template: str
reasoning_config: Optional["ReasoningToggleConfig"]
force_reasoning: bool
vocab: set[str]
def has_text(self, needle: str) -> bool:
return needle in self.template
def has_vocab(self, token: str) -> bool:
return token in self.vocab
def has_pattern(self, pattern: str, flags: int = 0) -> bool:
return re.search(pattern, self.template, flags) is not None
@dataclass(frozen=True)
class DetectionRule:
name: str
value: object
predicate: Callable[[TemplateDetectionContext], bool]
@dataclass(frozen=True)
class ReasoningToggleConfig:
toggle_param: Optional[str] = None
default_enabled: Optional[bool] = None
special_case: Optional[str] = None
@property
def always_on(self) -> bool:
return self.special_case == "always"
# ---------------------------------------------------------------------------
# Reasoning mode rules (detect toggle config from template)
# ---------------------------------------------------------------------------
REASONING_MODE_RULES = (
DetectionRule(
name="gpt_oss_channel_markers",
value=ReasoningToggleConfig(special_case="always"),
predicate=lambda ctx: ctx.has_text("<|channel|>"),
),
DetectionRule(
name="force_reasoning_pattern",
value=ReasoningToggleConfig(special_case="always"),
predicate=lambda ctx: ctx.has_pattern(r"<\|im_start\|>assistant\\n<think>\\n")
and not ctx.has_text("enable_thinking")
and not ctx.has_text("thinking"),
),
DetectionRule(
name="mistral_reasoning_effort",
value=ReasoningToggleConfig(special_case="mistral"),
predicate=lambda ctx: ctx.has_text("reasoning_effort")
and ctx.has_text("[THINK]"),
),
DetectionRule(
name="explicit_enable_thinking_default_false",
value=ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=False
),
predicate=lambda ctx: ctx.has_pattern(
r"{%\s*if\s+not\s+enable_thinking\s+is\s+defined\s*%}.*?"
r"{%\s*set\s+enable_thinking\s*=\s*(?:false|False)\s*%}",
re.DOTALL,
),
),
DetectionRule(
name="enable_thinking_default_true",
value=ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
),
predicate=lambda ctx: ctx.has_pattern(
r"{%\s*if\s+not\s+enable_thinking\s+is\s+defined\s*%}.*?"
r"{%\s*set\s+enable_thinking\s*=\s*(?:true|True)\s*%}",
re.DOTALL,
)
or ctx.has_pattern(
r"set\s+enable_thinking\s*=\s*enable_thinking\s+if\s+enable_thinking\s+is\s+defined\s+else\s+(?:true|True)"
)
or ctx.has_pattern(
r"enable_thinking\s+is\s+defined\s+and\s+(?:enable_thinking\s+is\s+false|not\s+enable_thinking)"
)
or ctx.has_pattern(
r"enable_thinking\s+is\s+not\s+defined\s+or\s+enable_thinking"
)
or ctx.has_pattern(r"namespace\([^)]*enable_thinking\s*=\s*true"),
),
DetectionRule(
name="explicit_thinking_default_false",
value=ReasoningToggleConfig(toggle_param="thinking", default_enabled=False),
predicate=lambda ctx: ctx.has_pattern(
r"{%\s*if\s+not\s+thinking\s+is\s+defined\s*%}.*?"
r"{%\s*set\s+thinking\s*=\s*(?:false|False)\s*%}",
re.DOTALL,
),
),
DetectionRule(
name="thinking_default_true",
value=ReasoningToggleConfig(toggle_param="thinking", default_enabled=True),
predicate=lambda ctx: ctx.has_pattern(
r"{%\s*if\s+not\s+thinking\s+is\s+defined\s*%}.*?"
r"{%\s*set\s+thinking\s*=\s*(?:true|True)\s*%}",
re.DOTALL,
)
or ctx.has_pattern(
r"set\s+thinking\s*=\s*thinking\s+if\s+thinking\s+is\s+defined\s+else\s+(?:true|True)"
)
or ctx.has_pattern(
r"thinking\s+is\s+defined\s+and\s+(?:thinking\s+is\s+false|not\s+thinking)"
)
or ctx.has_pattern(r"thinking\s+is\s+not\s+defined\s+or\s+thinking")
or ctx.has_pattern(r"namespace\([^)]*thinking\s*=\s*true"),
),
)
# ---------------------------------------------------------------------------
# Shared predicates for model-family detection
# ---------------------------------------------------------------------------
def _is_gemma4(ctx):
return ctx.has_text("<|channel>")
def _is_kimi(ctx):
return ctx.has_text("◁think▷")
def _is_interns1(ctx):
return ctx.has_text("default_thinking_sys") and ctx.reasoning_config == (
ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True)
)
def _is_mistral(ctx):
return (
ctx.reasoning_config is not None
and ctx.reasoning_config.special_case == "mistral"
)
def _is_gpt_oss(ctx):
return ctx.has_text("<|channel|>")
def _is_kimi_k2(ctx):
return ctx.has_vocab("<|tool_calls_section_begin|>")
def _is_nemotron_3(ctx):
return ctx.has_text("truncate_history_thinking") and ctx.reasoning_config == (
ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True)
)
def _is_glm45(ctx):
return (
(
ctx.has_text("[gMASK]<sop>")
or ctx.has_pattern(r"(?<!<)/nothink")
or ctx.has_pattern(r"(?<!<)/think")
)
and ctx.has_vocab("<tool_call>")
and ctx.reasoning_config
== ReasoningToggleConfig(toggle_param="enable_thinking", default_enabled=True)
and (ctx.has_vocab("<|user|>") or ctx.has_vocab("<|endoftext|>"))
)
def _is_mimo(ctx):
return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=False
)
def _is_minimax(ctx):
return ctx.has_text("<minimax:tool_call>")
def _is_qwen3(ctx):
return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
)
def _is_deepseek_v3(ctx):
return ctx.reasoning_config == ReasoningToggleConfig(
toggle_param="thinking", default_enabled=False
)
def _is_deepseek_r1(ctx):
return ctx.force_reasoning
def _is_deepseek_r1_think_tags(ctx):
return ctx.has_text("<think>") or ctx.has_text("</think>")
# ---------------------------------------------------------------------------
# Reasoning parser rules
# ---------------------------------------------------------------------------
REASONING_PARSER_RULES = (
DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
DetectionRule(name="kimi", value="kimi", predicate=_is_kimi),
DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
DetectionRule(name="gpt_oss", value="gpt-oss", predicate=_is_gpt_oss),
DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
DetectionRule(name="nemotron_3", value="nemotron_3", predicate=_is_nemotron_3),
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
DetectionRule(name="minimax", value="minimax", predicate=_is_minimax),
DetectionRule(name="qwen3", value="qwen3", predicate=_is_qwen3),
DetectionRule(name="deepseek_v3", value="deepseek-v3", predicate=_is_deepseek_v3),
DetectionRule(
name="deepseek_r1_force", value="deepseek-r1", predicate=_is_deepseek_r1
),
DetectionRule(
name="deepseek_r1_think_tags",
value="deepseek-r1",
predicate=_is_deepseek_r1_think_tags,
),
)
# ---------------------------------------------------------------------------
# Tool-call parser rules (reuse shared predicates, different values)
# ---------------------------------------------------------------------------
TOOL_CALL_PARSER_RULES = (
DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
DetectionRule(name="gpt_oss", value="gpt-oss", predicate=_is_gpt_oss),
DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2),
DetectionRule(name="minimax", value="minimax-m2", predicate=_is_minimax),
DetectionRule(name="interns1", value="interns1", predicate=_is_interns1),
DetectionRule(name="mistral", value="mistral", predicate=_is_mistral),
DetectionRule(name="glm45", value="glm45", predicate=_is_glm45),
DetectionRule(name="mimo", value="mimo", predicate=_is_mimo),
DetectionRule(name="qwen", value="qwen", predicate=_is_qwen3),
DetectionRule(name="deepseek_v3", value="deepseekv3", predicate=_is_deepseek_v3),
DetectionRule(name="deepseek_r1", value="deepseekv3", predicate=_is_deepseek_r1),
)
# ---------------------------------------------------------------------------
# Detection functions
# ---------------------------------------------------------------------------
def build_detection_context(
template: Optional[str],
tokenizer,
reasoning_config: Optional[ReasoningToggleConfig] = None,
force_reasoning: bool = False,
) -> Optional[TemplateDetectionContext]:
if template is None:
return None
vocab = set()
if tokenizer is not None:
try:
vocab = set(tokenizer.get_vocab().keys())
except Exception as e:
logger.warning(
"Failed to load tokenizer vocab for template detection: %s. "
"Vocab-dependent detection rules will be skipped.",
e,
)
return TemplateDetectionContext(
template=template,
reasoning_config=reasoning_config,
force_reasoning=force_reasoning,
vocab=vocab,
)
def match_rules(
ctx: TemplateDetectionContext,
rules: Tuple[DetectionRule, ...],
label: str,
) -> Optional[str]:
for rule in rules:
try:
if rule.predicate(ctx):
logger.info(
"Detected %s '%s' from template rule '%s'.",
label,
rule.value,
rule.name,
)
return rule.value
except Exception as e:
logger.warning(
"Detection rule '%s' for %s raised an exception: %s. Skipping.",
rule.name,
label,
e,
exc_info=True,
)
return None
def detect_reasoning_pattern(
template: Optional[str],
) -> Tuple[bool, Optional[ReasoningToggleConfig]]:
"""Detect if the chat template contains reasoning/thinking patterns."""
if template is None:
return False, None
ctx = TemplateDetectionContext(
template=template,
reasoning_config=None,
force_reasoning=False,
vocab=set(),
)
for rule in REASONING_MODE_RULES:
if rule.predicate(ctx):
logger.info(
"Detected reasoning config '%s' from template rule '%s'.",
rule.value,
rule.name,
)
return rule.value.always_on, rule.value
return False, None
def detect_reasoning_parser(
template: Optional[str],
tokenizer,
reasoning_config: Optional[ReasoningToggleConfig] = None,
force_reasoning: bool = False,
) -> Optional[str]:
"""Auto-detect which reasoning parser to use from the chat template."""
ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning
)
if ctx is None:
return None
return match_rules(ctx, REASONING_PARSER_RULES, "reasoning parser")
def detect_tool_call_parser(
template: Optional[str],
tokenizer,
reasoning_config: Optional[ReasoningToggleConfig] = None,
force_reasoning: bool = False,
) -> Optional[str]:
"""Auto-detect which tool-call parser to use from the chat template."""
ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning
)
if ctx is None:
return None
return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser")
def _resolve_auto_parser(
server_args,
attr: str,
ctx: TemplateDetectionContext,
rules: Tuple[DetectionRule, ...],
label: str,
) -> None:
"""Resolve a single auto parser, updating server_args in place."""
detected = match_rules(ctx, rules, label)
if detected:
setattr(server_args, attr, detected)
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template"
)
else:
logger.warning(
f"--{attr.replace('_', '-')}=auto specified but could not detect "
f"{label} from chat template. Disabling {label}."
)
setattr(server_args, attr, None)
def resolve_auto_parsers(server_args) -> None:
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
This performs a lightweight tokenizer load to detect parsers from the chat
template. Called early in engine init before scheduler subprocesses are spawned.
"""
needs_reasoning = server_args.reasoning_parser == "auto"
needs_tool_call = server_args.tool_call_parser == "auto"
if not needs_reasoning and not needs_tool_call:
return
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
try:
tokenizer = get_tokenizer(
server_args.model_path,
trust_remote_code=server_args.trust_remote_code,
)
template = getattr(tokenizer, "chat_template", None)
except Exception as e:
logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
if needs_reasoning:
logger.warning(
"--reasoning-parser=auto specified but could not detect "
"reasoning parser from chat template. Disabling reasoning parser."
)
server_args.reasoning_parser = None
if needs_tool_call:
logger.warning(
"--tool-call-parser=auto specified but could not detect "
"tool-call parser from chat template. Disabling tool-call parser."
)
server_args.tool_call_parser = None
return
force_reasoning, reasoning_config = detect_reasoning_pattern(template)
ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning
)
if ctx is None:
return
if needs_reasoning:
_resolve_auto_parser(
server_args,
"reasoning_parser",
ctx,
REASONING_PARSER_RULES,
"reasoning parser",
)
if needs_tool_call:
_resolve_auto_parser(
server_args,
"tool_call_parser",
ctx,
TOOL_CALL_PARSER_RULES,
"tool-call parser",
)
python/sglang/srt/managers/template_manager.py
+52 -17
View File
@@ -21,9 +21,16 @@ and code completion templates, eliminating global state and improving modularity
import json
import logging
import os
import re
from typing import Dict, Optional
from sglang.srt.managers.template_detection import (
REASONING_PARSER_RULES,
TOOL_CALL_PARSER_RULES,
ReasoningToggleConfig,
build_detection_context,
detect_reasoning_pattern,
match_rules,
)
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.parser.code_completion_parser import (
CompletionTemplate,
@@ -58,6 +65,9 @@ class TemplateManager:
self._completion_template_name: Optional[str] = None
self._jinja_template_content_format: Optional[str] = "openai"
self._force_reasoning: bool = False
self._reasoning_config: Optional[ReasoningToggleConfig] = None
self._suggested_reasoning_parser: Optional[str] = None
self._suggested_tool_call_parser: Optional[str] = None
@property
def chat_template_name(self) -> Optional[str]:
@@ -84,21 +94,39 @@ class TemplateManager:
"""
return self._force_reasoning
def _detect_reasoning_pattern(self, template: str) -> bool:
"""
Detect if the chat template contains reasoning/thinking patterns.
"""
if template is None:
return False
@property
def reasoning_config(self) -> Optional[ReasoningToggleConfig]:
"""Get the reasoning toggle config inferred from chat template."""
return self._reasoning_config
# TODO: remove this hard code the reasoning pattern
force_reasoning_pattern = r"<\|im_start\|>assistant\\n<think>\\n"
has_reasoning = re.search(force_reasoning_pattern, template) is not None
@property
def suggested_reasoning_parser(self) -> Optional[str]:
"""Get the auto-detected reasoning parser name, or None."""
return self._suggested_reasoning_parser
if has_reasoning:
logger.info("Detected the force reasoning pattern in chat template.")
@property
def suggested_tool_call_parser(self) -> Optional[str]:
"""Get the auto-detected tool-call parser name, or None."""
return self._suggested_tool_call_parser
return has_reasoning
def _run_template_detection(self, template, tokenizer) -> None:
"""Run reasoning pattern and parser detection on a template."""
self._force_reasoning, self._reasoning_config = detect_reasoning_pattern(
template
)
# Build context once, reuse for both parser detections (avoids
# duplicate tokenizer.get_vocab() calls).
ctx = build_detection_context(
template, tokenizer, self._reasoning_config, self._force_reasoning
)
if ctx is None:
return
self._suggested_reasoning_parser = match_rules(
ctx, REASONING_PARSER_RULES, "reasoning parser"
)
self._suggested_tool_call_parser = match_rules(
ctx, TOOL_CALL_PARSER_RULES, "tool-call parser"
)
def load_chat_template(
self,
@@ -141,11 +169,18 @@ class TemplateManager:
"No chat template found, defaulting to 'string' content format"
)
# Detect reasoning pattern from chat template
# Detect reasoning pattern and suggest parser from chat template
if tokenizer_manager.tokenizer:
self._force_reasoning = self._detect_reasoning_pattern(
tokenizer_manager.tokenizer.chat_template
)
template = tokenizer_manager.tokenizer.chat_template
self._run_template_detection(template, tokenizer_manager.tokenizer)
if self._suggested_reasoning_parser:
logger.info(
f"Auto-detected reasoning parser: {self._suggested_reasoning_parser}"
)
if self._suggested_tool_call_parser:
logger.info(
f"Auto-detected tool-call parser: {self._suggested_tool_call_parser}"
)
def _load_explicit_chat_template(
self, tokenizer_manager: TokenizerManager, chat_template_arg: str
python/sglang/srt/parser/reasoning_parser.py
+36 -3
View File
@@ -28,12 +28,17 @@ class BaseReasoningFormatDetector:
tool_start_token: Optional[str] = None,
continue_final_message: bool = False,
previous_content: str = "",
thinks_internally: bool = False,
reasoning_default: str = "always",
):
self.think_start_token = think_start_token
self.think_end_token = think_end_token
self.tool_start_token = tool_start_token
self.force_reasoning = force_reasoning
self._in_reasoning = force_reasoning
self.stream_reasoning = stream_reasoning
self.thinks_internally = thinks_internally
self.reasoning_default = reasoning_default
self._buffer = ""
self.stripped_think_start = False
@@ -244,6 +249,8 @@ class Qwen3Detector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
thinks_internally=True,
reasoning_default="enable_thinking",
)
@@ -298,6 +305,7 @@ class KimiK2Detector(BaseReasoningFormatDetector):
tool_start_token="<|tool_calls_section_begin|>",
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="thinking",
)
@@ -321,6 +329,8 @@ class Glm45Detector(BaseReasoningFormatDetector):
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token="<tool_call>",
thinks_internally=True,
reasoning_default="enable_thinking",
)
@@ -445,6 +455,7 @@ class Nemotron3Detector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="enable_thinking",
)
self._force_nonempty_content = force_nonempty_content
@@ -479,6 +490,7 @@ class MistralDetector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="mistral",
)
@@ -524,10 +536,27 @@ class Gemma4Detector(BaseReasoningFormatDetector):
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="explicit_enable_thinking",
)
self.think_start_self_label = "thought\n"
class _DeepSeekV3Detector(Qwen3Detector):
"""DeepSeek-V3 reuses Qwen3 tokens but requires explicit thinking=True to enable."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reasoning_default = "explicit_thinking"
class _MimoDetector(Qwen3Detector):
"""MIMO reuses Qwen3 tokens but requires explicit enable_thinking=True to enable."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reasoning_default = "explicit_enable_thinking"
class ReasoningParser:
"""
Parser that handles both streaming and non-streaming scenarios for extracting
@@ -541,13 +570,13 @@ class ReasoningParser:
DetectorMap: Dict[str, Type[BaseReasoningFormatDetector]] = {
"deepseek-r1": DeepSeekR1Detector,
"deepseek-v3": Qwen3Detector,
"deepseek-v3": _DeepSeekV3Detector,
"glm45": Glm45Detector,
"hunyuan": HunyuanDetector,
"gpt-oss": GptOssDetector,
"kimi": KimiDetector,
"kimi_k2": KimiK2Detector,
"mimo": Qwen3Detector,
"mimo": _MimoDetector,
"qwen3": Qwen3Detector,
"qwen3-thinking": Qwen3Detector,
"minimax": Qwen3Detector,
@@ -575,7 +604,11 @@ class ReasoningParser:
raise ValueError(f"Unsupported model type: {model_type}")
# Special cases where we override force_reasoning
if model_type.lower() in {"qwen3-thinking", "gpt-oss", "minimax"}:
if model_type.lower() in {
"qwen3-thinking",
"gpt-oss",
"minimax",
}:
force_reasoning = True
# Only pass force_reasoning if explicitly set, let detectors use their defaults
python/sglang/srt/server_args.py
+9 -4
View File
@@ -5110,12 +5110,15 @@ class ServerArgs:
action="store_true",
help="Return number of cached tokens in usage.prompt_tokens_details for each openai request.",
)
reasoning_parser_choices = list(ReasoningParser.DetectorMap.keys())
parser.add_argument(
"--reasoning-parser",
type=str,
choices=list(ReasoningParser.DetectorMap.keys()),
choices=["auto"] + reasoning_parser_choices,
default=ServerArgs.reasoning_parser,
help=f"Specify the parser for reasoning models, supported parsers are: {list(ReasoningParser.DetectorMap.keys())}.",
help=f"Specify the parser for reasoning models. "
f"Use 'auto' to detect from chat template. "
f"Options include: {reasoning_parser_choices}.",
)
parser.add_argument(
"--strip-thinking-cache",
@@ -5128,9 +5131,11 @@ class ServerArgs:
parser.add_argument(
"--tool-call-parser",
type=str,
choices=tool_call_parser_choices,
choices=["auto"] + tool_call_parser_choices,
default=ServerArgs.tool_call_parser,
help=f"Specify the parser for handling tool-call interactions. Options include: {tool_call_parser_choices}.",
help=f"Specify the parser for handling tool-call interactions. "
f"Use 'auto' to detect from chat template. "
f"Options include: {tool_call_parser_choices}.",
)
parser.add_argument(
"--tool-server",