minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

feat(parser): resolve special-token suffix at runtime for compatibility (#29920)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-07-05 00:13:46 +08:00
committed by GitHub
parent 763c6bf372
commit 854b46be99
10 changed files with 171 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/constrained/base_grammar_backend.py
+3 -1
View File
@@ -300,7 +300,9 @@ def create_grammar_backend(
)
reasoning_parser = ReasoningParser(
model_type=server_args.reasoning_parser, stream_reasoning=False
model_type=server_args.reasoning_parser,
stream_reasoning=False,
tokenizer=tokenizer,
)
grammar_backend = ReasonerGrammarBackend(
python/sglang/srt/entrypoints/http_server.py
+10 -2
View File
@@ -1517,7 +1517,11 @@ async def parse_function_call_request(
A native API endpoint to parse function calls from a text.
"""
# 1) Initialize the parser based on the request body
parser = FunctionCallParser(tools=obj.tools, tool_call_parser=obj.tool_call_parser)
parser = FunctionCallParser(
tools=obj.tools,
tool_call_parser=obj.tool_call_parser,
tokenizer=get_global_state().tokenizer_manager.tokenizer,
)
# 2) Call the non-stream parsing method (non-stream)
normal_text, calls = parser.parse_non_stream(obj.text)
@@ -1541,7 +1545,11 @@ async def separate_reasoning_request(
A native API endpoint to separate reasoning from a text.
"""
# 1) Initialize the parser based on the request body
parser = ReasoningParser(model_type=obj.reasoning_parser, request=request)
parser = ReasoningParser(
model_type=obj.reasoning_parser,
request=request,
tokenizer=get_global_state().tokenizer_manager.tokenizer,
)
# 2) Call the non-stream parsing method (non-stream)
if obj.return_blocks:
python/sglang/srt/entrypoints/openai/serving_chat.py
+15 -3
View File
@@ -175,7 +175,9 @@ class OpenAIServingChat(OpenAIServingBase):
if self.reasoning_parser:
try:
rp = ReasoningParser(
model_type=self.reasoning_parser, stream_reasoning=True
model_type=self.reasoning_parser,
stream_reasoning=True,
tokenizer=self.tokenizer_manager.tokenizer,
)
self._reasoning_detector = rp.detector
except ValueError as e:
@@ -668,7 +670,11 @@ class OpenAIServingChat(OpenAIServingBase):
else:
tools = [item.model_dump() for item in request.tools]
if self.tool_call_parser:
parser = FunctionCallParser(request.tools, self.tool_call_parser)
parser = FunctionCallParser(
request.tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
tool_call_constraint = parser.get_structure_constraint(
request.tool_choice,
parallel_tool_calls=request.parallel_tool_calls,
@@ -1327,6 +1333,7 @@ class OpenAIServingChat(OpenAIServingBase):
stream_reasoning=False,
force_reasoning=force_reasoning,
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
)
reasoning_text, text = parser.parse_non_stream(text)
except Exception as e:
@@ -1510,7 +1517,9 @@ class OpenAIServingChat(OpenAIServingBase):
# For required/named: only use parser when structural_tag was used
# as constraint (mirrors the streaming path). For auto: always try.
if self.tool_call_parser:
parser = FunctionCallParser(tools, self.tool_call_parser)
parser = FunctionCallParser(
tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer
)
should_try_parser = (
not is_required or parser.detector.supports_structural_tag()
)
@@ -1623,6 +1632,7 @@ class OpenAIServingChat(OpenAIServingBase):
request.stream_reasoning,
is_force_reasoning,
request,
tokenizer=self.tokenizer_manager.tokenizer,
)
reasoning_parser = reasoning_parser_dict[index]
return reasoning_parser.parse_stream_chunk(delta)
@@ -1867,6 +1877,7 @@ class OpenAIServingChat(OpenAIServingBase):
probe = FunctionCallParser(
tools=request.tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
use_native_parser = probe.detector.supports_structural_tag()
if use_native_parser:
@@ -1877,6 +1888,7 @@ class OpenAIServingChat(OpenAIServingBase):
parser_dict[index] = FunctionCallParser(
tools=request.tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
parser = parser_dict[index]
python/sglang/srt/entrypoints/openai/serving_responses.py
+17 -3
View File
@@ -672,6 +672,7 @@ class OpenAIServingResponses(OpenAIServingChat):
stream_reasoning=False,
force_reasoning=self._is_thinking_enabled_for_request(request),
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
)
reasoning_content, content = reasoning_parser.parse_non_stream(final_output)
else:
@@ -714,7 +715,11 @@ class OpenAIServingResponses(OpenAIServingChat):
and self.tool_call_parser
and request.tool_choice != "none"
):
parser = FunctionCallParser(chat_tools, self.tool_call_parser)
parser = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
should_try_native = (
not is_required or parser.detector.supports_structural_tag()
)
@@ -1799,14 +1804,22 @@ class OpenAIServingResponses(OpenAIServingChat):
if chat_tools and request.tool_choice != "none":
native_supports_structural_tag = False
if self.tool_call_parser:
probe = FunctionCallParser(chat_tools, self.tool_call_parser)
probe = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
native_supports_structural_tag = (
probe.detector.supports_structural_tag()
)
if is_required and not native_supports_structural_tag:
tool_parser = JsonArrayParser()
elif self.tool_call_parser:
tool_parser = FunctionCallParser(chat_tools, self.tool_call_parser)
tool_parser = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
reasoning_parser_obj: Optional[ReasoningParser] = None
if self.reasoning_parser:
reasoning_parser_obj = ReasoningParser(
@@ -1814,6 +1827,7 @@ class OpenAIServingResponses(OpenAIServingChat):
stream_reasoning=True,
force_reasoning=self._is_thinking_enabled_for_request(request),
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
)
current_output_index = -1
python/sglang/srt/function_call/function_call_parser.py
+8 -2
View File
@@ -1,3 +1,4 @@
import inspect
import logging
from typing import Dict, List, Literal, Optional, Set, Tuple, Type, Union
@@ -89,10 +90,15 @@ class FunctionCallParser:
"gemma4": Gemma4Detector,
}
def __init__(self, tools: List[Tool], tool_call_parser: str):
def __init__(self, tools: List[Tool], tool_call_parser: str, tokenizer=None):
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
if detector_class:
detector = detector_class()
kwargs = {}
if tokenizer is not None:
sig = inspect.signature(detector_class)
if "tokenizer" in sig.parameters:
kwargs["tokenizer"] = tokenizer
detector = detector_class(**kwargs)
else:
raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")
python/sglang/srt/function_call/hunyuan_detector.py
+82 -15
View File
@@ -16,6 +16,50 @@ from sglang.srt.function_call.core_types import (
logger = logging.getLogger(__name__)
# Bare (suffix-less) Hunyuan special tokens. The shipping Hy3 tokenizer appends
# a shared suffix to each (e.g. ``<tool_calls:opensource>``); resolve the real
# token string from the vocab at runtime and fall back to these literals.
_HUNYUAN_TOKEN_NAMES = (
"tool_calls",
"tool_call",
"tool_sep",
"arg_key",
"arg_value",
"think",
)
_HUNYUAN_TOKEN_RE = re.compile(
r"^<(?P<name>" + "|".join(_HUNYUAN_TOKEN_NAMES) + r")(?::[^>]+)?>$"
)
def resolve_hunyuan_tokens(tokenizer) -> Dict[str, str]:
"""Map bare token names to their real (possibly suffixed) strings in vocab.
Returns ``{name: token_str}`` for each name found. A bare literal is used
when the tokenizer carries no suffixed form, so the same detector serves
both the preview (suffix-less) and shipping (suffixed) Hy3 tokenizers.
"""
tokens: Dict[str, str] = {}
vocab = None
if tokenizer is not None:
try:
vocab = tokenizer.get_vocab()
except Exception as e:
logger.warning("Failed to read Hunyuan tokenizer vocab: %s", e)
vocab = None
if isinstance(vocab, dict):
for tok in vocab:
if not isinstance(tok, str):
continue
m = _HUNYUAN_TOKEN_RE.match(tok)
if m:
tokens[m.group("name")] = tok
for name in _HUNYUAN_TOKEN_NAMES:
tokens.setdefault(name, f"<{name}>")
return tokens
class HunyuanDetector(BaseFormatDetector):
"""
Detector for Hunyuan (HYV3) tool call format.
@@ -55,26 +99,49 @@ class HunyuanDetector(BaseFormatDetector):
_INTEGER_PREFIXES = ("int", "uint", "long", "short", "unsigned")
_NUMBER_PREFIXES = ("num", "float")
def __init__(self):
def __init__(self, tokenizer=None):
super().__init__()
self.bot_token = "<tool_calls>"
self.eot_token = "</tool_calls>"
t = resolve_hunyuan_tokens(tokenizer)
tool_calls = t["tool_calls"]
tool_call = t["tool_call"]
tool_sep = t["tool_sep"]
arg_key = t["arg_key"]
arg_value = t["arg_value"]
self.tool_call_start_token = "<tool_call>"
self.tool_call_end_token = "</tool_call>"
self.tool_sep_token = "<tool_sep>"
def _close(open_tok: str) -> str:
return "</" + open_tok[1:] if open_tok.startswith("<") else open_tok
self.arg_key_start_token = "<arg_key>"
self.arg_key_end_token = "</arg_key>"
self.arg_value_start_token = "<arg_value>"
self.arg_value_end_token = "</arg_value>"
self.bot_token = tool_calls
self.eot_token = _close(tool_calls)
self.tool_call_start_token = tool_call
self.tool_call_end_token = _close(tool_call)
self.tool_sep_token = tool_sep
self.arg_key_start_token = arg_key
self.arg_key_end_token = _close(arg_key)
self.arg_value_start_token = arg_value
self.arg_value_end_token = _close(arg_value)
tc_end = _close(tool_call)
ak_end = _close(arg_key)
av_end = _close(arg_value)
self.tool_call_regex = re.compile(
r"<tool_call>(.*?)<tool_sep>(.*?)</tool_call>", re.DOTALL
re.escape(tool_call)
+ r"(.*?)"
+ re.escape(tool_sep)
+ r"(.*?)"
+ re.escape(tc_end),
re.DOTALL,
)
self.func_args_regex = re.compile(
r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.DOTALL
re.escape(arg_key)
+ r"(.*?)"
+ re.escape(ak_end)
+ r"\s*"
+ re.escape(arg_value)
+ r"(.*?)"
+ re.escape(av_end),
re.DOTALL,
)
# Streaming state
@@ -467,9 +534,9 @@ class HunyuanDetector(BaseFormatDetector):
def structure_info(self) -> _GetInfoFunc:
return lambda name: StructureInfo(
begin=f"<tool_calls>\n<tool_call>{name}<tool_sep>",
end="</tool_call>\n</tool_calls>",
trigger="<tool_calls>",
begin=f"{self.bot_token}\n{self.tool_call_start_token}{name}{self.tool_sep_token}",
end=f"{self.tool_call_end_token}\n{self.eot_token}",
trigger=self.bot_token,
)
def supports_structural_tag(self) -> bool:
python/sglang/srt/managers/scheduler.py
+3 -1
View File
@@ -718,7 +718,9 @@ class Scheduler(
# Set reasoning_parser and think_end_id if --reasoning_parser is enabled
if self.server_args.reasoning_parser and self.tokenizer:
reasoning_parser = ReasoningParser(
model_type=self.server_args.reasoning_parser, stream_reasoning=False
model_type=self.server_args.reasoning_parser,
stream_reasoning=False,
tokenizer=self.tokenizer,
)
self.model_config.think_end_id = self.tokenizer.encode(
reasoning_parser.detector.think_end_token, add_special_tokens=False
python/sglang/srt/managers/template_detection.py
+15 -4
View File
@@ -47,6 +47,10 @@ class TemplateDetectionContext:
def has_pattern(self, pattern: str, flags: int = 0) -> bool:
return re.search(pattern, self.template, flags) is not None
def has_vocab_pattern(self, pattern: str) -> bool:
compiled = re.compile(pattern)
return any(isinstance(tok, str) and compiled.search(tok) for tok in self.vocab)
@dataclass(frozen=True)
class DetectionRule:
@@ -237,10 +241,17 @@ def _is_deepseek_v4(ctx):
def _is_hunyuan(ctx):
return (
(ctx.has_text("<tool_calls>") or ctx.has_vocab("<tool_calls>"))
and (ctx.has_text("<tool_sep>") or ctx.has_vocab("<tool_sep>"))
) or (ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking"))
# The shipping Hy3 tokenizer appends a shared suffix to each special token
# (e.g. ``<tool_calls:opensource>``), so match the bare or suffixed form.
tc = ctx.has_text("<tool_calls>") or ctx.has_vocab_pattern(
r"^<tool_calls(?::[^>]+)?>$"
)
sep = ctx.has_text("<tool_sep>") or ctx.has_vocab_pattern(
r"^<tool_sep(?::[^>]+)?>$"
)
return (tc and sep) or (
ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking")
)
def _is_poolside_v1(ctx):
python/sglang/srt/parser/reasoning_parser.py
+17 -3
View File
@@ -1,6 +1,8 @@
import inspect
from typing import Dict, List, Optional, Tuple, Type
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens
from sglang.srt.parser.harmony_parser import HarmonyParser
@@ -541,13 +543,19 @@ class HunyuanDetector(BaseReasoningFormatDetector):
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
tokenizer=None,
):
t = resolve_hunyuan_tokens(tokenizer)
think_open = t["think"]
think_close = (
"</" + think_open[1:] if think_open.startswith("<") else think_open
)
super().__init__(
"<think>",
"</think>",
think_open,
think_close,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token="<tool_calls>",
tool_start_token=t["tool_calls"],
continue_final_message=continue_final_message,
previous_content=previous_content,
)
@@ -1096,6 +1104,7 @@ class ReasoningParser:
stream_reasoning: bool = True,
force_reasoning: Optional[bool] = None,
request: ChatCompletionRequest = None,
tokenizer=None,
):
if not model_type:
raise ValueError("Model type must be specified")
@@ -1130,6 +1139,11 @@ class ReasoningParser:
if chat_template_kwargs.get("force_nonempty_content") is True:
kwargs["force_nonempty_content"] = True
if tokenizer is not None:
sig = inspect.signature(detector_class)
if "tokenizer" in sig.parameters:
kwargs["tokenizer"] = tokenizer
self.detector = detector_class(**kwargs)
def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]:
python/sglang/test/kits/reasoning_kit.py
+1 -1
View File
@@ -27,7 +27,7 @@ class ReasoningTokenUsageMixin:
def init_reasoning_token_verifier(cls):
assert cls.reasoning_parser_name, "reasoning_parser_name must be set"
cls.tokenizer = get_tokenizer(cls.model)
parser = ReasoningParser(cls.reasoning_parser_name)
parser = ReasoningParser(cls.reasoning_parser_name, tokenizer=cls.tokenizer)
cls.think_end_token_id = cls.tokenizer.convert_tokens_to_ids(
parser.detector.think_end_token
)