diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py index 4f907b983..4943d17f4 100644 --- a/python/sglang/srt/constrained/base_grammar_backend.py +++ b/python/sglang/srt/constrained/base_grammar_backend.py @@ -300,7 +300,9 @@ def create_grammar_backend( ) reasoning_parser = ReasoningParser( - model_type=server_args.reasoning_parser, stream_reasoning=False + model_type=server_args.reasoning_parser, + stream_reasoning=False, + tokenizer=tokenizer, ) grammar_backend = ReasonerGrammarBackend( diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 829274b76..bd9d7eafa 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -1517,7 +1517,11 @@ async def parse_function_call_request( A native API endpoint to parse function calls from a text. """ # 1) Initialize the parser based on the request body - parser = FunctionCallParser(tools=obj.tools, tool_call_parser=obj.tool_call_parser) + parser = FunctionCallParser( + tools=obj.tools, + tool_call_parser=obj.tool_call_parser, + tokenizer=get_global_state().tokenizer_manager.tokenizer, + ) # 2) Call the non-stream parsing method (non-stream) normal_text, calls = parser.parse_non_stream(obj.text) @@ -1541,7 +1545,11 @@ async def separate_reasoning_request( A native API endpoint to separate reasoning from a text. """ # 1) Initialize the parser based on the request body - parser = ReasoningParser(model_type=obj.reasoning_parser, request=request) + parser = ReasoningParser( + model_type=obj.reasoning_parser, + request=request, + tokenizer=get_global_state().tokenizer_manager.tokenizer, + ) # 2) Call the non-stream parsing method (non-stream) if obj.return_blocks: diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 7d1601e17..37062a502 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -175,7 +175,9 @@ class OpenAIServingChat(OpenAIServingBase): if self.reasoning_parser: try: rp = ReasoningParser( - model_type=self.reasoning_parser, stream_reasoning=True + model_type=self.reasoning_parser, + stream_reasoning=True, + tokenizer=self.tokenizer_manager.tokenizer, ) self._reasoning_detector = rp.detector except ValueError as e: @@ -668,7 +670,11 @@ class OpenAIServingChat(OpenAIServingBase): else: tools = [item.model_dump() for item in request.tools] if self.tool_call_parser: - parser = FunctionCallParser(request.tools, self.tool_call_parser) + parser = FunctionCallParser( + request.tools, + self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, + ) tool_call_constraint = parser.get_structure_constraint( request.tool_choice, parallel_tool_calls=request.parallel_tool_calls, @@ -1327,6 +1333,7 @@ class OpenAIServingChat(OpenAIServingBase): stream_reasoning=False, force_reasoning=force_reasoning, request=request, + tokenizer=self.tokenizer_manager.tokenizer, ) reasoning_text, text = parser.parse_non_stream(text) except Exception as e: @@ -1510,7 +1517,9 @@ class OpenAIServingChat(OpenAIServingBase): # For required/named: only use parser when structural_tag was used # as constraint (mirrors the streaming path). For auto: always try. if self.tool_call_parser: - parser = FunctionCallParser(tools, self.tool_call_parser) + parser = FunctionCallParser( + tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer + ) should_try_parser = ( not is_required or parser.detector.supports_structural_tag() ) @@ -1623,6 +1632,7 @@ class OpenAIServingChat(OpenAIServingBase): request.stream_reasoning, is_force_reasoning, request, + tokenizer=self.tokenizer_manager.tokenizer, ) reasoning_parser = reasoning_parser_dict[index] return reasoning_parser.parse_stream_chunk(delta) @@ -1867,6 +1877,7 @@ class OpenAIServingChat(OpenAIServingBase): probe = FunctionCallParser( tools=request.tools, tool_call_parser=self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, ) use_native_parser = probe.detector.supports_structural_tag() if use_native_parser: @@ -1877,6 +1888,7 @@ class OpenAIServingChat(OpenAIServingBase): parser_dict[index] = FunctionCallParser( tools=request.tools, tool_call_parser=self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, ) parser = parser_dict[index] diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index 5d569df26..9d2b3fe1a 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -672,6 +672,7 @@ class OpenAIServingResponses(OpenAIServingChat): stream_reasoning=False, force_reasoning=self._is_thinking_enabled_for_request(request), request=request, + tokenizer=self.tokenizer_manager.tokenizer, ) reasoning_content, content = reasoning_parser.parse_non_stream(final_output) else: @@ -714,7 +715,11 @@ class OpenAIServingResponses(OpenAIServingChat): and self.tool_call_parser and request.tool_choice != "none" ): - parser = FunctionCallParser(chat_tools, self.tool_call_parser) + parser = FunctionCallParser( + chat_tools, + self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, + ) should_try_native = ( not is_required or parser.detector.supports_structural_tag() ) @@ -1799,14 +1804,22 @@ class OpenAIServingResponses(OpenAIServingChat): if chat_tools and request.tool_choice != "none": native_supports_structural_tag = False if self.tool_call_parser: - probe = FunctionCallParser(chat_tools, self.tool_call_parser) + probe = FunctionCallParser( + chat_tools, + self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, + ) native_supports_structural_tag = ( probe.detector.supports_structural_tag() ) if is_required and not native_supports_structural_tag: tool_parser = JsonArrayParser() elif self.tool_call_parser: - tool_parser = FunctionCallParser(chat_tools, self.tool_call_parser) + tool_parser = FunctionCallParser( + chat_tools, + self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, + ) reasoning_parser_obj: Optional[ReasoningParser] = None if self.reasoning_parser: reasoning_parser_obj = ReasoningParser( @@ -1814,6 +1827,7 @@ class OpenAIServingResponses(OpenAIServingChat): stream_reasoning=True, force_reasoning=self._is_thinking_enabled_for_request(request), request=request, + tokenizer=self.tokenizer_manager.tokenizer, ) current_output_index = -1 diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 834aad9d5..b934eb5d1 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -1,3 +1,4 @@ +import inspect import logging from typing import Dict, List, Literal, Optional, Set, Tuple, Type, Union @@ -89,10 +90,15 @@ class FunctionCallParser: "gemma4": Gemma4Detector, } - def __init__(self, tools: List[Tool], tool_call_parser: str): + def __init__(self, tools: List[Tool], tool_call_parser: str, tokenizer=None): detector_class = self.ToolCallParserEnum.get(tool_call_parser) if detector_class: - detector = detector_class() + kwargs = {} + if tokenizer is not None: + sig = inspect.signature(detector_class) + if "tokenizer" in sig.parameters: + kwargs["tokenizer"] = tokenizer + detector = detector_class(**kwargs) else: raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}") diff --git a/python/sglang/srt/function_call/hunyuan_detector.py b/python/sglang/srt/function_call/hunyuan_detector.py index 67b0187e5..269142bd6 100644 --- a/python/sglang/srt/function_call/hunyuan_detector.py +++ b/python/sglang/srt/function_call/hunyuan_detector.py @@ -16,6 +16,50 @@ from sglang.srt.function_call.core_types import ( logger = logging.getLogger(__name__) +# Bare (suffix-less) Hunyuan special tokens. The shipping Hy3 tokenizer appends +# a shared suffix to each (e.g. ````); resolve the real +# token string from the vocab at runtime and fall back to these literals. +_HUNYUAN_TOKEN_NAMES = ( + "tool_calls", + "tool_call", + "tool_sep", + "arg_key", + "arg_value", + "think", +) + +_HUNYUAN_TOKEN_RE = re.compile( + r"^<(?P" + "|".join(_HUNYUAN_TOKEN_NAMES) + r")(?::[^>]+)?>$" +) + + +def resolve_hunyuan_tokens(tokenizer) -> Dict[str, str]: + """Map bare token names to their real (possibly suffixed) strings in vocab. + + Returns ``{name: token_str}`` for each name found. A bare literal is used + when the tokenizer carries no suffixed form, so the same detector serves + both the preview (suffix-less) and shipping (suffixed) Hy3 tokenizers. + """ + tokens: Dict[str, str] = {} + vocab = None + if tokenizer is not None: + try: + vocab = tokenizer.get_vocab() + except Exception as e: + logger.warning("Failed to read Hunyuan tokenizer vocab: %s", e) + vocab = None + if isinstance(vocab, dict): + for tok in vocab: + if not isinstance(tok, str): + continue + m = _HUNYUAN_TOKEN_RE.match(tok) + if m: + tokens[m.group("name")] = tok + for name in _HUNYUAN_TOKEN_NAMES: + tokens.setdefault(name, f"<{name}>") + return tokens + + class HunyuanDetector(BaseFormatDetector): """ Detector for Hunyuan (HYV3) tool call format. @@ -55,26 +99,49 @@ class HunyuanDetector(BaseFormatDetector): _INTEGER_PREFIXES = ("int", "uint", "long", "short", "unsigned") _NUMBER_PREFIXES = ("num", "float") - def __init__(self): + def __init__(self, tokenizer=None): super().__init__() - self.bot_token = "" - self.eot_token = "" + t = resolve_hunyuan_tokens(tokenizer) + tool_calls = t["tool_calls"] + tool_call = t["tool_call"] + tool_sep = t["tool_sep"] + arg_key = t["arg_key"] + arg_value = t["arg_value"] - self.tool_call_start_token = "" - self.tool_call_end_token = "" - self.tool_sep_token = "" + def _close(open_tok: str) -> str: + return "" - self.arg_key_end_token = "" - self.arg_value_start_token = "" - self.arg_value_end_token = "" + self.bot_token = tool_calls + self.eot_token = _close(tool_calls) + self.tool_call_start_token = tool_call + self.tool_call_end_token = _close(tool_call) + self.tool_sep_token = tool_sep + self.arg_key_start_token = arg_key + self.arg_key_end_token = _close(arg_key) + self.arg_value_start_token = arg_value + self.arg_value_end_token = _close(arg_value) + tc_end = _close(tool_call) + ak_end = _close(arg_key) + av_end = _close(arg_value) self.tool_call_regex = re.compile( - r"(.*?)(.*?)", re.DOTALL + re.escape(tool_call) + + r"(.*?)" + + re.escape(tool_sep) + + r"(.*?)" + + re.escape(tc_end), + re.DOTALL, ) self.func_args_regex = re.compile( - r"(.*?)\s*(.*?)", re.DOTALL + re.escape(arg_key) + + r"(.*?)" + + re.escape(ak_end) + + r"\s*" + + re.escape(arg_value) + + r"(.*?)" + + re.escape(av_end), + re.DOTALL, ) # Streaming state @@ -467,9 +534,9 @@ class HunyuanDetector(BaseFormatDetector): def structure_info(self) -> _GetInfoFunc: return lambda name: StructureInfo( - begin=f"\n{name}", - end="\n", - trigger="", + begin=f"{self.bot_token}\n{self.tool_call_start_token}{name}{self.tool_sep_token}", + end=f"{self.tool_call_end_token}\n{self.eot_token}", + trigger=self.bot_token, ) def supports_structural_tag(self) -> bool: diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 43ddeb397..1987ad172 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -718,7 +718,9 @@ class Scheduler( # Set reasoning_parser and think_end_id if --reasoning_parser is enabled if self.server_args.reasoning_parser and self.tokenizer: reasoning_parser = ReasoningParser( - model_type=self.server_args.reasoning_parser, stream_reasoning=False + model_type=self.server_args.reasoning_parser, + stream_reasoning=False, + tokenizer=self.tokenizer, ) self.model_config.think_end_id = self.tokenizer.encode( reasoning_parser.detector.think_end_token, add_special_tokens=False diff --git a/python/sglang/srt/managers/template_detection.py b/python/sglang/srt/managers/template_detection.py index 4c6f367f0..74050fae1 100644 --- a/python/sglang/srt/managers/template_detection.py +++ b/python/sglang/srt/managers/template_detection.py @@ -47,6 +47,10 @@ class TemplateDetectionContext: def has_pattern(self, pattern: str, flags: int = 0) -> bool: return re.search(pattern, self.template, flags) is not None + def has_vocab_pattern(self, pattern: str) -> bool: + compiled = re.compile(pattern) + return any(isinstance(tok, str) and compiled.search(tok) for tok in self.vocab) + @dataclass(frozen=True) class DetectionRule: @@ -237,10 +241,17 @@ def _is_deepseek_v4(ctx): def _is_hunyuan(ctx): - return ( - (ctx.has_text("") or ctx.has_vocab("")) - and (ctx.has_text("") or ctx.has_vocab("")) - ) or (ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking")) + # The shipping Hy3 tokenizer appends a shared suffix to each special token + # (e.g. ````), so match the bare or suffixed form. + tc = ctx.has_text("") or ctx.has_vocab_pattern( + r"^]+)?>$" + ) + sep = ctx.has_text("") or ctx.has_vocab_pattern( + r"^]+)?>$" + ) + return (tc and sep) or ( + ctx.has_text("reasoning_effort") and ctx.has_text("interleaved_thinking") + ) def _is_poolside_v1(ctx): diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index d4fe2eacf..531e4eee3 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -1,6 +1,8 @@ +import inspect from typing import Dict, List, Optional, Tuple, Type from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens from sglang.srt.parser.harmony_parser import HarmonyParser @@ -541,13 +543,19 @@ class HunyuanDetector(BaseReasoningFormatDetector): force_reasoning: bool = False, continue_final_message: bool = False, previous_content: str = "", + tokenizer=None, ): + t = resolve_hunyuan_tokens(tokenizer) + think_open = t["think"] + think_close = ( + "", - "", + think_open, + think_close, force_reasoning=force_reasoning, stream_reasoning=stream_reasoning, - tool_start_token="", + tool_start_token=t["tool_calls"], continue_final_message=continue_final_message, previous_content=previous_content, ) @@ -1096,6 +1104,7 @@ class ReasoningParser: stream_reasoning: bool = True, force_reasoning: Optional[bool] = None, request: ChatCompletionRequest = None, + tokenizer=None, ): if not model_type: raise ValueError("Model type must be specified") @@ -1130,6 +1139,11 @@ class ReasoningParser: if chat_template_kwargs.get("force_nonempty_content") is True: kwargs["force_nonempty_content"] = True + if tokenizer is not None: + sig = inspect.signature(detector_class) + if "tokenizer" in sig.parameters: + kwargs["tokenizer"] = tokenizer + self.detector = detector_class(**kwargs) def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]: diff --git a/python/sglang/test/kits/reasoning_kit.py b/python/sglang/test/kits/reasoning_kit.py index 55274b759..59f8bae68 100644 --- a/python/sglang/test/kits/reasoning_kit.py +++ b/python/sglang/test/kits/reasoning_kit.py @@ -27,7 +27,7 @@ class ReasoningTokenUsageMixin: def init_reasoning_token_verifier(cls): assert cls.reasoning_parser_name, "reasoning_parser_name must be set" cls.tokenizer = get_tokenizer(cls.model) - parser = ReasoningParser(cls.reasoning_parser_name) + parser = ReasoningParser(cls.reasoning_parser_name, tokenizer=cls.tokenizer) cls.think_end_token_id = cls.tokenizer.convert_tokens_to_ids( parser.detector.think_end_token )