From 27ac614f051f18de577f6f0f980d1c5cd83d7963 Mon Sep 17 00:00:00 2001 From: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:05:54 -0400 Subject: [PATCH] [Feat] parser: Add checkpoint-driven response-template adapters (#40479) Signed-off-by: Yoni Gozlan Co-authored-by: Yoni Gozlan Co-authored-by: Xinyuan Tong Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> --- .../sglang/srt/entrypoints/openai/protocol.py | 10 + .../srt/entrypoints/openai/serving_chat.py | 122 +- .../entrypoints/openai/serving_responses.py | 70 +- .../srt/function_call/function_call_parser.py | 17 +- .../srt/function_call/json_array_parser.py | 12 + .../sglang/srt/function_call/parser_names.py | 1 + .../srt/managers/detokenizer_manager.py | 6 + python/sglang/srt/parser/reasoning_parser.py | 16 +- .../srt/parser/reasoning_parser_names.py | 1 + python/sglang/srt/parser/response_template.py | 787 +++++++++++ .../sglang/srt/parser/template_detection.py | 47 +- python/sglang/srt/server_args.py | 4 +- .../unit/beam_search/test_output_decode.py | 4 +- .../entrypoints/openai/test_serving_chat.py | 432 ++++++ .../openai/test_serving_responses.py | 178 +++ .../openai/test_serving_responses_stream.py | 59 + .../test_function_call_parser.py | 27 + .../unit/managers/test_trim_matched_stop.py | 10 +- .../unit/parser/test_response_template.py | 1238 +++++++++++++++++ .../unit/parser/test_template_manager.py | 155 +++ 20 files changed, 3173 insertions(+), 23 deletions(-) create mode 100644 python/sglang/srt/parser/response_template.py create mode 100644 test/registered/unit/parser/test_response_template.py diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 641ccde476..110a0528bc 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -58,6 +58,7 @@ from pydantic import ( BaseModel, ConfigDict, Field, + PrivateAttr, StrictBool, field_serializer, field_validator, @@ -866,6 +867,10 @@ def _has_message_level_tools(messages: Any) -> bool: class ChatCompletionRequest(PDRoutingFields): + _response_parser_prefix: str = PrivateAttr(default="") + _response_parser_grammar_start: Optional[str] = PrivateAttr(default=None) + _response_parser_json_response_format: bool = PrivateAttr(default=False) + # Ordered by official OpenAI API documentation # https://platform.openai.com/docs/api-reference/chat/create messages: List[ChatCompletionMessageParam] @@ -1833,6 +1838,10 @@ ResponseInputOutputItem: TypeAlias = Union[ class ResponsesRequest(PDRoutingFields): """Request body for v1/responses endpoint.""" + _response_parser_prefix: str = PrivateAttr(default="") + _response_parser_grammar_start: Optional[str] = PrivateAttr(default=None) + _response_parser_json_response_format: bool = PrivateAttr(default=False) + # Core OpenAI API fields (ordered by official documentation) background: Optional[bool] = False include: Optional[ @@ -2322,6 +2331,7 @@ class MessageProcessingResult: skip_special_tokens: bool = True require_reasoning: bool = False reasoning_end_token_ids: Optional[List[int]] = None + uses_response_template: bool = False class ToolCallProcessingResult(NamedTuple): diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 9f37232989..12378585e7 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -68,9 +68,11 @@ from sglang.srt.entrypoints.openai.protocol import ( MessageProcessingResult, PromptTokensDetails, ResponseParserProtocol, + ResponsesRequest, SglExt, Tool, ToolCall, + ToolCallConstraint, ToolCallProcessingResult, ToolChoice, TopLogprob, @@ -113,6 +115,14 @@ from sglang.srt.parser.reasoning_parser import ( IQuestQ1ReasoningDetector, ReasoningParser, ) +from sglang.srt.parser.response_template import ( + GRAMMAR_AFTER_REASONING, + GRAMMAR_FROM_START, + ResponseTemplateReasoningDetector, + ResponseTemplateToolDetector, + configure_response_template_request, + tool_close_tokens, +) from sglang.srt.parser.template_detection import detect_inline_system_support from sglang.srt.sampling.sampling_params import ( set_request_reasoning_end_token_ids, @@ -130,6 +140,11 @@ _MEDIA_CONTENT_PART_TYPES = frozenset({"image_url", "video_url", "audio_url"}) _CHAT_TEMPLATE_CACHE_MAX_SIZE = 128 +def _incomplete_tool_call_indices(parser) -> set[int]: + detector = getattr(parser, "detector", parser) + return getattr(detector, "incomplete_tool_call_indices", set()) + + def normalize_tool_content(role: str, content): """Normalize tool message content from OpenAI array format to plain string. @@ -846,6 +861,10 @@ class OpenAIServingChat(OpenAIServingBase): else: delta = content["text"][offset:] stream_offsets[index] = len(content["text"]) + if finish_reason_type is not None: + delta = self._drop_kept_tool_close( + delta, content["meta_info"].get("finish_reason"), request + ) # Attach logprobs to the first chunk emitted this step (reasoning, # tool-call, or content) so they aren't dropped when a parser is active @@ -967,6 +986,7 @@ class OpenAIServingChat(OpenAIServingBase): request.tool_choice != "none" and self._effective_tools(request) and self.tool_call_parser + and not request._response_parser_json_response_format ) def _validate_request(self, request: ChatCompletionRequest) -> str | None: @@ -1285,8 +1305,89 @@ class OpenAIServingChat(OpenAIServingBase): ): apply_header_overrides(adapted_request, raw_request.headers) + if processed_messages.uses_response_template: + self._set_response_parser_context( + request, + adapted_request, + tool_call_constraint=processed_messages.tool_call_constraint, + ) return adapted_request, request + def _set_response_parser_context( + self, + request: ChatCompletionRequest | ResponsesRequest, + adapted_request: GenerateReqInput, + *, + tool_call_constraint: ToolCallConstraint | None, + ) -> None: + """Give response-template parsers the rendered assistant prefill.""" + prefix = adapted_request.text + if not isinstance(prefix, str) and adapted_request.input_ids: + prefix = self.tokenizer_manager.tokenizer.decode( + adapted_request.input_ids, + skip_special_tokens=False, + spaces_between_special_tokens=False, + ) + request._response_parser_prefix = prefix or "" + # Structural tags, regex and EBNF can describe the template's own + # framing, so only JSON-schema output bypasses the template. + if (adapted_request.sampling_params or {}).get("json_schema"): + # The scheduler defers the grammar to the reasoning's end only when + # the reasoning parser's think_end_token encodes to token ids. + reasoning_gated = ( + adapted_request.require_reasoning + and self._reasoning_detector is not None + and bool( + self._reasoning_detector.get_think_end_token_ids( + self.tokenizer_manager.tokenizer + ) + ) + ) + request._response_parser_grammar_start = ( + GRAMMAR_AFTER_REASONING if reasoning_gated else GRAMMAR_FROM_START + ) + # Unless the schema is the required/named tool call itself, the JSON + # is response data, and template syntax inside its strings is literal. + request._response_parser_json_response_format = ( + tool_call_constraint is None or tool_call_constraint[0] != "json_schema" + ) + + def _drop_kept_tool_close( + self, + text: str, + finish_reason: dict[str, Any] | None, + request: ChatCompletionRequest | ResponsesRequest, + ) -> str: + # Detokenization keeps a tool-call closer that stops generation; after + # JSON-schema output it ends no native call, so trim it like any stop. + if ( + not text + or request._response_parser_grammar_start is None + or (isinstance(request, ChatCompletionRequest) and request.no_stop_trim) + or not isinstance(finish_reason, dict) + ): + return text + closer = tool_close_tokens( + self.tool_call_parser, self.tokenizer_manager.tokenizer + ).get(finish_reason.get("matched")) + if closer and text.endswith(closer): + return text[: -len(closer)] + return text + + def _requires_response_template_detokenization( + self, request: ChatCompletionRequest + ) -> bool: + if isinstance(self._reasoning_detector, ResponseTemplateReasoningDetector): + return True + + tool_detector = FunctionCallParser.ToolCallParserEnum.get(self.tool_call_parser) + return ( + request.tool_choice != "none" + and bool(self._effective_tools(request)) + and tool_detector is not None + and issubclass(tool_detector, ResponseTemplateToolDetector) + ) + def _process_messages( self, request: ChatCompletionRequest, is_multimodal: bool ) -> MessageProcessingResult: @@ -1427,6 +1528,11 @@ class OpenAIServingChat(OpenAIServingBase): if tool_call_stop not in result.stop: result.stop.append(tool_call_stop) + # Configured after rendering so the detokenization options never reach + # the chat template. + if self._requires_response_template_detokenization(request): + configure_response_template_request(request) + result.uses_response_template = True result.tool_call_constraint = tool_call_constraint result.require_reasoning = thinking_mode result.skip_special_tokens = request.skip_special_tokens @@ -2094,7 +2200,11 @@ class OpenAIServingChat(OpenAIServingBase): # Change finish_reason to "tool_calls" if we had tool calls and stopped naturally final_finish_reason = finish_reason_type - if has_tool_calls.get(idx, False) and finish_reason_type == "stop": + if ( + has_tool_calls.get(idx, False) + and finish_reason_type == "stop" + and not _incomplete_tool_call_indices(parser_dict.get(idx)) + ): final_finish_reason = "tool_calls" matched_stop = finish_reason_data.get("matched") @@ -2357,6 +2467,7 @@ class OpenAIServingChat(OpenAIServingBase): text = self._decode_response(ret_item) if isinstance(text, ErrorResponse): return ORJSONResponse(content=text.model_dump(), status_code=text.code) + text = self._drop_kept_tool_close(text, finish_reason, request) # Handle reasoning content reasoning_text = None @@ -2402,6 +2513,7 @@ class OpenAIServingChat(OpenAIServingBase): finish_reason, request.tool_choice, history_tool_calls_cnt, + request._response_parser_prefix, ) # Extract prompt_token_ids if requested @@ -2586,6 +2698,7 @@ class OpenAIServingChat(OpenAIServingBase): finish_reason: dict[str, Any], tool_choice: str | ToolChoice | None = None, history_tool_calls_cnt: int = 0, + response_parser_prefix: str = "", ) -> ToolCallProcessingResult: """Process tool calls in the response""" @@ -2600,6 +2713,7 @@ class OpenAIServingChat(OpenAIServingBase): self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer, tool_choice=tool_choice, + prefix=response_parser_prefix, ) detector_owns_format = parser.owns_tool_format() should_try_parser = not is_required or detector_owns_format @@ -3055,6 +3169,7 @@ class OpenAIServingChat(OpenAIServingBase): tool_call_parser=self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer, tool_choice=request.tool_choice, + prefix=request._response_parser_prefix, ) use_native_parser = probe.owns_tool_format() if use_native_parser: @@ -3066,6 +3181,7 @@ class OpenAIServingChat(OpenAIServingBase): tools=effective_tools, tool_call_parser=self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer, + prefix=request._response_parser_prefix, ) parser = parser_dict[index] @@ -3074,6 +3190,10 @@ class OpenAIServingChat(OpenAIServingBase): if isinstance(parser, JsonArrayParser): result = parser.parse_streaming_increment(delta, effective_tools) normal_text, calls = result.normal_text, result.calls + if flush: + end = parser.finish(effective_tools) + normal_text = (normal_text or "") + end.normal_text + calls = list(calls) + end.calls else: normal_text, calls = parser.parse_stream_chunk(delta) if flush: diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index 43f980da6a..b85d4da840 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -84,7 +84,10 @@ from sglang.srt.entrypoints.openai.responses_adapters import ( encode_reasoning_state, label_developer_content, ) -from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat +from sglang.srt.entrypoints.openai.serving_chat import ( + OpenAIServingChat, + _incomplete_tool_call_indices, +) from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs from sglang.srt.function_call.core_types import ToolCallItem @@ -488,6 +491,11 @@ class OpenAIServingResponses(OpenAIServingChat): not processed_messages.skip_special_tokens ): sampling_params["skip_special_tokens"] = False + if ( + processed_messages is not None + and processed_messages.uses_response_template + ): + sampling_params["spaces_between_special_tokens"] = False context: ConversationContext if self.use_harmony: @@ -573,6 +581,15 @@ class OpenAIServingResponses(OpenAIServingChat): assert len(generators) == 1 (result_generator,) = generators + if ( + processed_messages is not None + and processed_messages.uses_response_template + ): + self._set_response_parser_context( + request, + adapted_request, + tool_call_constraint=processed_messages.tool_call_constraint, + ) # Store the input messages persist = self.enable_response_store and bool(request.store) @@ -796,7 +813,15 @@ class OpenAIServingResponses(OpenAIServingChat): ) output = self._make_response_output_items( request, - final_res["text"], + self._drop_kept_tool_close( + final_res["text"], + ( + meta_info.get("finish_reason") + if isinstance(meta_info, dict) + else None + ), + request, + ), tokenizer, output_logprobs=output_logprobs, require_reasoning=require_reasoning, @@ -1115,9 +1140,8 @@ class OpenAIServingResponses(OpenAIServingChat): request=request, tokenizer=self.tokenizer_manager.tokenizer, tool_call_parser_active=bool( - chat_tools - and self.tool_call_parser - and request.tool_choice != "none" + self.tool_call_parser + and self._tool_parsing_active(request, chat_tools) ), ) reasoning_content, content = reasoning_parser.parse_non_stream(final_output) @@ -1134,15 +1158,15 @@ class OpenAIServingResponses(OpenAIServingChat): detector_owns_format = False if ( content - and chat_tools and self.tool_call_parser - and request.tool_choice != "none" + and self._tool_parsing_active(request, chat_tools) ): parser = FunctionCallParser( chat_tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer, tool_choice=tool_choice, + prefix=request._response_parser_prefix, ) detector_owns_format = self._tool_parser_owns_format(parser) should_try_native = not is_required or detector_owns_format @@ -1163,7 +1187,7 @@ class OpenAIServingResponses(OpenAIServingChat): if ( content - and chat_tools + and self._tool_parsing_active(request, chat_tools) and is_required and not parsed_via_native and not detector_owns_format @@ -1220,6 +1244,14 @@ class OpenAIServingResponses(OpenAIServingChat): chat_tools and self.tool_call_parser and request.tool_choice != "none" ) + @staticmethod + def _tool_parsing_active(request: ResponsesRequest, chat_tools: list) -> bool: + return bool( + chat_tools + and request.tool_choice != "none" + and not request._response_parser_json_response_format + ) + @staticmethod def _tool_parser_owns_format(parser: FunctionCallParser) -> bool: return parser.owns_tool_format() @@ -2109,14 +2141,17 @@ class OpenAIServingResponses(OpenAIServingChat): tool_parser: Optional[Union[FunctionCallParser, JsonArrayParser]] = None if required_json: tool_parser = JsonArrayParser() - elif token_stream is None and self._tool_call_parsing_enabled( - request, chat_tools + elif ( + token_stream is None + and self._tool_call_parsing_enabled(request, chat_tools) + and not request._response_parser_json_response_format ): tool_parser = FunctionCallParser( chat_tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer, tool_choice=tool_choice, + prefix=request._response_parser_prefix, ) reasoning_parser_obj: Optional[ReasoningParser] = None if token_stream is None and self.reasoning_parser: @@ -2355,13 +2390,18 @@ class OpenAIServingResponses(OpenAIServingChat): ) ) else: + item_status = ( + "incomplete" + if tool_index in _incomplete_tool_call_indices(tool_parser) + else "completed" + ) completed_item = ResponseFunctionToolCall( arguments=arguments, call_id=state["call_id"], name=state["name"] or "", type="function_call", id=state["item_id"], - status="completed", + status=item_status, ) events.append( _send_event( @@ -2420,6 +2460,10 @@ class OpenAIServingResponses(OpenAIServingChat): else: delta = text[stream_offset:] stream_offset = len(text) + if finish_reason is not None: + delta = self._drop_kept_tool_close( + delta, finish_reason, request + ) if not delta and finish_reason is None: continue # finish_reason is sticky, so it would otherwise re-flush. @@ -2527,6 +2571,10 @@ class OpenAIServingResponses(OpenAIServingChat): normal_text, tool_calls = self._drain_json_array( tool_parser, delta, chat_tools ) + if flush: + end = tool_parser.finish(chat_tools) + normal_text += end.normal_text + tool_calls = list(tool_calls) + end.calls elif token_stream is not None: normal_text, tool_calls = delta, token_calls elif tool_parser is not None: diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index 3184ac7dc3..ec942aae5d 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -59,6 +59,7 @@ from sglang.srt.function_call.utils import ( _get_tool_schema_defs, get_json_schema_constraint, ) +from sglang.srt.parser.response_template import ResponseTemplateToolDetector logger = logging.getLogger(__name__) @@ -115,10 +116,16 @@ class FunctionCallParser: "gemma4": Gemma4Detector, "inkling": InklingDetector, "iquest_q1": IQuestQ1Detector, + "response_template": ResponseTemplateToolDetector, } def __init__( - self, tools: List[Tool], tool_call_parser: str, tokenizer=None, tool_choice=None + self, + tools: List[Tool], + tool_call_parser: str, + tokenizer=None, + tool_choice=None, + prefix: str = "", ): detector_class = self.ToolCallParserEnum.get(tool_call_parser) if detector_class: @@ -127,6 +134,8 @@ class FunctionCallParser: sig = inspect.signature(detector_class) if "tokenizer" in sig.parameters: kwargs["tokenizer"] = tokenizer + if prefix and issubclass(detector_class, ResponseTemplateToolDetector): + kwargs["prefix"] = prefix detector = detector_class(**kwargs) else: raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}") @@ -297,6 +306,12 @@ class FunctionCallParser: any(tool.function.strict for tool in self.tools) or self.tool_strict_level >= ToolStrictLevel.FUNCTION ) + if isinstance(self.detector, ResponseTemplateToolDetector): + self.detector.validate_structure_constraint_request( + tool_choice, + parallel_tool_calls, + should_constrain_auto, + ) try: if ( diff --git a/python/sglang/srt/function_call/json_array_parser.py b/python/sglang/srt/function_call/json_array_parser.py index e38e3b1b05..4fb61ea023 100644 --- a/python/sglang/srt/function_call/json_array_parser.py +++ b/python/sglang/srt/function_call/json_array_parser.py @@ -42,6 +42,18 @@ class JsonArrayParser(BaseFormatDetector): """ return super().parse_streaming_increment(new_text, tools) + def finish(self, tools: List[Tool]) -> StreamingParseResult: + # An increment sends a call's name before its arguments, so drain the + # arguments of calls that arrived whole in the last chunk. + normal_text, calls = "", [] + while True: + result = self.parse_streaming_increment("", tools) + normal_text += result.normal_text + progress = [call for call in result.calls if call.name or call.parameters] + if not progress: + return StreamingParseResult(normal_text=normal_text, calls=calls) + calls.extend(progress) + def structure_info(self) -> callable: """ Return a function that creates StructureInfo for constrained generation. diff --git a/python/sglang/srt/function_call/parser_names.py b/python/sglang/srt/function_call/parser_names.py index 3da09d6b59..69fc1a0962 100644 --- a/python/sglang/srt/function_call/parser_names.py +++ b/python/sglang/srt/function_call/parser_names.py @@ -50,4 +50,5 @@ TOOL_CALL_PARSER_NAMES = [ "gemma4", "inkling", "iquest_q1", + "response_template", ] diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py index 0e5f9a6d08..7cb5de0353 100644 --- a/python/sglang/srt/managers/detokenizer_manager.py +++ b/python/sglang/srt/managers/detokenizer_manager.py @@ -43,6 +43,7 @@ from sglang.srt.managers.io_struct import ( ) from sglang.srt.managers.multi_tokenizer_mixin import MultiHttpWorkerDetokenizerMixin from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread +from sglang.srt.parser.response_template import tool_close_token_ids from sglang.srt.plugins import load_plugins from sglang.srt.runtime_context import ( get_device, @@ -156,6 +157,9 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): get_serving().disable_tokenizer_batch_decode ) self.is_tool_call_parser_gpt_oss = get_serving().tool_call_parser == "gpt-oss" + self.tool_close_token_ids = tool_close_token_ids( + get_serving().tool_call_parser, self.tokenizer + ) self.soft_watchdog = Watchdog.create( debug_name="DetokenizerManager", @@ -214,6 +218,8 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): # 200012 <|call|> is the tool call token and one of eos tokens for gpt-oss model if output[-1] == 200012 and self.is_tool_call_parser_gpt_oss: return output + if output[-1] in self.tool_close_token_ids: + return output assert len(output) > 0 # NOTE: We can always assume the last token is the matched stop token return output[:-1] diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index a5f9ddeafe..c149eb8faf 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -12,7 +12,10 @@ from sglang.srt.entrypoints.openai.encoding_dsv4 import ( from sglang.srt.entrypoints.openai.encoding_dsv4 import ( thinking_start_token as dsv4_thinking_start_token, ) -from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + ResponsesRequest, +) from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens from sglang.srt.function_call.kimik3_format import ( MESSAGE_CLOSE, @@ -49,6 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import ( INKLING_SPECIAL_TOKEN_IDS, MESSAGE_MODEL, ) +from sglang.srt.parser.response_template import ResponseTemplateReasoningDetector class StreamingParseResult: @@ -2299,6 +2303,7 @@ class ReasoningParser: "inkling": InklingDetector, "iquest_q1": IQuestQ1ReasoningDetector, "cohere_command4": CohereCommand4Detector, + "response_template": ResponseTemplateReasoningDetector, } def __init__( @@ -2309,6 +2314,7 @@ class ReasoningParser: request: ChatCompletionRequest = None, tokenizer=None, tool_call_parser_active: bool = False, + prefix: str | None = None, ): if not model_type: raise ValueError("Model type must be specified") @@ -2386,6 +2392,14 @@ class ReasoningParser: if "tool_call_parser_active" in sig.parameters: kwargs["tool_call_parser_active"] = True + if prefix is None: + prefix = getattr(request, "_response_parser_prefix", "") + if issubclass(detector_class, ResponseTemplateReasoningDetector): + if prefix: + kwargs["prefix"] = prefix + if isinstance(request, (ChatCompletionRequest, ResponsesRequest)): + kwargs["grammar_start"] = request._response_parser_grammar_start + self.detector = detector_class(**kwargs) def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]: diff --git a/python/sglang/srt/parser/reasoning_parser_names.py b/python/sglang/srt/parser/reasoning_parser_names.py index c061572129..23227dd9f3 100644 --- a/python/sglang/srt/parser/reasoning_parser_names.py +++ b/python/sglang/srt/parser/reasoning_parser_names.py @@ -38,4 +38,5 @@ REASONING_PARSER_NAMES = [ "iquest_q1", "cohere_command4", "gigachat35", + "response_template", ] diff --git a/python/sglang/srt/parser/response_template.py b/python/sglang/srt/parser/response_template.py new file mode 100644 index 0000000000..77fcc96c36 --- /dev/null +++ b/python/sglang/srt/parser/response_template.py @@ -0,0 +1,787 @@ +# Copyright 2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""SGLang serving adapters for checkpoint-defined `response_template` grammars.""" + +from __future__ import annotations + +import copy +import functools +import json +import logging +from collections.abc import Callable, Sequence +from typing import Any + +import msgspec + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.environ import envs +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult as ToolStreamingParseResult, +) +from sglang.srt.function_call.core_types import ( + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.parser.chat_parsing.content_parsers import _apply_transform +from sglang.srt.parser.chat_parsing.response_parser import ResponseParser +from sglang.srt.parser.chat_parsing.response_templates import ( + ResponseTemplate, + load_response_template, +) + +logger = logging.getLogger(__name__) + +_THINKING_FIELD = "thinking" +_CONTENT_FIELD = "content" +_TOOL_FIELD = "tool_calls" +_PASSTHROUGH_FIELD = "normal" +_RAW_MEMBERS_KEY = "members" +_SUPPORTED_RESPONSE_TEMPLATE_FIELDS = frozenset( + {_THINKING_FIELD, _CONTENT_FIELD, _TOOL_FIELD} +) +# Where JSON-schema output replaces the template's framing, following the +# reasoning gate of ReasonerGrammarBackend. +GRAMMAR_AFTER_REASONING = "after_reasoning" +GRAMMAR_FROM_START = "from_start" + + +class _ServingTemplates(msgspec.Struct, frozen=True): + """A validated template and the variants the serving detectors parse with.""" + + source: ResponseTemplate + streaming: ResponseTemplate + reasoning_only: ResponseTemplate | None + tool_extraction: ResponseTemplate | None + + +def validate_response_template_for_serving(template: dict) -> ResponseTemplate: + """Validate the template and every variant the serving detectors build.""" + return _derive_serving_templates(template).source + + +def _derive_serving_templates(template: dict) -> _ServingTemplates: + # Detectors are built several times per request; derive each template once. + return _derive_serving_templates_from_json(json.dumps(template)) + + +@functools.lru_cache(maxsize=8) +def _derive_serving_templates_from_json(template_json: str) -> _ServingTemplates: + template = json.loads(template_json) + loaded = load_response_template(template) + _check_serving_fields(loaded) + return _ServingTemplates( + source=loaded, + streaming=_streaming_template(template), + reasoning_only=( + _reasoning_only_template(template) + if _THINKING_FIELD in loaded.fields + else None + ), + tool_extraction=( + _tool_extraction_template(template) + if _TOOL_FIELD in loaded.fields + else None + ), + ) + + +def _check_serving_fields(loaded: ResponseTemplate) -> None: + fields = set(loaded.fields) + unsupported = (fields - _SUPPORTED_RESPONSE_TEMPLATE_FIELDS) | ( + set(loaded.defaults) - {"role"} + ) + if unsupported: + raise ValueError( + "response_template contains unsupported semantic fields: " + f"{sorted(unsupported)}. Supported fields are: " + f"{sorted(_SUPPORTED_RESPONSE_TEMPLATE_FIELDS)}" + ) + for name in (_THINKING_FIELD, _CONTENT_FIELD): + field = loaded.fields.get(name) + if field is None: + continue + if ( + field.content != "text" + or field.transform is not None + or field.join is not None + ): + raise ValueError( + f"response_template field {name!r} uses semantics that cannot " + "be streamed by the OpenAI serving adapter" + ) + tool_field = loaded.fields.get(_TOOL_FIELD) + if tool_field is not None and tool_field.open_re is None: + raise ValueError( + f"response_template field {_TOOL_FIELD!r} needs 'open' or " + "'open_pattern' so tool calls can be told apart from other text" + ) + + +def resolve_response_template( + tokenizer: Any | None, + template: dict | None = None, +) -> dict | None: + """Return `template` if given, otherwise the tokenizer's metadata.""" + if template is not None or tokenizer is None: + return template + template = getattr(tokenizer, "response_template", None) + if not isinstance(template, dict): + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + template = init_kwargs.get("response_template") + return template if isinstance(template, dict) else None + + +def configure_response_template_request(request: Any) -> None: + """Preserve response-template delimiters during detokenization.""" + request.skip_special_tokens = False + template_kwargs = dict(getattr(request, "chat_template_kwargs", None) or {}) + template_kwargs["spaces_between_special_tokens"] = False + request.chat_template_kwargs = template_kwargs + + +def _load_serving_templates( + tokenizer: Any | None, template: dict | None, fallback: dict | None +) -> tuple[dict, _ServingTemplates]: + """Use the explicit template, then checkpoint metadata, then the detector's + built-in fallback.""" + template = resolve_response_template(tokenizer, template) or fallback + if template is None: + raise ValueError("response_template is required") + return template, _derive_serving_templates(template) + + +def _streaming_template(template: dict[str, Any]) -> ResponseTemplate: + result = copy.deepcopy(template) + fields = result["fields"] + if _CONTENT_FIELD not in fields and all( + "open" in field or "open_pattern" in field for field in fields.values() + ): + fields[_CONTENT_FIELD] = {} + for field in fields.values(): + field["optional"] = True + return load_response_template(result) + + +def _reasoning_only_template(template: dict[str, Any]) -> ResponseTemplate: + thinking = copy.deepcopy(template["fields"][_THINKING_FIELD]) + thinking["optional"] = True + anchor_name = ( + "start_anchor_pattern" if "start_anchor_pattern" in template else "start_anchor" + ) + return load_response_template( + {anchor_name: template[anchor_name], "fields": {_THINKING_FIELD: thinking}} + ) + + +def _tool_extraction_template(template: dict[str, Any]) -> ResponseTemplate: + tool_field = copy.deepcopy(template["fields"][_TOOL_FIELD]) + tool_field["optional"] = True + # The detector transforms each member itself, so a malformed member drops alone; + # the envelope keeps delimiter captures valid and hides raw members from schema + # coercion, which must only see the call each member's transform produces. + if tool_field.pop("transform_each", False): + tool_field["transform"] = {_RAW_MEMBERS_KEY: "{content}"} + anchor_name = ( + "start_anchor_pattern" if "start_anchor_pattern" in template else "start_anchor" + ) + return load_response_template( + { + anchor_name: template[anchor_name], + "fields": {_PASSTHROUGH_FIELD: {}, _TOOL_FIELD: tool_field}, + } + ) + + +def _passthrough_chunk(text: str) -> dict: + return {"type": "region_chunk", "field": _PASSTHROUGH_FIELD, "text": text} + + +class _ReasoningResult: + """Duck-types `reasoning_parser.StreamingParseResult`.""" + + def __init__(self, normal_text: str = "", reasoning_text: str = ""): + self.normal_text = normal_text + self.reasoning_text = reasoning_text + + +class ResponseTemplateStreamAdapter: + """Parse one response with a template and route its generic region events.""" + + def __init__( + self, + template: ResponseTemplate, + *, + prefix: str = "", + grammar_start: str | None = None, + each_transform: Any = None, + ): + self._template = template + self._prefix = prefix + self._parser: ResponseParser | None = None + self._prefix_end = 0 + self._finalized = False + tool_field = template.fields.get(_TOOL_FIELD) + self._tool_close_re = None if tool_field is None else tool_field.close_re + self._each_transform = each_transform + self._pending_reasoning = "" + self._pending_tool_streamed = False + self._open_tool_captures: dict = {} + self._grammar_start = grammar_start + # Once JSON-schema output has started, text passes through unparsed. + self._verbatim = grammar_start == GRAMMAR_FROM_START + + def _start(self, tools: Sequence[Any] | None) -> list[dict]: + """Create the parser and replay a tool region left open by the prefix.""" + if self._parser is not None: + return [] + self._parser = ResponseParser( + self._template, + prefix=self._prefix, + tools=[ + tool if isinstance(tool, dict) else tool.model_dump() + for tool in tools or [] + ], + ) + self._prefix_end = len(self._parser.input_text) + for event in reversed(self._parser.initial_events): + if event.get("field") == _TOOL_FIELD and event["type"] != "region_chunk": + return [event] if event["type"] == "region_open" else [] + return [] + + def feed(self, text: str, tools: Sequence[Any] | None = None) -> list[dict]: + if self._finalized: + return [] + if self._verbatim: + return [_passthrough_chunk(text)] if text else [] + events = self._start(tools) + return self._end_reasoning(events + self._parser.feed(text)) + + def finalize(self, tools: Sequence[Any] | None = None) -> list[dict]: + if self._finalized: + return [] + events = self._start(tools) + self._finalized = True + if self._verbatim: + return events + return self._end_reasoning(events + self._parser.finalize()[1]) + + def _end_reasoning(self, events: list[dict]) -> list[dict]: + # Stop parsing at the reasoning closer, where the grammar backend starts + # the JSON schema, and pass the rest through unchanged. + if self._grammar_start != GRAMMAR_AFTER_REASONING: + return events + for index, event in enumerate(events): + if event.get("field") == _THINKING_FIELD and event["type"] in ( + "region_close", + "region_malformed", + ): + self._verbatim = True + text = self._parser.input_text[event["end"] :] + return events[: index + 1] + ( + [_passthrough_chunk(text)] if text else [] + ) + return events + + def parse(self, text: str, tools: Sequence[Any] | None = None) -> list[dict]: + return self.feed(text, tools) + self.finalize(tools) + + def _generated_text(self, event: dict) -> str: + start = max(event["start"], self._prefix_end) + return self._parser.input_text[start : event["end"]] + + def _open_tool_name(self, captures: dict | None) -> str | None: + """Name already fixed by the opener, before the region body exists.""" + field = self._template.fields.get(_TOOL_FIELD) + transform = None if field is None or field.transform_each else field.transform + function = transform.get("function") if isinstance(transform, dict) else None + if not isinstance(function, dict): + return None + try: + name = _apply_transform(function.get("name"), captures or {}) + except (KeyError, ValueError): + return None + return name if isinstance(name, str) else None + + def route_reasoning_events( + self, events: list[dict], *, stream_reasoning: bool + ) -> _ReasoningResult: + reasoning_parts: list[str] = [] + normal_parts: list[str] = [] + for event in events: + field = event.get("field") + etype = event["type"] + if field == _THINKING_FIELD: + if etype == "region_chunk": + if stream_reasoning: + reasoning_parts.append(event["text"]) + else: + self._pending_reasoning += event["text"] + elif etype == "region_close" and not stream_reasoning: + reasoning_parts.append(self._pending_reasoning) + self._pending_reasoning = "" + elif ( + field in (_CONTENT_FIELD, _PASSTHROUGH_FIELD) + and etype == "region_chunk" + ): + normal_parts.append(event["text"]) + elif field == _TOOL_FIELD: + # Hand tool regions back verbatim so the tool detector can parse them. + normal_parts.append( + event["text"] + if etype == "region_chunk" + else self._generated_text(event) + ) + return _ReasoningResult("".join(normal_parts), "".join(reasoning_parts)) + + def route_tool_events( + self, + events: list[dict], + *, + on_tool_open: Callable[[str], bool] | None = None, + on_tool_close: Callable[[Any], bool], + on_tool_dropped: Callable[[], None] | None = None, + ) -> str: + """Route tool regions to the callbacks and return the passthrough text. + + A tool call is emitted only when it ends with its closer, parses, and is + accepted by `on_tool_close`. Otherwise it is dropped, and a name that was + already streamed stays an incomplete call.""" + normal_parts: list[str] = [] + unusable_opens: set[int] = set() + open_index = None + for index, event in enumerate(events): + if event.get("field") == _TOOL_FIELD: + if event["type"] == "region_open": + open_index = index + elif self._is_unusable_tool_end(event): + unusable_opens.add(open_index) + for index, event in enumerate(events): + field = event.get("field") + etype = event["type"] + if field == _PASSTHROUGH_FIELD and etype == "region_chunk": + normal_parts.append(event["text"]) + elif field != _TOOL_FIELD or etype == "region_chunk": + continue + elif etype == "region_open": + self._open_tool_captures = event.get("captures") or {} + name = self._open_tool_name(event.get("captures")) + self._pending_tool_streamed = bool( + index not in unusable_opens + and name is not None + and on_tool_open is not None + and on_tool_open(name) + ) + else: + if self._is_unusable_tool_end(event): + logger.warning( + "response_template: dropping malformed or cut-off tool call" + ) + accepted = False + else: + accepted = on_tool_close(self._transform_members(event["value"])) + if not accepted and self._pending_tool_streamed and on_tool_dropped: + on_tool_dropped() + self._pending_tool_streamed = False + return "".join(normal_parts) + + def _transform_members(self, value: Any) -> Any: + """Apply a `transform_each` template to each member of a tool region, + dropping the members it cannot transform, and cast their arguments to + the tool schemas as the parser does for transformed regions.""" + if self._each_transform is None: + return value + value = value[_RAW_MEMBERS_KEY] + if not isinstance(value, list): + logger.warning("response_template: malformed tool call value: %r", value) + return [] + members = [] + for member in value: + try: + scope = {**self._open_tool_captures, **member} + call = _apply_transform(self._each_transform, scope) + members.append(self._parser._coerce_tool_calls(call)) + except (KeyError, TypeError, ValueError): + logger.warning( + "response_template: malformed tool call value: %r", member + ) + return members + + def _is_unusable_tool_end(self, event: dict) -> bool: + """Whether a tool region ended malformed, or at end of stream without its + closer because the call was cut off.""" + if event["type"] == "region_malformed": + return True + text = self._parser.input_text + # End-of-stream finalization and a matched zero-width closer such as `\Z` + # emit the same close event, so check whether the closer matches there. + return ( + event["type"] == "region_close" + and self._tool_close_re is not None + and event["start"] == len(text) + and self._tool_close_re.match(text, len(text)) is None + ) + + +class ResponseTemplateReasoningDetector: + """Reasoning detector driven by a `response_template` grammar.""" + + response_template: dict | None = None + _default_think_start = "" + _default_think_end = "" + thinks_internally = False + reasoning_default = "explicit_enable_thinking" + + def __init__( + self, + stream_reasoning: bool = True, + tokenizer=None, + response_template: dict | None = None, + prefix: str = "", + force_nonempty_content: bool = False, + grammar_start: str | None = None, + **_kwargs, + ): + template, templates = _load_serving_templates( + tokenizer, response_template, self.response_template + ) + self.response_template = template + self.stream_reasoning = stream_reasoning + self.think_excluded_tokens = None + self._force_nonempty_content = force_nonempty_content + self._open_reasoning = "" + thinking = templates.source.fields.get(_THINKING_FIELD) + self.think_start_token = ( + thinking.open_literals[0] + if thinking and thinking.open_literals + else self._default_think_start + ) + self.think_end_token = ( + thinking.close_literals[0] + if thinking and thinking.close_literals + else self._default_think_end + ) + self.think_start_self_label = "" + if grammar_start == GRAMMAR_AFTER_REASONING and thinking is None: + grammar_start = GRAMMAR_FROM_START + self._grammar_start = grammar_start + if grammar_start == GRAMMAR_AFTER_REASONING: + self._template = templates.reasoning_only + else: + self._template = templates.streaming + self._prefix = prefix + self._adapter = self._new_adapter() + + def _new_adapter(self) -> ResponseTemplateStreamAdapter: + return ResponseTemplateStreamAdapter( + self._template, prefix=self._prefix, grammar_start=self._grammar_start + ) + + def get_think_end_token_ids(self, tokenizer) -> list[int]: + return tokenizer.encode(self.think_end_token, add_special_tokens=False) + + def get_answer_frame(self) -> None: + return None + + def detect_and_parse(self, text: str) -> _ReasoningResult: + adapter = self._new_adapter() + result = adapter.route_reasoning_events( + adapter.parse(text), stream_reasoning=True + ) + if self._force_nonempty_content and not result.normal_text: + result.normal_text, result.reasoning_text = result.reasoning_text, "" + return result + + def parse_streaming_increment(self, new_text: str) -> _ReasoningResult: + events = self._adapter.feed(new_text) + if self._force_nonempty_content: + self._track_open_reasoning(events) + return self._adapter.route_reasoning_events( + events, stream_reasoning=self.stream_reasoning + ) + + def finish(self) -> _ReasoningResult: + events = self._adapter.finalize() + result = self._adapter.route_reasoning_events( + events, stream_reasoning=self.stream_reasoning + ) + # Under force_nonempty_content, reasoning left open at the end of the + # stream is also returned as content. + if self._force_nonempty_content: + unclosed_reasoning = self._track_open_reasoning(events) + if unclosed_reasoning is not None: + result.normal_text += unclosed_reasoning + result.reasoning_text = "" + return result + + def _track_open_reasoning(self, events: list[dict]) -> str | None: + """Accumulate open thinking text; return the text of a region `events` close.""" + closed = None + for event in events: + if event.get("field") != _THINKING_FIELD: + continue + if event["type"] == "region_chunk": + self._open_reasoning += event["text"] + elif event["type"] in ("region_close", "region_malformed"): + closed, self._open_reasoning = self._open_reasoning, "" + return closed + + +class ResponseTemplateToolDetector(BaseFormatDetector): + """Tool-call detector driven by a `response_template` grammar.""" + + response_template: dict | None = None + + def __init__( + self, + tokenizer=None, + response_template: dict | None = None, + prefix: str = "", + ): + super().__init__() + template, templates = _load_serving_templates( + tokenizer, response_template, self.response_template + ) + self.response_template = template + if templates.tool_extraction is None: + raise ValueError(f"response_template has no {_TOOL_FIELD!r} field") + self._template = templates.tool_extraction + self._prefix = prefix + tool_field = templates.source.fields[_TOOL_FIELD] + self._each_transform = ( + tool_field.transform if tool_field.transform_each else None + ) + self._adapter = ResponseTemplateStreamAdapter( + self._template, prefix=prefix, each_transform=self._each_transform + ) + self.tool_close_literals = tool_field.close_literals or [] + if self.tool_close_literals: + self.eot_token = self.tool_close_literals[0] + self.incomplete_tool_call_indices: set[int] = set() + self._tool_indices: dict[str, int] | None = None + + def has_tool_call(self, text: str) -> bool: + adapter = ResponseTemplateStreamAdapter(self._template, prefix=self._prefix) + return any( + event["type"] == "region_open" and event.get("field") == _TOOL_FIELD + for event in adapter.parse(text) + ) + + def _to_tool_call( + self, value: Any, tool_indices: dict[str, int] + ) -> tuple[str, str] | None: + try: + function = value["function"] + name = function["name"] + arguments = function["arguments"] + except (TypeError, KeyError): + logger.warning("response_template: malformed tool call value: %r", value) + return None + + if not isinstance(name, str): + logger.warning("response_template: malformed tool name: %r", name) + return None + + if name not in tool_indices: + logger.warning("Model attempted to call undefined function: %s", name) + if not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get(): + return None + + if not isinstance(arguments, str): + arguments = json.dumps(arguments, ensure_ascii=False) + return name, arguments + + def _to_tool_call_items( + self, + value: Any, + tool_indices: dict[str, int], + first_tool_index: int, + ) -> list[ToolCallItem]: + """The region's accepted calls, numbered from `first_tool_index`.""" + values = value if isinstance(value, list) else [value] + calls = [ + call + for call in (self._to_tool_call(item, tool_indices) for item in values) + if call is not None + ] + return [ + ToolCallItem( + tool_index=first_tool_index + offset, name=name, parameters=arguments + ) + for offset, (name, arguments) in enumerate(calls) + ] + + def _emit_tool_call( + self, item: ToolCallItem, pending_calls: list[ToolCallItem] + ) -> None: + name_was_streamed = self.current_tool_name_sent + self._ensure_tool_slot() + try: + parsed_args = json.loads(item.parameters) + except json.JSONDecodeError: + parsed_args = item.parameters + self.prev_tool_call_arr[self.current_tool_id] = { + "name": item.name, + "arguments": parsed_args, + } + self.streamed_args_for_tool[self.current_tool_id] = item.parameters + if name_was_streamed: + item = item.model_copy(update={"name": None}) + pending_calls.append(item) + self.incomplete_tool_call_indices.discard(self.current_tool_id) + self.current_tool_id += 1 + self.current_tool_name_sent = False + + def _ensure_tool_slot(self) -> None: + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + def _emit_tool_name(self, name: str, pending_calls: list[ToolCallItem]) -> None: + self._ensure_tool_slot() + pending_calls.append( + ToolCallItem(tool_index=self.current_tool_id, name=name, parameters="") + ) + self.prev_tool_call_arr[self.current_tool_id] = {"name": name, "arguments": ""} + self.incomplete_tool_call_indices.add(self.current_tool_id) + self.current_tool_name_sent = True + + def _drop_streamed_tool_call(self) -> None: + """Leave the streamed name as an incomplete call and move to the next.""" + self.current_tool_id += 1 + self.current_tool_name_sent = False + + def detect_and_parse( + self, text: str, tools: list[Tool] + ) -> ToolStreamingParseResult: + adapter = ResponseTemplateStreamAdapter( + self._template, prefix=self._prefix, each_transform=self._each_transform + ) + tool_indices = self._get_tool_indices(tools) + calls: list[ToolCallItem] = [] + + def on_close(value: Any) -> bool: + items = self._to_tool_call_items(value, tool_indices, len(calls)) + if not items: + return False + calls.extend(items) + return True + + normal_text = adapter.route_tool_events( + adapter.parse(text, tools), on_tool_close=on_close + ) + return ToolStreamingParseResult(normal_text=normal_text, calls=calls) + + def parse_streaming_increment( + self, new_text: str, tools: list[Tool] + ) -> ToolStreamingParseResult: + return self._emit_streaming_events(self._adapter.feed(new_text, tools), tools) + + def finish(self, tools: list[Tool]) -> ToolStreamingParseResult: + return self._emit_streaming_events(self._adapter.finalize(tools), tools) + + def _emit_streaming_events( + self, events: list[dict], tools: list[Tool] + ) -> ToolStreamingParseResult: + if self._tool_indices is None: + self._tool_indices = self._get_tool_indices(tools) + tool_indices = self._tool_indices + pending_calls: list[ToolCallItem] = [] + + def on_open(name: str) -> bool: + if name not in tool_indices and not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get(): + return False + self._emit_tool_name(name, pending_calls) + return True + + def on_close(value: Any) -> bool: + items = self._to_tool_call_items( + value, tool_indices, max(self.current_tool_id, 0) + ) + if not items: + return False + for item in items: + self._emit_tool_call(item, pending_calls) + return True + + normal_text = self._adapter.route_tool_events( + events, + on_tool_open=on_open, + on_tool_close=on_close, + on_tool_dropped=self._drop_streamed_tool_call, + ) + return ToolStreamingParseResult(normal_text=normal_text, calls=pending_calls) + + def supports_structural_tag(self) -> bool: + return False + + def validate_structure_constraint_request( + self, + tool_choice: Any, + parallel_tool_calls: bool, + strict_requested: bool, + ) -> None: + if tool_choice == "auto" and not parallel_tool_calls: + raise ValueError( + f"{type(self).__name__} cannot enforce " + "parallel_tool_calls=False with automatic tool choice" + ) + if tool_choice == "auto" and strict_requested: + raise ValueError( + f"{type(self).__name__} does not support strict tool constraints" + ) + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError( + "structure_info not used with the response_template tool parser" + ) + + +def tool_close_tokens( + tool_call_parser: str | None, tokenizer: Any | None +) -> dict[int, str]: + """Single-token tool-call closers of a response-template tool parser, by id. + + Detokenization keeps these when they stop generation, so a tool call that + reaches end of stream without its closer was cut off.""" + from sglang.srt.function_call.function_call_parser import FunctionCallParser + + detector_class = FunctionCallParser.ToolCallParserEnum.get(tool_call_parser) + if ( + tokenizer is None + or detector_class is None + or not issubclass(detector_class, ResponseTemplateToolDetector) + ): + return {} + try: + detector = detector_class(tokenizer=tokenizer) + except ValueError: + return {} + closers = {} + for literal in detector.tool_close_literals: + ids = tokenizer.encode(literal, add_special_tokens=False) + if len(ids) == 1: + closers[ids[0]] = literal + return closers + + +def tool_close_token_ids( + tool_call_parser: str | None, tokenizer: Any | None +) -> frozenset[int]: + return frozenset(tool_close_tokens(tool_call_parser, tokenizer)) diff --git a/python/sglang/srt/parser/template_detection.py b/python/sglang/srt/parser/template_detection.py index 42384d97ed..c46c4c652f 100644 --- a/python/sglang/srt/parser/template_detection.py +++ b/python/sglang/srt/parser/template_detection.py @@ -773,7 +773,6 @@ def _detect_auto_parser( f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template" ) return detected - _log_undetected_parser(attr, label) return None @@ -924,7 +923,8 @@ def _architecture_parser_defaults(server_args) -> Dict[str, str]: def resolve_auto_parsers(server_args) -> None: - """Resolve architecture defaults and explicit ``auto`` parser requests. + """Resolve architecture defaults and explicit ``auto`` parser requests; + ``auto`` is detected from the chat template or checkpoint metadata. Performs a lightweight tokenizer load, so it runs once in engine init. The decision goes to this instance's declaration stash, so every holder of it @@ -982,8 +982,11 @@ def resolve_auto_parsers(server_args) -> None: tokenizer = None try: tokenizer = get_tokenizer( - cfg.model_path, + cfg.tokenizer_path, + tokenizer_mode=cfg.tokenizer_mode, trust_remote_code=cfg.trust_remote_code, + revision=cfg.revision, + tokenizer_backend=cfg.tokenizer_backend, ) except Exception as e: logger.warning(f"Failed to load tokenizer for auto-detection: {e}") @@ -1026,7 +1029,6 @@ def resolve_auto_parsers(server_args) -> None: ("tool_call_parser", "tool-call parser"), ): if attr in needs and attr not in detected: - _log_undetected_parser(attr, label) detected[attr] = None else: for attr, rules, label in ( @@ -1036,5 +1038,42 @@ def resolve_auto_parsers(server_args) -> None: if attr in needs: detected[attr] = _detect_auto_parser(attr, ctx, rules, label) + unresolved = tuple(attr for attr in needs if detected.get(attr) is None) + if unresolved and tokenizer is not None and chat_template_arg is None: + from sglang.srt.parser.response_template import ( + resolve_response_template, + validate_response_template_for_serving, + ) + + response_template = resolve_response_template(tokenizer) + if response_template is not None: + try: + response_template = validate_response_template_for_serving( + response_template + ) + except (TypeError, ValueError) as exc: + logger.warning( + "Ignoring invalid response_template from tokenizer configuration: %s", + exc, + ) + else: + for attr, field in ( + ("reasoning_parser", "thinking"), + ("tool_call_parser", "tool_calls"), + ): + if attr in unresolved and field in response_template.fields: + detected[attr] = "response_template" + logger.info( + "Auto-detected --%s as 'response_template' from tokenizer configuration", + attr.replace("_", "-"), + ) + + for attr, label in ( + ("reasoning_parser", "reasoning parser"), + ("tool_call_parser", "tool-call parser"), + ): + if attr in needs and detected.get(attr) is None: + _log_undetected_parser(attr, label) + if detected: declare_resolution(server_args, "template-detection", **detected) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 72fb9d54ce..f2af23bd65 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -370,7 +370,7 @@ class ServerArgs: choices=["auto"] + reasoning_parser_choices, default=_declared_default("reasoning_parser"), help=f"Specify the parser for reasoning models. " - f"Use 'auto' to detect from chat template. " + f"Use 'auto' to detect from chat template or checkpoint metadata. " f"Options include: {reasoning_parser_choices}.", ) tool_call_parser_choices = _tool_call_parser_choices() @@ -380,7 +380,7 @@ class ServerArgs: choices=["auto"] + tool_call_parser_choices, default=_declared_default("tool_call_parser"), help=f"Specify the parser for handling tool-call interactions. " - f"Use 'auto' to detect from chat template. " + f"Use 'auto' to detect from chat template or checkpoint metadata. " f"Options include: {tool_call_parser_choices}.", ) parser.add_argument( diff --git a/test/registered/unit/beam_search/test_output_decode.py b/test/registered/unit/beam_search/test_output_decode.py index 21a8f0e596..e8b3c62faf 100644 --- a/test/registered/unit/beam_search/test_output_decode.py +++ b/test/registered/unit/beam_search/test_output_decode.py @@ -25,7 +25,9 @@ STOP_ID = 99 def _trim(output, finished_reason, no_stop_trim): - stub = SimpleNamespace(is_tool_call_parser_gpt_oss=False) + stub = SimpleNamespace( + is_tool_call_parser_gpt_oss=False, tool_close_token_ids=frozenset() + ) return DetokenizerManager.trim_matched_stop( stub, output, finished_reason, no_stop_trim ) diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index b50b2acc87..a8bb48b9c1 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -30,7 +30,9 @@ from sglang.srt.entrypoints.openai.chat_encoding import ( ) from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, + Function, MessageProcessingResult, + Tool, ToolChoice, ToolChoiceFuncName, ) @@ -39,11 +41,18 @@ from sglang.srt.entrypoints.openai.serving_chat import ( normalize_tool_content, ) from sglang.srt.environ import envs +from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN from sglang.srt.managers.io_struct import GenerateReqInput from sglang.srt.parser.jinja_template_utils import ( jinja_template_may_reorder_tool_results, ) +from sglang.srt.parser.response_template import ( + GRAMMAR_AFTER_REASONING, + GRAMMAR_FROM_START, + ResponseTemplateReasoningDetector, + ResponseTemplateToolDetector, +) from sglang.srt.parser.template_detection import ReasoningToggleConfig from sglang.srt.runtime_context import get_context, publish, reset_context from sglang.srt.sampling.sampling_params import ( @@ -58,6 +67,45 @@ register_cpu_ci(est_time=13, suite="base-a-test-cpu") # Every spec resolve_chat_encoding_spec can return; pinned by the guard below. _ALL_CHAT_ENCODING_SPECS = ("dsv41", "dsv4", "dsv32", "inkling", "kimi_k3") +_RESPONSE_TEMPLATE = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}", + }, + }, + }, + }, +} +# Every field is delimited, so the template gives no field to text outside them. +_DELIMITED_RESPONSE_TEMPLATE = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + "content": {"open": "", "close": ""}, + "tool_calls": _RESPONSE_TEMPLATE["fields"]["tool_calls"], + }, +} +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, +} + def _spec_result(index): return { @@ -621,6 +669,329 @@ class ServingChatTestCase(CustomTestCase): self.assertEqual(adapted.session_id, "session-1") self.assertEqual(processed, self.basic_req) + def test_response_template_prefix_uses_prompt_token_ids(self): + class ResponseTemplateAlias(ResponseTemplateToolDetector): + pass + + processed_messages = MessageProcessingResult( + "prompt decoded with default spacing", + [1, 2, 3], + None, + None, + [], + [], + None, + uses_response_template=True, + ) + self.chat.tool_call_parser = "response-template-alias" + self.basic_req.input_ids = [1, 2, 3] + self.basic_req.tools = [ + Tool(function=Function(name="get_weather", parameters={})) + ] + self.basic_req.tool_choice = "auto" + self.tm.tokenizer.decode.return_value = "" + + with ( + patch.dict( + FunctionCallParser.ToolCallParserEnum, + {"response-template-alias": ResponseTemplateAlias}, + ), + patch.object( + self.chat, + "_process_messages", + return_value=processed_messages, + ), + ): + self.assertTrue( + self.chat._requires_response_template_detokenization(self.basic_req) + ) + _, request = self.chat._convert_to_internal_request(self.basic_req) + + self.assertEqual(request._response_parser_prefix, "") + self.tm.tokenizer.decode.assert_called_once_with( + [1, 2, 3], + skip_special_tokens=False, + spaces_between_special_tokens=False, + ) + + def test_response_template_prefix_uses_engine_prompt_text(self): + processed_messages = MessageProcessingResult( + "exact engine prompt", + [], + None, + None, + [], + [], + None, + ) + self.chat.reasoning_parser = "response_template" + self.chat._reasoning_detector = ResponseTemplateReasoningDetector( + response_template=_RESPONSE_TEMPLATE, + prefix="", + ) + self.tm.model_config.is_multimodal = True + + def render(request, _): + self.assertIsNone(request.chat_template_kwargs) + return processed_messages + + with patch.object( + self.chat, + "_apply_conversation_template", + side_effect=render, + ): + adapted, request = self.chat._convert_to_internal_request(self.basic_req) + + self.assertEqual(request._response_parser_prefix, "exact engine prompt") + self.assertFalse(request.skip_special_tokens) + self.assertFalse(adapted.sampling_params["spaces_between_special_tokens"]) + self.tm.tokenizer.decode.assert_not_called() + + def _use_delimited_response_template( + self, reasoning_parser, template=_DELIMITED_RESPONSE_TEMPLATE + ): + self.tm.tokenizer.response_template = template + self.tm.tokenizer.encode.side_effect = lambda text, **_: ( + [7] if text == "" else [1, 2, 3] if text else [] + ) + self.tm.model_config.is_multimodal = True + self.chat.tool_call_parser = "response_template" + self.chat.reasoning_parser = reasoning_parser + self.chat._reasoning_detector = ( + ResponseTemplateReasoningDetector(response_template=template) + if reasoning_parser + else None + ) + # Always-on reasoning: the output grammar starts after the reasoning. + self.template_manager.reasoning_config = ( + ReasoningToggleConfig(special_case="always") if reasoning_parser else None + ) + + @staticmethod + def _stopped_on_token(text, token_id): + return { + "text": text, + "meta_info": { + "id": "chatcmpl-stopped", + "prompt_tokens": 5, + "completion_tokens": 9, + "cached_tokens": 0, + "weight_version": "default", + "finish_reason": {"type": "stop", "matched": token_id}, + }, + "index": 0, + } + + def _convert_rendered_as(self, prompt, request): + rendered = MessageProcessingResult( + prompt=prompt, + prompt_ids=[], + image_data=None, + audio_data=None, + video_data=None, + modalities=[], + stop=[], + ) + with patch.object( + self.chat, "_apply_conversation_template", return_value=rendered + ): + return self.chat._convert_to_internal_request(request) + + def test_response_template_parser_context_records_grammar_start(self): + """JSON-schema output starts after the reasoning only when the grammar + backend waits for it, which a pattern-only reasoning closer cannot.""" + named = {"type": "function", "function": {"name": "get_weather"}} + required = {"tools": [_WEATHER_TOOL], "tool_choice": "required"} + pattern_closed = { + **_DELIMITED_RESPONSE_TEMPLATE, + "fields": { + **_DELIMITED_RESPONSE_TEMPLATE["fields"], + "thinking": {"open": "", "close_pattern": ""}, + }, + } + cases = { + "auto tool choice": ("response_template", {"tools": [_WEATHER_TOOL]}, None), + "named tool after reasoning": ( + "response_template", + {"tools": [_WEATHER_TOOL], "tool_choice": named}, + GRAMMAR_AFTER_REASONING, + ), + "response_format after reasoning": ( + "response_template", + {"response_format": {"type": "json_object"}}, + GRAMMAR_AFTER_REASONING, + ), + "required tool without reasoning": (None, required, GRAMMAR_FROM_START), + "required tool, reasoning without an end token": ( + "response_template", + required, + GRAMMAR_FROM_START, + pattern_closed, + ), + } + for name, (reasoning_parser, fields, grammar_start, *template) in cases.items(): + with self.subTest(name): + self._use_delimited_response_template(reasoning_parser, *template) + _, request = self._convert_rendered_as( + "", + ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Weather?"}], + **fields, + ), + ) + self.assertEqual(request._response_parser_grammar_start, grammar_start) + + def _chat_outcome(self, adapted, request, output): + """(calls, reasoning, content, finish reasons) as a client reconstructs them.""" + if not request.stream: + choice = self.chat._build_chat_response(request, [output], 0).choices[0] + calls = [ + (call.function.name, call.function.arguments) + for call in choice.message.tool_calls or [] + ] + return ( + calls, + choice.message.reasoning_content or "", + choice.message.content or "", + [choice.finish_reason], + ) + + async def generate(): + yield output + + self.tm.generate_request = Mock(return_value=generate()) + choices = [ + choice + for chunk in self._parse_chunks(self._run_chat_stream(adapted, request)) + for choice in chunk.get("choices", []) + ] + deltas = [choice.get("delta") or {} for choice in choices] + calls: dict[int, list[str]] = {} + for delta in deltas: + for call in delta.get("tool_calls") or []: + name, arguments = calls.setdefault(call["index"], ["", ""]) + calls[call["index"]] = [ + name + (call["function"].get("name") or ""), + arguments + (call["function"].get("arguments") or ""), + ] + return ( + [tuple(calls[index]) for index in sorted(calls)], + "".join(delta.get("reasoning_content") or "" for delta in deltas), + "".join(delta.get("content") or "" for delta in deltas), + [ + choice["finish_reason"] + for choice in choices + if choice.get("finish_reason") + ], + ) + + def test_required_tool_json_is_parsed_under_delimited_template(self): + """Required-tool JSON after the reasoning becomes the call, also when it + ends on the tool closer that detokenization keeps.""" + payload = '[{"name": "get_weather", "parameters": {"city": "Paris"}}]' + for reasoning_parser in (None, "response_template"): + self._use_delimited_response_template(reasoning_parser) + reasoning = "Check the forecast." if reasoning_parser else "" + text = (f"{reasoning}" if reasoning else "") + payload + text += "" + for stream in (False, True): + with self.subTest(reasoning_parser=reasoning_parser, stream=stream): + adapted, request = self._convert_rendered_as( + "", + ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Weather?"}], + tools=[_WEATHER_TOOL], + tool_choice="required", + stream=stream, + ), + ) + calls, got_reasoning, content, finish_reasons = self._chat_outcome( + adapted, request, self._stopped_on_token(text, 7) + ) + + ((name, arguments),) = calls + self.assertEqual(name, "get_weather") + self.assertEqual(json.loads(arguments), {"city": "Paris"}) + self.assertEqual(got_reasoning, reasoning) + self.assertEqual(content, "") + self.assertEqual(finish_reasons, ["tool_calls"]) + + def test_json_response_format_with_auto_tools_stays_content(self): + """A JSON response format with automatic tools must return the JSON + intact; template tool syntax inside its strings is never a call.""" + payload = '{"example": "{}"}' + for reasoning_parser in (None, "response_template"): + self._use_delimited_response_template(reasoning_parser) + reasoning = "Plan" if reasoning_parser else "" + text = (f"{reasoning}" if reasoning else "") + payload + for stream in (False, True): + with self.subTest(reasoning_parser=reasoning_parser, stream=stream): + adapted, request = self._convert_rendered_as( + "", + ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Weather?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + response_format={"type": "json_object"}, + stream=stream, + ), + ) + outcome = self._chat_outcome( + adapted, request, self._stopped_on_token(text, 2) + ) + + self.assertEqual(outcome, ([], reasoning, payload, ["stop"])) + + def test_native_output_under_other_constraints_is_parsed_by_the_template(self): + """A structural tag, regex or EBNF can spell the template's own framing, + so its content and a call ending on the kept tool closer still parse.""" + native = 'Sunny{"city": "Paris"}' + structural_tag = { + "type": "structural_tag", + "structures": [ + { + "begin": "", + "schema": _WEATHER_TOOL["function"]["parameters"], + "end": "", + } + ], + "triggers": ["", + ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Weather?"}], + tools=[_WEATHER_TOOL], + **fields, + ), + ) + output = self._stopped_on_token("Plan" + native, 7) + choice = self.chat._build_chat_response(request, [output], 0).choices[0] + (call,) = choice.message.tool_calls + self.assertEqual( + (choice.message.reasoning_content, choice.message.content), + ("Plan", "Sunny"), + ) + self.assertEqual( + (call.function.name, json.loads(call.function.arguments)), + ("get_weather", {"city": "Paris"}), + ) + self.assertEqual(choice.finish_reason, "tool_calls") + def test_text_only_prompt_reuses_rendered_token_ids(self): self.tm.model_config.is_multimodal = True self.chat.chat_encoding_spec = None @@ -1449,6 +1820,7 @@ class ServingChatTestCase(CustomTestCase): with patch( "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" ) as parser_cls: + parser_cls.ToolCallParserEnum = FunctionCallParser.ToolCallParserEnum parser = parser_cls.return_value parser.get_structure_constraint.return_value = ("structural_tag", "tag") @@ -1491,6 +1863,7 @@ class ServingChatTestCase(CustomTestCase): "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" ) as parser_cls, ): + parser_cls.ToolCallParserEnum = FunctionCallParser.ToolCallParserEnum parser = parser_cls.return_value parser.detector.eot_token = TOOLS_CLOSE parser.detector.parses_required_natively.return_value = False @@ -3439,6 +3812,65 @@ class ServingChatTestCase(CustomTestCase): "usage chunk dropped after error abort", ) + def test_truncated_response_template_call_keeps_stop_without_arguments(self): + self.chat.tool_call_parser = "response_template" + self.tm.tokenizer.response_template = _RESPONSE_TEMPLATE + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + stream=True, + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object"}, + }, + } + ], + ) + request._response_parser_prefix = "" + + async def generate(): + yield { + "text": '{"city":', + "meta_info": { + "id": "chatcmpl-truncated-call", + "prompt_tokens": 5, + "completion_tokens": 3, + "cached_tokens": 0, + "reasoning_tokens": 0, + "finish_reason": {"type": "stop", "matched": "CUSTOM_STOP"}, + }, + "index": 0, + } + + self.tm.generate_request = Mock(return_value=generate()) + adapted_request = GenerateReqInput( + text="", + sampling_params={}, + stream=True, + ) + + chunks = self._run_chat_stream(adapted_request, request) + parsed = self._parse_chunks(chunks) + arguments = "".join( + tool_call["function"]["arguments"] + for chunk in parsed + for choice in chunk.get("choices", []) + for tool_call in choice.get("delta", {}).get("tool_calls", []) + ) + finish = next( + choice + for chunk in parsed + for choice in chunk.get("choices", []) + if choice.get("finish_reason") is not None + ) + + self.assertEqual(arguments, "") + self.assertEqual(finish["finish_reason"], "stop") + self.assertEqual(finish["matched_stop"], "CUSTOM_STOP") + def _run_chat_stream(self, adapted_request, req): async def run_stream(): chunks = [] diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses.py b/test/registered/unit/entrypoints/openai/test_serving_responses.py index f5adfc67ab..93d5fefda5 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_responses.py +++ b/test/registered/unit/entrypoints/openai/test_serving_responses.py @@ -41,6 +41,7 @@ from sglang.srt.entrypoints.openai.serving_responses import ( _build_output_text_logprobs, ) from sglang.srt.function_call.core_types import ToolCallItem +from sglang.srt.parser.response_template import ResponseTemplateReasoningDetector from sglang.srt.parser.template_detection import ReasoningToggleConfig from sglang.srt.runtime_context import get_serving, publish, reset_context from sglang.srt.sampling.sampling_params import ( @@ -656,6 +657,183 @@ class SkipSpecialTokensForwardingTestCase(CustomTestCase): # no such field), so leave it unset for --preferred-sampling-params. self.assertNotIn("skip_special_tokens", params) + def test_response_template_parser_keeps_special_tokens_unspaced(self): + serving = make_serving() + serving.reasoning_parser = "response_template" + serving._reasoning_detector = ResponseTemplateReasoningDetector( + response_template={ + "start_anchor": "", + "fields": {"content": {"content": "text"}}, + }, + ) + params = self._create_responses_sampling_params(serving) + self.assertFalse(params["skip_special_tokens"]) + self.assertFalse(params["spaces_between_special_tokens"]) + + +class ResponseTemplateRequiredToolTestCase(CustomTestCase): + # Every field is delimited, so the template gives no field to the JSON. + TEMPLATE = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + "content": {"open": "", "close": ""}, + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + }, + } + + @staticmethod + def _output(text, matched): + return { + "text": text, + "meta_info": { + "prompt_tokens": 3, + "completion_tokens": 9, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": matched}, + }, + } + + def _respond( + self, stream, *, output, tool_call_parser="response_template", **request_fields + ): + serving = make_serving() + serving.default_chat_template_kwargs = None + serving.reasoning_parser = "response_template" + serving.tool_call_parser = tool_call_parser + serving._reasoning_detector = ResponseTemplateReasoningDetector( + response_template=self.TEMPLATE + ) + # Always-on reasoning: the output grammar starts after the reasoning. + serving.template_manager.reasoning_config = ReasoningToggleConfig( + special_case="always" + ) + tokenizer = serving.tokenizer_manager.tokenizer + tokenizer.response_template = self.TEMPLATE + tokenizer.encode.side_effect = lambda literal, **_: ( + [7] if literal == "" else [1, 2, 3] + ) + tokenizer.decode.return_value = "" + + async def fake_generate( + request_id, request_prompt, adapted_request, sampling_params, context, **_ + ): + context.append_output(output) + yield context + + serving._generate_with_builtin_tools = fake_generate + request = ResponsesRequest( + model="x", + input="Weather?", + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + } + ], + stream=stream, + store=False, + **request_fields, + ) + rendered = MessageProcessingResult( + prompt="", + prompt_ids=[1, 2, 3], + image_data=None, + audio_data=None, + video_data=None, + modalities=[], + stop=[], + ) + + async def respond(): + response = await serving.create_responses(request) + if stream: + events = [event async for event in response] + return event_payloads(events)[-1]["response"] + return response.model_dump() + + with patch.object( + serving, "_apply_conversation_template", return_value=rendered + ): + return asyncio.run(respond()) + + def test_required_tool_json_is_parsed_under_delimited_template(self): + """Required-tool JSON after the reasoning becomes a completed function + call, also when it ends on the tool closer that detokenization keeps.""" + output = self._output( + "Check the forecast." + '[{"name": "get_weather", "parameters": {"city": "Paris"}}]', + matched=7, + ) + for stream in (False, True): + with self.subTest(stream=stream): + response = self._respond(stream, output=output, tool_choice="required") + + self.assertEqual(response["status"], "completed") + self.assertEqual( + [item["type"] for item in response["output"]], + ["reasoning", "function_call"], + ) + call = response["output"][1] + self.assertEqual(call["name"], "get_weather") + self.assertEqual(orjson.loads(call["arguments"]), {"city": "Paris"}) + + def test_json_text_format_with_auto_tools_stays_message(self): + """A JSON text format with automatic tools must return the JSON intact; + template tool syntax inside its strings is never a function call.""" + payload = '{"example": "{}"}' + output = self._output(f"Plan{payload}", matched=2) + for stream in (False, True): + with self.subTest(stream=stream): + response = self._respond( + stream, + output=output, + tool_choice="auto", + text={"format": {"type": "json_object"}}, + ) + + self.assertEqual(response["status"], "completed") + self.assertEqual( + [item["type"] for item in response["output"]], + ["reasoning", "message"], + ) + self.assertEqual(response["output"][1]["content"][0]["text"], payload) + + def test_json_text_format_with_natively_required_tools_stays_message(self): + """A required or named choice that a tool parser owns natively accepts a + JSON text format; the JSON answer stays a message, never a tool call.""" + payload = '{"name": "get_weather", "parameters": {"city": "Paris"}}' + output = self._output(f"Plan{payload}", matched=2) + for tool_choice in ("required", {"type": "function", "name": "get_weather"}): + with self.subTest(tool_choice=tool_choice): + response = self._respond( + False, + output=output, + tool_call_parser="glm47", + tool_choice=tool_choice, + text={"format": {"type": "json_object"}}, + ) + + self.assertEqual(response["status"], "completed") + self.assertEqual( + [item["type"] for item in response["output"]], + ["reasoning", "message"], + ) + self.assertEqual(response["output"][1]["content"][0]["text"], payload) + class InputItemNormalizationTestCase(CustomTestCase): def test_function_call_becomes_assistant_tool_call(self): diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py b/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py index 69912aa0f1..6c519f8a39 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py +++ b/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py @@ -26,6 +26,27 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") +RESPONSE_TEMPLATE = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}", + }, + }, + }, + }, +} + + class NonHarmonyStreamTestCase(CustomTestCase): def test_reasoning_content_part_lifecycle_matches_terminal_output(self): for finish_reason, status, suffix in ( @@ -302,6 +323,44 @@ class NonHarmonyStreamTestCase(CustomTestCase): ] self.assertIn("function_call", added_kinds) + def test_truncated_response_template_call_is_incomplete_without_arguments(self): + serving = make_serving() + serving.reasoning_parser = None + serving.tool_call_parser = "response_template" + serving.tokenizer_manager.tokenizer.response_template = RESPONSE_TEMPLATE + request = ResponsesRequest( + model="x", + input="hi", + stream=True, + store=False, + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + } + ], + ) + request._response_parser_prefix = "" + chunk = engine_chunk('{"city":', finish=True) + chunk["meta_info"]["finish_reason"] = {"type": "length"} + + events = StreamFixture(serving, request).run([chunk]) + payloads = event_payloads(events) + deltas = "".join( + payload["delta"] + for payload in payloads + if payload["type"] == "response.function_call_arguments.delta" + ) + terminal = payloads[-1] + + self.assertEqual(deltas, "") + self.assertEqual(terminal["type"], "response.incomplete") + self.assertEqual(terminal["response"]["status"], "incomplete") + self.assertEqual(terminal["response"]["output"][0]["name"], "get_weather") + self.assertEqual(terminal["response"]["output"][0]["arguments"], "") + self.assertEqual(terminal["response"]["output"][0]["status"], "incomplete") + def test_required_native_parser_matches_full_response(self): serving = make_serving() serving.reasoning_parser = None diff --git a/test/registered/unit/function_call/test_function_call_parser.py b/test/registered/unit/function_call/test_function_call_parser.py index b7556e52e0..1865b74d1a 100644 --- a/test/registered/unit/function_call/test_function_call_parser.py +++ b/test/registered/unit/function_call/test_function_call_parser.py @@ -4049,6 +4049,33 @@ class TestJsonArrayParser(unittest.TestCase): self.assertEqual(len(result.calls), 0) self.assertEqual(result.normal_text, "") + def test_finish_streams_arguments_of_calls_completed_in_final_chunk(self): + """Calls that arrive whole in the last chunk still stream their + arguments at stream end, exactly once.""" + text = ( + '[{"name": "get_weather", "parameters": {"location": "Tokyo"}}, ' + '{"name": "search", "parameters": {"query": "restaurants"}}]' + ) + calls = self.detector.parse_streaming_increment(text, self.tools).calls + calls += self.detector.finish(self.tools).calls + + streamed = {} + for call in calls: + entry = streamed.setdefault(call.tool_index, {"name": "", "arguments": ""}) + entry["name"] += call.name or "" + entry["arguments"] += call.parameters or "" + self.assertEqual( + [ + (e["name"], json.loads(e["arguments"])) + for _, e in sorted(streamed.items()) + ], + [ + ("get_weather", {"location": "Tokyo"}), + ("search", {"query": "restaurants"}), + ], + ) + self.assertEqual(self.detector.finish(self.tools).calls, []) + def test_braces_in_strings(self): """Test that JSON with } characters inside strings works correctly""" # Test case: JSON array with } inside string values - streamed across chunks diff --git a/test/registered/unit/managers/test_trim_matched_stop.py b/test/registered/unit/managers/test_trim_matched_stop.py index cc3be717b1..24a251b115 100644 --- a/test/registered/unit/managers/test_trim_matched_stop.py +++ b/test/registered/unit/managers/test_trim_matched_stop.py @@ -17,8 +17,10 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") GPT_OSS_CALL_TOKEN = 200012 -def _trim(output, matched, no_stop_trim, *, gpt_oss=False): - stub = SimpleNamespace(is_tool_call_parser_gpt_oss=gpt_oss) +def _trim(output, matched, no_stop_trim, *, gpt_oss=False, tool_close=()): + stub = SimpleNamespace( + is_tool_call_parser_gpt_oss=gpt_oss, tool_close_token_ids=frozenset(tool_close) + ) finished_reason = None if matched is None else {"matched": matched} return DetokenizerManager.trim_matched_stop( stub, output, finished_reason, no_stop_trim @@ -69,6 +71,10 @@ class TestTrimMatchedStop(unittest.TestCase): [1, 2, GPT_OSS_CALL_TOKEN], ) + def test_token_tool_close_kept(self): + self.assertEqual(_trim([1, 2, 3], 3, False, tool_close=[3]), [1, 2, 3]) + self.assertEqual(_trim([1, 2, 4], 4, False, tool_close=[3]), [1, 2]) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/parser/test_response_template.py b/test/registered/unit/parser/test_response_template.py new file mode 100644 index 0000000000..0f9d2f4be0 --- /dev/null +++ b/test/registered/unit/parser/test_response_template.py @@ -0,0 +1,1238 @@ +"""Tests for checkpoint-driven response-template adapters.""" + +import copy +import json +import unittest +from types import SimpleNamespace + +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + Function, + ResponsesRequest, + Tool, + ToolChoice, +) +from sglang.srt.function_call.function_call_parser import FunctionCallParser +from sglang.srt.function_call.gemma4_detector import ( + Gemma4Detector as Gemma4ToolDetector, +) +from sglang.srt.parser.reasoning_parser import Gemma4Detector as Gemma4ReasoningDetector +from sglang.srt.parser.reasoning_parser import ReasoningParser +from sglang.srt.parser.response_template import ( + GRAMMAR_AFTER_REASONING, + GRAMMAR_FROM_START, + ResponseTemplateReasoningDetector, + ResponseTemplateToolDetector, + configure_response_template_request, + resolve_response_template, + tool_close_token_ids, + validate_response_template_for_serving, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +# Public checkpoint metadata shared by google/gemma-4-12B-it and +# google/gemma-4-26B-A4B-it. +GEMMA4_RESPONSE_TEMPLATE = { + "defaults": {"role": "assistant"}, + "fields": { + "content": { + "close": ["", "<|tool_response>", ""], + "content": "text", + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n", + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [['<|"|>', '<|"|>']], + "unquoted_keys": True, + }, + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "repeats": True, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}", + }, + "type": "function", + }, + }, + }, + "start_anchor": ["<|turn>model\n", ""], +} + +PREFIX = "<|turn>model\n" +THINKING = "<|channel>thought\nI should check the weather." +TOOL_CALL = ( + '<|tool_call>call:get_weather{location:<|"|>New York<|"|>,days:3,' + "details:{metric:true},hours:[1,2]}" +) +TOOL_CALL_ARGUMENTS = { + "location": "New York", + "days": 3, + "details": {"metric": True}, + "hours": [1, 2], +} + +# Arguments parse even when the call is cut off before its closer. +XML_TOOL_TEMPLATE = { + "start_anchor": "", + "fields": { + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "<|call|>", + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>(?P.*?)", + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + }, +} +XML_CALL_WITHOUT_CLOSER = "Paris3" + +# Every field is delimited, so the template itself gives no field to text +# outside them. +DELIMITED_TEMPLATE = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + "content": {"open": "", "close": ""}, + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + }, +} + +TRANSFORM_EACH_RESPONSE_TEMPLATE = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform_each": True, + "transform": { + "id": "{id}", + "type": "function", + "function": { + "name": "{name}", + "arguments": "{arguments}", + }, + }, + }, + }, +} + + +def _tool(name: str = "get_weather") -> Tool: + return Tool( + type="function", + function=Function( + name=name, + parameters={ + "type": "object", + "properties": { + "location": {"type": "string"}, + "days": {"type": "integer"}, + "details": {"type": "object"}, + "hours": { + "type": "array", + "items": {"type": "integer"}, + }, + }, + }, + ), + ) + + +def _call_values(calls): + return [(call.name, json.loads(call.parameters)) for call in calls] + + +def _chunks(text, size): + return [text[index : index + size] for index in range(0, len(text), size)] + + +def _collect_tool_stream(detector, chunks): + results = [detector.parse_streaming_increment(chunk, [_tool()]) for chunk in chunks] + results.append(detector.finish([_tool()])) + calls = {} + for result in results: + for call in result.calls: + entry = calls.setdefault(call.tool_index, {"name": "", "parameters": ""}) + if call.name: + entry["name"] = call.name + if call.parameters: + entry["parameters"] += call.parameters + parsed_calls = [ + (call["name"], json.loads(call["parameters"])) + for _, call in sorted(calls.items()) + ] + return "".join(result.normal_text for result in results), parsed_calls + + +class TestResponseTemplateLoading(CustomTestCase): + def test_loads_response_template_from_tokenizer_config(self): + tokenizer = SimpleNamespace( + init_kwargs={"response_template": GEMMA4_RESPONSE_TEMPLATE} + ) + + self.assertEqual( + resolve_response_template(tokenizer), + GEMMA4_RESPONSE_TEMPLATE, + ) + explicit = copy.deepcopy(GEMMA4_RESPONSE_TEMPLATE) + self.assertIs(resolve_response_template(tokenizer, explicit), explicit) + + def test_rejects_unsupported_semantic_fields(self): + template = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "citations": { + "open": "", + "close": "", + "content": "text", + }, + }, + } + + with self.assertRaisesRegex(ValueError, "unsupported semantic fields"): + validate_response_template_for_serving(template) + + for field_name, update in ( + ("content", {"content": "json"}), + ("thinking", {"transform": "{content}"}), + ): + with self.subTest(field=field_name, update=update): + template = copy.deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"][field_name].update(update) + with self.assertRaisesRegex(ValueError, "cannot be streamed"): + validate_response_template_for_serving(template) + + def test_checkpoint_metadata_takes_precedence_over_subclass_fallback(self): + class FallbackDetector(ResponseTemplateToolDetector): + response_template = XML_TOOL_TEMPLATE + + tokenizer = SimpleNamespace(response_template=GEMMA4_RESPONSE_TEMPLATE) + self.assertIs( + FallbackDetector(tokenizer=tokenizer).response_template, + GEMMA4_RESPONSE_TEMPLATE, + ) + self.assertIs(FallbackDetector().response_template, XML_TOOL_TEMPLATE) + + def test_tool_detector_requires_tool_calls_field(self): + template = copy.deepcopy(GEMMA4_RESPONSE_TEMPLATE) + del template["fields"]["tool_calls"] + tokenizer = SimpleNamespace(response_template=template) + + with self.assertRaisesRegex(ValueError, "no 'tool_calls' field"): + ResponseTemplateToolDetector(response_template=template) + self.assertEqual( + tool_close_token_ids("response_template", tokenizer), frozenset() + ) + + +class TestGemma4ResponseTemplateParity(CustomTestCase): + def test_non_streaming_reasoning_parity(self): + for text in ("It will be sunny.", THINKING + "It will be sunny."): + with self.subTest(text=text): + existing = Gemma4ReasoningDetector().detect_and_parse(text) + template = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ).detect_and_parse(text) + + self.assertEqual(template.reasoning_text, existing.reasoning_text) + self.assertEqual(template.normal_text, existing.normal_text) + + def test_non_streaming_tool_call_parity(self): + for text in ("Some text before " + TOOL_CALL, TOOL_CALL + TOOL_CALL): + with self.subTest(text=text): + existing = Gemma4ToolDetector().detect_and_parse(text, [_tool()]) + template = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ).detect_and_parse(text, [_tool()]) + + self.assertEqual(template.normal_text, existing.normal_text) + self.assertEqual( + _call_values(template.calls), _call_values(existing.calls) + ) + + def test_content_between_tool_calls_is_preserved(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ) + + parsed = detector.detect_and_parse( + TOOL_CALL + " between calls " + TOOL_CALL, + [_tool()], + ) + + self.assertEqual(parsed.normal_text, " between calls ") + self.assertEqual(len(parsed.calls), 2) + + def test_malformed_tool_body_does_not_expose_reasoning(self): + malformed = TOOL_CALL.replace("", "unexpected") + parsed = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ).detect_and_parse(THINKING + malformed) + + self.assertEqual(parsed.reasoning_text, "I should check the weather.") + self.assertEqual(parsed.normal_text, malformed) + + def test_required_tool_field_does_not_expose_reasoning(self): + template = { + "start_anchor": "", + "fields": { + "thinking": { + "open": "", + "close": "", + "content": "text", + }, + "content": {"content": "text"}, + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "optional": False, + }, + }, + } + + parsed = ResponseTemplateReasoningDetector( + response_template=template, + prefix="", + ).detect_and_parse("reasonanswer") + + self.assertEqual(parsed.reasoning_text, "reason") + self.assertEqual(parsed.normal_text, "answer") + + def test_thinking_only_template_preserves_plain_content(self): + template = { + "start_anchor": "", + "fields": { + "thinking": { + "open": "", + "close": "", + "content": "text", + }, + }, + } + + parsed = ResponseTemplateReasoningDetector( + response_template=template, + prefix="", + ).detect_and_parse("answer") + + self.assertEqual(parsed.reasoning_text, "") + self.assertEqual(parsed.normal_text, "answer") + + def test_streaming_reasoning_parity(self): + text = THINKING + "It will be sunny." + for chunk_size in (1, 7, len(text)): + with self.subTest(chunk_size=chunk_size): + existing = Gemma4ReasoningDetector() + template = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ) + existing_reasoning = [] + existing_normal = [] + template_reasoning = [] + template_normal = [] + for chunk in _chunks(text, chunk_size): + existing_result = existing.parse_streaming_increment(chunk) + existing_reasoning.append(existing_result.reasoning_text) + existing_normal.append(existing_result.normal_text) + template_result = template.parse_streaming_increment(chunk) + template_reasoning.append(template_result.reasoning_text) + template_normal.append(template_result.normal_text) + + existing_end = existing.finish() + template_end = template.finish() + existing_reasoning.append(existing_end.reasoning_text) + existing_normal.append(existing_end.normal_text) + template_reasoning.append(template_end.reasoning_text) + template_normal.append(template_end.normal_text) + + self.assertEqual( + "".join(template_reasoning), + "".join(existing_reasoning), + ) + self.assertEqual( + "".join(template_normal), + "".join(existing_normal), + ) + + def test_streaming_tool_call_parity(self): + for chunk_size in (1, 7, len(TOOL_CALL)): + with self.subTest(chunk_size=chunk_size): + existing = Gemma4ToolDetector() + template = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ) + chunks = _chunks(TOOL_CALL, chunk_size) + + self.assertEqual( + _collect_tool_stream(template, chunks), + _collect_tool_stream(existing, chunks), + ) + + def test_streaming_tool_name_timing_parity(self): + opening = "<|tool_call>call:get_weather{" + for detector in ( + Gemma4ToolDetector(), + ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ), + ): + with self.subTest(detector=type(detector).__name__): + parsed = detector.parse_streaming_increment(opening, [_tool()]) + + self.assertEqual(parsed.normal_text, "") + self.assertEqual( + [ + (call.tool_index, call.name, call.parameters) + for call in parsed.calls + ], + [(0, "get_weather", "")], + ) + + +class TestResponseTemplateAdapters(CustomTestCase): + def test_parser_prefix_request_state_is_private(self): + requests = ( + ChatCompletionRequest(messages=[]), + ResponsesRequest(input="hello"), + ) + + for request in requests: + with self.subTest(request=type(request).__name__): + request._response_parser_prefix = PREFIX + request._response_parser_grammar_start = GRAMMAR_AFTER_REASONING + + self.assertEqual(request._response_parser_prefix, PREFIX) + self.assertNotIn("response_parser_prefix", request.model_dump()) + self.assertNotIn("response_parser_grammar_start", request.model_dump()) + self.assertEqual( + request.model_copy()._response_parser_prefix, + PREFIX, + ) + self.assertEqual( + request.model_copy()._response_parser_grammar_start, + GRAMMAR_AFTER_REASONING, + ) + + def test_adapters_use_checkpoint_metadata(self): + tokenizer = SimpleNamespace(response_template=GEMMA4_RESPONSE_TEMPLATE) + reasoning = ReasoningParser( + model_type="response_template", + tokenizer=tokenizer, + prefix=PREFIX, + ) + tools = FunctionCallParser( + tools=[_tool()], + tool_call_parser="response_template", + tokenizer=tokenizer, + prefix=PREFIX, + ) + + reasoning_text, tool_wire = reasoning.parse_non_stream(THINKING + TOOL_CALL) + normal_text, calls = tools.parse_non_stream(tool_wire) + + self.assertEqual(reasoning_text, "I should check the weather.") + self.assertEqual(normal_text, "") + self.assertEqual(_call_values(calls), [("get_weather", TOOL_CALL_ARGUMENTS)]) + + def test_empty_thinking_prefill_routes_generated_content(self): + detector = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|channel>thought\n", + ) + + parsed = detector.parse_streaming_increment("Hello") + finished = detector.finish() + + self.assertEqual(parsed.reasoning_text + finished.reasoning_text, "") + self.assertEqual(parsed.normal_text + finished.normal_text, "Hello") + + def test_open_thinking_prefill_routes_reasoning_and_content(self): + detector = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|channel>thought\n", + ) + + parsed = detector.parse_streaming_increment( + "Still thinkingDone" + ) + finished = detector.finish() + + self.assertEqual( + parsed.reasoning_text + finished.reasoning_text, + "Still thinking", + ) + self.assertEqual(parsed.normal_text + finished.normal_text, "Done") + + def test_nonempty_prefix_content_is_not_replayed(self): + reasoning = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|channel>thought\nExisting reasoning", + ) + content = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "Existing answer", + ) + non_streaming = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "Existing answer", + ) + + reasoning_parsed = reasoning.parse_streaming_increment( + " continuedDone" + ) + content_parsed = content.parse_streaming_increment(" continued") + non_streaming_parsed = non_streaming.detect_and_parse(" continued") + + self.assertEqual(reasoning_parsed.reasoning_text, " continued") + self.assertEqual(reasoning_parsed.normal_text, "Done") + self.assertEqual(content_parsed.normal_text, " continued") + self.assertEqual(non_streaming_parsed.normal_text, " continued") + + def test_held_prefix_bytes_are_not_replayed(self): + detector = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "Existing answer <", + ) + + parsed = detector.parse_streaming_increment("b> tail") + + self.assertEqual(parsed.normal_text, "b> tail") + + def test_parser_config_disables_generated_special_token_spacing(self): + request = ChatCompletionRequest(messages=[]) + + configure_response_template_request(request) + sampling_params = request.to_sampling_params([], {}) + + self.assertFalse(sampling_params["skip_special_tokens"]) + self.assertFalse(sampling_params["spaces_between_special_tokens"]) + self.assertFalse(sampling_params["no_stop_trim"]) + + def test_output_grammar_output_is_passed_through_verbatim(self): + """JSON-schema output passes through unchanged from where its grammar + starts, including template delimiters inside its strings.""" + plain = '[{"name": "get_weather", "parameters": {"days": 3}}]' + delimited = ( + '[{"name": "get_weather", "parameters": ' + '{"location": "Paris", "details": "x"}}]' + ) + cases = ( + # Without JSON-schema output the template frames the whole output, + # so text outside its fields has no field. + (None, "Plan" + plain, ("Plan", "")), + ( + GRAMMAR_AFTER_REASONING, + "Plan" + delimited, + ("Plan", delimited), + ), + (GRAMMAR_FROM_START, delimited, ("", delimited)), + ) + for grammar_start, output, expected in cases: + with self.subTest(grammar_start=grammar_start): + request = ChatCompletionRequest(messages=[]) + request._response_parser_grammar_start = grammar_start + one_shot, streaming = ( + ReasoningParser( + model_type="response_template", + tokenizer=SimpleNamespace(response_template=DELIMITED_TEMPLATE), + request=request, + prefix="", + ) + for _ in range(2) + ) + parts = [streaming.parse_stream_chunk(c) for c in _chunks(output, 5)] + parts.append(streaming.parse_stream_end()) + + self.assertEqual(one_shot.parse_non_stream(output), expected) + self.assertEqual( + ( + "".join(reasoning or "" for reasoning, _ in parts), + "".join(normal or "" for _, normal in parts), + ), + expected, + ) + + framed = ReasoningParser( + model_type="response_template", + tokenizer=SimpleNamespace(response_template=DELIMITED_TEMPLATE), + prefix="", + ) + self.assertEqual( + framed.parse_non_stream("PlanSunny"), + ("Plan", "Sunny"), + ) + + def test_reasoning_requires_explicit_enable_without_template_policy(self): + detector = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + + self.assertEqual(detector.reasoning_default, "explicit_enable_thinking") + + def test_reasoning_detector_supports_think_end_token_lookups(self): + detector = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + tokenizer = SimpleNamespace( + encode=lambda text, add_special_tokens: [len(text), add_special_tokens] + ) + + self.assertIsNone(detector.think_excluded_tokens) + self.assertIsNone(detector.get_answer_frame()) + self.assertEqual(detector.get_think_end_token_ids(tokenizer), [10, False]) + + def test_force_nonempty_content_returns_reasoning_only_output_as_content(self): + request = ChatCompletionRequest( + messages=[], + chat_template_kwargs={"force_nonempty_content": True}, + ) + reasoning = ReasoningParser( + model_type="response_template", + tokenizer=SimpleNamespace(response_template=GEMMA4_RESPONSE_TEMPLATE), + request=request, + prefix=PREFIX, + ) + + self.assertEqual( + reasoning.parse_non_stream(THINKING + ""), + ("", "I should check the weather."), + ) + self.assertEqual( + reasoning.parse_non_stream(THINKING + "Sunny"), + ("I should check the weather.", "Sunny"), + ) + + def test_force_nonempty_content_streams_unclosed_reasoning_as_content(self): + for stream_reasoning in (True, False): + with self.subTest(stream_reasoning=stream_reasoning): + detector = ResponseTemplateReasoningDetector( + stream_reasoning=stream_reasoning, + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|channel>thought\n", + force_nonempty_content=True, + ) + + parsed = [ + detector.parse_streaming_increment(chunk) + for chunk in _chunks("Still thinking", 4) + ] + finished = detector.finish() + + self.assertEqual("".join(result.normal_text for result in parsed), "") + self.assertEqual(finished.normal_text, "Still thinking") + self.assertEqual(finished.reasoning_text, "") + + def test_force_nonempty_content_keeps_closed_streamed_reasoning(self): + detector = ResponseTemplateReasoningDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + force_nonempty_content=True, + ) + + parsed = [ + detector.parse_streaming_increment(chunk) + for chunk in _chunks(THINKING + "", 5) + ] + finished = detector.finish() + + self.assertEqual( + "".join(result.reasoning_text for result in parsed + [finished]), + "I should check the weather.", + ) + self.assertEqual( + "".join(result.normal_text for result in parsed + [finished]), "" + ) + + def test_reasoning_subclass_can_define_regex_delimiters_and_policy(self): + class PatternReasoningDetector(ResponseTemplateReasoningDetector): + response_template = { + "start_anchor": "", + "fields": { + "thinking": { + "open_pattern": r"", + "close_pattern": r"", + }, + "content": {"content": "text"}, + }, + } + _default_think_start = "" + _default_think_end = "" + thinks_internally = True + reasoning_default = "always" + + detector = PatternReasoningDetector() + + self.assertEqual(detector.think_start_token, "") + self.assertEqual(detector.think_end_token, "") + self.assertTrue(detector.thinks_internally) + self.assertEqual(detector.reasoning_default, "always") + + def test_transform_each_tool_region_emits_all_calls(self): + """Each member is transformed with the opener's captures, and its + arguments are cast to the tool schema.""" + template = TRANSFORM_EACH_RESPONSE_TEMPLATE + tools = [_tool(), _tool("get_forecast")] + opener = "" + text = ( + opener + '[{"name":"get_weather","arguments":{"location":"Paris"}},' + '{"name":"get_forecast","arguments":{"days":"3"}}]' + ) + detector = ResponseTemplateToolDetector( + response_template=template, + prefix="", + ) + + parsed = detector.detect_and_parse(text, tools) + + self.assertEqual(parsed.normal_text, "") + self.assertEqual( + [(call.name, json.loads(call.parameters)) for call in parsed.calls], + [ + ("get_weather", {"location": "Paris"}), + ("get_forecast", {"days": 3}), + ], + ) + + streaming = ResponseTemplateToolDetector( + response_template=template, + prefix="", + ) + opened = streaming.parse_streaming_increment(opener, tools) + closed = streaming.parse_streaming_increment( + text.removeprefix(opener), + tools, + ) + self.assertEqual(opened.calls, []) + self.assertEqual( + [(call.name, json.loads(call.parameters)) for call in closed.calls], + [ + ("get_weather", {"location": "Paris"}), + ("get_forecast", {"days": 3}), + ], + ) + + def test_unknown_function_in_array_keeps_valid_siblings(self): + """An undefined or malformed member of a parallel-call array drops only + itself; its valid siblings and later calls keep consecutive indices.""" + tools = [_tool(), _tool("get_forecast")] + text = ( + '[{"name":"get_weather","arguments":{"location":"Paris"}},' + '{"name":"unknown","arguments":{}},' + '{"name":"get_weather"},{"arguments":{}},null,' + '{"name":"get_forecast","arguments":{"days":3}}]' + '[{"name":"get_weather","arguments":{"location":"Rome"}}]' + ) + expected = [ + (0, "get_weather", {"location": "Paris"}), + (1, "get_forecast", {"days": 3}), + (2, "get_weather", {"location": "Rome"}), + ] + + parsed = ResponseTemplateToolDetector( + response_template=TRANSFORM_EACH_RESPONSE_TEMPLATE, + prefix="", + ).detect_and_parse(text, tools) + streaming = ResponseTemplateToolDetector( + response_template=TRANSFORM_EACH_RESPONSE_TEMPLATE, + prefix="", + ) + streamed = streaming.parse_streaming_increment(text, tools).calls + streamed += streaming.finish(tools).calls + + for calls in (parsed.calls, streamed): + self.assertEqual( + [ + (call.tool_index, call.name, json.loads(call.parameters)) + for call in calls + ], + expected, + ) + self.assertEqual(streaming.incomplete_tool_call_indices, set()) + + def test_transform_each_casts_arguments_to_the_transformed_call(self): + """A member's arguments are cast to the schema of the call its transform + produces, never to the schema of the function named in the raw member.""" + template = { + "start_anchor": "", + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform_each": True, + "transform": { + "type": "function", + "function": { + "name": "echo", + "arguments": "{function.arguments}", + }, + }, + }, + }, + } + tools = [ + Tool( + type="function", + function=Function( + name=name, + parameters={ + "type": "object", + "properties": {"days": {"type": days_type}}, + }, + ), + ) + for name, days_type in (("get_forecast", "integer"), ("echo", "string")) + ] + text = ( + '[{"function":{"name":"get_forecast",' + '"arguments":{"days":"003"}}}]' + ) + + def streamed(chunk_size): + detector = ResponseTemplateToolDetector( + response_template=template, prefix="" + ) + calls = [] + for start in range(0, len(text), chunk_size): + chunk = text[start : start + chunk_size] + calls += detector.parse_streaming_increment(chunk, tools).calls + return calls + detector.finish(tools).calls + + parsed = ResponseTemplateToolDetector( + response_template=template, prefix="" + ).detect_and_parse(text, tools) + for mode, calls in ( + ("non-streaming", parsed.calls), + ("whole output", streamed(len(text))), + ("per character", streamed(1)), + ): + with self.subTest(mode=mode): + ((name, arguments),) = [ + (call.name, json.loads(call.parameters)) for call in calls + ] + self.assertEqual( + (name, arguments, type(arguments["days"])), + ("echo", {"days": "003"}, str), + ) + + def test_malformed_call_is_dropped(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + malformed = TOOL_CALL.replace("", "unexpected") + + result = detector.detect_and_parse("hello" + malformed + TOOL_CALL, [_tool()]) + + self.assertEqual(result.normal_text, "hello") + self.assertEqual([call.name for call in result.calls], ["get_weather"]) + + def test_call_cut_off_after_its_opener_is_detected(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + opener = TOOL_CALL.split("{", 1)[0] + + self.assertTrue(detector.has_tool_call(opener)) + result = detector.detect_and_parse(opener, [_tool()]) + self.assertEqual(result.normal_text, "") + self.assertEqual(result.calls, []) + + def test_streaming_malformed_call_is_dropped_before_emission(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + malformed = TOOL_CALL.replace("", "unexpected") + + content = detector.parse_streaming_increment("hello", [_tool()]) + parsed = detector.parse_streaming_increment(malformed, [_tool()]) + + self.assertEqual(content.normal_text, "hello") + self.assertEqual(parsed.normal_text, "") + self.assertEqual(parsed.calls, []) + self.assertFalse(detector.incomplete_tool_call_indices) + + def test_streaming_malformed_call_leaves_emitted_name_incomplete(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + opening, body = TOOL_CALL.split("{", 1) + + opened = detector.parse_streaming_increment(opening + "{", [_tool()]) + failed = detector.parse_streaming_increment( + body.replace("", "unexpected"), + [_tool()], + ) + + self.assertEqual( + [(call.name, call.parameters) for call in opened.calls], + [("get_weather", "")], + ) + self.assertEqual(failed.normal_text, "") + self.assertEqual(failed.calls, []) + self.assertEqual(detector.incomplete_tool_call_indices, {0}) + later = detector.parse_streaming_increment(" trailing bytes", [_tool()]) + self.assertEqual(later.normal_text, " trailing bytes") + self.assertEqual(later.calls, []) + next_call = detector.parse_streaming_increment(TOOL_CALL, [_tool()]) + self.assertEqual({call.tool_index for call in next_call.calls}, {1}) + + def test_streaming_invalid_call_value_leaves_emitted_name_incomplete(self): + template = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform": {"function": {"name": "{name}"}}, + }, + }, + } + detector = ResponseTemplateToolDetector(response_template=template) + + opened = detector.parse_streaming_increment("{", [_tool()]) + detector.parse_streaming_increment("}", [_tool()]) + closed = detector.parse_streaming_increment("", [_tool()]) + + self.assertEqual( + [(call.name, call.parameters) for call in opened.calls], + [("get_weather", "")], + ) + self.assertEqual(closed.calls, []) + self.assertEqual(detector.incomplete_tool_call_indices, {0}) + + def test_one_shot_parse_does_not_disturb_active_stream(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX, + ) + opening, body = TOOL_CALL.split("{", 1) + + opened = detector.parse_streaming_increment(opening + "{", [_tool()]) + one_shot = detector.detect_and_parse(TOOL_CALL, [_tool()]) + closed = detector.parse_streaming_increment(body, [_tool()]) + + self.assertEqual([call.name for call in opened.calls], ["get_weather"]) + self.assertEqual(len(one_shot.calls), 1) + self.assertEqual( + [(call.tool_index, call.name) for call in closed.calls], [(0, None)] + ) + self.assertEqual(detector.incomplete_tool_call_indices, set()) + + def test_streaming_waits_when_tool_name_depends_on_content(self): + template = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "repeats": True, + "transform": { + "type": "function", + "function": { + "name": "{content.name}", + "arguments": "{content.arguments}", + }, + }, + }, + }, + } + detector = ResponseTemplateToolDetector( + response_template=template, + prefix="", + ) + + opened = detector.parse_streaming_increment("", [_tool()]) + closed = detector.parse_streaming_increment( + '{"name":"get_weather","arguments":{"location":"Paris"}}', + [_tool()], + ) + + self.assertEqual(opened.calls, []) + self.assertEqual( + [(call.name, json.loads(call.parameters)) for call in closed.calls], + [("get_weather", {"location": "Paris"})], + ) + + def test_prefilled_malformed_call_is_dropped(self): + opening, body = TOOL_CALL.split("{", 1) + malformed_body = "{" + body.replace( + "", + "unexpected", + ) + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + opening, + ) + + parsed = detector.parse_streaming_increment(malformed_body, [_tool()]) + + self.assertEqual(parsed.normal_text, "") + self.assertEqual(parsed.calls, []) + + def test_non_streaming_call_opened_in_prefix_is_parsed(self): + opening, body = TOOL_CALL.split("{", 1) + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + opening, + ) + + parsed = detector.detect_and_parse("{" + body, [_tool()]) + + self.assertEqual(parsed.normal_text, "") + self.assertEqual( + _call_values(parsed.calls), [("get_weather", TOOL_CALL_ARGUMENTS)] + ) + + def test_streaming_call_opened_in_prefix_emits_name_then_arguments(self): + opening, body = TOOL_CALL.split("{", 1) + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + opening, + ) + + parsed = detector.parse_streaming_increment("{" + body, [_tool()]) + + self.assertEqual( + [(call.tool_index, call.name) for call in parsed.calls], + [(0, "get_weather"), (0, None)], + ) + self.assertEqual(json.loads(parsed.calls[1].parameters), TOOL_CALL_ARGUMENTS) + + def test_call_without_closer_is_dropped(self): + def parse(template, text): + parser = FunctionCallParser( + tools=[_tool()], + tool_call_parser="response_template", + tokenizer=SimpleNamespace(response_template=template), + ) + return parser.parse_non_stream("hello" + text) + + text, calls = parse(XML_TOOL_TEMPLATE, XML_CALL_WITHOUT_CLOSER + "<|call|>") + self.assertEqual(text, "hello") + self.assertEqual(_call_values(calls), [("get_weather", {"location": "Paris"})]) + + for template, text in ( + (XML_TOOL_TEMPLATE, XML_CALL_WITHOUT_CLOSER), + (GEMMA4_RESPONSE_TEMPLATE, TOOL_CALL.removesuffix("")), + (GEMMA4_RESPONSE_TEMPLATE, TOOL_CALL.removesuffix("}")), + ): + with self.subTest(text=text): + self.assertEqual(parse(template, text), ("hello", [])) + + def test_zero_width_closer_is_not_mistaken_for_cut_off(self): + template = copy.deepcopy(XML_TOOL_TEMPLATE) + del template["fields"]["tool_calls"]["close"] + template["fields"]["tool_calls"]["close_pattern"] = r"(?=)" + parser = FunctionCallParser( + tools=[_tool()], + tool_call_parser="response_template", + tokenizer=SimpleNamespace(response_template=template), + ) + + text, calls = parser.parse_non_stream( + "helloParis" + ) + + self.assertEqual(text, "hello") + self.assertEqual(_call_values(calls), [("get_weather", {"location": "Paris"})]) + + def test_end_of_input_closer_completes_the_call(self): + """A closer that matches only at end of input, such as `\\Z`, closes the + last call instead of marking it cut off.""" + template = copy.deepcopy(XML_TOOL_TEMPLATE) + del template["fields"]["tool_calls"]["close"] + template["fields"]["tool_calls"]["close_pattern"] = r"\Z" + text = "helloParis" + + def new_parser(): + return FunctionCallParser( + tools=[_tool()], + tool_call_parser="response_template", + tokenizer=SimpleNamespace(response_template=template), + ) + + normal_text, calls = new_parser().parse_non_stream(text) + self.assertEqual(normal_text, "hello") + self.assertEqual(_call_values(calls), [("get_weather", {"location": "Paris"})]) + + streaming = new_parser() + streamed = [streaming.parse_stream_chunk(char) for char in text] + streamed.append(streaming.parse_stream_end()) + calls = [call for _, chunk_calls in streamed for call in chunk_calls] + self.assertEqual("".join(chunk_text for chunk_text, _ in streamed), "hello") + self.assertEqual("".join(call.name or "" for call in calls), "get_weather") + self.assertEqual( + json.loads("".join(call.parameters for call in calls)), + {"location": "Paris"}, + ) + self.assertEqual(streaming.detector.incomplete_tool_call_indices, set()) + + def test_streamed_call_without_closer_stays_incomplete(self): + for template, first_call, cut_call in ( + ( + XML_TOOL_TEMPLATE, + XML_CALL_WITHOUT_CLOSER + "<|call|>", + XML_CALL_WITHOUT_CLOSER, + ), + ( + GEMMA4_RESPONSE_TEMPLATE, + TOOL_CALL, + TOOL_CALL.removesuffix("") + "": [7]}[text], + ) + + self.assertEqual( + tool_close_token_ids("response_template", tokenizer), frozenset({7}) + ) + self.assertEqual(tool_close_token_ids("qwen", tokenizer), frozenset()) + self.assertEqual( + tool_close_token_ids("response_template", SimpleNamespace()), + frozenset(), + ) + + def test_unknown_tool_is_dropped_when_forwarding_is_disabled(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + unknown = TOOL_CALL.replace("get_weather", "unknown") + + result = detector.detect_and_parse(unknown, [_tool()]) + + self.assertEqual(result.normal_text, "") + self.assertEqual(result.calls, []) + + def test_streaming_unknown_tool_is_not_emitted_early(self): + detector = ResponseTemplateToolDetector( + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + unknown = TOOL_CALL.replace("get_weather", "unknown") + opening, body = unknown.split("{", 1) + + opened = detector.parse_streaming_increment(opening + "{", [_tool()]) + closed = detector.parse_streaming_increment(body, [_tool()]) + + self.assertEqual(opened.calls, []) + self.assertEqual(closed.normal_text, "") + self.assertEqual(closed.calls, []) + + def test_strict_tools_require_native_constraint_support(self): + tokenizer = SimpleNamespace(response_template=GEMMA4_RESPONSE_TEMPLATE) + tool = _tool() + tool.function.strict = True + parser = FunctionCallParser( + tools=[tool], + tool_call_parser="response_template", + tokenizer=tokenizer, + ) + + with self.assertRaisesRegex(ValueError, "does not support strict"): + parser.get_structure_constraint("auto") + + def test_required_and_named_tools_use_generic_json_schema(self): + tokenizer = SimpleNamespace(response_template=GEMMA4_RESPONSE_TEMPLATE) + parser = FunctionCallParser( + tools=[_tool()], + tool_call_parser="response_template", + tokenizer=tokenizer, + ) + + for choice in ( + "required", + ToolChoice(function={"name": "get_weather"}), + ): + with self.subTest(choice=choice): + constraint = parser.get_structure_constraint(choice) + self.assertEqual(constraint[0], "json_schema") + + strict_tool = _tool() + strict_tool.function.strict = True + strict_parser = FunctionCallParser( + tools=[strict_tool], + tool_call_parser="response_template", + tokenizer=tokenizer, + ) + self.assertEqual( + strict_parser.get_structure_constraint("required")[0], + "json_schema", + ) + + def test_parallel_false_auto_is_rejected_without_native_constraint(self): + tokenizer = SimpleNamespace(response_template=GEMMA4_RESPONSE_TEMPLATE) + parser = FunctionCallParser( + tools=[_tool()], + tool_call_parser="response_template", + tokenizer=tokenizer, + ) + + with self.assertRaisesRegex(ValueError, "parallel_tool_calls=False"): + parser.get_structure_constraint("auto", parallel_tool_calls=False) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/parser/test_template_manager.py b/test/registered/unit/parser/test_template_manager.py index 8d00219284..6f33310a42 100644 --- a/test/registered/unit/parser/test_template_manager.py +++ b/test/registered/unit/parser/test_template_manager.py @@ -978,6 +978,27 @@ class TestResolveAutoParsers(CustomTestCase): """Tests for resolve_auto_parsers().""" qwen3_template = "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}" + response_template = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "thinking": { + "open": "", + "close": "", + }, + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform": { + "function": { + "name": "tool", + "arguments": "{content}", + }, + }, + }, + }, + } def _make_server_args( self, reasoning_parser=None, tool_call_parser=None, chat_template=None @@ -1002,6 +1023,129 @@ class TestResolveAutoParsers(CustomTestCase): self.assertEqual(_declared(args, "reasoning_parser"), "qwen3") self.assertEqual(_declared(args, "tool_call_parser"), "qwen") + def test_uses_runtime_tokenizer_configuration(self): + args = ServerArgs( + model_path="weights-path", + tokenizer_path="tokenizer-path", + tokenizer_mode="slow", + tokenizer_backend="fastokens", + revision="tokenizer-revision", + reasoning_parser="auto", + tool_call_parser="auto", + trust_remote_code=True, + ) + tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template) + get_tokenizer = Mock(return_value=tokenizer) + + with _patch_hf_transformers_utils(get_tokenizer): + resolve_auto_parsers(args) + + get_tokenizer.assert_called_once_with( + "tokenizer-path", + tokenizer_mode="slow", + trust_remote_code=True, + revision="tokenizer-revision", + tokenizer_backend="fastokens", + ) + + def test_existing_detection_takes_precedence_over_response_template(self): + args = self._make_server_args( + reasoning_parser="auto", + tool_call_parser="auto", + ) + tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template) + tokenizer.response_template = self.response_template + + with _patch_hf_transformers_utils(Mock(return_value=tokenizer)): + resolve_auto_parsers(args) + + self.assertEqual(_declared(args, "reasoning_parser"), "qwen3") + self.assertEqual(_declared(args, "tool_call_parser"), "qwen") + + def test_response_template_fills_undetected_parsers(self): + args = self._make_server_args( + reasoning_parser="auto", + tool_call_parser="auto", + ) + tokenizer = _DummyTokenizer([], chat_template="unrecognized") + tokenizer.response_template = self.response_template + + with _patch_hf_transformers_utils(Mock(return_value=tokenizer)): + resolve_auto_parsers(args) + + self.assertEqual(_declared(args, "reasoning_parser"), "response_template") + self.assertEqual(_declared(args, "tool_call_parser"), "response_template") + + def test_response_template_resolves_only_supported_parser_fields(self): + args = self._make_server_args( + reasoning_parser="auto", + tool_call_parser="auto", + ) + tokenizer = _DummyTokenizer([], chat_template="unrecognized") + tokenizer.response_template = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + }, + } + + with _patch_hf_transformers_utils(Mock(return_value=tokenizer)): + resolve_auto_parsers(args) + + self.assertEqual( + _declared(args, "reasoning_parser"), + "response_template", + ) + self.assertIsNone(_declared(args, "tool_call_parser")) + + def test_unsupported_response_template_is_ignored(self): + args = self._make_server_args( + reasoning_parser="auto", + tool_call_parser="auto", + ) + tokenizer = _DummyTokenizer([], chat_template="unrecognized") + tokenizer.response_template = { + "start_anchor": "", + "fields": { + "content": {"content": "text"}, + "metadata": { + "open": "", + "close": "", + "content": "json", + }, + }, + } + + with _patch_hf_transformers_utils(Mock(return_value=tokenizer)): + resolve_auto_parsers(args) + + self.assertIsNone(_declared(args, "reasoning_parser")) + self.assertIsNone(_declared(args, "tool_call_parser")) + + def test_response_template_with_undelimited_tool_field_is_ignored(self): + """A tool field without an opener leaves no text outside the calls, so + the tool parser could never be constructed; auto must not select it.""" + args = self._make_server_args( + reasoning_parser="auto", + tool_call_parser="auto", + ) + tokenizer = _DummyTokenizer([], chat_template="unrecognized") + tokenizer.response_template = { + "start_anchor": "", + "fields": { + "tool_calls": { + "content": "json", + "transform": {"type": "function", "function": "{content}"}, + }, + }, + } + + with _patch_hf_transformers_utils(Mock(return_value=tokenizer)): + resolve_auto_parsers(args) + + self.assertIsNone(_declared(args, "reasoning_parser")) + self.assertIsNone(_declared(args, "tool_call_parser")) + def test_resolves_reasoning_parser_only(self): args = self._make_server_args(reasoning_parser="auto", tool_call_parser=None) tokenizer = _DummyTokenizer([], chat_template=self.qwen3_template) @@ -1198,6 +1342,17 @@ class TestResolveAutoParsers(CustomTestCase): def test_explicit_jinja_template_takes_precedence(self): tokenizer = _DummyTokenizer([], chat_template=None) + tokenizer.response_template = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + "tool_calls": { + "open": "", + "close": "", + "content": "json", + }, + }, + } with tempfile.NamedTemporaryFile("w", suffix=".jinja") as f: f.write(