[Feat] parser: Add checkpoint-driven response-template adapters (#40479)

Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Co-authored-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
Yoni Gozlan
2026-10-23 14:05:54 -07:00
committed by GitHub
co-authored by Yoni Gozlan Xinyuan Tong Jimmy Shong
parent 4542fc8f59
commit 27ac614f05
20 changed files with 3173 additions and 23 deletions
@@ -58,6 +58,7 @@ from pydantic import (
BaseModel,
ConfigDict,
Field,
PrivateAttr,
StrictBool,
field_serializer,
field_validator,
@@ -866,6 +867,10 @@ def _has_message_level_tools(messages: Any) -> bool:
class ChatCompletionRequest(PDRoutingFields):
_response_parser_prefix: str = PrivateAttr(default="")
_response_parser_grammar_start: Optional[str] = PrivateAttr(default=None)
_response_parser_json_response_format: bool = PrivateAttr(default=False)
# Ordered by official OpenAI API documentation
# https://platform.openai.com/docs/api-reference/chat/create
messages: List[ChatCompletionMessageParam]
@@ -1833,6 +1838,10 @@ ResponseInputOutputItem: TypeAlias = Union[
class ResponsesRequest(PDRoutingFields):
"""Request body for v1/responses endpoint."""
_response_parser_prefix: str = PrivateAttr(default="")
_response_parser_grammar_start: Optional[str] = PrivateAttr(default=None)
_response_parser_json_response_format: bool = PrivateAttr(default=False)
# Core OpenAI API fields (ordered by official documentation)
background: Optional[bool] = False
include: Optional[
@@ -2322,6 +2331,7 @@ class MessageProcessingResult:
skip_special_tokens: bool = True
require_reasoning: bool = False
reasoning_end_token_ids: Optional[List[int]] = None
uses_response_template: bool = False
class ToolCallProcessingResult(NamedTuple):
@@ -68,9 +68,11 @@ from sglang.srt.entrypoints.openai.protocol import (
MessageProcessingResult,
PromptTokensDetails,
ResponseParserProtocol,
ResponsesRequest,
SglExt,
Tool,
ToolCall,
ToolCallConstraint,
ToolCallProcessingResult,
ToolChoice,
TopLogprob,
@@ -113,6 +115,14 @@ from sglang.srt.parser.reasoning_parser import (
IQuestQ1ReasoningDetector,
ReasoningParser,
)
from sglang.srt.parser.response_template import (
GRAMMAR_AFTER_REASONING,
GRAMMAR_FROM_START,
ResponseTemplateReasoningDetector,
ResponseTemplateToolDetector,
configure_response_template_request,
tool_close_tokens,
)
from sglang.srt.parser.template_detection import detect_inline_system_support
from sglang.srt.sampling.sampling_params import (
set_request_reasoning_end_token_ids,
@@ -130,6 +140,11 @@ _MEDIA_CONTENT_PART_TYPES = frozenset({"image_url", "video_url", "audio_url"})
_CHAT_TEMPLATE_CACHE_MAX_SIZE = 128
def _incomplete_tool_call_indices(parser) -> set[int]:
detector = getattr(parser, "detector", parser)
return getattr(detector, "incomplete_tool_call_indices", set())
def normalize_tool_content(role: str, content):
"""Normalize tool message content from OpenAI array format to plain string.
@@ -846,6 +861,10 @@ class OpenAIServingChat(OpenAIServingBase):
else:
delta = content["text"][offset:]
stream_offsets[index] = len(content["text"])
if finish_reason_type is not None:
delta = self._drop_kept_tool_close(
delta, content["meta_info"].get("finish_reason"), request
)
# Attach logprobs to the first chunk emitted this step (reasoning,
# tool-call, or content) so they aren't dropped when a parser is active
@@ -967,6 +986,7 @@ class OpenAIServingChat(OpenAIServingBase):
request.tool_choice != "none"
and self._effective_tools(request)
and self.tool_call_parser
and not request._response_parser_json_response_format
)
def _validate_request(self, request: ChatCompletionRequest) -> str | None:
@@ -1285,8 +1305,89 @@ class OpenAIServingChat(OpenAIServingBase):
):
apply_header_overrides(adapted_request, raw_request.headers)
if processed_messages.uses_response_template:
self._set_response_parser_context(
request,
adapted_request,
tool_call_constraint=processed_messages.tool_call_constraint,
)
return adapted_request, request
def _set_response_parser_context(
self,
request: ChatCompletionRequest | ResponsesRequest,
adapted_request: GenerateReqInput,
*,
tool_call_constraint: ToolCallConstraint | None,
) -> None:
"""Give response-template parsers the rendered assistant prefill."""
prefix = adapted_request.text
if not isinstance(prefix, str) and adapted_request.input_ids:
prefix = self.tokenizer_manager.tokenizer.decode(
adapted_request.input_ids,
skip_special_tokens=False,
spaces_between_special_tokens=False,
)
request._response_parser_prefix = prefix or ""
# Structural tags, regex and EBNF can describe the template's own
# framing, so only JSON-schema output bypasses the template.
if (adapted_request.sampling_params or {}).get("json_schema"):
# The scheduler defers the grammar to the reasoning's end only when
# the reasoning parser's think_end_token encodes to token ids.
reasoning_gated = (
adapted_request.require_reasoning
and self._reasoning_detector is not None
and bool(
self._reasoning_detector.get_think_end_token_ids(
self.tokenizer_manager.tokenizer
)
)
)
request._response_parser_grammar_start = (
GRAMMAR_AFTER_REASONING if reasoning_gated else GRAMMAR_FROM_START
)
# Unless the schema is the required/named tool call itself, the JSON
# is response data, and template syntax inside its strings is literal.
request._response_parser_json_response_format = (
tool_call_constraint is None or tool_call_constraint[0] != "json_schema"
)
def _drop_kept_tool_close(
self,
text: str,
finish_reason: dict[str, Any] | None,
request: ChatCompletionRequest | ResponsesRequest,
) -> str:
# Detokenization keeps a tool-call closer that stops generation; after
# JSON-schema output it ends no native call, so trim it like any stop.
if (
not text
or request._response_parser_grammar_start is None
or (isinstance(request, ChatCompletionRequest) and request.no_stop_trim)
or not isinstance(finish_reason, dict)
):
return text
closer = tool_close_tokens(
self.tool_call_parser, self.tokenizer_manager.tokenizer
).get(finish_reason.get("matched"))
if closer and text.endswith(closer):
return text[: -len(closer)]
return text
def _requires_response_template_detokenization(
self, request: ChatCompletionRequest
) -> bool:
if isinstance(self._reasoning_detector, ResponseTemplateReasoningDetector):
return True
tool_detector = FunctionCallParser.ToolCallParserEnum.get(self.tool_call_parser)
return (
request.tool_choice != "none"
and bool(self._effective_tools(request))
and tool_detector is not None
and issubclass(tool_detector, ResponseTemplateToolDetector)
)
def _process_messages(
self, request: ChatCompletionRequest, is_multimodal: bool
) -> MessageProcessingResult:
@@ -1427,6 +1528,11 @@ class OpenAIServingChat(OpenAIServingBase):
if tool_call_stop not in result.stop:
result.stop.append(tool_call_stop)
# Configured after rendering so the detokenization options never reach
# the chat template.
if self._requires_response_template_detokenization(request):
configure_response_template_request(request)
result.uses_response_template = True
result.tool_call_constraint = tool_call_constraint
result.require_reasoning = thinking_mode
result.skip_special_tokens = request.skip_special_tokens
@@ -2094,7 +2200,11 @@ class OpenAIServingChat(OpenAIServingBase):
# Change finish_reason to "tool_calls" if we had tool calls and stopped naturally
final_finish_reason = finish_reason_type
if has_tool_calls.get(idx, False) and finish_reason_type == "stop":
if (
has_tool_calls.get(idx, False)
and finish_reason_type == "stop"
and not _incomplete_tool_call_indices(parser_dict.get(idx))
):
final_finish_reason = "tool_calls"
matched_stop = finish_reason_data.get("matched")
@@ -2357,6 +2467,7 @@ class OpenAIServingChat(OpenAIServingBase):
text = self._decode_response(ret_item)
if isinstance(text, ErrorResponse):
return ORJSONResponse(content=text.model_dump(), status_code=text.code)
text = self._drop_kept_tool_close(text, finish_reason, request)
# Handle reasoning content
reasoning_text = None
@@ -2402,6 +2513,7 @@ class OpenAIServingChat(OpenAIServingBase):
finish_reason,
request.tool_choice,
history_tool_calls_cnt,
request._response_parser_prefix,
)
# Extract prompt_token_ids if requested
@@ -2586,6 +2698,7 @@ class OpenAIServingChat(OpenAIServingBase):
finish_reason: dict[str, Any],
tool_choice: str | ToolChoice | None = None,
history_tool_calls_cnt: int = 0,
response_parser_prefix: str = "",
) -> ToolCallProcessingResult:
"""Process tool calls in the response"""
@@ -2600,6 +2713,7 @@ class OpenAIServingChat(OpenAIServingBase):
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
prefix=response_parser_prefix,
)
detector_owns_format = parser.owns_tool_format()
should_try_parser = not is_required or detector_owns_format
@@ -3055,6 +3169,7 @@ class OpenAIServingChat(OpenAIServingBase):
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=request.tool_choice,
prefix=request._response_parser_prefix,
)
use_native_parser = probe.owns_tool_format()
if use_native_parser:
@@ -3066,6 +3181,7 @@ class OpenAIServingChat(OpenAIServingBase):
tools=effective_tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
prefix=request._response_parser_prefix,
)
parser = parser_dict[index]
@@ -3074,6 +3190,10 @@ class OpenAIServingChat(OpenAIServingBase):
if isinstance(parser, JsonArrayParser):
result = parser.parse_streaming_increment(delta, effective_tools)
normal_text, calls = result.normal_text, result.calls
if flush:
end = parser.finish(effective_tools)
normal_text = (normal_text or "") + end.normal_text
calls = list(calls) + end.calls
else:
normal_text, calls = parser.parse_stream_chunk(delta)
if flush:
@@ -84,7 +84,10 @@ from sglang.srt.entrypoints.openai.responses_adapters import (
encode_reasoning_state,
label_developer_content,
)
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.serving_chat import (
OpenAIServingChat,
_incomplete_tool_call_indices,
)
from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer
from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs
from sglang.srt.function_call.core_types import ToolCallItem
@@ -488,6 +491,11 @@ class OpenAIServingResponses(OpenAIServingChat):
not processed_messages.skip_special_tokens
):
sampling_params["skip_special_tokens"] = False
if (
processed_messages is not None
and processed_messages.uses_response_template
):
sampling_params["spaces_between_special_tokens"] = False
context: ConversationContext
if self.use_harmony:
@@ -573,6 +581,15 @@ class OpenAIServingResponses(OpenAIServingChat):
assert len(generators) == 1
(result_generator,) = generators
if (
processed_messages is not None
and processed_messages.uses_response_template
):
self._set_response_parser_context(
request,
adapted_request,
tool_call_constraint=processed_messages.tool_call_constraint,
)
# Store the input messages
persist = self.enable_response_store and bool(request.store)
@@ -796,7 +813,15 @@ class OpenAIServingResponses(OpenAIServingChat):
)
output = self._make_response_output_items(
request,
final_res["text"],
self._drop_kept_tool_close(
final_res["text"],
(
meta_info.get("finish_reason")
if isinstance(meta_info, dict)
else None
),
request,
),
tokenizer,
output_logprobs=output_logprobs,
require_reasoning=require_reasoning,
@@ -1115,9 +1140,8 @@ class OpenAIServingResponses(OpenAIServingChat):
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
tool_call_parser_active=bool(
chat_tools
and self.tool_call_parser
and request.tool_choice != "none"
self.tool_call_parser
and self._tool_parsing_active(request, chat_tools)
),
)
reasoning_content, content = reasoning_parser.parse_non_stream(final_output)
@@ -1134,15 +1158,15 @@ class OpenAIServingResponses(OpenAIServingChat):
detector_owns_format = False
if (
content
and chat_tools
and self.tool_call_parser
and request.tool_choice != "none"
and self._tool_parsing_active(request, chat_tools)
):
parser = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
prefix=request._response_parser_prefix,
)
detector_owns_format = self._tool_parser_owns_format(parser)
should_try_native = not is_required or detector_owns_format
@@ -1163,7 +1187,7 @@ class OpenAIServingResponses(OpenAIServingChat):
if (
content
and chat_tools
and self._tool_parsing_active(request, chat_tools)
and is_required
and not parsed_via_native
and not detector_owns_format
@@ -1220,6 +1244,14 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools and self.tool_call_parser and request.tool_choice != "none"
)
@staticmethod
def _tool_parsing_active(request: ResponsesRequest, chat_tools: list) -> bool:
return bool(
chat_tools
and request.tool_choice != "none"
and not request._response_parser_json_response_format
)
@staticmethod
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
return parser.owns_tool_format()
@@ -2109,14 +2141,17 @@ class OpenAIServingResponses(OpenAIServingChat):
tool_parser: Optional[Union[FunctionCallParser, JsonArrayParser]] = None
if required_json:
tool_parser = JsonArrayParser()
elif token_stream is None and self._tool_call_parsing_enabled(
request, chat_tools
elif (
token_stream is None
and self._tool_call_parsing_enabled(request, chat_tools)
and not request._response_parser_json_response_format
):
tool_parser = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
prefix=request._response_parser_prefix,
)
reasoning_parser_obj: Optional[ReasoningParser] = None
if token_stream is None and self.reasoning_parser:
@@ -2355,13 +2390,18 @@ class OpenAIServingResponses(OpenAIServingChat):
)
)
else:
item_status = (
"incomplete"
if tool_index in _incomplete_tool_call_indices(tool_parser)
else "completed"
)
completed_item = ResponseFunctionToolCall(
arguments=arguments,
call_id=state["call_id"],
name=state["name"] or "",
type="function_call",
id=state["item_id"],
status="completed",
status=item_status,
)
events.append(
_send_event(
@@ -2420,6 +2460,10 @@ class OpenAIServingResponses(OpenAIServingChat):
else:
delta = text[stream_offset:]
stream_offset = len(text)
if finish_reason is not None:
delta = self._drop_kept_tool_close(
delta, finish_reason, request
)
if not delta and finish_reason is None:
continue
# finish_reason is sticky, so it would otherwise re-flush.
@@ -2527,6 +2571,10 @@ class OpenAIServingResponses(OpenAIServingChat):
normal_text, tool_calls = self._drain_json_array(
tool_parser, delta, chat_tools
)
if flush:
end = tool_parser.finish(chat_tools)
normal_text += end.normal_text
tool_calls = list(tool_calls) + end.calls
elif token_stream is not None:
normal_text, tool_calls = delta, token_calls
elif tool_parser is not None:
@@ -59,6 +59,7 @@ from sglang.srt.function_call.utils import (
_get_tool_schema_defs,
get_json_schema_constraint,
)
from sglang.srt.parser.response_template import ResponseTemplateToolDetector
logger = logging.getLogger(__name__)
@@ -115,10 +116,16 @@ class FunctionCallParser:
"gemma4": Gemma4Detector,
"inkling": InklingDetector,
"iquest_q1": IQuestQ1Detector,
"response_template": ResponseTemplateToolDetector,
}
def __init__(
self, tools: List[Tool], tool_call_parser: str, tokenizer=None, tool_choice=None
self,
tools: List[Tool],
tool_call_parser: str,
tokenizer=None,
tool_choice=None,
prefix: str = "",
):
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
if detector_class:
@@ -127,6 +134,8 @@ class FunctionCallParser:
sig = inspect.signature(detector_class)
if "tokenizer" in sig.parameters:
kwargs["tokenizer"] = tokenizer
if prefix and issubclass(detector_class, ResponseTemplateToolDetector):
kwargs["prefix"] = prefix
detector = detector_class(**kwargs)
else:
raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")
@@ -297,6 +306,12 @@ class FunctionCallParser:
any(tool.function.strict for tool in self.tools)
or self.tool_strict_level >= ToolStrictLevel.FUNCTION
)
if isinstance(self.detector, ResponseTemplateToolDetector):
self.detector.validate_structure_constraint_request(
tool_choice,
parallel_tool_calls,
should_constrain_auto,
)
try:
if (
@@ -42,6 +42,18 @@ class JsonArrayParser(BaseFormatDetector):
"""
return super().parse_streaming_increment(new_text, tools)
def finish(self, tools: List[Tool]) -> StreamingParseResult:
# An increment sends a call's name before its arguments, so drain the
# arguments of calls that arrived whole in the last chunk.
normal_text, calls = "", []
while True:
result = self.parse_streaming_increment("", tools)
normal_text += result.normal_text
progress = [call for call in result.calls if call.name or call.parameters]
if not progress:
return StreamingParseResult(normal_text=normal_text, calls=calls)
calls.extend(progress)
def structure_info(self) -> callable:
"""
Return a function that creates StructureInfo for constrained generation.
@@ -50,4 +50,5 @@ TOOL_CALL_PARSER_NAMES = [
"gemma4",
"inkling",
"iquest_q1",
"response_template",
]
@@ -43,6 +43,7 @@ from sglang.srt.managers.io_struct import (
)
from sglang.srt.managers.multi_tokenizer_mixin import MultiHttpWorkerDetokenizerMixin
from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread
from sglang.srt.parser.response_template import tool_close_token_ids
from sglang.srt.plugins import load_plugins
from sglang.srt.runtime_context import (
get_device,
@@ -156,6 +157,9 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
get_serving().disable_tokenizer_batch_decode
)
self.is_tool_call_parser_gpt_oss = get_serving().tool_call_parser == "gpt-oss"
self.tool_close_token_ids = tool_close_token_ids(
get_serving().tool_call_parser, self.tokenizer
)
self.soft_watchdog = Watchdog.create(
debug_name="DetokenizerManager",
@@ -214,6 +218,8 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
# 200012 <|call|> is the tool call token and one of eos tokens for gpt-oss model
if output[-1] == 200012 and self.is_tool_call_parser_gpt_oss:
return output
if output[-1] in self.tool_close_token_ids:
return output
assert len(output) > 0
# NOTE: We can always assume the last token is the matched stop token
return output[:-1]
+15 -1
View File
@@ -12,7 +12,10 @@ from sglang.srt.entrypoints.openai.encoding_dsv4 import (
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
thinking_start_token as dsv4_thinking_start_token,
)
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
ResponsesRequest,
)
from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens
from sglang.srt.function_call.kimik3_format import (
MESSAGE_CLOSE,
@@ -49,6 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import (
INKLING_SPECIAL_TOKEN_IDS,
MESSAGE_MODEL,
)
from sglang.srt.parser.response_template import ResponseTemplateReasoningDetector
class StreamingParseResult:
@@ -2299,6 +2303,7 @@ class ReasoningParser:
"inkling": InklingDetector,
"iquest_q1": IQuestQ1ReasoningDetector,
"cohere_command4": CohereCommand4Detector,
"response_template": ResponseTemplateReasoningDetector,
}
def __init__(
@@ -2309,6 +2314,7 @@ class ReasoningParser:
request: ChatCompletionRequest = None,
tokenizer=None,
tool_call_parser_active: bool = False,
prefix: str | None = None,
):
if not model_type:
raise ValueError("Model type must be specified")
@@ -2386,6 +2392,14 @@ class ReasoningParser:
if "tool_call_parser_active" in sig.parameters:
kwargs["tool_call_parser_active"] = True
if prefix is None:
prefix = getattr(request, "_response_parser_prefix", "")
if issubclass(detector_class, ResponseTemplateReasoningDetector):
if prefix:
kwargs["prefix"] = prefix
if isinstance(request, (ChatCompletionRequest, ResponsesRequest)):
kwargs["grammar_start"] = request._response_parser_grammar_start
self.detector = detector_class(**kwargs)
def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]:
@@ -38,4 +38,5 @@ REASONING_PARSER_NAMES = [
"iquest_q1",
"cohere_command4",
"gigachat35",
"response_template",
]
@@ -0,0 +1,787 @@
# Copyright 2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""SGLang serving adapters for checkpoint-defined `response_template` grammars."""
from __future__ import annotations
import copy
import functools
import json
import logging
from collections.abc import Callable, Sequence
from typing import Any
import msgspec
from sglang.srt.entrypoints.openai.protocol import Tool
from sglang.srt.environ import envs
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
from sglang.srt.function_call.core_types import (
StreamingParseResult as ToolStreamingParseResult,
)
from sglang.srt.function_call.core_types import (
ToolCallItem,
_GetInfoFunc,
)
from sglang.srt.parser.chat_parsing.content_parsers import _apply_transform
from sglang.srt.parser.chat_parsing.response_parser import ResponseParser
from sglang.srt.parser.chat_parsing.response_templates import (
ResponseTemplate,
load_response_template,
)
logger = logging.getLogger(__name__)
_THINKING_FIELD = "thinking"
_CONTENT_FIELD = "content"
_TOOL_FIELD = "tool_calls"
_PASSTHROUGH_FIELD = "normal"
_RAW_MEMBERS_KEY = "members"
_SUPPORTED_RESPONSE_TEMPLATE_FIELDS = frozenset(
{_THINKING_FIELD, _CONTENT_FIELD, _TOOL_FIELD}
)
# Where JSON-schema output replaces the template's framing, following the
# reasoning gate of ReasonerGrammarBackend.
GRAMMAR_AFTER_REASONING = "after_reasoning"
GRAMMAR_FROM_START = "from_start"
class _ServingTemplates(msgspec.Struct, frozen=True):
"""A validated template and the variants the serving detectors parse with."""
source: ResponseTemplate
streaming: ResponseTemplate
reasoning_only: ResponseTemplate | None
tool_extraction: ResponseTemplate | None
def validate_response_template_for_serving(template: dict) -> ResponseTemplate:
"""Validate the template and every variant the serving detectors build."""
return _derive_serving_templates(template).source
def _derive_serving_templates(template: dict) -> _ServingTemplates:
# Detectors are built several times per request; derive each template once.
return _derive_serving_templates_from_json(json.dumps(template))
@functools.lru_cache(maxsize=8)
def _derive_serving_templates_from_json(template_json: str) -> _ServingTemplates:
template = json.loads(template_json)
loaded = load_response_template(template)
_check_serving_fields(loaded)
return _ServingTemplates(
source=loaded,
streaming=_streaming_template(template),
reasoning_only=(
_reasoning_only_template(template)
if _THINKING_FIELD in loaded.fields
else None
),
tool_extraction=(
_tool_extraction_template(template)
if _TOOL_FIELD in loaded.fields
else None
),
)
def _check_serving_fields(loaded: ResponseTemplate) -> None:
fields = set(loaded.fields)
unsupported = (fields - _SUPPORTED_RESPONSE_TEMPLATE_FIELDS) | (
set(loaded.defaults) - {"role"}
)
if unsupported:
raise ValueError(
"response_template contains unsupported semantic fields: "
f"{sorted(unsupported)}. Supported fields are: "
f"{sorted(_SUPPORTED_RESPONSE_TEMPLATE_FIELDS)}"
)
for name in (_THINKING_FIELD, _CONTENT_FIELD):
field = loaded.fields.get(name)
if field is None:
continue
if (
field.content != "text"
or field.transform is not None
or field.join is not None
):
raise ValueError(
f"response_template field {name!r} uses semantics that cannot "
"be streamed by the OpenAI serving adapter"
)
tool_field = loaded.fields.get(_TOOL_FIELD)
if tool_field is not None and tool_field.open_re is None:
raise ValueError(
f"response_template field {_TOOL_FIELD!r} needs 'open' or "
"'open_pattern' so tool calls can be told apart from other text"
)
def resolve_response_template(
tokenizer: Any | None,
template: dict | None = None,
) -> dict | None:
"""Return `template` if given, otherwise the tokenizer's metadata."""
if template is not None or tokenizer is None:
return template
template = getattr(tokenizer, "response_template", None)
if not isinstance(template, dict):
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
template = init_kwargs.get("response_template")
return template if isinstance(template, dict) else None
def configure_response_template_request(request: Any) -> None:
"""Preserve response-template delimiters during detokenization."""
request.skip_special_tokens = False
template_kwargs = dict(getattr(request, "chat_template_kwargs", None) or {})
template_kwargs["spaces_between_special_tokens"] = False
request.chat_template_kwargs = template_kwargs
def _load_serving_templates(
tokenizer: Any | None, template: dict | None, fallback: dict | None
) -> tuple[dict, _ServingTemplates]:
"""Use the explicit template, then checkpoint metadata, then the detector's
built-in fallback."""
template = resolve_response_template(tokenizer, template) or fallback
if template is None:
raise ValueError("response_template is required")
return template, _derive_serving_templates(template)
def _streaming_template(template: dict[str, Any]) -> ResponseTemplate:
result = copy.deepcopy(template)
fields = result["fields"]
if _CONTENT_FIELD not in fields and all(
"open" in field or "open_pattern" in field for field in fields.values()
):
fields[_CONTENT_FIELD] = {}
for field in fields.values():
field["optional"] = True
return load_response_template(result)
def _reasoning_only_template(template: dict[str, Any]) -> ResponseTemplate:
thinking = copy.deepcopy(template["fields"][_THINKING_FIELD])
thinking["optional"] = True
anchor_name = (
"start_anchor_pattern" if "start_anchor_pattern" in template else "start_anchor"
)
return load_response_template(
{anchor_name: template[anchor_name], "fields": {_THINKING_FIELD: thinking}}
)
def _tool_extraction_template(template: dict[str, Any]) -> ResponseTemplate:
tool_field = copy.deepcopy(template["fields"][_TOOL_FIELD])
tool_field["optional"] = True
# The detector transforms each member itself, so a malformed member drops alone;
# the envelope keeps delimiter captures valid and hides raw members from schema
# coercion, which must only see the call each member's transform produces.
if tool_field.pop("transform_each", False):
tool_field["transform"] = {_RAW_MEMBERS_KEY: "{content}"}
anchor_name = (
"start_anchor_pattern" if "start_anchor_pattern" in template else "start_anchor"
)
return load_response_template(
{
anchor_name: template[anchor_name],
"fields": {_PASSTHROUGH_FIELD: {}, _TOOL_FIELD: tool_field},
}
)
def _passthrough_chunk(text: str) -> dict:
return {"type": "region_chunk", "field": _PASSTHROUGH_FIELD, "text": text}
class _ReasoningResult:
"""Duck-types `reasoning_parser.StreamingParseResult`."""
def __init__(self, normal_text: str = "", reasoning_text: str = ""):
self.normal_text = normal_text
self.reasoning_text = reasoning_text
class ResponseTemplateStreamAdapter:
"""Parse one response with a template and route its generic region events."""
def __init__(
self,
template: ResponseTemplate,
*,
prefix: str = "",
grammar_start: str | None = None,
each_transform: Any = None,
):
self._template = template
self._prefix = prefix
self._parser: ResponseParser | None = None
self._prefix_end = 0
self._finalized = False
tool_field = template.fields.get(_TOOL_FIELD)
self._tool_close_re = None if tool_field is None else tool_field.close_re
self._each_transform = each_transform
self._pending_reasoning = ""
self._pending_tool_streamed = False
self._open_tool_captures: dict = {}
self._grammar_start = grammar_start
# Once JSON-schema output has started, text passes through unparsed.
self._verbatim = grammar_start == GRAMMAR_FROM_START
def _start(self, tools: Sequence[Any] | None) -> list[dict]:
"""Create the parser and replay a tool region left open by the prefix."""
if self._parser is not None:
return []
self._parser = ResponseParser(
self._template,
prefix=self._prefix,
tools=[
tool if isinstance(tool, dict) else tool.model_dump()
for tool in tools or []
],
)
self._prefix_end = len(self._parser.input_text)
for event in reversed(self._parser.initial_events):
if event.get("field") == _TOOL_FIELD and event["type"] != "region_chunk":
return [event] if event["type"] == "region_open" else []
return []
def feed(self, text: str, tools: Sequence[Any] | None = None) -> list[dict]:
if self._finalized:
return []
if self._verbatim:
return [_passthrough_chunk(text)] if text else []
events = self._start(tools)
return self._end_reasoning(events + self._parser.feed(text))
def finalize(self, tools: Sequence[Any] | None = None) -> list[dict]:
if self._finalized:
return []
events = self._start(tools)
self._finalized = True
if self._verbatim:
return events
return self._end_reasoning(events + self._parser.finalize()[1])
def _end_reasoning(self, events: list[dict]) -> list[dict]:
# Stop parsing at the reasoning closer, where the grammar backend starts
# the JSON schema, and pass the rest through unchanged.
if self._grammar_start != GRAMMAR_AFTER_REASONING:
return events
for index, event in enumerate(events):
if event.get("field") == _THINKING_FIELD and event["type"] in (
"region_close",
"region_malformed",
):
self._verbatim = True
text = self._parser.input_text[event["end"] :]
return events[: index + 1] + (
[_passthrough_chunk(text)] if text else []
)
return events
def parse(self, text: str, tools: Sequence[Any] | None = None) -> list[dict]:
return self.feed(text, tools) + self.finalize(tools)
def _generated_text(self, event: dict) -> str:
start = max(event["start"], self._prefix_end)
return self._parser.input_text[start : event["end"]]
def _open_tool_name(self, captures: dict | None) -> str | None:
"""Name already fixed by the opener, before the region body exists."""
field = self._template.fields.get(_TOOL_FIELD)
transform = None if field is None or field.transform_each else field.transform
function = transform.get("function") if isinstance(transform, dict) else None
if not isinstance(function, dict):
return None
try:
name = _apply_transform(function.get("name"), captures or {})
except (KeyError, ValueError):
return None
return name if isinstance(name, str) else None
def route_reasoning_events(
self, events: list[dict], *, stream_reasoning: bool
) -> _ReasoningResult:
reasoning_parts: list[str] = []
normal_parts: list[str] = []
for event in events:
field = event.get("field")
etype = event["type"]
if field == _THINKING_FIELD:
if etype == "region_chunk":
if stream_reasoning:
reasoning_parts.append(event["text"])
else:
self._pending_reasoning += event["text"]
elif etype == "region_close" and not stream_reasoning:
reasoning_parts.append(self._pending_reasoning)
self._pending_reasoning = ""
elif (
field in (_CONTENT_FIELD, _PASSTHROUGH_FIELD)
and etype == "region_chunk"
):
normal_parts.append(event["text"])
elif field == _TOOL_FIELD:
# Hand tool regions back verbatim so the tool detector can parse them.
normal_parts.append(
event["text"]
if etype == "region_chunk"
else self._generated_text(event)
)
return _ReasoningResult("".join(normal_parts), "".join(reasoning_parts))
def route_tool_events(
self,
events: list[dict],
*,
on_tool_open: Callable[[str], bool] | None = None,
on_tool_close: Callable[[Any], bool],
on_tool_dropped: Callable[[], None] | None = None,
) -> str:
"""Route tool regions to the callbacks and return the passthrough text.
A tool call is emitted only when it ends with its closer, parses, and is
accepted by `on_tool_close`. Otherwise it is dropped, and a name that was
already streamed stays an incomplete call."""
normal_parts: list[str] = []
unusable_opens: set[int] = set()
open_index = None
for index, event in enumerate(events):
if event.get("field") == _TOOL_FIELD:
if event["type"] == "region_open":
open_index = index
elif self._is_unusable_tool_end(event):
unusable_opens.add(open_index)
for index, event in enumerate(events):
field = event.get("field")
etype = event["type"]
if field == _PASSTHROUGH_FIELD and etype == "region_chunk":
normal_parts.append(event["text"])
elif field != _TOOL_FIELD or etype == "region_chunk":
continue
elif etype == "region_open":
self._open_tool_captures = event.get("captures") or {}
name = self._open_tool_name(event.get("captures"))
self._pending_tool_streamed = bool(
index not in unusable_opens
and name is not None
and on_tool_open is not None
and on_tool_open(name)
)
else:
if self._is_unusable_tool_end(event):
logger.warning(
"response_template: dropping malformed or cut-off tool call"
)
accepted = False
else:
accepted = on_tool_close(self._transform_members(event["value"]))
if not accepted and self._pending_tool_streamed and on_tool_dropped:
on_tool_dropped()
self._pending_tool_streamed = False
return "".join(normal_parts)
def _transform_members(self, value: Any) -> Any:
"""Apply a `transform_each` template to each member of a tool region,
dropping the members it cannot transform, and cast their arguments to
the tool schemas as the parser does for transformed regions."""
if self._each_transform is None:
return value
value = value[_RAW_MEMBERS_KEY]
if not isinstance(value, list):
logger.warning("response_template: malformed tool call value: %r", value)
return []
members = []
for member in value:
try:
scope = {**self._open_tool_captures, **member}
call = _apply_transform(self._each_transform, scope)
members.append(self._parser._coerce_tool_calls(call))
except (KeyError, TypeError, ValueError):
logger.warning(
"response_template: malformed tool call value: %r", member
)
return members
def _is_unusable_tool_end(self, event: dict) -> bool:
"""Whether a tool region ended malformed, or at end of stream without its
closer because the call was cut off."""
if event["type"] == "region_malformed":
return True
text = self._parser.input_text
# End-of-stream finalization and a matched zero-width closer such as `\Z`
# emit the same close event, so check whether the closer matches there.
return (
event["type"] == "region_close"
and self._tool_close_re is not None
and event["start"] == len(text)
and self._tool_close_re.match(text, len(text)) is None
)
class ResponseTemplateReasoningDetector:
"""Reasoning detector driven by a `response_template` grammar."""
response_template: dict | None = None
_default_think_start = ""
_default_think_end = ""
thinks_internally = False
reasoning_default = "explicit_enable_thinking"
def __init__(
self,
stream_reasoning: bool = True,
tokenizer=None,
response_template: dict | None = None,
prefix: str = "",
force_nonempty_content: bool = False,
grammar_start: str | None = None,
**_kwargs,
):
template, templates = _load_serving_templates(
tokenizer, response_template, self.response_template
)
self.response_template = template
self.stream_reasoning = stream_reasoning
self.think_excluded_tokens = None
self._force_nonempty_content = force_nonempty_content
self._open_reasoning = ""
thinking = templates.source.fields.get(_THINKING_FIELD)
self.think_start_token = (
thinking.open_literals[0]
if thinking and thinking.open_literals
else self._default_think_start
)
self.think_end_token = (
thinking.close_literals[0]
if thinking and thinking.close_literals
else self._default_think_end
)
self.think_start_self_label = ""
if grammar_start == GRAMMAR_AFTER_REASONING and thinking is None:
grammar_start = GRAMMAR_FROM_START
self._grammar_start = grammar_start
if grammar_start == GRAMMAR_AFTER_REASONING:
self._template = templates.reasoning_only
else:
self._template = templates.streaming
self._prefix = prefix
self._adapter = self._new_adapter()
def _new_adapter(self) -> ResponseTemplateStreamAdapter:
return ResponseTemplateStreamAdapter(
self._template, prefix=self._prefix, grammar_start=self._grammar_start
)
def get_think_end_token_ids(self, tokenizer) -> list[int]:
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
def get_answer_frame(self) -> None:
return None
def detect_and_parse(self, text: str) -> _ReasoningResult:
adapter = self._new_adapter()
result = adapter.route_reasoning_events(
adapter.parse(text), stream_reasoning=True
)
if self._force_nonempty_content and not result.normal_text:
result.normal_text, result.reasoning_text = result.reasoning_text, ""
return result
def parse_streaming_increment(self, new_text: str) -> _ReasoningResult:
events = self._adapter.feed(new_text)
if self._force_nonempty_content:
self._track_open_reasoning(events)
return self._adapter.route_reasoning_events(
events, stream_reasoning=self.stream_reasoning
)
def finish(self) -> _ReasoningResult:
events = self._adapter.finalize()
result = self._adapter.route_reasoning_events(
events, stream_reasoning=self.stream_reasoning
)
# Under force_nonempty_content, reasoning left open at the end of the
# stream is also returned as content.
if self._force_nonempty_content:
unclosed_reasoning = self._track_open_reasoning(events)
if unclosed_reasoning is not None:
result.normal_text += unclosed_reasoning
result.reasoning_text = ""
return result
def _track_open_reasoning(self, events: list[dict]) -> str | None:
"""Accumulate open thinking text; return the text of a region `events` close."""
closed = None
for event in events:
if event.get("field") != _THINKING_FIELD:
continue
if event["type"] == "region_chunk":
self._open_reasoning += event["text"]
elif event["type"] in ("region_close", "region_malformed"):
closed, self._open_reasoning = self._open_reasoning, ""
return closed
class ResponseTemplateToolDetector(BaseFormatDetector):
"""Tool-call detector driven by a `response_template` grammar."""
response_template: dict | None = None
def __init__(
self,
tokenizer=None,
response_template: dict | None = None,
prefix: str = "",
):
super().__init__()
template, templates = _load_serving_templates(
tokenizer, response_template, self.response_template
)
self.response_template = template
if templates.tool_extraction is None:
raise ValueError(f"response_template has no {_TOOL_FIELD!r} field")
self._template = templates.tool_extraction
self._prefix = prefix
tool_field = templates.source.fields[_TOOL_FIELD]
self._each_transform = (
tool_field.transform if tool_field.transform_each else None
)
self._adapter = ResponseTemplateStreamAdapter(
self._template, prefix=prefix, each_transform=self._each_transform
)
self.tool_close_literals = tool_field.close_literals or []
if self.tool_close_literals:
self.eot_token = self.tool_close_literals[0]
self.incomplete_tool_call_indices: set[int] = set()
self._tool_indices: dict[str, int] | None = None
def has_tool_call(self, text: str) -> bool:
adapter = ResponseTemplateStreamAdapter(self._template, prefix=self._prefix)
return any(
event["type"] == "region_open" and event.get("field") == _TOOL_FIELD
for event in adapter.parse(text)
)
def _to_tool_call(
self, value: Any, tool_indices: dict[str, int]
) -> tuple[str, str] | None:
try:
function = value["function"]
name = function["name"]
arguments = function["arguments"]
except (TypeError, KeyError):
logger.warning("response_template: malformed tool call value: %r", value)
return None
if not isinstance(name, str):
logger.warning("response_template: malformed tool name: %r", name)
return None
if name not in tool_indices:
logger.warning("Model attempted to call undefined function: %s", name)
if not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get():
return None
if not isinstance(arguments, str):
arguments = json.dumps(arguments, ensure_ascii=False)
return name, arguments
def _to_tool_call_items(
self,
value: Any,
tool_indices: dict[str, int],
first_tool_index: int,
) -> list[ToolCallItem]:
"""The region's accepted calls, numbered from `first_tool_index`."""
values = value if isinstance(value, list) else [value]
calls = [
call
for call in (self._to_tool_call(item, tool_indices) for item in values)
if call is not None
]
return [
ToolCallItem(
tool_index=first_tool_index + offset, name=name, parameters=arguments
)
for offset, (name, arguments) in enumerate(calls)
]
def _emit_tool_call(
self, item: ToolCallItem, pending_calls: list[ToolCallItem]
) -> None:
name_was_streamed = self.current_tool_name_sent
self._ensure_tool_slot()
try:
parsed_args = json.loads(item.parameters)
except json.JSONDecodeError:
parsed_args = item.parameters
self.prev_tool_call_arr[self.current_tool_id] = {
"name": item.name,
"arguments": parsed_args,
}
self.streamed_args_for_tool[self.current_tool_id] = item.parameters
if name_was_streamed:
item = item.model_copy(update={"name": None})
pending_calls.append(item)
self.incomplete_tool_call_indices.discard(self.current_tool_id)
self.current_tool_id += 1
self.current_tool_name_sent = False
def _ensure_tool_slot(self) -> None:
if self.current_tool_id == -1:
self.current_tool_id = 0
self.prev_tool_call_arr = []
self.streamed_args_for_tool = [""]
while len(self.prev_tool_call_arr) <= self.current_tool_id:
self.prev_tool_call_arr.append({})
while len(self.streamed_args_for_tool) <= self.current_tool_id:
self.streamed_args_for_tool.append("")
def _emit_tool_name(self, name: str, pending_calls: list[ToolCallItem]) -> None:
self._ensure_tool_slot()
pending_calls.append(
ToolCallItem(tool_index=self.current_tool_id, name=name, parameters="")
)
self.prev_tool_call_arr[self.current_tool_id] = {"name": name, "arguments": ""}
self.incomplete_tool_call_indices.add(self.current_tool_id)
self.current_tool_name_sent = True
def _drop_streamed_tool_call(self) -> None:
"""Leave the streamed name as an incomplete call and move to the next."""
self.current_tool_id += 1
self.current_tool_name_sent = False
def detect_and_parse(
self, text: str, tools: list[Tool]
) -> ToolStreamingParseResult:
adapter = ResponseTemplateStreamAdapter(
self._template, prefix=self._prefix, each_transform=self._each_transform
)
tool_indices = self._get_tool_indices(tools)
calls: list[ToolCallItem] = []
def on_close(value: Any) -> bool:
items = self._to_tool_call_items(value, tool_indices, len(calls))
if not items:
return False
calls.extend(items)
return True
normal_text = adapter.route_tool_events(
adapter.parse(text, tools), on_tool_close=on_close
)
return ToolStreamingParseResult(normal_text=normal_text, calls=calls)
def parse_streaming_increment(
self, new_text: str, tools: list[Tool]
) -> ToolStreamingParseResult:
return self._emit_streaming_events(self._adapter.feed(new_text, tools), tools)
def finish(self, tools: list[Tool]) -> ToolStreamingParseResult:
return self._emit_streaming_events(self._adapter.finalize(tools), tools)
def _emit_streaming_events(
self, events: list[dict], tools: list[Tool]
) -> ToolStreamingParseResult:
if self._tool_indices is None:
self._tool_indices = self._get_tool_indices(tools)
tool_indices = self._tool_indices
pending_calls: list[ToolCallItem] = []
def on_open(name: str) -> bool:
if name not in tool_indices and not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get():
return False
self._emit_tool_name(name, pending_calls)
return True
def on_close(value: Any) -> bool:
items = self._to_tool_call_items(
value, tool_indices, max(self.current_tool_id, 0)
)
if not items:
return False
for item in items:
self._emit_tool_call(item, pending_calls)
return True
normal_text = self._adapter.route_tool_events(
events,
on_tool_open=on_open,
on_tool_close=on_close,
on_tool_dropped=self._drop_streamed_tool_call,
)
return ToolStreamingParseResult(normal_text=normal_text, calls=pending_calls)
def supports_structural_tag(self) -> bool:
return False
def validate_structure_constraint_request(
self,
tool_choice: Any,
parallel_tool_calls: bool,
strict_requested: bool,
) -> None:
if tool_choice == "auto" and not parallel_tool_calls:
raise ValueError(
f"{type(self).__name__} cannot enforce "
"parallel_tool_calls=False with automatic tool choice"
)
if tool_choice == "auto" and strict_requested:
raise ValueError(
f"{type(self).__name__} does not support strict tool constraints"
)
def structure_info(self) -> _GetInfoFunc:
raise NotImplementedError(
"structure_info not used with the response_template tool parser"
)
def tool_close_tokens(
tool_call_parser: str | None, tokenizer: Any | None
) -> dict[int, str]:
"""Single-token tool-call closers of a response-template tool parser, by id.
Detokenization keeps these when they stop generation, so a tool call that
reaches end of stream without its closer was cut off."""
from sglang.srt.function_call.function_call_parser import FunctionCallParser
detector_class = FunctionCallParser.ToolCallParserEnum.get(tool_call_parser)
if (
tokenizer is None
or detector_class is None
or not issubclass(detector_class, ResponseTemplateToolDetector)
):
return {}
try:
detector = detector_class(tokenizer=tokenizer)
except ValueError:
return {}
closers = {}
for literal in detector.tool_close_literals:
ids = tokenizer.encode(literal, add_special_tokens=False)
if len(ids) == 1:
closers[ids[0]] = literal
return closers
def tool_close_token_ids(
tool_call_parser: str | None, tokenizer: Any | None
) -> frozenset[int]:
return frozenset(tool_close_tokens(tool_call_parser, tokenizer))
+43 -4
View File
@@ -773,7 +773,6 @@ def _detect_auto_parser(
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template"
)
return detected
_log_undetected_parser(attr, label)
return None
@@ -924,7 +923,8 @@ def _architecture_parser_defaults(server_args) -> Dict[str, str]:
def resolve_auto_parsers(server_args) -> None:
"""Resolve architecture defaults and explicit ``auto`` parser requests.
"""Resolve architecture defaults and explicit ``auto`` parser requests;
``auto`` is detected from the chat template or checkpoint metadata.
Performs a lightweight tokenizer load, so it runs once in engine init. The
decision goes to this instance's declaration stash, so every holder of it
@@ -982,8 +982,11 @@ def resolve_auto_parsers(server_args) -> None:
tokenizer = None
try:
tokenizer = get_tokenizer(
cfg.model_path,
cfg.tokenizer_path,
tokenizer_mode=cfg.tokenizer_mode,
trust_remote_code=cfg.trust_remote_code,
revision=cfg.revision,
tokenizer_backend=cfg.tokenizer_backend,
)
except Exception as e:
logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
@@ -1026,7 +1029,6 @@ def resolve_auto_parsers(server_args) -> None:
("tool_call_parser", "tool-call parser"),
):
if attr in needs and attr not in detected:
_log_undetected_parser(attr, label)
detected[attr] = None
else:
for attr, rules, label in (
@@ -1036,5 +1038,42 @@ def resolve_auto_parsers(server_args) -> None:
if attr in needs:
detected[attr] = _detect_auto_parser(attr, ctx, rules, label)
unresolved = tuple(attr for attr in needs if detected.get(attr) is None)
if unresolved and tokenizer is not None and chat_template_arg is None:
from sglang.srt.parser.response_template import (
resolve_response_template,
validate_response_template_for_serving,
)
response_template = resolve_response_template(tokenizer)
if response_template is not None:
try:
response_template = validate_response_template_for_serving(
response_template
)
except (TypeError, ValueError) as exc:
logger.warning(
"Ignoring invalid response_template from tokenizer configuration: %s",
exc,
)
else:
for attr, field in (
("reasoning_parser", "thinking"),
("tool_call_parser", "tool_calls"),
):
if attr in unresolved and field in response_template.fields:
detected[attr] = "response_template"
logger.info(
"Auto-detected --%s as 'response_template' from tokenizer configuration",
attr.replace("_", "-"),
)
for attr, label in (
("reasoning_parser", "reasoning parser"),
("tool_call_parser", "tool-call parser"),
):
if attr in needs and detected.get(attr) is None:
_log_undetected_parser(attr, label)
if detected:
declare_resolution(server_args, "template-detection", **detected)
+2 -2
View File
@@ -370,7 +370,7 @@ class ServerArgs:
choices=["auto"] + reasoning_parser_choices,
default=_declared_default("reasoning_parser"),
help=f"Specify the parser for reasoning models. "
f"Use 'auto' to detect from chat template. "
f"Use 'auto' to detect from chat template or checkpoint metadata. "
f"Options include: {reasoning_parser_choices}.",
)
tool_call_parser_choices = _tool_call_parser_choices()
@@ -380,7 +380,7 @@ class ServerArgs:
choices=["auto"] + tool_call_parser_choices,
default=_declared_default("tool_call_parser"),
help=f"Specify the parser for handling tool-call interactions. "
f"Use 'auto' to detect from chat template. "
f"Use 'auto' to detect from chat template or checkpoint metadata. "
f"Options include: {tool_call_parser_choices}.",
)
parser.add_argument(