[Feat] parser: Add checkpoint-driven response-template adapters (#40479)
Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Co-authored-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
co-authored by
Yoni Gozlan
Xinyuan Tong
Jimmy Shong
parent
4542fc8f59
commit
27ac614f05
@@ -58,6 +58,7 @@ from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
PrivateAttr,
|
||||
StrictBool,
|
||||
field_serializer,
|
||||
field_validator,
|
||||
@@ -866,6 +867,10 @@ def _has_message_level_tools(messages: Any) -> bool:
|
||||
|
||||
|
||||
class ChatCompletionRequest(PDRoutingFields):
|
||||
_response_parser_prefix: str = PrivateAttr(default="")
|
||||
_response_parser_grammar_start: Optional[str] = PrivateAttr(default=None)
|
||||
_response_parser_json_response_format: bool = PrivateAttr(default=False)
|
||||
|
||||
# Ordered by official OpenAI API documentation
|
||||
# https://platform.openai.com/docs/api-reference/chat/create
|
||||
messages: List[ChatCompletionMessageParam]
|
||||
@@ -1833,6 +1838,10 @@ ResponseInputOutputItem: TypeAlias = Union[
|
||||
class ResponsesRequest(PDRoutingFields):
|
||||
"""Request body for v1/responses endpoint."""
|
||||
|
||||
_response_parser_prefix: str = PrivateAttr(default="")
|
||||
_response_parser_grammar_start: Optional[str] = PrivateAttr(default=None)
|
||||
_response_parser_json_response_format: bool = PrivateAttr(default=False)
|
||||
|
||||
# Core OpenAI API fields (ordered by official documentation)
|
||||
background: Optional[bool] = False
|
||||
include: Optional[
|
||||
@@ -2322,6 +2331,7 @@ class MessageProcessingResult:
|
||||
skip_special_tokens: bool = True
|
||||
require_reasoning: bool = False
|
||||
reasoning_end_token_ids: Optional[List[int]] = None
|
||||
uses_response_template: bool = False
|
||||
|
||||
|
||||
class ToolCallProcessingResult(NamedTuple):
|
||||
|
||||
@@ -68,9 +68,11 @@ from sglang.srt.entrypoints.openai.protocol import (
|
||||
MessageProcessingResult,
|
||||
PromptTokensDetails,
|
||||
ResponseParserProtocol,
|
||||
ResponsesRequest,
|
||||
SglExt,
|
||||
Tool,
|
||||
ToolCall,
|
||||
ToolCallConstraint,
|
||||
ToolCallProcessingResult,
|
||||
ToolChoice,
|
||||
TopLogprob,
|
||||
@@ -113,6 +115,14 @@ from sglang.srt.parser.reasoning_parser import (
|
||||
IQuestQ1ReasoningDetector,
|
||||
ReasoningParser,
|
||||
)
|
||||
from sglang.srt.parser.response_template import (
|
||||
GRAMMAR_AFTER_REASONING,
|
||||
GRAMMAR_FROM_START,
|
||||
ResponseTemplateReasoningDetector,
|
||||
ResponseTemplateToolDetector,
|
||||
configure_response_template_request,
|
||||
tool_close_tokens,
|
||||
)
|
||||
from sglang.srt.parser.template_detection import detect_inline_system_support
|
||||
from sglang.srt.sampling.sampling_params import (
|
||||
set_request_reasoning_end_token_ids,
|
||||
@@ -130,6 +140,11 @@ _MEDIA_CONTENT_PART_TYPES = frozenset({"image_url", "video_url", "audio_url"})
|
||||
_CHAT_TEMPLATE_CACHE_MAX_SIZE = 128
|
||||
|
||||
|
||||
def _incomplete_tool_call_indices(parser) -> set[int]:
|
||||
detector = getattr(parser, "detector", parser)
|
||||
return getattr(detector, "incomplete_tool_call_indices", set())
|
||||
|
||||
|
||||
def normalize_tool_content(role: str, content):
|
||||
"""Normalize tool message content from OpenAI array format to plain string.
|
||||
|
||||
@@ -846,6 +861,10 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
else:
|
||||
delta = content["text"][offset:]
|
||||
stream_offsets[index] = len(content["text"])
|
||||
if finish_reason_type is not None:
|
||||
delta = self._drop_kept_tool_close(
|
||||
delta, content["meta_info"].get("finish_reason"), request
|
||||
)
|
||||
|
||||
# Attach logprobs to the first chunk emitted this step (reasoning,
|
||||
# tool-call, or content) so they aren't dropped when a parser is active
|
||||
@@ -967,6 +986,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
request.tool_choice != "none"
|
||||
and self._effective_tools(request)
|
||||
and self.tool_call_parser
|
||||
and not request._response_parser_json_response_format
|
||||
)
|
||||
|
||||
def _validate_request(self, request: ChatCompletionRequest) -> str | None:
|
||||
@@ -1285,8 +1305,89 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
):
|
||||
apply_header_overrides(adapted_request, raw_request.headers)
|
||||
|
||||
if processed_messages.uses_response_template:
|
||||
self._set_response_parser_context(
|
||||
request,
|
||||
adapted_request,
|
||||
tool_call_constraint=processed_messages.tool_call_constraint,
|
||||
)
|
||||
return adapted_request, request
|
||||
|
||||
def _set_response_parser_context(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
adapted_request: GenerateReqInput,
|
||||
*,
|
||||
tool_call_constraint: ToolCallConstraint | None,
|
||||
) -> None:
|
||||
"""Give response-template parsers the rendered assistant prefill."""
|
||||
prefix = adapted_request.text
|
||||
if not isinstance(prefix, str) and adapted_request.input_ids:
|
||||
prefix = self.tokenizer_manager.tokenizer.decode(
|
||||
adapted_request.input_ids,
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=False,
|
||||
)
|
||||
request._response_parser_prefix = prefix or ""
|
||||
# Structural tags, regex and EBNF can describe the template's own
|
||||
# framing, so only JSON-schema output bypasses the template.
|
||||
if (adapted_request.sampling_params or {}).get("json_schema"):
|
||||
# The scheduler defers the grammar to the reasoning's end only when
|
||||
# the reasoning parser's think_end_token encodes to token ids.
|
||||
reasoning_gated = (
|
||||
adapted_request.require_reasoning
|
||||
and self._reasoning_detector is not None
|
||||
and bool(
|
||||
self._reasoning_detector.get_think_end_token_ids(
|
||||
self.tokenizer_manager.tokenizer
|
||||
)
|
||||
)
|
||||
)
|
||||
request._response_parser_grammar_start = (
|
||||
GRAMMAR_AFTER_REASONING if reasoning_gated else GRAMMAR_FROM_START
|
||||
)
|
||||
# Unless the schema is the required/named tool call itself, the JSON
|
||||
# is response data, and template syntax inside its strings is literal.
|
||||
request._response_parser_json_response_format = (
|
||||
tool_call_constraint is None or tool_call_constraint[0] != "json_schema"
|
||||
)
|
||||
|
||||
def _drop_kept_tool_close(
|
||||
self,
|
||||
text: str,
|
||||
finish_reason: dict[str, Any] | None,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> str:
|
||||
# Detokenization keeps a tool-call closer that stops generation; after
|
||||
# JSON-schema output it ends no native call, so trim it like any stop.
|
||||
if (
|
||||
not text
|
||||
or request._response_parser_grammar_start is None
|
||||
or (isinstance(request, ChatCompletionRequest) and request.no_stop_trim)
|
||||
or not isinstance(finish_reason, dict)
|
||||
):
|
||||
return text
|
||||
closer = tool_close_tokens(
|
||||
self.tool_call_parser, self.tokenizer_manager.tokenizer
|
||||
).get(finish_reason.get("matched"))
|
||||
if closer and text.endswith(closer):
|
||||
return text[: -len(closer)]
|
||||
return text
|
||||
|
||||
def _requires_response_template_detokenization(
|
||||
self, request: ChatCompletionRequest
|
||||
) -> bool:
|
||||
if isinstance(self._reasoning_detector, ResponseTemplateReasoningDetector):
|
||||
return True
|
||||
|
||||
tool_detector = FunctionCallParser.ToolCallParserEnum.get(self.tool_call_parser)
|
||||
return (
|
||||
request.tool_choice != "none"
|
||||
and bool(self._effective_tools(request))
|
||||
and tool_detector is not None
|
||||
and issubclass(tool_detector, ResponseTemplateToolDetector)
|
||||
)
|
||||
|
||||
def _process_messages(
|
||||
self, request: ChatCompletionRequest, is_multimodal: bool
|
||||
) -> MessageProcessingResult:
|
||||
@@ -1427,6 +1528,11 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if tool_call_stop not in result.stop:
|
||||
result.stop.append(tool_call_stop)
|
||||
|
||||
# Configured after rendering so the detokenization options never reach
|
||||
# the chat template.
|
||||
if self._requires_response_template_detokenization(request):
|
||||
configure_response_template_request(request)
|
||||
result.uses_response_template = True
|
||||
result.tool_call_constraint = tool_call_constraint
|
||||
result.require_reasoning = thinking_mode
|
||||
result.skip_special_tokens = request.skip_special_tokens
|
||||
@@ -2094,7 +2200,11 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
# Change finish_reason to "tool_calls" if we had tool calls and stopped naturally
|
||||
final_finish_reason = finish_reason_type
|
||||
if has_tool_calls.get(idx, False) and finish_reason_type == "stop":
|
||||
if (
|
||||
has_tool_calls.get(idx, False)
|
||||
and finish_reason_type == "stop"
|
||||
and not _incomplete_tool_call_indices(parser_dict.get(idx))
|
||||
):
|
||||
final_finish_reason = "tool_calls"
|
||||
|
||||
matched_stop = finish_reason_data.get("matched")
|
||||
@@ -2357,6 +2467,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
text = self._decode_response(ret_item)
|
||||
if isinstance(text, ErrorResponse):
|
||||
return ORJSONResponse(content=text.model_dump(), status_code=text.code)
|
||||
text = self._drop_kept_tool_close(text, finish_reason, request)
|
||||
|
||||
# Handle reasoning content
|
||||
reasoning_text = None
|
||||
@@ -2402,6 +2513,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
finish_reason,
|
||||
request.tool_choice,
|
||||
history_tool_calls_cnt,
|
||||
request._response_parser_prefix,
|
||||
)
|
||||
|
||||
# Extract prompt_token_ids if requested
|
||||
@@ -2586,6 +2698,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
finish_reason: dict[str, Any],
|
||||
tool_choice: str | ToolChoice | None = None,
|
||||
history_tool_calls_cnt: int = 0,
|
||||
response_parser_prefix: str = "",
|
||||
) -> ToolCallProcessingResult:
|
||||
"""Process tool calls in the response"""
|
||||
|
||||
@@ -2600,6 +2713,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
prefix=response_parser_prefix,
|
||||
)
|
||||
detector_owns_format = parser.owns_tool_format()
|
||||
should_try_parser = not is_required or detector_owns_format
|
||||
@@ -3055,6 +3169,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=request.tool_choice,
|
||||
prefix=request._response_parser_prefix,
|
||||
)
|
||||
use_native_parser = probe.owns_tool_format()
|
||||
if use_native_parser:
|
||||
@@ -3066,6 +3181,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
tools=effective_tools,
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
prefix=request._response_parser_prefix,
|
||||
)
|
||||
|
||||
parser = parser_dict[index]
|
||||
@@ -3074,6 +3190,10 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if isinstance(parser, JsonArrayParser):
|
||||
result = parser.parse_streaming_increment(delta, effective_tools)
|
||||
normal_text, calls = result.normal_text, result.calls
|
||||
if flush:
|
||||
end = parser.finish(effective_tools)
|
||||
normal_text = (normal_text or "") + end.normal_text
|
||||
calls = list(calls) + end.calls
|
||||
else:
|
||||
normal_text, calls = parser.parse_stream_chunk(delta)
|
||||
if flush:
|
||||
|
||||
@@ -84,7 +84,10 @@ from sglang.srt.entrypoints.openai.responses_adapters import (
|
||||
encode_reasoning_state,
|
||||
label_developer_content,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
from sglang.srt.entrypoints.openai.serving_chat import (
|
||||
OpenAIServingChat,
|
||||
_incomplete_tool_call_indices,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer
|
||||
from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
@@ -488,6 +491,11 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
not processed_messages.skip_special_tokens
|
||||
):
|
||||
sampling_params["skip_special_tokens"] = False
|
||||
if (
|
||||
processed_messages is not None
|
||||
and processed_messages.uses_response_template
|
||||
):
|
||||
sampling_params["spaces_between_special_tokens"] = False
|
||||
|
||||
context: ConversationContext
|
||||
if self.use_harmony:
|
||||
@@ -573,6 +581,15 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
|
||||
assert len(generators) == 1
|
||||
(result_generator,) = generators
|
||||
if (
|
||||
processed_messages is not None
|
||||
and processed_messages.uses_response_template
|
||||
):
|
||||
self._set_response_parser_context(
|
||||
request,
|
||||
adapted_request,
|
||||
tool_call_constraint=processed_messages.tool_call_constraint,
|
||||
)
|
||||
|
||||
# Store the input messages
|
||||
persist = self.enable_response_store and bool(request.store)
|
||||
@@ -796,7 +813,15 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
output = self._make_response_output_items(
|
||||
request,
|
||||
final_res["text"],
|
||||
self._drop_kept_tool_close(
|
||||
final_res["text"],
|
||||
(
|
||||
meta_info.get("finish_reason")
|
||||
if isinstance(meta_info, dict)
|
||||
else None
|
||||
),
|
||||
request,
|
||||
),
|
||||
tokenizer,
|
||||
output_logprobs=output_logprobs,
|
||||
require_reasoning=require_reasoning,
|
||||
@@ -1115,9 +1140,8 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
request=request,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_call_parser_active=bool(
|
||||
chat_tools
|
||||
and self.tool_call_parser
|
||||
and request.tool_choice != "none"
|
||||
self.tool_call_parser
|
||||
and self._tool_parsing_active(request, chat_tools)
|
||||
),
|
||||
)
|
||||
reasoning_content, content = reasoning_parser.parse_non_stream(final_output)
|
||||
@@ -1134,15 +1158,15 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
detector_owns_format = False
|
||||
if (
|
||||
content
|
||||
and chat_tools
|
||||
and self.tool_call_parser
|
||||
and request.tool_choice != "none"
|
||||
and self._tool_parsing_active(request, chat_tools)
|
||||
):
|
||||
parser = FunctionCallParser(
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
prefix=request._response_parser_prefix,
|
||||
)
|
||||
detector_owns_format = self._tool_parser_owns_format(parser)
|
||||
should_try_native = not is_required or detector_owns_format
|
||||
@@ -1163,7 +1187,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
|
||||
if (
|
||||
content
|
||||
and chat_tools
|
||||
and self._tool_parsing_active(request, chat_tools)
|
||||
and is_required
|
||||
and not parsed_via_native
|
||||
and not detector_owns_format
|
||||
@@ -1220,6 +1244,14 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_tools and self.tool_call_parser and request.tool_choice != "none"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _tool_parsing_active(request: ResponsesRequest, chat_tools: list) -> bool:
|
||||
return bool(
|
||||
chat_tools
|
||||
and request.tool_choice != "none"
|
||||
and not request._response_parser_json_response_format
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
|
||||
return parser.owns_tool_format()
|
||||
@@ -2109,14 +2141,17 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
tool_parser: Optional[Union[FunctionCallParser, JsonArrayParser]] = None
|
||||
if required_json:
|
||||
tool_parser = JsonArrayParser()
|
||||
elif token_stream is None and self._tool_call_parsing_enabled(
|
||||
request, chat_tools
|
||||
elif (
|
||||
token_stream is None
|
||||
and self._tool_call_parsing_enabled(request, chat_tools)
|
||||
and not request._response_parser_json_response_format
|
||||
):
|
||||
tool_parser = FunctionCallParser(
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
prefix=request._response_parser_prefix,
|
||||
)
|
||||
reasoning_parser_obj: Optional[ReasoningParser] = None
|
||||
if token_stream is None and self.reasoning_parser:
|
||||
@@ -2355,13 +2390,18 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
)
|
||||
else:
|
||||
item_status = (
|
||||
"incomplete"
|
||||
if tool_index in _incomplete_tool_call_indices(tool_parser)
|
||||
else "completed"
|
||||
)
|
||||
completed_item = ResponseFunctionToolCall(
|
||||
arguments=arguments,
|
||||
call_id=state["call_id"],
|
||||
name=state["name"] or "",
|
||||
type="function_call",
|
||||
id=state["item_id"],
|
||||
status="completed",
|
||||
status=item_status,
|
||||
)
|
||||
events.append(
|
||||
_send_event(
|
||||
@@ -2420,6 +2460,10 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
else:
|
||||
delta = text[stream_offset:]
|
||||
stream_offset = len(text)
|
||||
if finish_reason is not None:
|
||||
delta = self._drop_kept_tool_close(
|
||||
delta, finish_reason, request
|
||||
)
|
||||
if not delta and finish_reason is None:
|
||||
continue
|
||||
# finish_reason is sticky, so it would otherwise re-flush.
|
||||
@@ -2527,6 +2571,10 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
normal_text, tool_calls = self._drain_json_array(
|
||||
tool_parser, delta, chat_tools
|
||||
)
|
||||
if flush:
|
||||
end = tool_parser.finish(chat_tools)
|
||||
normal_text += end.normal_text
|
||||
tool_calls = list(tool_calls) + end.calls
|
||||
elif token_stream is not None:
|
||||
normal_text, tool_calls = delta, token_calls
|
||||
elif tool_parser is not None:
|
||||
|
||||
@@ -59,6 +59,7 @@ from sglang.srt.function_call.utils import (
|
||||
_get_tool_schema_defs,
|
||||
get_json_schema_constraint,
|
||||
)
|
||||
from sglang.srt.parser.response_template import ResponseTemplateToolDetector
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -115,10 +116,16 @@ class FunctionCallParser:
|
||||
"gemma4": Gemma4Detector,
|
||||
"inkling": InklingDetector,
|
||||
"iquest_q1": IQuestQ1Detector,
|
||||
"response_template": ResponseTemplateToolDetector,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self, tools: List[Tool], tool_call_parser: str, tokenizer=None, tool_choice=None
|
||||
self,
|
||||
tools: List[Tool],
|
||||
tool_call_parser: str,
|
||||
tokenizer=None,
|
||||
tool_choice=None,
|
||||
prefix: str = "",
|
||||
):
|
||||
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
|
||||
if detector_class:
|
||||
@@ -127,6 +134,8 @@ class FunctionCallParser:
|
||||
sig = inspect.signature(detector_class)
|
||||
if "tokenizer" in sig.parameters:
|
||||
kwargs["tokenizer"] = tokenizer
|
||||
if prefix and issubclass(detector_class, ResponseTemplateToolDetector):
|
||||
kwargs["prefix"] = prefix
|
||||
detector = detector_class(**kwargs)
|
||||
else:
|
||||
raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")
|
||||
@@ -297,6 +306,12 @@ class FunctionCallParser:
|
||||
any(tool.function.strict for tool in self.tools)
|
||||
or self.tool_strict_level >= ToolStrictLevel.FUNCTION
|
||||
)
|
||||
if isinstance(self.detector, ResponseTemplateToolDetector):
|
||||
self.detector.validate_structure_constraint_request(
|
||||
tool_choice,
|
||||
parallel_tool_calls,
|
||||
should_constrain_auto,
|
||||
)
|
||||
|
||||
try:
|
||||
if (
|
||||
|
||||
@@ -42,6 +42,18 @@ class JsonArrayParser(BaseFormatDetector):
|
||||
"""
|
||||
return super().parse_streaming_increment(new_text, tools)
|
||||
|
||||
def finish(self, tools: List[Tool]) -> StreamingParseResult:
|
||||
# An increment sends a call's name before its arguments, so drain the
|
||||
# arguments of calls that arrived whole in the last chunk.
|
||||
normal_text, calls = "", []
|
||||
while True:
|
||||
result = self.parse_streaming_increment("", tools)
|
||||
normal_text += result.normal_text
|
||||
progress = [call for call in result.calls if call.name or call.parameters]
|
||||
if not progress:
|
||||
return StreamingParseResult(normal_text=normal_text, calls=calls)
|
||||
calls.extend(progress)
|
||||
|
||||
def structure_info(self) -> callable:
|
||||
"""
|
||||
Return a function that creates StructureInfo for constrained generation.
|
||||
|
||||
@@ -50,4 +50,5 @@ TOOL_CALL_PARSER_NAMES = [
|
||||
"gemma4",
|
||||
"inkling",
|
||||
"iquest_q1",
|
||||
"response_template",
|
||||
]
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.managers.io_struct import (
|
||||
)
|
||||
from sglang.srt.managers.multi_tokenizer_mixin import MultiHttpWorkerDetokenizerMixin
|
||||
from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread
|
||||
from sglang.srt.parser.response_template import tool_close_token_ids
|
||||
from sglang.srt.plugins import load_plugins
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
@@ -156,6 +157,9 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
|
||||
get_serving().disable_tokenizer_batch_decode
|
||||
)
|
||||
self.is_tool_call_parser_gpt_oss = get_serving().tool_call_parser == "gpt-oss"
|
||||
self.tool_close_token_ids = tool_close_token_ids(
|
||||
get_serving().tool_call_parser, self.tokenizer
|
||||
)
|
||||
|
||||
self.soft_watchdog = Watchdog.create(
|
||||
debug_name="DetokenizerManager",
|
||||
@@ -214,6 +218,8 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
|
||||
# 200012 <|call|> is the tool call token and one of eos tokens for gpt-oss model
|
||||
if output[-1] == 200012 and self.is_tool_call_parser_gpt_oss:
|
||||
return output
|
||||
if output[-1] in self.tool_close_token_ids:
|
||||
return output
|
||||
assert len(output) > 0
|
||||
# NOTE: We can always assume the last token is the matched stop token
|
||||
return output[:-1]
|
||||
|
||||
@@ -12,7 +12,10 @@ from sglang.srt.entrypoints.openai.encoding_dsv4 import (
|
||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
|
||||
thinking_start_token as dsv4_thinking_start_token,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ResponsesRequest,
|
||||
)
|
||||
from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens
|
||||
from sglang.srt.function_call.kimik3_format import (
|
||||
MESSAGE_CLOSE,
|
||||
@@ -49,6 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import (
|
||||
INKLING_SPECIAL_TOKEN_IDS,
|
||||
MESSAGE_MODEL,
|
||||
)
|
||||
from sglang.srt.parser.response_template import ResponseTemplateReasoningDetector
|
||||
|
||||
|
||||
class StreamingParseResult:
|
||||
@@ -2299,6 +2303,7 @@ class ReasoningParser:
|
||||
"inkling": InklingDetector,
|
||||
"iquest_q1": IQuestQ1ReasoningDetector,
|
||||
"cohere_command4": CohereCommand4Detector,
|
||||
"response_template": ResponseTemplateReasoningDetector,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
@@ -2309,6 +2314,7 @@ class ReasoningParser:
|
||||
request: ChatCompletionRequest = None,
|
||||
tokenizer=None,
|
||||
tool_call_parser_active: bool = False,
|
||||
prefix: str | None = None,
|
||||
):
|
||||
if not model_type:
|
||||
raise ValueError("Model type must be specified")
|
||||
@@ -2386,6 +2392,14 @@ class ReasoningParser:
|
||||
if "tool_call_parser_active" in sig.parameters:
|
||||
kwargs["tool_call_parser_active"] = True
|
||||
|
||||
if prefix is None:
|
||||
prefix = getattr(request, "_response_parser_prefix", "")
|
||||
if issubclass(detector_class, ResponseTemplateReasoningDetector):
|
||||
if prefix:
|
||||
kwargs["prefix"] = prefix
|
||||
if isinstance(request, (ChatCompletionRequest, ResponsesRequest)):
|
||||
kwargs["grammar_start"] = request._response_parser_grammar_start
|
||||
|
||||
self.detector = detector_class(**kwargs)
|
||||
|
||||
def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
|
||||
@@ -38,4 +38,5 @@ REASONING_PARSER_NAMES = [
|
||||
"iquest_q1",
|
||||
"cohere_command4",
|
||||
"gigachat35",
|
||||
"response_template",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,787 @@
|
||||
# Copyright 2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""SGLang serving adapters for checkpoint-defined `response_template` grammars."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import Tool
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.function_call.base_format_detector import BaseFormatDetector
|
||||
from sglang.srt.function_call.core_types import (
|
||||
StreamingParseResult as ToolStreamingParseResult,
|
||||
)
|
||||
from sglang.srt.function_call.core_types import (
|
||||
ToolCallItem,
|
||||
_GetInfoFunc,
|
||||
)
|
||||
from sglang.srt.parser.chat_parsing.content_parsers import _apply_transform
|
||||
from sglang.srt.parser.chat_parsing.response_parser import ResponseParser
|
||||
from sglang.srt.parser.chat_parsing.response_templates import (
|
||||
ResponseTemplate,
|
||||
load_response_template,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_THINKING_FIELD = "thinking"
|
||||
_CONTENT_FIELD = "content"
|
||||
_TOOL_FIELD = "tool_calls"
|
||||
_PASSTHROUGH_FIELD = "normal"
|
||||
_RAW_MEMBERS_KEY = "members"
|
||||
_SUPPORTED_RESPONSE_TEMPLATE_FIELDS = frozenset(
|
||||
{_THINKING_FIELD, _CONTENT_FIELD, _TOOL_FIELD}
|
||||
)
|
||||
# Where JSON-schema output replaces the template's framing, following the
|
||||
# reasoning gate of ReasonerGrammarBackend.
|
||||
GRAMMAR_AFTER_REASONING = "after_reasoning"
|
||||
GRAMMAR_FROM_START = "from_start"
|
||||
|
||||
|
||||
class _ServingTemplates(msgspec.Struct, frozen=True):
|
||||
"""A validated template and the variants the serving detectors parse with."""
|
||||
|
||||
source: ResponseTemplate
|
||||
streaming: ResponseTemplate
|
||||
reasoning_only: ResponseTemplate | None
|
||||
tool_extraction: ResponseTemplate | None
|
||||
|
||||
|
||||
def validate_response_template_for_serving(template: dict) -> ResponseTemplate:
|
||||
"""Validate the template and every variant the serving detectors build."""
|
||||
return _derive_serving_templates(template).source
|
||||
|
||||
|
||||
def _derive_serving_templates(template: dict) -> _ServingTemplates:
|
||||
# Detectors are built several times per request; derive each template once.
|
||||
return _derive_serving_templates_from_json(json.dumps(template))
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=8)
|
||||
def _derive_serving_templates_from_json(template_json: str) -> _ServingTemplates:
|
||||
template = json.loads(template_json)
|
||||
loaded = load_response_template(template)
|
||||
_check_serving_fields(loaded)
|
||||
return _ServingTemplates(
|
||||
source=loaded,
|
||||
streaming=_streaming_template(template),
|
||||
reasoning_only=(
|
||||
_reasoning_only_template(template)
|
||||
if _THINKING_FIELD in loaded.fields
|
||||
else None
|
||||
),
|
||||
tool_extraction=(
|
||||
_tool_extraction_template(template)
|
||||
if _TOOL_FIELD in loaded.fields
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _check_serving_fields(loaded: ResponseTemplate) -> None:
|
||||
fields = set(loaded.fields)
|
||||
unsupported = (fields - _SUPPORTED_RESPONSE_TEMPLATE_FIELDS) | (
|
||||
set(loaded.defaults) - {"role"}
|
||||
)
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"response_template contains unsupported semantic fields: "
|
||||
f"{sorted(unsupported)}. Supported fields are: "
|
||||
f"{sorted(_SUPPORTED_RESPONSE_TEMPLATE_FIELDS)}"
|
||||
)
|
||||
for name in (_THINKING_FIELD, _CONTENT_FIELD):
|
||||
field = loaded.fields.get(name)
|
||||
if field is None:
|
||||
continue
|
||||
if (
|
||||
field.content != "text"
|
||||
or field.transform is not None
|
||||
or field.join is not None
|
||||
):
|
||||
raise ValueError(
|
||||
f"response_template field {name!r} uses semantics that cannot "
|
||||
"be streamed by the OpenAI serving adapter"
|
||||
)
|
||||
tool_field = loaded.fields.get(_TOOL_FIELD)
|
||||
if tool_field is not None and tool_field.open_re is None:
|
||||
raise ValueError(
|
||||
f"response_template field {_TOOL_FIELD!r} needs 'open' or "
|
||||
"'open_pattern' so tool calls can be told apart from other text"
|
||||
)
|
||||
|
||||
|
||||
def resolve_response_template(
|
||||
tokenizer: Any | None,
|
||||
template: dict | None = None,
|
||||
) -> dict | None:
|
||||
"""Return `template` if given, otherwise the tokenizer's metadata."""
|
||||
if template is not None or tokenizer is None:
|
||||
return template
|
||||
template = getattr(tokenizer, "response_template", None)
|
||||
if not isinstance(template, dict):
|
||||
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
|
||||
template = init_kwargs.get("response_template")
|
||||
return template if isinstance(template, dict) else None
|
||||
|
||||
|
||||
def configure_response_template_request(request: Any) -> None:
|
||||
"""Preserve response-template delimiters during detokenization."""
|
||||
request.skip_special_tokens = False
|
||||
template_kwargs = dict(getattr(request, "chat_template_kwargs", None) or {})
|
||||
template_kwargs["spaces_between_special_tokens"] = False
|
||||
request.chat_template_kwargs = template_kwargs
|
||||
|
||||
|
||||
def _load_serving_templates(
|
||||
tokenizer: Any | None, template: dict | None, fallback: dict | None
|
||||
) -> tuple[dict, _ServingTemplates]:
|
||||
"""Use the explicit template, then checkpoint metadata, then the detector's
|
||||
built-in fallback."""
|
||||
template = resolve_response_template(tokenizer, template) or fallback
|
||||
if template is None:
|
||||
raise ValueError("response_template is required")
|
||||
return template, _derive_serving_templates(template)
|
||||
|
||||
|
||||
def _streaming_template(template: dict[str, Any]) -> ResponseTemplate:
|
||||
result = copy.deepcopy(template)
|
||||
fields = result["fields"]
|
||||
if _CONTENT_FIELD not in fields and all(
|
||||
"open" in field or "open_pattern" in field for field in fields.values()
|
||||
):
|
||||
fields[_CONTENT_FIELD] = {}
|
||||
for field in fields.values():
|
||||
field["optional"] = True
|
||||
return load_response_template(result)
|
||||
|
||||
|
||||
def _reasoning_only_template(template: dict[str, Any]) -> ResponseTemplate:
|
||||
thinking = copy.deepcopy(template["fields"][_THINKING_FIELD])
|
||||
thinking["optional"] = True
|
||||
anchor_name = (
|
||||
"start_anchor_pattern" if "start_anchor_pattern" in template else "start_anchor"
|
||||
)
|
||||
return load_response_template(
|
||||
{anchor_name: template[anchor_name], "fields": {_THINKING_FIELD: thinking}}
|
||||
)
|
||||
|
||||
|
||||
def _tool_extraction_template(template: dict[str, Any]) -> ResponseTemplate:
|
||||
tool_field = copy.deepcopy(template["fields"][_TOOL_FIELD])
|
||||
tool_field["optional"] = True
|
||||
# The detector transforms each member itself, so a malformed member drops alone;
|
||||
# the envelope keeps delimiter captures valid and hides raw members from schema
|
||||
# coercion, which must only see the call each member's transform produces.
|
||||
if tool_field.pop("transform_each", False):
|
||||
tool_field["transform"] = {_RAW_MEMBERS_KEY: "{content}"}
|
||||
anchor_name = (
|
||||
"start_anchor_pattern" if "start_anchor_pattern" in template else "start_anchor"
|
||||
)
|
||||
return load_response_template(
|
||||
{
|
||||
anchor_name: template[anchor_name],
|
||||
"fields": {_PASSTHROUGH_FIELD: {}, _TOOL_FIELD: tool_field},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _passthrough_chunk(text: str) -> dict:
|
||||
return {"type": "region_chunk", "field": _PASSTHROUGH_FIELD, "text": text}
|
||||
|
||||
|
||||
class _ReasoningResult:
|
||||
"""Duck-types `reasoning_parser.StreamingParseResult`."""
|
||||
|
||||
def __init__(self, normal_text: str = "", reasoning_text: str = ""):
|
||||
self.normal_text = normal_text
|
||||
self.reasoning_text = reasoning_text
|
||||
|
||||
|
||||
class ResponseTemplateStreamAdapter:
|
||||
"""Parse one response with a template and route its generic region events."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
template: ResponseTemplate,
|
||||
*,
|
||||
prefix: str = "",
|
||||
grammar_start: str | None = None,
|
||||
each_transform: Any = None,
|
||||
):
|
||||
self._template = template
|
||||
self._prefix = prefix
|
||||
self._parser: ResponseParser | None = None
|
||||
self._prefix_end = 0
|
||||
self._finalized = False
|
||||
tool_field = template.fields.get(_TOOL_FIELD)
|
||||
self._tool_close_re = None if tool_field is None else tool_field.close_re
|
||||
self._each_transform = each_transform
|
||||
self._pending_reasoning = ""
|
||||
self._pending_tool_streamed = False
|
||||
self._open_tool_captures: dict = {}
|
||||
self._grammar_start = grammar_start
|
||||
# Once JSON-schema output has started, text passes through unparsed.
|
||||
self._verbatim = grammar_start == GRAMMAR_FROM_START
|
||||
|
||||
def _start(self, tools: Sequence[Any] | None) -> list[dict]:
|
||||
"""Create the parser and replay a tool region left open by the prefix."""
|
||||
if self._parser is not None:
|
||||
return []
|
||||
self._parser = ResponseParser(
|
||||
self._template,
|
||||
prefix=self._prefix,
|
||||
tools=[
|
||||
tool if isinstance(tool, dict) else tool.model_dump()
|
||||
for tool in tools or []
|
||||
],
|
||||
)
|
||||
self._prefix_end = len(self._parser.input_text)
|
||||
for event in reversed(self._parser.initial_events):
|
||||
if event.get("field") == _TOOL_FIELD and event["type"] != "region_chunk":
|
||||
return [event] if event["type"] == "region_open" else []
|
||||
return []
|
||||
|
||||
def feed(self, text: str, tools: Sequence[Any] | None = None) -> list[dict]:
|
||||
if self._finalized:
|
||||
return []
|
||||
if self._verbatim:
|
||||
return [_passthrough_chunk(text)] if text else []
|
||||
events = self._start(tools)
|
||||
return self._end_reasoning(events + self._parser.feed(text))
|
||||
|
||||
def finalize(self, tools: Sequence[Any] | None = None) -> list[dict]:
|
||||
if self._finalized:
|
||||
return []
|
||||
events = self._start(tools)
|
||||
self._finalized = True
|
||||
if self._verbatim:
|
||||
return events
|
||||
return self._end_reasoning(events + self._parser.finalize()[1])
|
||||
|
||||
def _end_reasoning(self, events: list[dict]) -> list[dict]:
|
||||
# Stop parsing at the reasoning closer, where the grammar backend starts
|
||||
# the JSON schema, and pass the rest through unchanged.
|
||||
if self._grammar_start != GRAMMAR_AFTER_REASONING:
|
||||
return events
|
||||
for index, event in enumerate(events):
|
||||
if event.get("field") == _THINKING_FIELD and event["type"] in (
|
||||
"region_close",
|
||||
"region_malformed",
|
||||
):
|
||||
self._verbatim = True
|
||||
text = self._parser.input_text[event["end"] :]
|
||||
return events[: index + 1] + (
|
||||
[_passthrough_chunk(text)] if text else []
|
||||
)
|
||||
return events
|
||||
|
||||
def parse(self, text: str, tools: Sequence[Any] | None = None) -> list[dict]:
|
||||
return self.feed(text, tools) + self.finalize(tools)
|
||||
|
||||
def _generated_text(self, event: dict) -> str:
|
||||
start = max(event["start"], self._prefix_end)
|
||||
return self._parser.input_text[start : event["end"]]
|
||||
|
||||
def _open_tool_name(self, captures: dict | None) -> str | None:
|
||||
"""Name already fixed by the opener, before the region body exists."""
|
||||
field = self._template.fields.get(_TOOL_FIELD)
|
||||
transform = None if field is None or field.transform_each else field.transform
|
||||
function = transform.get("function") if isinstance(transform, dict) else None
|
||||
if not isinstance(function, dict):
|
||||
return None
|
||||
try:
|
||||
name = _apply_transform(function.get("name"), captures or {})
|
||||
except (KeyError, ValueError):
|
||||
return None
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
def route_reasoning_events(
|
||||
self, events: list[dict], *, stream_reasoning: bool
|
||||
) -> _ReasoningResult:
|
||||
reasoning_parts: list[str] = []
|
||||
normal_parts: list[str] = []
|
||||
for event in events:
|
||||
field = event.get("field")
|
||||
etype = event["type"]
|
||||
if field == _THINKING_FIELD:
|
||||
if etype == "region_chunk":
|
||||
if stream_reasoning:
|
||||
reasoning_parts.append(event["text"])
|
||||
else:
|
||||
self._pending_reasoning += event["text"]
|
||||
elif etype == "region_close" and not stream_reasoning:
|
||||
reasoning_parts.append(self._pending_reasoning)
|
||||
self._pending_reasoning = ""
|
||||
elif (
|
||||
field in (_CONTENT_FIELD, _PASSTHROUGH_FIELD)
|
||||
and etype == "region_chunk"
|
||||
):
|
||||
normal_parts.append(event["text"])
|
||||
elif field == _TOOL_FIELD:
|
||||
# Hand tool regions back verbatim so the tool detector can parse them.
|
||||
normal_parts.append(
|
||||
event["text"]
|
||||
if etype == "region_chunk"
|
||||
else self._generated_text(event)
|
||||
)
|
||||
return _ReasoningResult("".join(normal_parts), "".join(reasoning_parts))
|
||||
|
||||
def route_tool_events(
|
||||
self,
|
||||
events: list[dict],
|
||||
*,
|
||||
on_tool_open: Callable[[str], bool] | None = None,
|
||||
on_tool_close: Callable[[Any], bool],
|
||||
on_tool_dropped: Callable[[], None] | None = None,
|
||||
) -> str:
|
||||
"""Route tool regions to the callbacks and return the passthrough text.
|
||||
|
||||
A tool call is emitted only when it ends with its closer, parses, and is
|
||||
accepted by `on_tool_close`. Otherwise it is dropped, and a name that was
|
||||
already streamed stays an incomplete call."""
|
||||
normal_parts: list[str] = []
|
||||
unusable_opens: set[int] = set()
|
||||
open_index = None
|
||||
for index, event in enumerate(events):
|
||||
if event.get("field") == _TOOL_FIELD:
|
||||
if event["type"] == "region_open":
|
||||
open_index = index
|
||||
elif self._is_unusable_tool_end(event):
|
||||
unusable_opens.add(open_index)
|
||||
for index, event in enumerate(events):
|
||||
field = event.get("field")
|
||||
etype = event["type"]
|
||||
if field == _PASSTHROUGH_FIELD and etype == "region_chunk":
|
||||
normal_parts.append(event["text"])
|
||||
elif field != _TOOL_FIELD or etype == "region_chunk":
|
||||
continue
|
||||
elif etype == "region_open":
|
||||
self._open_tool_captures = event.get("captures") or {}
|
||||
name = self._open_tool_name(event.get("captures"))
|
||||
self._pending_tool_streamed = bool(
|
||||
index not in unusable_opens
|
||||
and name is not None
|
||||
and on_tool_open is not None
|
||||
and on_tool_open(name)
|
||||
)
|
||||
else:
|
||||
if self._is_unusable_tool_end(event):
|
||||
logger.warning(
|
||||
"response_template: dropping malformed or cut-off tool call"
|
||||
)
|
||||
accepted = False
|
||||
else:
|
||||
accepted = on_tool_close(self._transform_members(event["value"]))
|
||||
if not accepted and self._pending_tool_streamed and on_tool_dropped:
|
||||
on_tool_dropped()
|
||||
self._pending_tool_streamed = False
|
||||
return "".join(normal_parts)
|
||||
|
||||
def _transform_members(self, value: Any) -> Any:
|
||||
"""Apply a `transform_each` template to each member of a tool region,
|
||||
dropping the members it cannot transform, and cast their arguments to
|
||||
the tool schemas as the parser does for transformed regions."""
|
||||
if self._each_transform is None:
|
||||
return value
|
||||
value = value[_RAW_MEMBERS_KEY]
|
||||
if not isinstance(value, list):
|
||||
logger.warning("response_template: malformed tool call value: %r", value)
|
||||
return []
|
||||
members = []
|
||||
for member in value:
|
||||
try:
|
||||
scope = {**self._open_tool_captures, **member}
|
||||
call = _apply_transform(self._each_transform, scope)
|
||||
members.append(self._parser._coerce_tool_calls(call))
|
||||
except (KeyError, TypeError, ValueError):
|
||||
logger.warning(
|
||||
"response_template: malformed tool call value: %r", member
|
||||
)
|
||||
return members
|
||||
|
||||
def _is_unusable_tool_end(self, event: dict) -> bool:
|
||||
"""Whether a tool region ended malformed, or at end of stream without its
|
||||
closer because the call was cut off."""
|
||||
if event["type"] == "region_malformed":
|
||||
return True
|
||||
text = self._parser.input_text
|
||||
# End-of-stream finalization and a matched zero-width closer such as `\Z`
|
||||
# emit the same close event, so check whether the closer matches there.
|
||||
return (
|
||||
event["type"] == "region_close"
|
||||
and self._tool_close_re is not None
|
||||
and event["start"] == len(text)
|
||||
and self._tool_close_re.match(text, len(text)) is None
|
||||
)
|
||||
|
||||
|
||||
class ResponseTemplateReasoningDetector:
|
||||
"""Reasoning detector driven by a `response_template` grammar."""
|
||||
|
||||
response_template: dict | None = None
|
||||
_default_think_start = ""
|
||||
_default_think_end = ""
|
||||
thinks_internally = False
|
||||
reasoning_default = "explicit_enable_thinking"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stream_reasoning: bool = True,
|
||||
tokenizer=None,
|
||||
response_template: dict | None = None,
|
||||
prefix: str = "",
|
||||
force_nonempty_content: bool = False,
|
||||
grammar_start: str | None = None,
|
||||
**_kwargs,
|
||||
):
|
||||
template, templates = _load_serving_templates(
|
||||
tokenizer, response_template, self.response_template
|
||||
)
|
||||
self.response_template = template
|
||||
self.stream_reasoning = stream_reasoning
|
||||
self.think_excluded_tokens = None
|
||||
self._force_nonempty_content = force_nonempty_content
|
||||
self._open_reasoning = ""
|
||||
thinking = templates.source.fields.get(_THINKING_FIELD)
|
||||
self.think_start_token = (
|
||||
thinking.open_literals[0]
|
||||
if thinking and thinking.open_literals
|
||||
else self._default_think_start
|
||||
)
|
||||
self.think_end_token = (
|
||||
thinking.close_literals[0]
|
||||
if thinking and thinking.close_literals
|
||||
else self._default_think_end
|
||||
)
|
||||
self.think_start_self_label = ""
|
||||
if grammar_start == GRAMMAR_AFTER_REASONING and thinking is None:
|
||||
grammar_start = GRAMMAR_FROM_START
|
||||
self._grammar_start = grammar_start
|
||||
if grammar_start == GRAMMAR_AFTER_REASONING:
|
||||
self._template = templates.reasoning_only
|
||||
else:
|
||||
self._template = templates.streaming
|
||||
self._prefix = prefix
|
||||
self._adapter = self._new_adapter()
|
||||
|
||||
def _new_adapter(self) -> ResponseTemplateStreamAdapter:
|
||||
return ResponseTemplateStreamAdapter(
|
||||
self._template, prefix=self._prefix, grammar_start=self._grammar_start
|
||||
)
|
||||
|
||||
def get_think_end_token_ids(self, tokenizer) -> list[int]:
|
||||
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
|
||||
|
||||
def get_answer_frame(self) -> None:
|
||||
return None
|
||||
|
||||
def detect_and_parse(self, text: str) -> _ReasoningResult:
|
||||
adapter = self._new_adapter()
|
||||
result = adapter.route_reasoning_events(
|
||||
adapter.parse(text), stream_reasoning=True
|
||||
)
|
||||
if self._force_nonempty_content and not result.normal_text:
|
||||
result.normal_text, result.reasoning_text = result.reasoning_text, ""
|
||||
return result
|
||||
|
||||
def parse_streaming_increment(self, new_text: str) -> _ReasoningResult:
|
||||
events = self._adapter.feed(new_text)
|
||||
if self._force_nonempty_content:
|
||||
self._track_open_reasoning(events)
|
||||
return self._adapter.route_reasoning_events(
|
||||
events, stream_reasoning=self.stream_reasoning
|
||||
)
|
||||
|
||||
def finish(self) -> _ReasoningResult:
|
||||
events = self._adapter.finalize()
|
||||
result = self._adapter.route_reasoning_events(
|
||||
events, stream_reasoning=self.stream_reasoning
|
||||
)
|
||||
# Under force_nonempty_content, reasoning left open at the end of the
|
||||
# stream is also returned as content.
|
||||
if self._force_nonempty_content:
|
||||
unclosed_reasoning = self._track_open_reasoning(events)
|
||||
if unclosed_reasoning is not None:
|
||||
result.normal_text += unclosed_reasoning
|
||||
result.reasoning_text = ""
|
||||
return result
|
||||
|
||||
def _track_open_reasoning(self, events: list[dict]) -> str | None:
|
||||
"""Accumulate open thinking text; return the text of a region `events` close."""
|
||||
closed = None
|
||||
for event in events:
|
||||
if event.get("field") != _THINKING_FIELD:
|
||||
continue
|
||||
if event["type"] == "region_chunk":
|
||||
self._open_reasoning += event["text"]
|
||||
elif event["type"] in ("region_close", "region_malformed"):
|
||||
closed, self._open_reasoning = self._open_reasoning, ""
|
||||
return closed
|
||||
|
||||
|
||||
class ResponseTemplateToolDetector(BaseFormatDetector):
|
||||
"""Tool-call detector driven by a `response_template` grammar."""
|
||||
|
||||
response_template: dict | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer=None,
|
||||
response_template: dict | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
template, templates = _load_serving_templates(
|
||||
tokenizer, response_template, self.response_template
|
||||
)
|
||||
self.response_template = template
|
||||
if templates.tool_extraction is None:
|
||||
raise ValueError(f"response_template has no {_TOOL_FIELD!r} field")
|
||||
self._template = templates.tool_extraction
|
||||
self._prefix = prefix
|
||||
tool_field = templates.source.fields[_TOOL_FIELD]
|
||||
self._each_transform = (
|
||||
tool_field.transform if tool_field.transform_each else None
|
||||
)
|
||||
self._adapter = ResponseTemplateStreamAdapter(
|
||||
self._template, prefix=prefix, each_transform=self._each_transform
|
||||
)
|
||||
self.tool_close_literals = tool_field.close_literals or []
|
||||
if self.tool_close_literals:
|
||||
self.eot_token = self.tool_close_literals[0]
|
||||
self.incomplete_tool_call_indices: set[int] = set()
|
||||
self._tool_indices: dict[str, int] | None = None
|
||||
|
||||
def has_tool_call(self, text: str) -> bool:
|
||||
adapter = ResponseTemplateStreamAdapter(self._template, prefix=self._prefix)
|
||||
return any(
|
||||
event["type"] == "region_open" and event.get("field") == _TOOL_FIELD
|
||||
for event in adapter.parse(text)
|
||||
)
|
||||
|
||||
def _to_tool_call(
|
||||
self, value: Any, tool_indices: dict[str, int]
|
||||
) -> tuple[str, str] | None:
|
||||
try:
|
||||
function = value["function"]
|
||||
name = function["name"]
|
||||
arguments = function["arguments"]
|
||||
except (TypeError, KeyError):
|
||||
logger.warning("response_template: malformed tool call value: %r", value)
|
||||
return None
|
||||
|
||||
if not isinstance(name, str):
|
||||
logger.warning("response_template: malformed tool name: %r", name)
|
||||
return None
|
||||
|
||||
if name not in tool_indices:
|
||||
logger.warning("Model attempted to call undefined function: %s", name)
|
||||
if not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get():
|
||||
return None
|
||||
|
||||
if not isinstance(arguments, str):
|
||||
arguments = json.dumps(arguments, ensure_ascii=False)
|
||||
return name, arguments
|
||||
|
||||
def _to_tool_call_items(
|
||||
self,
|
||||
value: Any,
|
||||
tool_indices: dict[str, int],
|
||||
first_tool_index: int,
|
||||
) -> list[ToolCallItem]:
|
||||
"""The region's accepted calls, numbered from `first_tool_index`."""
|
||||
values = value if isinstance(value, list) else [value]
|
||||
calls = [
|
||||
call
|
||||
for call in (self._to_tool_call(item, tool_indices) for item in values)
|
||||
if call is not None
|
||||
]
|
||||
return [
|
||||
ToolCallItem(
|
||||
tool_index=first_tool_index + offset, name=name, parameters=arguments
|
||||
)
|
||||
for offset, (name, arguments) in enumerate(calls)
|
||||
]
|
||||
|
||||
def _emit_tool_call(
|
||||
self, item: ToolCallItem, pending_calls: list[ToolCallItem]
|
||||
) -> None:
|
||||
name_was_streamed = self.current_tool_name_sent
|
||||
self._ensure_tool_slot()
|
||||
try:
|
||||
parsed_args = json.loads(item.parameters)
|
||||
except json.JSONDecodeError:
|
||||
parsed_args = item.parameters
|
||||
self.prev_tool_call_arr[self.current_tool_id] = {
|
||||
"name": item.name,
|
||||
"arguments": parsed_args,
|
||||
}
|
||||
self.streamed_args_for_tool[self.current_tool_id] = item.parameters
|
||||
if name_was_streamed:
|
||||
item = item.model_copy(update={"name": None})
|
||||
pending_calls.append(item)
|
||||
self.incomplete_tool_call_indices.discard(self.current_tool_id)
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
|
||||
def _ensure_tool_slot(self) -> None:
|
||||
if self.current_tool_id == -1:
|
||||
self.current_tool_id = 0
|
||||
self.prev_tool_call_arr = []
|
||||
self.streamed_args_for_tool = [""]
|
||||
while len(self.prev_tool_call_arr) <= self.current_tool_id:
|
||||
self.prev_tool_call_arr.append({})
|
||||
while len(self.streamed_args_for_tool) <= self.current_tool_id:
|
||||
self.streamed_args_for_tool.append("")
|
||||
|
||||
def _emit_tool_name(self, name: str, pending_calls: list[ToolCallItem]) -> None:
|
||||
self._ensure_tool_slot()
|
||||
pending_calls.append(
|
||||
ToolCallItem(tool_index=self.current_tool_id, name=name, parameters="")
|
||||
)
|
||||
self.prev_tool_call_arr[self.current_tool_id] = {"name": name, "arguments": ""}
|
||||
self.incomplete_tool_call_indices.add(self.current_tool_id)
|
||||
self.current_tool_name_sent = True
|
||||
|
||||
def _drop_streamed_tool_call(self) -> None:
|
||||
"""Leave the streamed name as an incomplete call and move to the next."""
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
|
||||
def detect_and_parse(
|
||||
self, text: str, tools: list[Tool]
|
||||
) -> ToolStreamingParseResult:
|
||||
adapter = ResponseTemplateStreamAdapter(
|
||||
self._template, prefix=self._prefix, each_transform=self._each_transform
|
||||
)
|
||||
tool_indices = self._get_tool_indices(tools)
|
||||
calls: list[ToolCallItem] = []
|
||||
|
||||
def on_close(value: Any) -> bool:
|
||||
items = self._to_tool_call_items(value, tool_indices, len(calls))
|
||||
if not items:
|
||||
return False
|
||||
calls.extend(items)
|
||||
return True
|
||||
|
||||
normal_text = adapter.route_tool_events(
|
||||
adapter.parse(text, tools), on_tool_close=on_close
|
||||
)
|
||||
return ToolStreamingParseResult(normal_text=normal_text, calls=calls)
|
||||
|
||||
def parse_streaming_increment(
|
||||
self, new_text: str, tools: list[Tool]
|
||||
) -> ToolStreamingParseResult:
|
||||
return self._emit_streaming_events(self._adapter.feed(new_text, tools), tools)
|
||||
|
||||
def finish(self, tools: list[Tool]) -> ToolStreamingParseResult:
|
||||
return self._emit_streaming_events(self._adapter.finalize(tools), tools)
|
||||
|
||||
def _emit_streaming_events(
|
||||
self, events: list[dict], tools: list[Tool]
|
||||
) -> ToolStreamingParseResult:
|
||||
if self._tool_indices is None:
|
||||
self._tool_indices = self._get_tool_indices(tools)
|
||||
tool_indices = self._tool_indices
|
||||
pending_calls: list[ToolCallItem] = []
|
||||
|
||||
def on_open(name: str) -> bool:
|
||||
if name not in tool_indices and not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get():
|
||||
return False
|
||||
self._emit_tool_name(name, pending_calls)
|
||||
return True
|
||||
|
||||
def on_close(value: Any) -> bool:
|
||||
items = self._to_tool_call_items(
|
||||
value, tool_indices, max(self.current_tool_id, 0)
|
||||
)
|
||||
if not items:
|
||||
return False
|
||||
for item in items:
|
||||
self._emit_tool_call(item, pending_calls)
|
||||
return True
|
||||
|
||||
normal_text = self._adapter.route_tool_events(
|
||||
events,
|
||||
on_tool_open=on_open,
|
||||
on_tool_close=on_close,
|
||||
on_tool_dropped=self._drop_streamed_tool_call,
|
||||
)
|
||||
return ToolStreamingParseResult(normal_text=normal_text, calls=pending_calls)
|
||||
|
||||
def supports_structural_tag(self) -> bool:
|
||||
return False
|
||||
|
||||
def validate_structure_constraint_request(
|
||||
self,
|
||||
tool_choice: Any,
|
||||
parallel_tool_calls: bool,
|
||||
strict_requested: bool,
|
||||
) -> None:
|
||||
if tool_choice == "auto" and not parallel_tool_calls:
|
||||
raise ValueError(
|
||||
f"{type(self).__name__} cannot enforce "
|
||||
"parallel_tool_calls=False with automatic tool choice"
|
||||
)
|
||||
if tool_choice == "auto" and strict_requested:
|
||||
raise ValueError(
|
||||
f"{type(self).__name__} does not support strict tool constraints"
|
||||
)
|
||||
|
||||
def structure_info(self) -> _GetInfoFunc:
|
||||
raise NotImplementedError(
|
||||
"structure_info not used with the response_template tool parser"
|
||||
)
|
||||
|
||||
|
||||
def tool_close_tokens(
|
||||
tool_call_parser: str | None, tokenizer: Any | None
|
||||
) -> dict[int, str]:
|
||||
"""Single-token tool-call closers of a response-template tool parser, by id.
|
||||
|
||||
Detokenization keeps these when they stop generation, so a tool call that
|
||||
reaches end of stream without its closer was cut off."""
|
||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
|
||||
detector_class = FunctionCallParser.ToolCallParserEnum.get(tool_call_parser)
|
||||
if (
|
||||
tokenizer is None
|
||||
or detector_class is None
|
||||
or not issubclass(detector_class, ResponseTemplateToolDetector)
|
||||
):
|
||||
return {}
|
||||
try:
|
||||
detector = detector_class(tokenizer=tokenizer)
|
||||
except ValueError:
|
||||
return {}
|
||||
closers = {}
|
||||
for literal in detector.tool_close_literals:
|
||||
ids = tokenizer.encode(literal, add_special_tokens=False)
|
||||
if len(ids) == 1:
|
||||
closers[ids[0]] = literal
|
||||
return closers
|
||||
|
||||
|
||||
def tool_close_token_ids(
|
||||
tool_call_parser: str | None, tokenizer: Any | None
|
||||
) -> frozenset[int]:
|
||||
return frozenset(tool_close_tokens(tool_call_parser, tokenizer))
|
||||
@@ -773,7 +773,6 @@ def _detect_auto_parser(
|
||||
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template"
|
||||
)
|
||||
return detected
|
||||
_log_undetected_parser(attr, label)
|
||||
return None
|
||||
|
||||
|
||||
@@ -924,7 +923,8 @@ def _architecture_parser_defaults(server_args) -> Dict[str, str]:
|
||||
|
||||
|
||||
def resolve_auto_parsers(server_args) -> None:
|
||||
"""Resolve architecture defaults and explicit ``auto`` parser requests.
|
||||
"""Resolve architecture defaults and explicit ``auto`` parser requests;
|
||||
``auto`` is detected from the chat template or checkpoint metadata.
|
||||
|
||||
Performs a lightweight tokenizer load, so it runs once in engine init. The
|
||||
decision goes to this instance's declaration stash, so every holder of it
|
||||
@@ -982,8 +982,11 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
tokenizer = None
|
||||
try:
|
||||
tokenizer = get_tokenizer(
|
||||
cfg.model_path,
|
||||
cfg.tokenizer_path,
|
||||
tokenizer_mode=cfg.tokenizer_mode,
|
||||
trust_remote_code=cfg.trust_remote_code,
|
||||
revision=cfg.revision,
|
||||
tokenizer_backend=cfg.tokenizer_backend,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load tokenizer for auto-detection: {e}")
|
||||
@@ -1026,7 +1029,6 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
("tool_call_parser", "tool-call parser"),
|
||||
):
|
||||
if attr in needs and attr not in detected:
|
||||
_log_undetected_parser(attr, label)
|
||||
detected[attr] = None
|
||||
else:
|
||||
for attr, rules, label in (
|
||||
@@ -1036,5 +1038,42 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
if attr in needs:
|
||||
detected[attr] = _detect_auto_parser(attr, ctx, rules, label)
|
||||
|
||||
unresolved = tuple(attr for attr in needs if detected.get(attr) is None)
|
||||
if unresolved and tokenizer is not None and chat_template_arg is None:
|
||||
from sglang.srt.parser.response_template import (
|
||||
resolve_response_template,
|
||||
validate_response_template_for_serving,
|
||||
)
|
||||
|
||||
response_template = resolve_response_template(tokenizer)
|
||||
if response_template is not None:
|
||||
try:
|
||||
response_template = validate_response_template_for_serving(
|
||||
response_template
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
logger.warning(
|
||||
"Ignoring invalid response_template from tokenizer configuration: %s",
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
for attr, field in (
|
||||
("reasoning_parser", "thinking"),
|
||||
("tool_call_parser", "tool_calls"),
|
||||
):
|
||||
if attr in unresolved and field in response_template.fields:
|
||||
detected[attr] = "response_template"
|
||||
logger.info(
|
||||
"Auto-detected --%s as 'response_template' from tokenizer configuration",
|
||||
attr.replace("_", "-"),
|
||||
)
|
||||
|
||||
for attr, label in (
|
||||
("reasoning_parser", "reasoning parser"),
|
||||
("tool_call_parser", "tool-call parser"),
|
||||
):
|
||||
if attr in needs and detected.get(attr) is None:
|
||||
_log_undetected_parser(attr, label)
|
||||
|
||||
if detected:
|
||||
declare_resolution(server_args, "template-detection", **detected)
|
||||
|
||||
@@ -370,7 +370,7 @@ class ServerArgs:
|
||||
choices=["auto"] + reasoning_parser_choices,
|
||||
default=_declared_default("reasoning_parser"),
|
||||
help=f"Specify the parser for reasoning models. "
|
||||
f"Use 'auto' to detect from chat template. "
|
||||
f"Use 'auto' to detect from chat template or checkpoint metadata. "
|
||||
f"Options include: {reasoning_parser_choices}.",
|
||||
)
|
||||
tool_call_parser_choices = _tool_call_parser_choices()
|
||||
@@ -380,7 +380,7 @@ class ServerArgs:
|
||||
choices=["auto"] + tool_call_parser_choices,
|
||||
default=_declared_default("tool_call_parser"),
|
||||
help=f"Specify the parser for handling tool-call interactions. "
|
||||
f"Use 'auto' to detect from chat template. "
|
||||
f"Use 'auto' to detect from chat template or checkpoint metadata. "
|
||||
f"Options include: {tool_call_parser_choices}.",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
Reference in New Issue
Block a user