diff --git a/python/sglang/srt/entrypoints/anthropic/protocol.py b/python/sglang/srt/entrypoints/anthropic/protocol.py index 8c5f986d78..6c46954554 100644 --- a/python/sglang/srt/entrypoints/anthropic/protocol.py +++ b/python/sglang/srt/entrypoints/anthropic/protocol.py @@ -56,6 +56,7 @@ class AnthropicUsage(BaseModel): class TextBlock(BaseModel): type: Literal["text"] = "text" text: str + cache_control: Optional[dict[str, Any]] = None class ImageBlock(BaseModel): @@ -91,6 +92,50 @@ class ToolReferenceBlock(BaseModel): id: Optional[str] = None +class McpToolReference(BaseModel): + type: Literal["mcp_tool_reference"] + server_name: str + name: str + + +class McpToolsetReference(BaseModel): + type: Literal["mcp_toolset_reference"] + server_name: str + + +class ToolDefinition(BaseModel): + """A tool given by value: `definition` is a `tools` entry.""" + + type: Literal["tool_definition"] + definition: dict[str, Any] + + +class ToolAdditionBlock(BaseModel): + """Anthropic beta block: a ``system`` message offers a tool from its position + on. Rendered through deepseek-recipe, which reads only a ``tool_reference`` + target.""" + + type: Literal["tool_addition"] = "tool_addition" + tool: Annotated[ + Union[ + ToolReferenceBlock, McpToolReference, McpToolsetReference, ToolDefinition + ], + Field(discriminator="type"), + ] + cache_control: Optional[dict[str, Any]] = None + + +class ToolRemovalBlock(BaseModel): + """Anthropic beta block: a ``system`` message withdraws a tool.""" + + type: Literal["tool_removal"] = "tool_removal" + tool: Annotated[ + Union[ToolReferenceBlock, McpToolReference, McpToolsetReference], + Field(discriminator="type"), + ] + cache_control: Optional[dict[str, Any]] = None + + class SearchResultBlock(BaseModel): type: Literal["search_result"] = "search_result" # ``source`` here is a URL/identifier string (unlike ImageBlock.source). @@ -117,6 +162,8 @@ AnthropicContentBlock = Annotated[ ToolUseBlock, ToolResultBlock, ToolReferenceBlock, + ToolAdditionBlock, + ToolRemovalBlock, SearchResultBlock, ThinkingBlock, RedactedThinkingBlock, @@ -128,6 +175,8 @@ AnthropicContentBlock = Annotated[ class AnthropicMessage(BaseModel): role: Literal["user", "assistant", "system"] content: Union[str, list[AnthropicContentBlock]] + clear_at: Optional[Literal["next_user_message", "never"]] = None + output_config: Optional[dict[str, Any]] = None # ---------- Tools (discriminated by ``type`` family) ---------- diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index 722389b74a..d5506354e3 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -20,8 +20,10 @@ from sglang.srt.entrypoints.anthropic.protocol import ( AnthropicContentBlock, AnthropicCountTokensRequest, AnthropicCountTokensResponse, + AnthropicCustomTool, AnthropicError, AnthropicErrorResponse, + AnthropicMessage, AnthropicMessageEndDelta, AnthropicMessagesRequest, AnthropicMessagesResponse, @@ -32,6 +34,8 @@ from sglang.srt.entrypoints.anthropic.protocol import ( ContentBlockStopEvent, ErrorEvent, InputJsonDelta, + McpToolReference, + McpToolsetReference, MessageDeltaEvent, MessageStartEvent, MessageStopEvent, @@ -40,6 +44,8 @@ from sglang.srt.entrypoints.anthropic.protocol import ( TextDelta, ThinkingBlock, ThinkingDelta, + ToolDefinition, + ToolReferenceBlock, ToolUseBlock, is_server_tool, ) @@ -152,6 +158,207 @@ def _extract_system_text( return "\n".join(texts) if texts else None +def _block_types(request: AnthropicMessagesRequest) -> set[str]: + """The block types of the request's system and messages, tool result contents + included.""" + types = set() + for content in [request.system, *(message.content for message in request.messages)]: + for block in content if isinstance(content, list) else []: + types.add(block.type) + if block.type == "tool_result" and isinstance(block.content, list): + types.update(item.type for item in block.content) + return types + + +def _without_cleared_messages( + request: AnthropicMessagesRequest, +) -> AnthropicMessagesRequest: + """`request` without the turn-scoped system messages (`clear_at: + "next_user_message"`) that a later user message has cleared.""" + last_user = max( + ( + index + for index, message in enumerate(request.messages) + if message.role == "user" + ), + default=-1, + ) + rendered = [] + for index, message in enumerate(request.messages): + if message.clear_at is not None and message.role != "system": + raise ValueError( + f"messages.{index}.clear_at: clear_at is only permitted on role " + "'system' messages" + ) + if message.clear_at == "next_user_message": + _check_turn_scoped(index=index, message=message) + _check_turn_scoped_placement(index=index, messages=request.messages) + if index < last_user: + continue + rendered.append(message) + return request.model_copy(update={"messages": rendered}) + + +def _check_turn_scoped(index: int, message: AnthropicMessage) -> None: + scope = "a turn-scoped system message (clear_at: 'next_user_message')" + if message.output_config is not None: + raise ValueError(f"messages.{index}: output_config is not permitted on {scope}") + for position, block in enumerate( + message.content if isinstance(message.content, list) else [] + ): + if block.type != "text": + raise ValueError( + f"messages.{index}: a turn-scoped system message supports text " + "blocks only (clear_at: 'next_user_message')" + ) + if block.cache_control is not None: + raise ValueError( + f"messages.{index}.content.{position}: cache_control is not " + f"permitted on {scope}" + ) + + +def _check_turn_scoped_placement(index: int, messages: list[AnthropicMessage]) -> None: + # Consecutive system messages are placed as one group. + before = next( + (m.role for m in reversed(messages[:index]) if m.role != "system"), None + ) + after = next((m.role for m in messages[index + 1 :] if m.role != "system"), None) + if before != "user" or after == "user": + raise ValueError( + f"messages.{index}: a turn-scoped system message (clear_at: " + "'next_user_message') must follow a user message and either end " + "messages or precede an assistant message" + ) + + +def _declare_inline_tools( + request: AnthropicMessagesRequest, +) -> AnthropicMessagesRequest: + """`request` with each custom tool a `tool_addition` block defines by value + declared in `tools` as deferred, and the block naming it by reference. A + definition of a tool already offered unchanged is dropped.""" + tools = list(request.tools or []) + declared = {tool.name: tool for tool in tools} + offered = {tool.name for tool in tools if not tool.defer_loading} + unresolved: dict[str, str] = {} + messages = [] + for index, message in enumerate(request.messages): + if isinstance(message.content, str): + messages.append(message) + continue + blocks = [] + for position, block in enumerate(message.content): + where = f"messages.{index}.content.{position}" + for name in _referenced_tool_names(block): + if name not in declared: + unresolved.setdefault(name, where) + if block.type == "tool_addition" and block.tool.type != "tool_reference": + if message.role != "system": + raise ValueError( + f"{where}: tool_addition blocks can only be in system messages" + ) + tool = _inline_custom_tool(where=where, target=block.tool) + # Declaring the tool makes it resolvable from the first message on. + if tool.name in unresolved: + raise ValueError( + f"{unresolved[tool.name]}: tool reference {tool.name!r} " + f"precedes the tool_definition that defines it at {where}" + ) + earlier = declared.setdefault(tool.name, tool) + if earlier is tool: + tools.append(tool) + elif is_server_tool(earlier) or _definition(earlier) != _definition( + tool + ): + raise ValueError( + f"{where}: tool_definition " + f"reuses the name {tool.name!r} with a different " + "definition; replacing a tool mid-conversation is not " + "supported" + ) + if tool.name in offered: + continue + offered.add(tool.name) + block = block.model_copy( + update={"tool": ToolReferenceBlock(name=tool.name)} + ) + blocks.append(block) + # recipe renders a system message left without blocks as an empty turn. + if blocks or not message.content: + messages.append(message.model_copy(update={"content": blocks})) + return request.model_copy(update={"tools": tools, "messages": messages}) + + +def _referenced_tool_names(block: AnthropicContentBlock) -> list[Optional[str]]: + if block.type == "tool_addition" and block.tool.type == "tool_reference": + return [block.tool.name or block.tool.tool_name] + if block.type == "tool_reference": + return [block.tool_name or block.name] + if block.type == "tool_result" and isinstance(block.content, list): + return [name for item in block.content for name in _referenced_tool_names(item)] + return [] + + +def _inline_custom_tool( + where: str, target: Union[McpToolReference, McpToolsetReference, ToolDefinition] +) -> AnthropicCustomTool: + if target.type != "tool_definition": + raise ValueError( + f"{where}: tool_addition with a {target.type} is not supported; this " + "server runs no MCP connector" + ) + kind = target.definition.get("type") + if kind not in (None, "custom"): + raise ValueError( + f"{where}: tool_definition supports custom tools only, got a " + f"definition of type {kind!r}" + ) + tool = AnthropicCustomTool.model_validate(target.definition) + if tool.defer_loading: + raise ValueError( + f"{where}: a tool_definition with defer_loading is not supported; " + "declare the tool in tools and add it by reference" + ) + return tool.model_copy(update={"defer_loading": True}) + + +def _definition(tool: AnthropicCustomTool) -> dict[str, Any]: + return tool.model_dump(exclude={"type", "defer_loading"}) + + +def _native_block(block: dict[str, Any]) -> dict[str, Any]: + """`block` with SGLang's accepted aliases spelled as deepseek-recipe reads them.""" + kind = block["type"] + if kind == "tool_result": + content = block.get("content") + fields = { + "type": kind, + "tool_use_id": block.get("tool_use_id") or block.get("id"), + "content": ( + [_native_block(item) for item in content] + if isinstance(content, list) + else content + ), + "is_error": block.get("is_error"), + } + return {key: value for key, value in fields.items() if value is not None} + if kind == "tool_reference": + return {"type": kind, "tool_name": block.get("tool_name") or block.get("name")} + if kind == "tool_addition": + tool = block["tool"] + name = tool.get("name") or tool.get("tool_name") + return {"type": kind, "tool": {"type": "tool_reference", "name": name}} + return block + + +def _native_message(message: AnthropicMessage) -> dict[str, Any]: + data = message.model_dump(exclude_none=True) + if isinstance(data["content"], list): + data["content"] = [_native_block(block) for block in data["content"]] + return data + + def _wrap_sse_event(data: str, event_type: str) -> str: """Format an Anthropic SSE event with event type and data lines.""" return f"event: {event_type}\ndata: {data}\n\n" @@ -218,6 +425,24 @@ class AnthropicServing: self, anthropic_request: AnthropicMessagesRequest ) -> ChatCompletionRequest: """Convert an Anthropic Messages request to an OpenAI ChatCompletion request.""" + anthropic_request = _declare_inline_tools( + _without_cleared_messages(anthropic_request) + ) + block_types = _block_types(anthropic_request) + if "tool_removal" in block_types: + raise ValueError( + "tool_removal blocks are not supported; a tool cannot be withdrawn " + "mid-conversation" + ) + native = self.openai_serving_chat.native_tool_loading and ( + bool(block_types & {"tool_reference", "tool_addition"}) + or any( + not is_server_tool(tool) and tool.defer_loading + for tool in anthropic_request.tools or [] + ) + ) + if not native and "tool_addition" in block_types: + raise ValueError("tool_addition blocks are not supported by this model") openai_messages = [] def _convert_anthropic_image_source_to_openai_part( @@ -719,8 +944,51 @@ class AnthropicServing: elif chat_request.tools: chat_request.tool_choice = "auto" + if native: + self._prepare_native_prompt(anthropic_request, chat_request) return chat_request + def _prepare_native_prompt( + self, + anthropic_request: AnthropicMessagesRequest, + chat_request: ChatCompletionRequest, + ) -> None: + """Render the request's own wire form into `chat_request`, whose tools + become the ones the history has loaded.""" + body = { + "messages": [ + _native_message(message) for message in anthropic_request.messages + ], + # recipe reads a tool with a `type` as a server tool. + "tools": [ + tool.model_dump(exclude_none=True, exclude={"type"}) + for tool in anthropic_request.tools or [] + if not is_server_tool(tool) + ], + } + system = anthropic_request.system + if system: + body["system"] = ( + system + if isinstance(system, str) + else [block.model_dump(exclude_none=True) for block in system] + ) + choice = anthropic_request.tool_choice + if choice is not None and choice.type == "none": + body["tool_choice"] = {"type": "none"} + tools = self.openai_serving_chat.prepare_native_prompt( + chat_request, protocol="messages", body=body + ).tools + if ( + choice is not None + and choice.type == "tool" + and not any(tool.function.name == choice.name for tool in tools) + ): + raise ValueError( + f"tool_choice names {choice.name!r}, a deferred tool no " + "tool_reference or tool_addition block has loaded" + ) + async def _handle_non_streaming( self, chat_request: ChatCompletionRequest, diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py b/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py index 0cffe3bae9..d559b088b0 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py @@ -3,7 +3,7 @@ concrete adapters load only when selected.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol, Union import msgspec @@ -15,8 +15,27 @@ if TYPE_CHECKING: ToolCall, ) from sglang.srt.function_call.core_types import ToolCallItem + from sglang.srt.multimodal.media_io import ImageData EncodingBackend = Literal["builtin", "recipe"] +HistoryToolCallArguments = Literal["strict", "lenient", "as_sent"] +NativeProtocol = Literal["responses", "messages"] + + +class NativePrompt(msgspec.Struct, frozen=True, kw_only=True): + """A Responses or Messages request rendered from its own wire form.""" + + prompt_ids: list[int] + tools: list[Tool] + """The tools the model can call after the request's history: those loaded from + the start and those the history has loaded since.""" + image_data: list[ImageData] = [] + """The images the prompt's placeholders stand for, in order.""" + namespace_names: dict[str, tuple[str, str]] = {} + """The call name of each namespace member, mapped to its namespace and the + name the request gives it there.""" + continued_turn: Optional[dict[str, Any]] = None + """The final assistant message the prompt continues, as a chat message.""" class ChatOutputHooks(msgspec.Struct, frozen=True, kw_only=True): @@ -78,6 +97,25 @@ class ChatEncodingOutputAdapter(Protocol): ) -> ResponsesOutputStream: ... +class NativeRequestEncoder(Protocol): + def encode_native( + self, + *, + protocol: NativeProtocol, + body: dict[str, Any], + thinking: bool, + effort: Union[str, int, float, None], + response_schema: Optional[dict[str, Any]], + continue_final_message: bool, + ) -> NativePrompt: + """``body`` carries the request's conversation, tools and, for + ``tool_choice="none"``, its choice; ``thinking`` and ``effort`` are the + handler's decisions, as for ``encode_messages``. ``response_schema`` is + the JSON schema the answer follows; a trailing assistant turn is continued + under ``continue_final_message`` and otherwise re-sent as a user turn.""" + ... + + class ChatEncodingAdapter(Protocol): collects_openai_media: bool """Whether media still has to be collected from the OpenAI messages after @@ -92,6 +130,12 @@ class ChatEncodingAdapter(Protocol): """None when the configured parsers leave output to the text parsers.""" ... + @property + def native_encoder(self) -> Optional[NativeRequestEncoder]: + """None when Responses and Messages requests that load tools mid-conversation + have no rendering of their own wire form.""" + ... + def encode_messages( self, *, diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index c854d7e560..9d378a7cdc 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -17,6 +17,7 @@ from __future__ import annotations import logging import time +import msgspec import unicodedata import uuid from dataclasses import dataclass @@ -743,6 +744,10 @@ class ChatCompletionMessageGenericParam(BaseModel): name: Optional[str] = None phase: Optional[Literal["commentary", "final_answer"]] = None reasoning_content: Optional[str] = None + action: Optional[str] = Field( + default=None, + description="Action tag for assistant messages (used by deepseek-recipe).", + ) tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None]) tools: Optional[List[Tool]] = Field(default=None, examples=[None]) # DeepSeek API prefix completion: continue this final assistant message. @@ -798,12 +803,18 @@ class Function(BaseModel): parameters: Optional[object] = None strict: bool = False defer_loading: Optional[bool] = None + namespace: Optional[str] = Field( + default=None, + description="Tool namespace for disambiguation (used by deepseek-recipe).", + ) @model_serializer(mode="wrap") def _serialize(self, handler): data = handler(self) if self.defer_loading is None: data.pop("defer_loading", None) + if self.namespace is None: + data.pop("namespace", None) return data @@ -913,6 +924,15 @@ class ChatCompletionRequest(PDRoutingFields): # renders these, while ``tools`` (flattened to ``.``) still # drives tool-call parsing. _template_tools: Optional[List[Dict[str, Any]]] = PrivateAttr(default=None) + # Set with ``input_ids`` by a caller that rendered them itself: the images the + # prompt's placeholders stand for, in order. + _input_image_data: Optional[List[Any]] = PrivateAttr(default=None) + # Whether ``tools`` are the only tools the rendered prompt shows, so a call + # to any other tool is dropped rather than returned. + _drops_unlisted_tool_calls: bool = PrivateAttr(default=False) + # Set with ``input_ids`` under ``continue_final_message``: the final assistant + # message the prompt continues, where output parsing resumes. + _continued_turn: Optional[Dict[str, Any]] = PrivateAttr(default=None) tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field( default="auto", examples=["none"] ) # noqa @@ -1252,6 +1272,7 @@ class ChatMessage(BaseModel): role: Optional[str] = None content: Optional[str] = None reasoning_content: Optional[str] = None + action: Optional[str] = None tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None]) @model_serializer(mode="wrap") @@ -1259,6 +1280,8 @@ class ChatMessage(BaseModel): data = handler(self) if self.tool_calls is None: data.pop("tool_calls", None) + if self.action is None: + data.pop("action", None) return data @@ -1830,6 +1853,40 @@ class ResponseTool(BaseModel): return self +class ResponseToolState(msgspec.Struct, frozen=True): + """Public tool metadata and name mappings for one translated request.""" + + public_tools: list[ResponseTool] | None = None + public_choice: str | dict[str, Any] | None = None + names: dict[str, dict[str, str]] = {} + # The tools the model can call after the history, from a request rendered in + # its own wire form; None when `tools` alone declares them. + chat_tools: list[Tool] | None = None + # ``custom`` tools a ``tool_search_output`` loaded, beside the declared ones. + discovered_custom_names: frozenset[str] = frozenset() + + +class ResponseToolSearchCall(BaseModel): + """A client-executed ``tool_search`` call; the pinned openai SDK has no type.""" + + id: str + type: Literal["tool_search_call"] = "tool_search_call" + call_id: str + execution: Literal["client"] = "client" + arguments: Dict[str, Any] + status: Literal["in_progress", "completed", "incomplete"] + + +class ResponseToolSearchCallEvent(BaseModel): + """``response.output_item.added`` / ``.done`` for a tool search call, which the + SDK's event types reject.""" + + type: Literal["response.output_item.added", "response.output_item.done"] + sequence_number: int + output_index: int + item: ResponseToolSearchCall + + class ResponseInputMessageParam(EasyInputMessageParam, total=False): phase: Optional[Literal["commentary", "final_answer"]] @@ -1856,6 +1913,10 @@ ResponseInputOutputItem: TypeAlias = Union[ class ResponsesRequest(PDRoutingFields): """Request body for v1/responses endpoint.""" + _tool_state: ResponseToolState = PrivateAttr(default_factory=ResponseToolState) + # By call_id, the name the model called a tool by when the output item + # reports another; the stored history replays the call under it. + _called_tool_names: Dict[str, str] = PrivateAttr(default_factory=dict) _response_parser_prefix: str = PrivateAttr(default="") _response_parser_grammar_start: Optional[str] = PrivateAttr(default=None) _response_parser_json_response_format: bool = PrivateAttr(default=False) @@ -2176,6 +2237,7 @@ class ResponsesResponse(BaseModel): ResponseOutputItem, ResponseReasoningItem, ResponseFunctionToolCall, + ResponseToolSearchCall, ] ] = Field(default_factory=list) status: Literal[ @@ -2241,6 +2303,7 @@ class ResponsesResponse(BaseModel): ResponseOutputItem, ResponseReasoningItem, ResponseFunctionToolCall, + ResponseToolSearchCall, ] ], status: str, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 426cc1a916..e67fb655e9 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -957,12 +957,13 @@ class OpenAIServingChat(OpenAIServingBase): The reasoning parser is told the same thing, so channel-framed formats keep their framing intact exactly when a tool-call parser consumes it. """ - return bool( - request.tool_choice != "none" - and self._effective_tools(request) - and self.tool_call_parser - and not request._response_parser_json_response_format - ) + if not self.tool_call_parser or request._response_parser_json_response_format: + return False + if request._drops_unlisted_tool_calls and request.tool_choice == "none": + # The history showed the model tools, so it can still write a call; + # parsing drops it, where skipping the parser would return its markup. + return True + return bool(request.tool_choice != "none" and self._effective_tools(request)) def _validate_request(self, request: ChatCompletionRequest) -> str | None: """Validate that the input is valid.""" @@ -1367,18 +1368,7 @@ class OpenAIServingChat(OpenAIServingBase): self, request: ChatCompletionRequest, is_multimodal: bool ) -> MessageProcessingResult: """Process chat messages and apply chat template""" - if self.default_chat_template_kwargs: - ctk = dict(request.chat_template_kwargs or {}) - for k, v in self.default_chat_template_kwargs.items(): - ctk.setdefault(k, v) - request.chat_template_kwargs = ctk - effort = ctk.get("reasoning_effort") - if effort is not None and request.reasoning_effort is None: - request.reasoning_effort = effort - - normalize_hunyuan_reasoning_effort( - request, self.reasoning_parser, self.template_manager.reasoning_config - ) + self._apply_request_defaults(request) # GptOss model needs to keep special tokens for harmony parsing if self.is_gpt_oss or self.is_gemma4: @@ -1484,7 +1474,7 @@ class OpenAIServingChat(OpenAIServingBase): result = MessageProcessingResult( prompt="", prompt_ids=request.input_ids, - image_data=None, + image_data=request._input_image_data, audio_data=None, video_data=None, modalities=[], @@ -1540,6 +1530,64 @@ class OpenAIServingChat(OpenAIServingBase): result.reasoning_end_token_ids = list(token_ids) return result + def _apply_request_defaults(self, request: ChatCompletionRequest) -> None: + """Settle the thinking and effort that rendering reads; repeating it + changes nothing.""" + if self.default_chat_template_kwargs: + ctk = dict(request.chat_template_kwargs or {}) + for k, v in self.default_chat_template_kwargs.items(): + ctk.setdefault(k, v) + request.chat_template_kwargs = ctk + effort = ctk.get("reasoning_effort") + if effort is not None and request.reasoning_effort is None: + request.reasoning_effort = effort + + normalize_hunyuan_reasoning_effort( + request, self.reasoning_parser, self.template_manager.reasoning_config + ) + + @property + def native_tool_loading(self) -> bool: + """Whether Responses and Messages requests that load tools mid-conversation + render from their own wire form (see `prepare_native_prompt`).""" + return ( + self._encoding_adapter is not None + and self._encoding_adapter.native_encoder is not None + ) + + def prepare_native_prompt( + self, + request: ChatCompletionRequest, + *, + protocol: chat_encoding_adapters.NativeProtocol, + body: dict[str, Any], + ) -> chat_encoding_adapters.NativePrompt: + """Render `body`, the Responses or Messages form of `request`, into its + `input_ids` and images, and set its `tools` to the tools the model can call + after the history; rendering then keeps only grammar, stops and parsing.""" + self._apply_request_defaults(request) + response_format = request.response_format + prompt = self._encoding_adapter.native_encoder.encode_native( + protocol=protocol, + body=body, + thinking=self._get_reasoning_from_request(request), + effort=request.reasoning_effort, + response_schema=( + response_format.json_schema.schema_ + if response_format is not None and response_format.json_schema + else None + ), + continue_final_message=request.continue_final_message, + ) + if prompt.image_data and not self.tokenizer_manager.model_config.is_multimodal: + raise ValueError("image input is not supported for this model") + request.input_ids = prompt.prompt_ids + request.tools = prompt.tools or None + request._input_image_data = prompt.image_data or None + request._drops_unlisted_tool_calls = True + request._continued_turn = prompt.continued_turn + return prompt + def _apply_jinja_template( self, request: ChatCompletionRequest, diff --git a/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py b/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py index 8bcf8b204e..dac66b6c48 100644 --- a/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py +++ b/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py @@ -5,15 +5,20 @@ request semantics.""" from __future__ import annotations from collections.abc import Sequence +import re from typing import TYPE_CHECKING, Any, Optional, Union from sglang.srt.entrypoints.openai.chat_encoding_adapters import ( ADAPTER_OUTPUT_PARSERS, + NativePrompt, + NativeProtocol, select_new_output_ids, token_output_sse_chunks, ) from sglang.srt.entrypoints.openai.protocol import ( + Function, FunctionResponse, + Tool, ToolCall, ) from sglang.srt.environ import envs @@ -26,7 +31,9 @@ from sglang.srt.parser.deepseek_recipe_output import ( whole_tool_calls, ) from sglang.srt.parser.deepseek_recipe_renderer import ( + BUDGET_TIER, DeepSeekRecipe, + loaded_tools, parse_effort_budget, ) from sglang.srt.runtime_context import get_serving @@ -37,7 +44,6 @@ if TYPE_CHECKING: from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, ResponsesRequest, - Tool, ) DEFAULT_EFFORT_ENVS = { @@ -45,6 +51,12 @@ DEFAULT_EFFORT_ENVS = { "dsv41": envs.SGLANG_DSV41_REASONING_EFFORT, } JSON_RESPONSE_FORMATS = ("json_object", "json_schema") +# V4 renders deferred tools up front and never loads one mid-conversation. +NATIVE_TOOL_LOADING_SPEC = "dsv41" +NATIVE_EFFORT_FIELDS: dict[str, str] = { + "responses": "reasoning", + "messages": "output_config", +} class DeepSeekRecipeChatEncodingAdapter: @@ -96,6 +108,13 @@ class DeepSeekRecipeChatEncodingAdapter: def output(self) -> Optional[DeepSeekRecipeChatEncodingAdapter]: return self if self._token_output else None + def validate_request(self, request: ChatCompletionRequest) -> Optional[str]: + return None + + @property + def native_encoder(self) -> Optional[DeepSeekRecipeChatEncodingAdapter]: + return self if self._spec == NATIVE_TOOL_LOADING_SPEC else None + def encode_messages( self, *, @@ -149,6 +168,66 @@ class DeepSeekRecipeChatEncodingAdapter: prompt_ids += self.deepseek_recipe.encode_text(task_prefix) return prompt_ids + def encode_native( + self, + *, + protocol: NativeProtocol, + body: dict[str, Any], + thinking: bool, + effort: Union[str, int, float, None], + response_schema: Optional[dict[str, Any]], + continue_final_message: bool, + ) -> NativePrompt: + tier, budget = self._effort(effort) + # recipe reads an effort as a request to think. + if thinking and tier is not None: + body = {**body, NATIVE_EFFORT_FIELDS[protocol]: {"effort": tier}} + conversation = self.deepseek_recipe.with_host_settings( + self.deepseek_recipe.convert_native(protocol, body, thinking=thinking), + response_schema=response_schema, + continue_final_message=continue_final_message, + ) + last = conversation.messages[-1] if conversation.messages else None + # recipe's Messages conversion drops `strict`, which SGLang's grammar enforces. + strict = {tool.get("name") for tool in body["tools"] if tool.get("strict")} + tools = [ + Tool( + type="function", + function=Function( + name=tool.name, + description=tool.description, + parameters=tool.parameters, + strict=True if tool.name in strict else tool.strict, + ), + ) + for tool in loaded_tools(conversation) + ] + return NativePrompt( + prompt_ids=self.deepseek_recipe.encode_conversation( + conversation, effort_budget=budget + ), + tools=tools, + image_data=[ + ImageData(url=url) + for url in self.deepseek_recipe.image_urls(conversation) + ], + namespace_names=( + self.deepseek_recipe.namespace_tool_names(_responses_namespaces(body)) + if protocol == "responses" + else {} + ), + continued_turn=( + {"role": "assistant", "content": last.content} + | ( + {"reasoning_content": last.reasoning_content} + if last.reasoning_content is not None + else {} + ) + if last is not None and last.prefix + else None + ), + ) + def _encode( self, body: dict[str, Any], @@ -157,21 +236,29 @@ class DeepSeekRecipeChatEncodingAdapter: task: Optional[str] = None, image_urls: Optional[list[str]] = None, ) -> list[int]: - if effort is None or effort == "none": - effort = self._default_effort - budget = None - if effort is not None and not isinstance(effort, str): - if self._spec != "dsv41": - raise ValueError( - f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}" - ) - budget = parse_effort_budget(effort) - elif effort is not None: - body["reasoning_effort"] = effort + tier, budget = self._effort(effort) + if budget is None and tier is not None: + body["reasoning_effort"] = tier return self.deepseek_recipe.encode( body, task=task, effort_budget=budget, image_urls=image_urls ) + def _effort( + self, effort: Union[str, int, float, None] + ) -> tuple[Optional[str], Optional[int]]: + """The named tier recipe renders and, for a V4.1 integer budget, the budget + that replaces the tier's line.""" + effort = _numeric_effort(effort) + if effort is None or effort == "none": + effort = self._default_effort + if effort is None or isinstance(effort, str): + return effort, None + if self._spec != "dsv41": + raise ValueError( + f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}" + ) + return BUDGET_TIER, parse_effort_budget(effort) + def parse_chat_response( self, *, @@ -320,7 +407,7 @@ class DeepSeekRecipeChatEncodingAdapter: self, request: ChatCompletionRequest ) -> DeepSeekRecipeOutputParser: response_format = request.response_format - continued = ( + continued = request._continued_turn or ( _trailing_assistant_turn( [ message.model_dump(exclude_unset=True, by_alias=True, mode="json") @@ -344,6 +431,11 @@ class DeepSeekRecipeChatEncodingAdapter: separate_reasoning=self._separate_reasoning and request.separate_reasoning, stream_reasoning=request.stream_reasoning, stop=_stop_strings(request.stop), + callable_names=( + _native_callable_names(request.tools or [], request.tool_choice) + if request._drops_unlisted_tool_calls + else None + ), ) def _new_responses_parser( @@ -353,12 +445,25 @@ class DeepSeekRecipeChatEncodingAdapter: separate_reasoning: bool, parse_tool_calls: bool, ) -> DeepSeekRecipeOutputParser: + native = request._tool_state.chat_tools is not None return DeepSeekRecipeOutputParser( self.deepseek_recipe, thinking_stage=self.deepseek_recipe.parse_start(thinking=require_reasoning), parse_tool_calls=parse_tool_calls, + # Under `none` the tool parser runs only to drop calls; a JSON answer + # must be read as JSON, or a call marker spelled in a string ends it. + parse_json_output=native + and request.tool_choice == "none" + and request.has_json_schema_constraint(), separate_reasoning=separate_reasoning, stop=_stop_strings(request.stop), + callable_names=( + _native_callable_names( + request._tool_state.chat_tools, request.tool_choice + ) + if native + else None + ), ) def _tool_call_id(self, index: int, name: str, history_tool_calls_cnt: int) -> str: @@ -390,6 +495,37 @@ class DeepSeekRecipeResponsesStream: return delta.reasoning, delta.content, _tool_call_items(delta.tool_calls) +def _responses_namespaces(body: dict[str, Any]) -> list[dict[str, Any]]: + """The namespace tools a Responses body declares and its history discovers, in + the order recipe's conversion names their members.""" + discovered = [ + tool + for item in body["input"] + if isinstance(item, dict) and item.get("type") == "tool_search_output" + for tool in item["tools"] + ] + return [ + tool for tool in [*body["tools"], *discovered] if tool["type"] == "namespace" + ] + + +def _null_content_as_empty(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + """As `_process_messages` renders them: a null content is "".""" + return [ + message if message.get("content") is not None else {**message, "content": ""} + for message in messages + ] + + +def _numeric_effort( + effort: Union[str, int, float, None], +) -> Union[str, int, float, None]: + """A numeric string effort ("30", "0.3") as the number it spells.""" + if isinstance(effort, str) and re.fullmatch(r"\d+(\.\d+)?", effort.strip()): + return float(effort) if "." in effort else int(effort) + return effort + + def _trailing_assistant_turn( messages: list[dict[str, Any]], ) -> Optional[dict[str, Any]]: @@ -405,6 +541,18 @@ def _stop_strings(stop: Union[str, list[str], None]) -> list[str]: return [stop] if isinstance(stop, str) else stop or [] +def _native_callable_names( + tools: list[Tool], tool_choice: Any +) -> Optional[frozenset[str]]: + """The names of `tools`, all that a natively rendered prompt shows the model; + None when calls to other tools are returned. `none` allows no call at all.""" + if tool_choice == "none": + return frozenset() + if envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get(): + return None + return frozenset(tool.function.name for tool in tools) + + def _tool_call_items( calls: Sequence[Union[RecipeToolCall, RecipeToolCallDelta]], ) -> list[ToolCallItem]: diff --git a/python/sglang/srt/entrypoints/openai/serving_inkling.py b/python/sglang/srt/entrypoints/openai/serving_inkling.py index 5545a0829f..3f379dbf24 100644 --- a/python/sglang/srt/entrypoints/openai/serving_inkling.py +++ b/python/sglang/srt/entrypoints/openai/serving_inkling.py @@ -36,7 +36,10 @@ if TYPE_CHECKING: class InklingChatEncodingAdapter: collects_openai_media = True - strict_tool_call_arguments = True + history_tool_call_arguments = "strict" + renders_response_format = False + uncounted_prompt_tokens = 0 + native_encoder = None def __init__( self, diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index 2f1f8279ab..0c7e72d104 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -13,6 +13,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterator, Optional, Union import jinja2 +import msgspec import openai.types.responses as openai_responses_types import orjson from fastapi import Request @@ -43,6 +44,7 @@ from openai.types.responses.response_reasoning_summary_part_done_event import ( Part as ResponseReasoningSummaryDonePart, ) from openai_harmony import Message as OpenAIMessage +from pydantic import BaseModel, TypeAdapter from sglang.srt.entrypoints.context import ( ConversationContext, @@ -70,7 +72,12 @@ from sglang.srt.entrypoints.openai.protocol import ( ResponseOutputMessage, ResponsesRequest, ResponsesResponse, + ResponseTool, + ResponseToolSearchCall, + ResponseToolSearchCallEvent, Tool, + ToolChoice, + ToolChoiceFuncName, UsageInfo, ) from sglang.srt.entrypoints.openai.responses_adapters import ( @@ -114,6 +121,100 @@ class _MediaInputValidationError(ValueError): pass +class ResponseToolChoiceError(ValueError): + """A response tool_choice that names a tool the model cannot call.""" + + +TOOL_SEARCH = "tool_search" +TOOL_SEARCH_ITEMS = ("tool_search_call", "tool_search_output") +APPLY_PATCH = "apply_patch" +NAMESPACE_MEMBERS_ERROR = ( + "namespace tools must list function tools, each with a type and a name" +) + + +def _holds_function_tools(members: Any) -> bool: + return isinstance(members, list) and all( + isinstance(member, dict) + and member.get("type") == "function" + and isinstance(member.get("name"), str) + for member in members + ) + + +def _defers_loading(tool: ResponseTool) -> bool: + if tool.type == "namespace": + return bool(tool.defer_loading) or any( + isinstance(member, dict) and member.get("defer_loading") + for member in tool.tools or [] + ) + return tool.type in ("function", "custom") and bool(tool.defer_loading) + + +def _defer_tool_loading( + tools: list[dict[str, Any]], items: list +) -> tuple[list[dict[str, Any]], list]: + """`tools` without those that defer loading, and `items` whose + `tool_search_output` entries naming a deferred tool take the fields they + leave out from its declaration, so a search result loads it where it returns. + recipe's Responses conversion renders a deferred tool up front and rejects a + search result that repeats a declared name.""" + deferred: dict[tuple[Optional[str], str], dict[str, Any]] = {} + shown = [] + for tool in tools: + if tool["type"] == "namespace": + members = [] + for member in tool["tools"]: + if tool.get("defer_loading") or member.get("defer_loading"): + deferred[(tool["name"], member["name"])] = member + else: + members.append(member) + if members: + shown.append({**tool, "tools": members}) + elif tool["type"] in ("function", "custom") and tool.get("defer_loading"): + deferred[(None, tool["name"])] = tool + else: + shown.append(tool) + + def loaded(tool: dict[str, Any], namespace: Optional[str] = None) -> dict: + if tool["type"] == "namespace": + members = [loaded(member, tool["name"]) for member in tool["tools"]] + return {**tool, "tools": members} + declared = deferred.get((namespace, tool.get("name"))) + if declared is None or declared["type"] != tool["type"]: + return tool + return declared | {k: v for k, v in tool.items() if v is not None} + + if deferred: + items = [ + ( + {**item, "tools": [loaded(tool) for tool in item["tools"]]} + if isinstance(item, dict) and item.get("type") == "tool_search_output" + else item + ) + for item in items + ] + return shown, items + + +def _tool_search_call( + item_id: str, call_id: str, arguments: str +) -> ResponseToolSearchCall: + """A search call reports object arguments; one whose arguments are not a JSON + object is incomplete.""" + try: + parsed = json.loads(arguments) + except ValueError: + parsed = None + complete = isinstance(parsed, dict) + return ResponseToolSearchCall( + id=item_id, + call_id=call_id, + arguments=parsed if complete else {}, + status="completed" if complete else "incomplete", + ) + + def _build_output_text_logprobs(meta_info: dict) -> list[Logprob]: """Reshape decoded ``meta_info`` logprobs into the Responses logprob type, covering every generated token.""" @@ -319,9 +420,21 @@ class OpenAIServingResponses(OpenAIServingChat): # FIXME: If the engine is dead, raise an error # This is required for the streaming case + # Such a request's callable tools are known once its history renders, + # so `_make_request` checks its tool_choice instead. + native = self._renders_natively(request) + if native: + tools_error = self._native_tools_error(request) + if tools_error is not None: + return self.create_error_response(tools_error, param="tools") + # ``tool_choice="required"`` needs a tool the parser can actually force. - if request.tool_choice == "required" and not any( - tool.type in ("function", "custom") for tool in (request.tools or []) + if ( + not native + and request.tool_choice == "required" + and not any( + tool.type in ("function", "custom") for tool in (request.tools or []) + ) ): return self.create_error_response( 'tool_choice="required" requires at least one tool with ' @@ -330,9 +443,13 @@ class OpenAIServingResponses(OpenAIServingChat): ) tool_choice = request.effective_tool_choice() - if isinstance(tool_choice, dict) and not any( - tool.type in ("function", "custom") and tool.name == tool_choice["name"] - for tool in request.tools or [] + if ( + not native + and isinstance(tool_choice, dict) + and not any( + tool.type in ("function", "custom") and tool.name == tool_choice["name"] + for tool in request.tools or [] + ) ): return self.create_error_response( f"Tool {tool_choice['name']!r} is not declared in tools", @@ -415,6 +532,8 @@ class OpenAIServingResponses(OpenAIServingChat): except _MediaInputValidationError as e: return self.create_error_response(str(e)) + except ResponseToolChoiceError as e: + return self.create_error_response(str(e), param="tool_choice") except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e: logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(f"{e} {e.__cause__}") @@ -684,7 +803,10 @@ class OpenAIServingResponses(OpenAIServingChat): prev_response: Optional[ResponsesResponse], tokenizer: Any, ): - messages = self._construct_input_messages(request, prev_response) + native = self._renders_natively(request) + messages = ( + [] if native else self._construct_input_messages(request, prev_response) + ) chat_tools = self._response_tools_to_chat_tools(request) chat_request = ChatCompletionRequest( @@ -707,6 +829,8 @@ class OpenAIServingResponses(OpenAIServingChat): chat_template_kwargs=request.chat_template_kwargs, ) chat_request._template_tools = self._response_tools_to_template_tools(request) + if native: + self._prepare_native_responses_prompt(request, chat_request) media_error = self._validate_media_content(chat_request) if media_error: @@ -906,10 +1030,7 @@ class OpenAIServingResponses(OpenAIServingChat): else: self.msg_store[response.id] = self._response_input_history( request - ) + [ - item.model_dump(exclude_none=True) - for item in response.output - ] + ) + self._replayed_output_items(request, response.output) return response @@ -952,12 +1073,18 @@ class OpenAIServingResponses(OpenAIServingChat): name: str, arguments: str, custom_names: set[str], - namespace_names: Optional[dict[str, tuple[str, str]]] = None, - ) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]: + tool_names: dict[str, dict[str, str]], + *, + client_tool_search: bool = False, + ) -> Union[ + ResponseFunctionToolCall, ResponseCustomToolCall, ResponseToolSearchCall + ]: """A call against a ``custom`` tool reports its freeform payload rather than the JSON arguments of the shim function tool. A call against a namespace member reports ``namespace`` and the bare function ``name``.""" call_id = f"call_{random_uuid()[:24]}" + if client_tool_search and name == TOOL_SEARCH: + return _tool_search_call(f"tsc_{random_uuid()[:8]}", call_id, arguments) if name in custom_names: return ResponseCustomToolCall( type="custom_tool_call", @@ -966,7 +1093,13 @@ class OpenAIServingResponses(OpenAIServingChat): name=name, input=decode_custom_tool_input(arguments), ) - namespace, name = (namespace_names or {}).get(name, (None, name)) + entry = (tool_names or {}).get(name) + if isinstance(entry, tuple): + namespace, name = entry + elif isinstance(entry, dict): + namespace, name = entry.get("namespace"), entry.get("name", name) + else: + namespace = None return ResponseFunctionToolCall( arguments=arguments, call_id=call_id, @@ -1063,12 +1196,17 @@ class OpenAIServingResponses(OpenAIServingChat): finish_reason: Optional[dict[str, Any]] = None, ): chat_tools = self._response_tools_to_chat_tools(request) - custom_names = custom_tool_names(request.tools) + custom_names = self._custom_tool_names(request) namespace_names = ( namespace_function_names(request.tools) if self.tool_call_parser == "muse" - else {} + else { + call_name: (entry.get("namespace"), entry.get("name", call_name)) + for call_name, entry in request._tool_state.names.items() + if entry.get("namespace") + } ) + client_tool_search = self._client_tool_search(request) encoding_output = self._encoding_output() if encoding_output is not None: required_json = self._uses_required_tool_json(request, chat_tools) @@ -1088,12 +1226,17 @@ class OpenAIServingResponses(OpenAIServingChat): content, custom_names, namespace_names ) else: - tool_call_items = [ - self._make_tool_call_item( - call.name, call.parameters, custom_names, namespace_names + tool_call_items = [] + for call in calls: + item = self._make_tool_call_item( + call.name, + call.parameters, + custom_names, + namespace_names, + client_tool_search=client_tool_search, ) - for call in calls - ] + self._record_called_name(request, item.call_id, call.name) + tool_call_items.append(item) else: reasoning_content, content, tool_call_items = self._parse_text_output( request, @@ -1194,14 +1337,15 @@ class OpenAIServingResponses(OpenAIServingChat): try: content, call_info_list = parser.parse_non_stream(content) for call_info in call_info_list: - tool_call_items.append( - self._make_tool_call_item( - call_info.name, - call_info.parameters or "", - custom_names, - namespace_names, - ) + item = self._make_tool_call_item( + call_info.name, + call_info.parameters or "", + custom_names, + namespace_names, + client_tool_search=self._client_tool_search(request), ) + self._record_called_name(request, item.call_id, call_info.name) + tool_call_items.append(item) parsed_via_native = bool(call_info_list) except Exception as e: logger.error("Tool call parsing error: %s", e) @@ -1268,6 +1412,10 @@ class OpenAIServingResponses(OpenAIServingChat): def _tool_call_parsing_enabled( self, request: ResponsesRequest, chat_tools: list[Tool] ) -> bool: + if request._tool_state.chat_tools is not None and request.tool_choice == "none": + # The history showed the model tools, so it can still write a call; + # parsing drops it, where skipping the parser would return its markup. + return bool(self.tool_call_parser) return bool( chat_tools and self.tool_call_parser and request.tool_choice != "none" ) @@ -1326,7 +1474,213 @@ class OpenAIServingResponses(OpenAIServingChat): return tool_choice return {"type": "function", "function": {"name": tool_choice["name"]}} + def _renders_natively(self, request: ResponsesRequest) -> bool: + """Whether the served encoding renders the request in its own wire form, + which tool search, namespace tools and deferred tools take.""" + return self.native_tool_loading and ( + any( + tool.type == "namespace" or _defers_loading(tool) + for tool in request.tools + ) + or self._uses_tool_search(request) + ) + + def _uses_tool_search(self, request: ResponsesRequest) -> bool: + """Whether the request searches for tools, or its history has.""" + if any(tool.type == TOOL_SEARCH for tool in request.tools): + return True + return any( + isinstance(item, dict) and item.get("type") in TOOL_SEARCH_ITEMS + for item in self._history_items(request) + ) + + def _history_items(self, request: ResponsesRequest) -> list: + """The input history, before `previous_response_id` is known to exist.""" + stored = self.msg_store.get(request.previous_response_id) or [] + return list(stored) + (request.input if isinstance(request.input, list) else []) + + def _native_tools_error(self, request: ResponsesRequest) -> Optional[str]: + deferred_types = { + tool.name: tool.type + for tool in request.tools + if tool.type in ("function", "custom") and tool.defer_loading + } + for tool in request.tools: + if ( + tool.type == TOOL_SEARCH + and tool.model_extra.get("execution") != "client" + ): + return ( + 'tool_search requires execution "client": this server runs no ' + "searches" + ) + if tool.type == "namespace" and not _holds_function_tools(tool.tools): + return NAMESPACE_MEMBERS_ERROR + for item in self._history_items(request): + if not isinstance(item, dict) or item.get("type") != "tool_search_output": + continue + tools = item.get("tools") + if not isinstance(tools, list) or not all( + isinstance(tool, dict) + and tool.get("type") in ("function", "namespace", "custom") + for tool in tools + ): + return ( + "tool_search_output tools must be a list of function, namespace " + "or custom tools" + ) + for tool in tools: + if tool["type"] == "namespace" and not _holds_function_tools( + tool.get("tools") + ): + return NAMESPACE_MEMBERS_ERROR + # The served encoding renders no other custom tool. + if tool["type"] == "custom" and tool.get("name") != APPLY_PATCH: + return ( + f"tool_search_output can load only the {APPLY_PATCH!r} " + "custom tool" + ) + # A deferred declaration is loaded as the tool it declares. + declared = deferred_types.get(tool.get("name"), tool["type"]) + if tool["type"] != "namespace" and declared != tool["type"]: + return ( + f"tool_search_output returns {tool.get('name')!r} as a " + f"{tool['type']} tool, but tools declares it as a deferred " + f"{declared} tool" + ) + return None + + def _prepare_native_responses_prompt( + self, request: ResponsesRequest, chat_request: ChatCompletionRequest + ) -> None: + """Render the request's own wire form into `chat_request`, whose tools and + choice become the ones the history has loaded.""" + choice = request.effective_tool_choice() + tools, items = _defer_tool_loading( + [tool.model_dump(exclude_unset=True) for tool in request.tools], + [ + item + for item in map( + self._native_input_item, self._response_input_history(request) + ) + if item is not None + ], + ) + body = {"input": items, "tools": tools} + if request.instructions: + body["instructions"] = request.instructions + if choice == "none": + body["tool_choice"] = "none" + prompt = self.prepare_native_prompt( + chat_request, protocol="responses", body=body + ) + tools = prompt.tools + request._tool_state = msgspec.structs.replace( + request._tool_state, + chat_tools=tools, + names={ + call_name: {"namespace": namespace, "name": name} + for call_name, (namespace, name) in prompt.namespace_names.items() + }, + discovered_custom_names=frozenset( + tool["name"] + for item in body["input"] + if isinstance(item, dict) and item.get("type") == "tool_search_output" + for tool in item["tools"] + if tool["type"] == "custom" + ), + ) + if isinstance(choice, dict): + name = choice["name"] + if "namespace" in choice: + # `tools` keeps conversion order, declared before discovered, so + # the last match is the member's latest definition. + member = (choice["namespace"], name) + name = next( + ( + tool.function.name + for tool in reversed(tools) + if prompt.namespace_names.get(tool.function.name) == member + ), + None, + ) + if not any(tool.function.name == name for tool in tools) or ( + choice["type"] == "custom" + and name not in self._custom_tool_names(request) + ): + raise ResponseToolChoiceError( + f"Tool {choice['name']!r} is not declared in tools without " + "defer_loading or loaded by a tool_search_output" + ) + chat_request.tool_choice = ToolChoice( + function=ToolChoiceFuncName(name=name) + ) + elif choice == "required" and not tools: + raise ResponseToolChoiceError( + 'tool_choice="required" needs a tool the model can call' + ) + else: + chat_request.tool_choice = choice if tools else "none" + + @classmethod + def _native_input_item(cls, item: Any) -> Any: + if isinstance(item, BaseModel): + item = item.model_dump(exclude_none=True) + if isinstance(item, dict) and item.get("type") == "reasoning": + # recipe reads only `content`; keep the text SGLang replays from a + # summary or a store=false client's state. + message = cls._normalize_response_message_for_chat(item) + if message is None: + return None + return { + "type": "reasoning", + "content": [ + {"type": "reasoning_text", "text": message["reasoning_content"]} + ], + } + return item + + @staticmethod + def _record_called_name(request: ResponsesRequest, call_id: str, name: str) -> None: + """Keep the name the model called a namespace member by: the item reports + its namespace and own name, which several call names can share.""" + if request._tool_state.chat_tools is not None and ( + name in request._tool_state.names + ): + request._called_tool_names[call_id] = name + + @staticmethod + def _replayed_output_items(request: ResponsesRequest, output: list) -> list[dict]: + """`output` as the history a later request replays: a namespace member's + call under the name the model called it by.""" + items = [] + for item in output: + item = item.model_dump(exclude_none=True) + called_name = request._called_tool_names.get(item.get("call_id")) + if called_name is not None: + item = {**item, "name": called_name} + del item["namespace"] + items.append(item) + return items + + @staticmethod + def _custom_tool_names(request: ResponsesRequest) -> set[str]: + declared = custom_tool_names(request.tools) + if request._tool_state.chat_tools is not None: + # A deferred declaration counts once a search result has loaded it. + declared -= {tool.name for tool in request.tools if tool.defer_loading} + return declared | request._tool_state.discovered_custom_names + + @staticmethod + def _client_tool_search(request: ResponsesRequest) -> bool: + """Whether a `tool_search` call is the client-executed search tool.""" + return request._tool_state.chat_tools is not None and any( + tool.type == TOOL_SEARCH for tool in request.tools + ) + def _response_tools_to_chat_tools(self, request: ResponsesRequest) -> list[Tool]: + if request._tool_state.chat_tools is not None: + return list(request._tool_state.chat_tools) # ``function`` and ``custom`` tools flow to chat; built-ins go through # harmony. A custom tool is shimmed into a single-string function tool. chat_tools = [] @@ -2133,6 +2487,7 @@ class OpenAIServingResponses(OpenAIServingChat): tool_follows = opens_item and item_type in ( "function_call", "custom_tool_call", + "tool_search_call", ) terminal = event_type in ( "response.completed", @@ -2215,14 +2570,19 @@ class OpenAIServingResponses(OpenAIServingChat): ) chat_tools = self._response_tools_to_chat_tools(request) - custom_names = custom_tool_names(request.tools) + custom_names = self._custom_tool_names(request) namespace_names = ( namespace_function_names(request.tools) if self.tool_call_parser == "muse" - else {} + else { + call_name: (entry.get("namespace"), entry.get("name", call_name)) + for call_name, entry in request._tool_state.names.items() + if entry.get("namespace") + } ) tool_choice = request.effective_tool_choice() required_json = self._uses_required_tool_json(request, chat_tools) + client_tool_search = self._client_tool_search(request) encoding_output = self._encoding_output() token_stream = ( encoding_output.new_responses_stream( @@ -2453,7 +2813,13 @@ class OpenAIServingResponses(OpenAIServingChat): return [] arguments = state["arguments"] events: list = [] - if state["custom"]: + done_event = openai_responses_types.ResponseOutputItemDoneEvent + if state["search"]: + completed_item = _tool_search_call( + state["item_id"], state["call_id"], arguments + ) + done_event = ResponseToolSearchCallEvent + elif state["custom"]: payload = decode_custom_tool_input(arguments) # Deltas cannot be retracted, so a payload that no longer # extends what was already streamed defers to the streamed text. @@ -2520,7 +2886,7 @@ class OpenAIServingResponses(OpenAIServingChat): ) events.append( _send_event( - openai_responses_types.ResponseOutputItemDoneEvent( + done_event( type="response.output_item.done", sequence_number=-1, output_index=state["output_index"], @@ -2717,27 +3083,43 @@ class OpenAIServingResponses(OpenAIServingChat): current_output_index += 1 name = call.name or "" is_custom = name in custom_names + is_search = client_tool_search and name == TOOL_SEARCH namespace, name = namespace_names.get(name, (None, name)) + prefix = ( + "ctc" if is_custom else "tsc" if is_search else "fc" + ) state = { - "item_id": ( - f"ctc_{random_uuid()[:8]}" - if is_custom - else f"fc_{random_uuid()[:8]}" - ), + "item_id": f"{prefix}_{random_uuid()[:8]}", "call_id": f"call_{random_uuid()[:24]}", "output_index": current_output_index, "name": name, "namespace": namespace, "arguments": "", "custom": is_custom, + "search": is_search, "payload": "", "added": False, "done": False, } tool_call_states[tool_index] = state + self._record_called_name( + request, state["call_id"], call.name or "" + ) if not state["added"]: state["added"] = True - if state["custom"]: + added_event = ( + openai_responses_types.ResponseOutputItemAddedEvent + ) + if state["search"]: + # Search arguments are reported whole when it completes. + added_item = ResponseToolSearchCall( + id=state["item_id"], + call_id=state["call_id"], + arguments={}, + status="in_progress", + ) + added_event = ResponseToolSearchCallEvent + elif state["custom"]: added_item = ResponseCustomToolCall( type="custom_tool_call", id=state["item_id"], @@ -2756,7 +3138,7 @@ class OpenAIServingResponses(OpenAIServingChat): **_namespace_field(state), ) yield _send_event( - openai_responses_types.ResponseOutputItemAddedEvent( + added_event( type="response.output_item.added", sequence_number=-1, output_index=state["output_index"], @@ -2786,7 +3168,7 @@ class OpenAIServingResponses(OpenAIServingChat): delta=delta, ) ) - else: + elif not state["search"]: yield _send_event( openai_responses_types.ResponseFunctionCallArgumentsDeltaEvent( type="response.function_call_arguments.delta", @@ -2936,10 +3318,7 @@ class OpenAIServingResponses(OpenAIServingChat): self.response_store[final_response.id] = final_response self.msg_store[final_response.id] = self._response_input_history( request - ) + [ - item.model_dump(exclude_none=True) - for item in final_response.output - ] + ) + self._replayed_output_items(request, final_response.output) response_dict = _sanitize_response_dict(final_response.model_dump()) diff --git a/python/sglang/srt/parser/deepseek_recipe_output.py b/python/sglang/srt/parser/deepseek_recipe_output.py index 74ff161215..91b298ca41 100644 --- a/python/sglang/srt/parser/deepseek_recipe_output.py +++ b/python/sglang/srt/parser/deepseek_recipe_output.py @@ -5,6 +5,7 @@ parses them, and tool-call arguments arrive as the API streams them.""" from __future__ import annotations import json +import logging from collections.abc import Sequence from typing import Any, Optional @@ -17,6 +18,11 @@ from sglang.srt.parser.deepseek_recipe_renderer import ( load_deepseek_recipe, ) +logger = logging.getLogger(__name__) + +# The most tokens one UTF-8 character can span, a byte each. +_MAX_UTF8_BYTES = 4 + class RecipeToolCallDelta(msgspec.Struct, frozen=True): """A piece of one call's JSON arguments; the first piece also names the call.""" @@ -74,6 +80,7 @@ class DeepSeekRecipeOutputParser: separate_reasoning: bool = True, stream_reasoning: bool = True, stop: Sequence[str] = (), + callable_names: Optional[frozenset[str]] = None, ): recipe = load_deepseek_recipe() options = recipe.ParsingOptions( @@ -101,6 +108,10 @@ class DeepSeekRecipeOutputParser: # out, so a matched stop is cut before recipe has emitted any of it. self._stop_hold_chars = max(map(len, stop), default=1) - 1 self._pending_ids: list[int] = [] + self._callable_names = callable_names + # recipe's index of each call so far, mapped to the index it is reported + # at; None for a dropped call. + self._call_indices: dict[int, Optional[int]] = {} self.num_consumed_tokens = 0 def feed(self, token_ids: Sequence[int]) -> RecipeOutputDelta: @@ -192,6 +203,27 @@ class DeepSeekRecipeOutputParser: + self._hf_tokenizer.encode(text[:cut], add_special_tokens=False).ids ) + def _reported_call( + self, call: RecipeToolCallDelta + ) -> Optional[RecipeToolCallDelta]: + """`call` at the index it is reported at; None when it belongs to a call + outside `callable_names`.""" + if self._callable_names is None: + return call + if call.name is not None: + index = None + if call.name in self._callable_names: + index = sum(kept is not None for kept in self._call_indices.values()) + else: + logger.warning( + "Model attempted to call undefined function: %s", call.name + ) + self._call_indices[call.index] = index + index = self._call_indices[call.index] + if index is None: + return None + return msgspec.structs.replace(call, index=index) + def _deltas(self, chunks: list[Any]) -> RecipeOutputDelta: delta = RecipeOutputDelta() for chunk in chunks: @@ -202,12 +234,19 @@ class DeepSeekRecipeOutputParser: reasoning = piece.get("reasoning_content") or "" content = piece.get("content") or "" tool_calls = tuple( - RecipeToolCallDelta( - index=call["index"], - arguments=call["function"]["arguments"], - name=call["function"].get("name"), + call + for call in map( + self._reported_call, + ( + RecipeToolCallDelta( + index=call["index"], + arguments=call["function"]["arguments"], + name=call["function"].get("name"), + ) + for call in piece.get("tool_calls") or () + ), ) - for call in piece.get("tool_calls") or () + if call is not None ) if self._reasoning_as_content: content, reasoning = reasoning + content, "" diff --git a/python/sglang/srt/parser/deepseek_recipe_renderer.py b/python/sglang/srt/parser/deepseek_recipe_renderer.py index 964c655455..00c271e06e 100644 --- a/python/sglang/srt/parser/deepseek_recipe_renderer.py +++ b/python/sglang/srt/parser/deepseek_recipe_renderer.py @@ -95,6 +95,23 @@ def pre_normalize(body: dict[str, Any]) -> dict[str, Any]: return request +def loaded_tools(conversation: Any) -> list[Any]: + """The tools the model can call after `conversation`'s history: those it does + not defer and the deferred ones a system message has loaded.""" + recipe = load_deepseek_recipe() + loaded = { + tool.name + for message in conversation.messages + if isinstance(message, recipe.SystemMessage) + for tool in message.tools or [] + } + return [ + tool + for tool in conversation.tools + if not tool.defer_loading or tool.name in loaded + ] + + def _error_message(body: Any) -> str: try: return json.loads(body)["error"]["message"] @@ -146,6 +163,19 @@ class DeepSeekRecipe: ) -> list[int]: """Prompt ids for `body` (see `pre_normalize`); a request recipe rejects raises ValueError. `image_urls` receives the images in placeholder order.""" + conversation = self.convert(body, effort_budget=effort_budget) + prompt_ids = self.encode_conversation(conversation, effort_budget=effort_budget) + if image_urls is not None: + image_urls.extend(self.image_urls(conversation)) + if task is not None: + prompt_ids = self._attach_task(prompt_ids, task, body["thinking"]) + return prompt_ids + + def convert( + self, body: dict[str, Any], *, effort_budget: Optional[int] = None + ) -> Any: + """Convert `body` to recipe's `Conversation`; invalid requests raise ValueError. + Pass the same `effort_budget` to `encode_conversation`.""" recipe = load_deepseek_recipe() normalized = pre_normalize(body) normalized["model"] = self.model @@ -155,16 +185,104 @@ class DeepSeekRecipe: converted = recipe.ChatCompletionRequest(normalized).convert(self._options) except recipe.ConversionError as error: raise ValueError(_error_message(error.body)) from error - prompt_ids = self._encoding.encode(converted.conversation) - if image_urls is not None: - rendered = self._encoding.render_conversation(converted.conversation) - image_urls.extend( - source.url or source.data_url for source in rendered.image_sources - ) - if effort_budget is not None and body["thinking"]: + return converted.conversation + + def convert_native( + self, protocol: str, body: dict[str, Any], *, thinking: bool + ) -> Any: + """Convert a Responses or Messages request to recipe's `Conversation` in the + host's thinking mode; invalid requests raise ValueError.""" + recipe = load_deepseek_recipe() + request_type = { + "responses": recipe.ResponsesRequest, + "messages": recipe.MessagesRequest, + }[protocol] + options = self._options.with_default_thinking_mode(thinking) + try: + converted = request_type({**body, "model": self.model}).convert(options) + except recipe.ConversionError as error: + raise ValueError(_error_message(error.body)) from error + return converted.conversation + + def with_host_settings( + self, + conversation: Any, + *, + response_schema: Optional[dict[str, Any]], + continue_final_message: bool, + ) -> Any: + """`conversation` with what recipe's Responses and Messages conversions + leave out: a JSON response schema, and SGLang's reading of a trailing + assistant turn, continued or else re-sent as a user turn.""" + recipe = load_deepseek_recipe() + messages = conversation.messages + last = messages[-1] if messages else None + if last is not None and last.role == "assistant": + # recipe closes the turn and opens another; SGLang continues it or, + # as `_handle_last_assistant_message`, re-sends it as a user turn. + if continue_final_message: + last = recipe.AssistantMessage( + last.content, last.reasoning_content, prefix=True + ) + else: + last = recipe.UserMessage(last.content) + messages = [*messages[:-1], last] + elif response_schema is None: + return conversation + return recipe.Conversation( + messages, + thinking_mode=conversation.thinking_mode, + tools=conversation.tools, + tool_choice=conversation.tool_choice, + reasoning_effort=conversation.reasoning_effort, + response_schema=( + response_schema + if response_schema is not None + else conversation.response_schema + ), + ) + + def namespace_tool_names( + self, namespaces: list[dict[str, Any]] + ) -> dict[str, tuple[str, str]]: + """The name recipe's Responses conversion calls each member of `namespaces` + by, mapped to its namespace and its name there. `namespaces` lists the + request's namespace tools in conversion order: declared, then discovered.""" + recipe = load_deepseek_recipe() + body = {"model": self.model, "input": "", "tools": namespaces} + try: + return recipe.ResponsesRequest(body).namespace_tool_map(self._options) + except recipe.ConversionError as error: + raise ValueError(_error_message(error.body)) from error + + def image_urls(self, conversation: Any) -> list[str]: + """The images of `conversation` in the order its prompt places them.""" + rendered = self._encoding.render_conversation(conversation) + return [source.url or source.data_url for source in rendered.image_sources] + + def encode_conversation( + self, conversation: Any, *, effort_budget: Optional[int] = None + ) -> list[int]: + """Encode a recipe `Conversation`, including a prefix of its messages.""" + if self._fast_prompt_tokenizer is None: + prompt_ids = self._encoding.encode(conversation) + else: + rendered = self._encoding.render_conversation(conversation) + prompt = rendered.prompt + segments = [] + cursor = 0 + # Recipe distinguishes control tokens from identical spellings in + # user text. Its Python ranges use code-point offsets, not bytes. + for start, end in rendered.special_ranges: + if start > cursor: + segments.append((prompt[cursor:start], False)) + segments.append((prompt[start:end], True)) + cursor = end + if cursor < len(prompt): + segments.append((prompt[cursor:], False)) + prompt_ids = self._fast_prompt_tokenizer.encode_segments(segments).ids + if effort_budget is not None and conversation.thinking_mode: prompt_ids = self._replace_effort_budget(prompt_ids, effort_budget) - if task is not None: - prompt_ids = self._attach_task(prompt_ids, task, body["thinking"]) return prompt_ids def parse_start( diff --git a/python/sglang/test/kits/anthropic_messages_kit.py b/python/sglang/test/kits/anthropic_messages_kit.py index 6795f05bf2..84304ed6e2 100644 --- a/python/sglang/test/kits/anthropic_messages_kit.py +++ b/python/sglang/test/kits/anthropic_messages_kit.py @@ -97,7 +97,9 @@ class AnthropicMessagesMixin: ) serving = AnthropicServing( - openai_serving_chat=SimpleNamespace(supports_inline_system=True) + openai_serving_chat=SimpleNamespace( + supports_inline_system=True, native_tool_loading=False + ) ) chat_request = serving._convert_to_chat_completion_request(anthropic_request) converted = chat_request.model_dump() diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index 37dfedd0e3..7fc5dd20ce 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -31,6 +31,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") class _FakeOpenAIServingChat: native_reasoning_history = False + native_tool_loading = False def __init__(self, stream_lines=None, chat_template=None): self.stream_lines = stream_lines or [] @@ -61,6 +62,7 @@ class _FakeNonStreamingErrorOpenAI: """Returns an error response or raises an exception from the OpenAI handler.""" supports_inline_system = False + native_tool_loading = False def __init__(self, status_code=400, body=None, content=None, exception=None): self._status_code = status_code @@ -496,6 +498,26 @@ class TestAnthropicServing(unittest.TestCase): ], ) + def test_tool_addition_fails_where_tools_cannot_load_mid_conversation(self): + """A tool_addition block must not vanish on a model that cannot render it.""" + request = self._anthropic_request( + messages=[ + {"role": "user", "content": "hello"}, + { + "role": "system", + "content": [ + { + "type": "tool_addition", + "tool": {"type": "tool_reference", "name": "Bash"}, + } + ], + }, + ] + ) + + with self.assertRaisesRegex(ValueError, "tool_addition"): + self._serving()._convert_to_chat_completion_request(request) + def test_builtin_web_search_tool_without_schema_is_skipped(self): request = AnthropicMessagesRequest.model_validate( { diff --git a/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py b/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py index a203efb391..387a8ddd2f 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py +++ b/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py @@ -1,4 +1,6 @@ +import asyncio import json +import re import unittest from types import SimpleNamespace @@ -23,7 +25,9 @@ def _tokenizer_file(): raise unittest.SkipTest(f"the tokenizer is not available: {error}") -def _serving(test, tokenizer, *, is_multimodal=False, **server_args): +def _serving( + test, tokenizer, *, is_multimodal=False, handler=None, engine=None, **server_args +): from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs @@ -53,6 +57,9 @@ def _serving(test, tokenizer, *, is_multimodal=False, **server_args): get_default_sampling_params=lambda: {}, ), config_value=lambda name: getattr(args, name), + lora_registry=None, + generate_request=engine, + create_abort_task=lambda request: None, ) template_manager = SimpleNamespace( chat_template_name=None, @@ -62,7 +69,7 @@ def _serving(test, tokenizer, *, is_multimodal=False, **server_args): python_chat_template=None, jinja_template_may_reorder_tool_results=False, ) - return OpenAIServingChat(manager, template_manager) + return (handler or OpenAIServingChat)(manager, template_manager) class TestAdapterRegistry(CustomTestCase): @@ -429,5 +436,1555 @@ class TestHandlerRequestSemantics(CustomTestCase): native_text_parser("deepseek-v41", chat_encoding_spec=None, adapter=None) +SEARCH_TOOL = {"type": "tool_search", "execution": "client"} +WEATHER_TOOL = { + "type": "function", + "name": "get_weather", + "description": "Weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, +} +SEARCH_DOCS_TOOL = { + "name": "search_docs", + "description": "Search", + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}, +} +TICKET_TOOL = { + "name": "create_ticket", + "description": "Ticket", + "input_schema": { + "type": "object", + "properties": {"order": {"type": "string"}}, + "required": ["order"], + }, + "strict": True, + "defer_loading": True, +} +CRM_NAMESPACE = { + "type": "namespace", + "name": "crm", + "description": "CRM", + "tools": [ + { + "type": "function", + "name": "create_ticket", + "description": "Ticket", + "parameters": { + "type": "object", + "properties": {"order": {"type": "string"}}, + }, + } + ], +} +APPLY_PATCH_TOOL = {"type": "custom", "name": "apply_patch"} +PATCH = "*** Begin Patch\n*** Delete File: a.txt\n*** End Patch" +ASK_TICKET = {"role": "user", "content": "Open a ticket for A-1"} +ADD_TICKET = { + "role": "system", + "content": [ + { + "type": "tool_addition", + "tool": {"type": "tool_reference", "name": "create_ticket"}, + } + ], +} +DEFINE_TICKET = { + "role": "system", + "content": [ + { + "type": "tool_addition", + "tool": { + "type": "tool_definition", + "definition": { + key: value + for key, value in TICKET_TOOL.items() + if key != "defer_loading" + }, + }, + } + ], +} + + +def _search_history(found_tools): + return [ + {"role": "user", "content": "Weather in Paris?"}, + { + "type": "tool_search_call", + "call_id": "search", + "execution": "client", + "arguments": {"query": "weather"}, + }, + { + "type": "tool_search_output", + "call_id": "search", + "execution": "client", + "tools": found_tools, + }, + ] + + +def _dsml_call(name, **arguments): + parameters = "".join( + f'<|DSML| parameter name="{key}" string="true">{value}\n' + for key, value in arguments.items() + ) + return ( + f'<|DSML| calls>\n<|DSML| invoke name="{name}">\n{parameters}' + "\n" + ) + + +def _schema_names(prompt): + """The tools whose schemas the prompt shows, initially or as updates.""" + return re.findall(r'^\{"name": "([^"]+)"', prompt, flags=re.MULTILINE) + + +async def _sse_payloads(stream): + return [ + json.loads(line.removeprefix("data: ")) + async for chunk in stream + for line in chunk.splitlines() + if line.startswith("data: ") + ] + + +class _Engine: + """Answers each request with its next scripted output and keeps its prompt. + `chunk_tokens` splits the output into chunks of that many tokens; `finish_type` + is the reason the last one carries.""" + + def __init__(self, tokenizer): + self.tokenizer = tokenizer + self.outputs = [] + self.prompts = [] + self.requests = [] + self.chunk_tokens = None + self.finish_type = "stop" + + def __call__(self, adapted_request, raw_request): + self.requests.append(adapted_request) + self.prompts.append(self.tokenizer.decode(adapted_request.input_ids)) + output_ids = self.tokenizer.encode( + self.outputs.pop(0), add_special_tokens=False + ) + if self.finish_type == "stop": + eos = self.tokenizer.convert_tokens_to_ids("<|end▁of▁sentence|>") + output_ids.append(eos) + finish_reason = {"type": "stop", "matched": eos} + else: + finish_reason = {"type": self.finish_type, "length": len(output_ids)} + step = self.chunk_tokens or len(output_ids) + ends = [*range(step, len(output_ids), step), len(output_ids)] + + async def generate(): + # The engine's default streaming: each chunk holds the output so far. + for end in ends: + yield { + "text": "", + "output_ids": output_ids[:end], + "meta_info": { + "id": "rid", + "prompt_tokens": len(adapted_request.input_ids), + "completion_tokens": end, + "cached_tokens": 0, + "weight_version": "default", + "finish_reason": ( + finish_reason if end == len(output_ids) else None + ), + }, + } + + return generate() + + +class TestNativeToolLoading(CustomTestCase): + """Responses and Messages requests that load tools mid-conversation render once, + through recipe's conversion of their own wire form.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + from transformers import PreTrainedTokenizerFast + + cls.tokenizer = PreTrainedTokenizerFast(tokenizer_file=_tokenizer_file()) + + def _responses(self, **model): + from sglang.srt.entrypoints.openai.serving_responses import ( + OpenAIServingResponses, + ) + + engine = _Engine(self.tokenizer) + serving = _serving( + self, + self.tokenizer, + handler=OpenAIServingResponses, + engine=engine, + enable_response_store=True, + **model, + ) + return serving, engine + + def _messages(self, **model): + from sglang.srt.entrypoints.anthropic.serving import AnthropicServing + + engine = _Engine(self.tokenizer) + chat = _serving(self, self.tokenizer, engine=engine, **model) + return AnthropicServing(chat), engine + + def _respond(self, serving, engine, output, **fields): + """The response to a Responses request the engine answers with `output`; + a streamed one as its events.""" + from sglang.srt.entrypoints.openai.protocol import ResponsesRequest + + engine.outputs.append(output) + request = ResponsesRequest(model="dummy", **fields) + response = asyncio.run(serving.create_responses(request)) + return asyncio.run(_sse_payloads(response)) if request.stream else response + + def _messages_request(self, messages, **fields): + from sglang.srt.entrypoints.anthropic.protocol import AnthropicMessagesRequest + + return AnthropicMessagesRequest( + **{ + "model": "dummy", + "max_tokens": 64, + "tools": [SEARCH_DOCS_TOOL, TICKET_TOOL], + "messages": messages, + **fields, + } + ) + + def test_responses_tool_search_discovery_to_call(self): + from sglang.srt.entrypoints.openai.protocol import ResponsesRequest + + serving, engine = self._responses() + + def respond(output, **fields): + engine.outputs.append(output) + request = ResponsesRequest(model="dummy", tools=[SEARCH_TOOL], **fields) + return request, asyncio.run(serving.create_responses(request)) + + # The search call streams as a typed item whose arguments arrive whole. + _, stream = respond( + _dsml_call("tool_search", query="weather"), + input="Weather in Paris?", + stream=True, + ) + payloads = asyncio.run(_sse_payloads(stream)) + items = [ + payload["item"] + for payload in payloads + if payload["type"].startswith("response.output_item.") + ] + self.assertEqual( + [(item["type"], item["status"], item["arguments"]) for item in items], + [ + ("tool_search_call", "in_progress", {}), + ("tool_search_call", "completed", {"query": "weather"}), + ], + ) + self.assertNotIn( + "response.function_call_arguments.delta", + [payload["type"] for payload in payloads], + ) + searched = payloads[-1]["response"] + self.assertEqual(searched["output"], [items[-1]]) + self.assertEqual(_schema_names(engine.prompts[-1]), ["tool_search"]) + + # The returned schema renders where the search returned it and is callable. + request, called = respond( + _dsml_call("get_weather", city="Paris"), + previous_response_id=searched["id"], + input=[ + { + "type": "tool_search_output", + "call_id": items[-1]["call_id"], + "execution": "client", + "status": "completed", + "tools": [WEATHER_TOOL], + } + ], + tool_choice={"type": "function", "name": "get_weather"}, + ) + prompt = engine.prompts[-1] + self.assertGreater( + prompt.index("### Updated Tool Schemas"), prompt.index("") + ) + self.assertEqual(_schema_names(prompt), ["tool_search", "get_weather"]) + self.assertEqual( + [ + tool.function.name + for tool in serving._response_tools_to_chat_tools(request) + ], + _schema_names(prompt), + ) + (call,) = called.output + self.assertEqual( + (call.type, call.name, json.loads(call.arguments)), + ("function_call", "get_weather", {"city": "Paris"}), + ) + + _, answered = respond( + "Sunny.", + previous_response_id=called.id, + input=[ + { + "type": "function_call_output", + "call_id": call.call_id, + "output": "sunny", + } + ], + ) + self.assertIn('<|DSML| invoke name="get_weather">', engine.prompts[-1]) + self.assertEqual(answered.output[-1].content[0].text, "Sunny.") + + def test_responses_tool_choice_follows_loaded_tools(self): + from sglang.srt.entrypoints.openai.protocol import ResponsesRequest + + serving, engine = self._responses() + for tools, choice, param in ( + ([SEARCH_TOOL], {"type": "function", "name": "get_weather"}, "tool_choice"), + ([{"type": "tool_search"}], "auto", "tools"), + ): + with self.subTest(param=param): + rejected = asyncio.run( + serving.create_responses( + ResponsesRequest( + model="dummy", tools=tools, input="hi", tool_choice=choice + ) + ) + ) + self.assertEqual(json.loads(rejected.body)["error"]["param"], param) + + # `none` hides even the tools the history loaded. + request = ResponsesRequest( + model="dummy", + tools=[SEARCH_TOOL], + tool_choice="none", + input=_search_history([WEATHER_TOOL]), + ) + engine.outputs.append("Sunny.") + answered = asyncio.run(serving.create_responses(request)) + self.assertEqual(answered.output[-1].content[0].text, "Sunny.") + self.assertIn("", engine.prompts[-1]) + self.assertEqual(_schema_names(engine.prompts[-1]), []) + self.assertEqual(serving._response_tools_to_chat_tools(request), []) + + def test_responses_search_output_loads_only_renderable_tools(self): + """A malformed search result, or one the encoding cannot render, is a client + error before generation, not a server error.""" + from sglang.srt.entrypoints.openai.protocol import ResponsesRequest + + serving, engine = self._responses() + for name, found_tools in ( + ("not a list", 42), + ("custom", [{"type": "custom", "name": "shell"}]), + ("custom member", [{**CRM_NAMESPACE, "tools": [APPLY_PATCH_TOOL]}]), + ): + with self.subTest(name): + rejected = asyncio.run( + serving.create_responses( + ResponsesRequest( + model="dummy", + tools=[SEARCH_TOOL], + input=_search_history(found_tools), + ) + ) + ) + self.assertEqual(rejected.status_code, 400) + self.assertEqual(json.loads(rejected.body)["error"]["param"], "tools") + self.assertEqual(engine.prompts, []) + + def test_responses_namespace_member_keeps_its_identity(self): + """A namespace member is called by the encoding's flat name and reported + by its namespace and own name, whether declared or discovered.""" + serving, engine = self._responses() + flat = "crm::create_ticket" + call = _dsml_call(flat, order="A-1") + identity = { + "type": "function_call", + "namespace": "crm", + "name": "create_ticket", + } + discovered = { + "tools": [SEARCH_TOOL], + "input": _search_history([CRM_NAMESPACE]), + } + for name, fields, schemas in ( + ("declared", {"tools": [CRM_NAMESPACE], "input": "hi"}, [flat]), + ("discovered", discovered, ["tool_search", flat]), + ): + with self.subTest(name): + response = self._respond( + serving, + engine, + call, + tool_choice={**identity, "type": "function"}, + **fields, + ) + self.assertEqual(_schema_names(engine.prompts[-1]), schemas) + # The named choice constrains the output to the flat name. + self.assertIn( + flat, json.dumps(engine.requests[-1].sampling_params, default=str) + ) + (item,) = response.output + self.assertEqual(item.model_dump(include=set(identity)), identity) + self.assertEqual(json.loads(item.arguments), {"order": "A-1"}) + + # The stored call replays under the name the model called. + self._respond( + serving, + engine, + "Done.", + previous_response_id=response.id, + input=[ + { + "type": "function_call_output", + "call_id": item.call_id, + "output": "ok", + } + ], + tools=fields["tools"], + ) + self.assertIn(f'invoke name="{flat}"', engine.prompts[-1]) + + events = self._respond(serving, engine, call, stream=True, **fields) + items = [ + event["item"] + for event in events + if event["type"].startswith("response.output_item.") + ] + self.assertEqual( + [{key: item[key] for key in identity} for item in items], + [identity, identity], + ) + + rejected = self._respond( + serving, + engine, + call, + tool_choice={ + "type": "function", + "namespace": "ops", + "name": "create_ticket", + }, + **discovered, + ) + self.assertEqual(json.loads(rejected.body)["error"]["param"], "tool_choice") + + def test_responses_replays_a_namespace_call_under_the_called_name(self): + """A member discovered again gets a suffixed call name beside the first + one. Both report the same namespace and name, so a call replayed from + those would render as the other definition's call.""" + rediscovered = { + **CRM_NAMESPACE, + "tools": [ + { + **CRM_NAMESPACE["tools"][0], + "parameters": { + "type": "object", + "properties": {"ticket_id": {"type": "string"}}, + }, + } + ], + } + fields = { + "tools": [CRM_NAMESPACE, SEARCH_TOOL], + "input": _search_history([rediscovered]), + } + called = "crm::create_ticket_1" + for stream in (False, True): + with self.subTest(stream=stream): + serving, engine = self._responses() + response = self._respond( + serving, + engine, + _dsml_call(called, ticket_id="B-2"), + stream=stream, + **fields, + ) + response = response[-1]["response"] if stream else response.model_dump() + self.assertEqual( + _schema_names(engine.prompts[-1]), + ["crm::create_ticket", "tool_search", called], + ) + (item,) = response["output"] + self.assertEqual( + (item["namespace"], item["name"]), ("crm", "create_ticket") + ) + + self._respond( + serving, + engine, + "Done.", + previous_response_id=response["id"], + input=[ + { + "type": "function_call_output", + "call_id": item["call_id"], + "output": "ok", + } + ], + tools=fields["tools"], + ) + self.assertEqual( + re.findall(r'invoke name="([^"]+)"', engine.prompts[-1])[-2:], + ["tool_search", called], + ) + self.assertIn( + '<|DSML| parameter name="ticket_id" string="true">B-2', + engine.prompts[-1], + ) + + def test_responses_named_choice_takes_the_latest_namespace_definition(self): + """Naming a member discovered again constrains the output to its latest + definition, in whatever order recipe lists the members' call names.""" + from unittest.mock import patch + + from sglang.srt.parser.deepseek_recipe_renderer import DeepSeekRecipe + + rediscovered = { + **CRM_NAMESPACE, + "tools": [ + { + **CRM_NAMESPACE["tools"][0], + "parameters": { + "type": "object", + "properties": {"ticket_id": {"type": "string"}}, + }, + } + ], + } + latest = "crm::create_ticket_1" + namespace_tool_names = DeepSeekRecipe.namespace_tool_names + for latest_first in (False, True): + + def ordered(recipe, namespaces, latest_first=latest_first): + names = namespace_tool_names(recipe, namespaces) + return dict(sorted(names.items(), reverse=latest_first)) + + for stream in (False, True): + with ( + self.subTest(latest_first=latest_first, stream=stream), + patch.object(DeepSeekRecipe, "namespace_tool_names", ordered), + ): + serving, engine = self._responses() + self._respond( + serving, + engine, + _dsml_call(latest, ticket_id="B-2"), + stream=stream, + tools=[CRM_NAMESPACE, SEARCH_TOOL], + input=_search_history([rediscovered]), + tool_choice={ + "type": "function", + "namespace": "crm", + "name": "create_ticket", + }, + ) + tag = json.loads( + engine.requests[-1].sampling_params["structural_tag"] + ) + forced = tag["format"]["elements"][1] + self.assertIn(f'invoke name="{latest}"', forced["begin"]) + self.assertEqual( + list(forced["content"]["json_schema"]["properties"]), + ["ticket_id"], + ) + + def test_responses_discovered_apply_patch_is_a_custom_tool(self): + serving, engine = self._responses() + fields = { + "tools": [SEARCH_TOOL], + "input": _search_history([APPLY_PATCH_TOOL]), + } + call = _dsml_call("apply_patch", input=PATCH) + response = self._respond(serving, engine, call, **fields) + self.assertEqual( + _schema_names(engine.prompts[-1]), ["tool_search", "apply_patch"] + ) + (item,) = response.output + self.assertEqual( + (item.type, item.name, item.input), + ("custom_tool_call", "apply_patch", PATCH), + ) + + events = self._respond(serving, engine, call, stream=True, **fields) + types = [event["type"] for event in events] + self.assertNotIn("response.function_call_arguments.delta", types) + self.assertEqual( + "".join( + event["delta"] + for event in events + if event["type"] == "response.custom_tool_call_input.delta" + ), + PATCH, + ) + (done,) = ( + event["item"] + for event in events + if event["type"] == "response.output_item.done" + ) + self.assertEqual( + (done["type"], done["name"], done["input"]), + ("custom_tool_call", "apply_patch", PATCH), + ) + + def test_images_reach_the_engine_in_prompt_order(self): + """Images of a request rendered from its own wire form are sent with the + prompt ids, one per placeholder and in placeholder order.""" + urls = [f"https://example.com/{index}.png" for index in range(4)] + placeholder = "<|deepseek_image|>" + + def image(url): + return {"type": "input_image", "image_url": url} + + search_call, search_output = _search_history([WEATHER_TOOL])[1:] + responses_fields = { + "tools": [SEARCH_TOOL], + "input": [ + {"role": "system", "content": [image(urls[0])]}, + { + "role": "user", + "content": [ + image(urls[1]), + {"type": "input_text", "text": "Weather here?"}, + image(urls[2]), + ], + }, + search_call, + search_output, + {"role": "user", "content": [image(urls[3])]}, + ], + } + serving, engine = self._responses(is_multimodal=True) + self._respond(serving, engine, "Sunny.", **responses_fields) + sent = engine.requests[-1] + self.assertEqual([data.url for data in sent.image_data], urls) + prompt = engine.prompts[-1] + self.assertEqual(prompt.count(placeholder), len(urls)) + self.assertLess( + prompt.index("### Updated Tool Schemas"), prompt.rindex(placeholder) + ) + + def block(url): + return {"type": "image", "source": {"type": "url", "url": url}} + + anthropic, engine = self._messages(is_multimodal=True) + engine.outputs.append("A ticket.") + messages_request = self._messages_request( + [ + {"role": "user", "content": [block(urls[0]), block(urls[1])]}, + ADD_TICKET, + {"role": "user", "content": [block(urls[2])]}, + ] + ) + asyncio.run(anthropic.handle_messages(messages_request, None)) + sent = engine.requests[-1] + self.assertEqual([data.url for data in sent.image_data], urls[:3]) + self.assertEqual(engine.prompts[-1].count(placeholder), 3) + + # A text-only checkpoint refuses the images instead of dropping them. + serving, engine = self._responses() + rejected = self._respond(serving, engine, "Sunny.", **responses_fields) + self.assertEqual(rejected.status_code, 400) + self.assertIn("image input is not supported", rejected.body.decode()) + self.assertEqual(engine.prompts, []) + + def test_auto_reports_only_calls_to_loaded_tools(self): + """Without a grammar, a call to a deferred tool the prompt never showed is + not reported; calls to the tools it shows are.""" + loaded, unloaded = ( + _dsml_call("search_docs", q="ticket"), + _dsml_call("create_ticket", order="A-1"), + ) + anthropic, engine = self._messages() + + def tool_uses(output): + engine.outputs.append(output) + request = self._messages_request([ASK_TICKET]) + response = json.loads( + asyncio.run(anthropic.handle_messages(request, None)).body + ) + self.assertEqual(_schema_names(engine.prompts[-1]), ["search_docs"]) + return response["stop_reason"], [ + block["name"] + for block in response["content"] + if block["type"] == "tool_use" + ] + + self.assertEqual(tool_uses(unloaded), ("end_turn", [])) + self.assertEqual(tool_uses(loaded), ("tool_use", ["search_docs"])) + both = unloaded.replace("", "") + loaded.partition("\n")[2] + self.assertEqual(tool_uses(both), ("tool_use", ["search_docs"])) + + serving, engine = self._responses() + fields = {"tools": [SEARCH_TOOL, WEATHER_TOOL], "input": "Open a ticket"} + for stream in (False, True): + with self.subTest(stream=stream): + response = self._respond( + serving, + engine, + both.replace("search_docs", "get_weather"), + stream=stream, + **fields, + ) + output = ( + response[-1]["response"]["output"] + if stream + else [item.model_dump() for item in response.output] + ) + self.assertEqual( + [(item["type"], item["name"]) for item in output], + [("function_call", "get_weather")], + ) + + def _none_answers(self, anthropic, serving, engine, output, **formats): + """What a natively rendered `tool_choice: none` request returns for + `output`, per API: its text, how it finished, and the whole response.""" + answers = {} + stream = engine.chunk_tokens is not None + engine.outputs.append(output) + response = asyncio.run( + anthropic.handle_messages( + self._messages_request( + [ASK_TICKET, ADD_TICKET], + tool_choice={"type": "none"}, + stream=stream, + **formats.get("messages", {}), + ), + None, + ) + ) + if stream: + events = asyncio.run(_sse_payloads(response.body_iterator)) + blocks = [ + event["content_block"] + for event in events + if event["type"] == "content_block_start" + ] + text = "".join( + event["delta"].get("text", "") + for event in events + if event["type"] == "content_block_delta" + ) + (finish,) = ( + event["delta"]["stop_reason"] + for event in events + if event["type"] == "message_delta" + ) + else: + events = json.loads(response.body) + blocks = events["content"] + text = "".join(block.get("text", "") for block in blocks) + finish = events["stop_reason"] + self.assertEqual({block["type"] for block in blocks} - {"text"}, set()) + answers["messages"] = (text, finish, events) + + response = self._respond( + serving, + engine, + output, + stream=stream, + tools=[SEARCH_TOOL], + tool_choice="none", + input=_search_history([WEATHER_TOOL]), + **formats.get("responses", {}), + ) + final = response[-1]["response"] if stream else response.model_dump() + self.assertEqual( + {item["type"] for item in final["output"]} - {"message"}, set() + ) + text = "".join( + part["text"] for item in final["output"] for part in item["content"] + ) + if stream: + self.assertEqual( + "".join( + event["delta"] + for event in response + if event["type"] == "response.output_text.delta" + ), + text, + ) + answers["responses"] = (text, final["status"], response if stream else final) + return answers + + def test_none_ends_the_answer_at_a_call_the_model_writes_anyway(self): + """Under `tool_choice: none` the answer ends where the model starts a call + block, as any answer does: the text before it is returned, the call and + what follows are not, and no call is reported. A JSON answer is read as + JSON, so a call marker spelled in one of its strings stays.""" + from sglang.srt.environ import envs + + anthropic, engine = self._messages() + serving, _ = self._responses() + # One engine serves both handlers, so both see the same chunking. + serving.tokenizer_manager.generate_request = engine + call = _dsml_call("create_ticket", order="A-1") + other = _dsml_call("get_weather", city="Paris") + unclosed = call[: call.index("")] + texts = ( + (call, ""), + (_dsml_call("tool_search", query="weather"), ""), + ("Sure." + call, "Sure."), + ("Grüße ☃ 你好" + unclosed, "Grüße ☃ 你好"), + ("BEFORE" + call + "BETWEEN" + other + "AFTER", "BEFORE"), + ("BEFORE" + call + "AFTER", "BEFORE"), + # The marker spelled in prose is the marker: the tokenizer has one + # token for it either way. + ("Write <|DSML| calls> to call.", "Write "), + ("No ticket needed.", "No ticket needed."), + ) + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, + } + spelled = '{"answer":"<|DSML| calls>"}' + json_formats = [ + { + "messages": { + "output_config": { + "format": {"type": "json_schema", "schema": schema} + } + }, + "responses": {"text": {"format": responses_format}}, + } + for responses_format in ( + { + "type": "json_schema", + "name": "out", + "strict": True, + "schema": schema, + }, + {"type": "json_object"}, + ) + ] + + def answers(output, **formats): + engine.chunk_tokens = chunk_tokens + return self._none_answers(anthropic, serving, engine, output, **formats) + + for forward_unknown in (False, True): + for chunk_tokens in (None, 10**6, 1, 2, 7): + with envs.SGLANG_FORWARD_UNKNOWN_TOOLS.override(forward_unknown): + for output, text in texts: + with self.subTest( + output=output, chunk=chunk_tokens, forward=forward_unknown + ): + for api, (answer, finish, response) in answers( + output + ).items(): + self.assertEqual((api, answer), (api, text)) + self.assertIn(finish, ("end_turn", "completed")) + if "DSML" not in text: + self.assertNotIn( + "DSML", json.dumps(response, ensure_ascii=False) + ) + for formats in json_formats: + with self.subTest( + json=formats["responses"]["text"]["format"]["type"], + chunk=chunk_tokens, + forward=forward_unknown, + ): + for api, (answer, finish, _) in answers( + spelled, **formats + ).items(): + self.assertEqual((api, answer), (api, spelled)) + self.assertIn(finish, ("end_turn", "completed")) + + # A dropped call does not turn a cut-off answer into a finished one. + engine.finish_type = "length" + for chunk_tokens in (None, 2): + with self.subTest("length", chunk=chunk_tokens): + cut = answers("Sure." + unclosed) + self.assertEqual(cut["messages"][:2], ("Sure.", "max_tokens")) + self.assertEqual(cut["responses"][:2], ("Sure.", "incomplete")) + engine.finish_type = "abort" + for chunk_tokens in (None, 2): + with self.subTest("abort", chunk=chunk_tokens): + aborted = answers("Sure." + unclosed) + self.assertEqual(aborted["messages"][0], "Sure.") + self.assertEqual(aborted["responses"][:2], ("Sure.", "failed")) + + def test_responses_deferred_tool_loads_from_a_search_result(self): + """A declared tool that defers loading is hidden and not callable until a + search result names it, by reference or with its definition.""" + serving, engine = self._responses() + deferred = {**WEATHER_TOOL, "defer_loading": True} + named = {"type": "function", "name": "get_weather"} + call = _dsml_call("get_weather", city="Paris") + + def output(response): + return [ + (item.type, getattr(item, "name", None)) for item in response.output + ] + + for tools, schemas in ( + ([deferred], []), + ([SEARCH_TOOL, deferred], ["tool_search"]), + ): + with self.subTest("unloaded", tools=len(tools)): + response = self._respond(serving, engine, call, tools=tools, input="hi") + self.assertEqual(_schema_names(engine.prompts[-1]), schemas) + self.assertNotIn( + "function_call", [kind for kind, _ in output(response)] + ) + rejected = self._respond( + serving, engine, call, tools=tools, input="hi", tool_choice=named + ) + self.assertEqual( + json.loads(rejected.body)["error"]["param"], "tool_choice" + ) + engine.outputs.clear() + + for name, found in (("reference", named), ("definition", deferred)): + with self.subTest("loaded", found=name): + response = self._respond( + serving, + engine, + call, + tools=[SEARCH_TOOL, deferred], + input=_search_history([found]), + tool_choice=named, + ) + prompt = engine.prompts[-1] + self.assertEqual(_schema_names(prompt), ["tool_search", "get_weather"]) + loaded = prompt.index("### Updated Tool Schemas") + self.assertGreater(loaded, prompt.index("")) + self.assertIn('"required": ["city"]', prompt[loaded:]) + tag = json.loads(engine.requests[-1].sampling_params["structural_tag"]) + forced = tag["format"]["elements"][1] + self.assertIn('invoke name="get_weather"', forced["begin"]) + self.assertEqual( + list(forced["content"]["json_schema"]["properties"]), ["city"] + ) + self.assertEqual(output(response), [("function_call", "get_weather")]) + + # A namespace member defers the same way and keeps its identity. + member = CRM_NAMESPACE["tools"][0] + response = self._respond( + serving, + engine, + _dsml_call("crm::create_ticket", order="A-1"), + tools=[ + SEARCH_TOOL, + {**CRM_NAMESPACE, "tools": [{**member, "defer_loading": True}]}, + ], + input=_search_history( + [ + { + **CRM_NAMESPACE, + "tools": [{"type": "function", "name": "create_ticket"}], + } + ] + ), + ) + prompt = engine.prompts[-1] + self.assertEqual(_schema_names(prompt), ["tool_search", "crm::create_ticket"]) + self.assertIn('"order"', prompt[prompt.index("### Updated Tool Schemas") :]) + (item,) = response.output + self.assertEqual((item.namespace, item.name), ("crm", "create_ticket")) + + def test_responses_deferred_tool_loads_as_the_type_it_declares(self): + """A search result that returns a deferred custom tool's name as a function + is rejected, so its call is never reported as a custom call; the custom + tool itself loads and is forced only by a custom choice of a custom tool.""" + from sglang.srt.entrypoints.openai.protocol import ResponsesRequest + + serving, engine = self._responses() + deferred = {**APPLY_PATCH_TOOL, "defer_loading": True} + function = { + "type": "function", + "name": "apply_patch", + "parameters": { + "type": "object", + "properties": {"file": {"type": "string"}}, + }, + } + custom_choice = {"type": "custom", "name": "apply_patch"} + for stream in (False, True): + for choice in ("auto", custom_choice): + with self.subTest("conflict", stream=stream, choice=choice): + request = ResponsesRequest( + model="dummy", + stream=stream, + tools=[SEARCH_TOOL, deferred], + input=_search_history([function]), + tool_choice=choice, + ) + rejected = asyncio.run(serving.create_responses(request)) + self.assertEqual(rejected.status_code, 400) + error = json.loads(rejected.body)["error"] + self.assertEqual(error["param"], "tools") + self.assertIn("deferred custom tool", error["message"]) + self.assertEqual(engine.prompts, []) + + for stream in (False, True): + with self.subTest("loaded", stream=stream): + response = self._respond( + serving, + engine, + _dsml_call("apply_patch", input=PATCH), + stream=stream, + tools=[SEARCH_TOOL, deferred], + input=_search_history([APPLY_PATCH_TOOL]), + tool_choice=custom_choice, + ) + output = ( + response[-1]["response"]["output"] + if stream + else [item.model_dump() for item in response.output] + ) + self.assertEqual( + [(item["type"], item["input"]) for item in output], + [("custom_tool_call", PATCH)], + ) + + # A custom choice does not force a function tool. + rejected = self._respond( + serving, + engine, + "", + tools=[SEARCH_TOOL, deferred, WEATHER_TOOL], + input="hi", + tool_choice={"type": "custom", "name": "get_weather"}, + ) + self.assertEqual(json.loads(rejected.body)["error"]["param"], "tool_choice") + + def test_messages_response_format_renders_with_a_deferred_load(self): + """`output_config.format` shows its schema in the prompt and constrains the + answer, beside a deferred tool that loads where it is added.""" + anthropic, engine = self._messages() + schema = { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + "additionalProperties": False, + } + + def respond(output, **fields): + engine.outputs.append(output) + request = self._messages_request( + [ASK_TICKET, ADD_TICKET], + output_config={"format": {"type": "json_schema", "schema": schema}}, + **fields, + ) + response = asyncio.run(anthropic.handle_messages(request, None)) + return response.status_code, json.loads(response.body) + + for choice, schemas in ( + ({}, ["search_docs", "create_ticket"]), + ({"tool_choice": {"type": "none"}}, []), + ): + with self.subTest(choice=choice): + status, response = respond('{"answer": "ok"}', **choice) + self.assertEqual(status, 200) + prompt = engine.prompts[-1] + self.assertEqual(_schema_names(prompt), schemas) + shown = prompt.index("## Response Format:") + self.assertLess(shown, prompt.index(ASK_TICKET["content"])) + self.assertIn( + json.dumps(schema), prompt[shown : prompt.index("<|User|>")] + ) + self.assertEqual( + json.loads(engine.requests[-1].sampling_params["json_schema"]), + schema, + ) + self.assertEqual( + response["content"], [{"type": "text", "text": '{"answer": "ok"}'}] + ) + + # A forced call and an answer schema cannot both constrain the output. + count = len(engine.prompts) + status, response = respond( + "", tool_choice={"type": "tool", "name": "create_ticket"} + ) + self.assertEqual(status, 400) + self.assertIn("cannot be combined", response["error"]["message"]) + self.assertEqual(len(engine.prompts), count) + + def test_trailing_assistant_keeps_the_loaded_tools(self): + """A final assistant message is re-sent as a user turn, as on requests that + load no tools, or continued under continue_final_message; either way the + prompt holds one open assistant turn and the tools the history loaded.""" + from unittest.mock import patch + + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + + prefill = {"role": "assistant", "content": "The answer is"} + loaded = '"name": "create_ticket"' + prepare = OpenAIServingChat.prepare_native_prompt + + def continued(chat, request, **fields): + request.continue_final_message = True + return prepare(chat, request, **fields) + + anthropic, engine = self._messages() + request = self._messages_request( + [ASK_TICKET, ADD_TICKET, prefill], + thinking={"type": "enabled", "budget_tokens": 1024}, + ) + + def respond(output): + engine.outputs.append(output) + response = asyncio.run(anthropic.handle_messages(request, None)) + prompt = engine.prompts[-1] + self.assertEqual(_schema_names(prompt), ["search_docs", "create_ticket"]) + return prompt.partition(loaded)[2], json.loads(response.body)["content"] + + tail, content = respond("Hm. 42.") + self.assertTrue(tail.endswith("<|User|>The answer is<|Assistant|>")) + self.assertEqual([block["type"] for block in content], ["thinking", "text"]) + # The continued turn stays open, and the output resumes its answer. + with patch.object(OpenAIServingChat, "prepare_native_prompt", continued): + tail, content = respond(" 42.") + self.assertTrue(tail.endswith("<|Assistant|>The answer is")) + self.assertEqual(content, [{"type": "text", "text": " 42."}]) + + serving, engine = self._responses() + self._respond( + serving, + engine, + "42.", + tools=[SEARCH_TOOL], + input=[*_search_history([WEATHER_TOOL]), prefill], + ) + prompt = engine.prompts[-1] + self.assertEqual(_schema_names(prompt), ["tool_search", "get_weather"]) + self.assertTrue( + prompt.endswith("<|User|>The answer is<|Assistant|>") + ) + + def test_messages_deferred_tool_loads_where_referenced(self): + anthropic, engine = self._messages() + referenced = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "found", + "name": "search_docs", + "input": {"q": "ticket"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "found", + "content": [ + {"type": "tool_reference", "tool_name": "create_ticket"} + ], + } + ], + }, + ] + for name, loading, anchor in ( + ("tool_reference", referenced, ""), + ("tool_addition", [ADD_TICKET], ASK_TICKET["content"]), + ): + with self.subTest(name): + request = self._messages_request( + [ASK_TICKET, *loading], + tool_choice={"type": "tool", "name": "create_ticket"}, + ) + chat_request = anthropic._convert_to_chat_completion_request(request) + prompt = self.tokenizer.decode(chat_request.input_ids) + self.assertGreater( + prompt.index("### Updated Tool Schemas"), prompt.index(anchor) + ) + # Calls are constrained to the tools shown, `strict` included. + self.assertEqual( + [ + (tool.function.name, bool(tool.function.strict)) + for tool in anthropic.openai_serving_chat._effective_tools( + chat_request + ) + ], + [("search_docs", False), ("create_ticket", True)], + ) + self.assertEqual( + _schema_names(prompt), ["search_docs", "create_ticket"] + ) + + engine.outputs.append(_dsml_call("create_ticket", order="A-1")) + response = json.loads( + asyncio.run(anthropic.handle_messages(request, None)).body + ) + self.assertEqual(engine.prompts[-1], prompt) + self.assertEqual(response["stop_reason"], "tool_use") + self.assertEqual( + [(block["name"], block["input"]) for block in response["content"]], + [("create_ticket", {"order": "A-1"})], + ) + + def test_messages_tool_choice_follows_loaded_tools(self): + anthropic, _ = self._messages() + for fields, error in ( + ({"tool_choice": {"type": "tool", "name": "create_ticket"}}, "deferred"), + ( + {"tools": [{**SEARCH_DOCS_TOOL, "defer_loading": True}, TICKET_TOOL]}, + "cannot be deferred", + ), + ): + with self.subTest(error=error), self.assertRaisesRegex(ValueError, error): + anthropic._convert_to_chat_completion_request( + self._messages_request([ASK_TICKET], **fields) + ) + + # `none` hides even the tools the history loaded. + chat_request = anthropic._convert_to_chat_completion_request( + self._messages_request( + [ASK_TICKET, ADD_TICKET], tool_choice={"type": "none"} + ) + ) + self.assertEqual( + _schema_names(self.tokenizer.decode(chat_request.input_ids)), [] + ) + self.assertEqual( + anthropic.openai_serving_chat._effective_tools(chat_request), [] + ) + + def test_messages_inline_tool_definition_loads_where_defined(self): + """A custom tool a `tool_addition` block defines by value renders as the + same tool declared deferred and added by reference.""" + anthropic, engine = self._messages() + named = {"tool_choice": {"type": "tool", "name": "create_ticket"}} + again = [{"role": "assistant", "content": "Done"}, ASK_TICKET] + + def prompt(messages, **fields): + chat_request = anthropic._convert_to_chat_completion_request( + self._messages_request(messages, **fields) + ) + return self.tokenizer.decode(chat_request.input_ids), chat_request + + by_reference, _ = prompt([ASK_TICKET, ADD_TICKET], **named) + request = self._messages_request( + [ASK_TICKET, DEFINE_TICKET], tools=[SEARCH_DOCS_TOOL], **named + ) + chat_request = anthropic._convert_to_chat_completion_request(request) + self.assertEqual(self.tokenizer.decode(chat_request.input_ids), by_reference) + self.assertGreater( + by_reference.index('{"name": "create_ticket"'), + by_reference.index(ASK_TICKET["content"]), + ) + self.assertEqual( + [ + (tool.function.name, bool(tool.function.strict)) + for tool in anthropic.openai_serving_chat._effective_tools(chat_request) + ], + [("search_docs", False), ("create_ticket", True)], + ) + engine.outputs.append(_dsml_call("create_ticket", order="A-1")) + response = json.loads( + asyncio.run(anthropic.handle_messages(request, None)).body + ) + self.assertEqual( + [(block["name"], block["input"]) for block in response["content"]], + [("create_ticket", {"order": "A-1"})], + ) + + # A definition of a tool already offered unchanged adds nothing; one equal + # to a deferred declaration loads that declaration. + once, _ = prompt([ASK_TICKET, ADD_TICKET, *again]) + self.assertEqual( + prompt( + [ASK_TICKET, DEFINE_TICKET, *again, DEFINE_TICKET], + tools=[SEARCH_DOCS_TOOL], + )[0], + once, + ) + self.assertEqual(prompt([ASK_TICKET, DEFINE_TICKET], **named)[0], by_reference) + shown = [ + SEARCH_DOCS_TOOL, + DEFINE_TICKET["content"][0]["tool"]["definition"], + {**TICKET_TOOL, "name": "other"}, + ] + unchanged, _ = prompt([ASK_TICKET, DEFINE_TICKET], tools=shown) + self.assertEqual(unchanged, prompt([ASK_TICKET], tools=shown)[0]) + self.assertNotIn("### Updated Tool Schemas", unchanged) + + def addition(tool): + return { + "role": "system", + "content": [{"type": "tool_addition", "tool": tool}], + } + + changed = addition( + { + "type": "tool_definition", + "definition": {**SEARCH_DOCS_TOOL, "name": "create_ticket"}, + } + ) + mcp_toolset = {"type": "mcp_toolset", "mcp_server_name": "crm"} + removal = { + "type": "tool_removal", + "tool": {"type": "tool_reference", "name": "search_docs"}, + } + generations = len(engine.requests) + for name, messages, tools, error in ( + ("redefined", [DEFINE_TICKET, changed], [SEARCH_DOCS_TOOL], "different"), + ("declared differently", [changed], None, "different"), + ( + "server tool definition", + [addition({"type": "tool_definition", "definition": mcp_toolset})], + None, + "custom tools only", + ), + ( + "mcp tool", + [ + addition( + { + "type": "mcp_tool_reference", + "server_name": "crm", + "name": "create_ticket", + } + ) + ], + None, + "mcp_tool_reference is not supported", + ), + ( + "mcp toolset", + [addition({"type": "mcp_toolset_reference", "server_name": "crm"})], + None, + "mcp_toolset_reference is not supported", + ), + ( + "removal", + [{"role": "system", "content": [removal]}], + None, + "tool_removal blocks are not supported", + ), + ("no tool shown initially", [DEFINE_TICKET], [], "cannot be deferred"), + ): + with self.subTest(name): + fields = {} if tools is None else {"tools": tools} + response = asyncio.run( + anthropic.handle_messages( + self._messages_request([ASK_TICKET, *messages], **fields), None + ) + ) + self.assertEqual(response.status_code, 400) + self.assertIn(error, json.loads(response.body)["error"]["message"]) + self.assertEqual(len(engine.requests), generations) + + def test_messages_inline_tool_definition_takes_effect_only_where_it_stands(self): + """A reference cannot load a tool an inline definition introduces later, and + an inline definition outside a system message is rejected even when it + repeats a tool already shown.""" + anthropic, engine = self._messages() + said = {"role": "assistant", "content": "Done"} + found = [ + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t", "name": "search_docs", "input": {}} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t", + "content": [ + {"type": "tool_reference", "tool_name": "create_ticket"} + ], + } + ], + }, + ] + definition = DEFINE_TICKET["content"][0]["tool"]["definition"] + cases = [ + ( + "addition before the definition", + [ASK_TICKET, ADD_TICKET, said, ASK_TICKET, DEFINE_TICKET], + [SEARCH_DOCS_TOOL], + "precedes the tool_definition", + ), + ( + "tool result reference before the definition", + [ASK_TICKET, *found, DEFINE_TICKET], + [SEARCH_DOCS_TOOL], + "precedes the tool_definition", + ), + ] + [ + ( + f"definition in a {role} message", + [ASK_TICKET, {"role": role, "content": DEFINE_TICKET["content"]}], + [SEARCH_DOCS_TOOL, definition], + "only be in system messages", + ) + for role in ("user", "assistant") + ] + for name, messages, tools, error in cases: + for stream in (False, True): + with self.subTest(name, stream=stream): + response = asyncio.run( + anthropic.handle_messages( + self._messages_request( + messages, tools=tools, stream=stream + ), + None, + ) + ) + self.assertEqual(response.status_code, 400) + self.assertIn(error, json.loads(response.body)["error"]["message"]) + self.assertEqual(engine.requests, []) + + def prompt(messages, **fields): + chat_request = anthropic._convert_to_chat_completion_request( + self._messages_request(messages, **fields) + ) + return self.tokenizer.decode(chat_request.input_ids) + + # A reference after the definition loads the tool, like a deferred declaration. + later = [said, ASK_TICKET, *found] + self.assertEqual( + prompt([ASK_TICKET, DEFINE_TICKET, *later], tools=[SEARCH_DOCS_TOOL]), + prompt([ASK_TICKET, ADD_TICKET, *later]), + ) + + def test_messages_clear_at_drops_a_reminder_once_a_user_message_follows(self): + anthropic, engine = self._messages() + reminder = { + "role": "system", + "clear_at": "next_user_message", + "content": [{"type": "text", "text": "REMINDER"}], + } + said = {"role": "assistant", "content": "Sure."} + called = { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t", "name": "search_docs", "input": {}} + ], + } + result = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "t", "content": "ok"}], + } + for native in (True, False): + for name, messages, rendered in ( + ("last message", [ASK_TICKET, reminder], True), + ("assistant only", [ASK_TICKET, reminder, said], True), + ("user follows", [ASK_TICKET, reminder, said, ASK_TICKET], False), + ("tool result follows", [ASK_TICKET, reminder, called, result], False), + ( + "never", + [ASK_TICKET, {**reminder, "clear_at": "never"}, called, result], + True, + ), + ): + with self.subTest(name, native=native): + engine.outputs.append("ok") + request = self._messages_request( + messages, + tools=[SEARCH_DOCS_TOOL, *([TICKET_TOOL] if native else [])], + ) + response = asyncio.run(anthropic.handle_messages(request, None)) + self.assertEqual(response.status_code, 200) + self.assertEqual("REMINDER" in engine.prompts[-1], rendered) + + generations = len(engine.requests) + text = reminder["content"][0] + for name, messages in ( + ("user role", [{**ASK_TICKET, "clear_at": "never"}]), + ( + "tool change", + [ASK_TICKET, {**reminder, "content": ADD_TICKET["content"]}], + ), + ( + "output_config", + [ASK_TICKET, {**reminder, "output_config": {"effort": "low"}}], + ), + ( + "cache_control", + [ + ASK_TICKET, + { + **reminder, + "content": [{**text, "cache_control": {"type": "ephemeral"}}], + }, + ], + ), + ("user directly after", [ASK_TICKET, reminder, ASK_TICKET]), + ("after an assistant", [ASK_TICKET, said, reminder]), + ): + with self.subTest(name): + response = asyncio.run( + anthropic.handle_messages(self._messages_request(messages), None) + ) + self.assertEqual(response.status_code, 400) + self.assertIn("clear_at", json.loads(response.body)["error"]["message"]) + self.assertEqual(len(engine.requests), generations) + + def test_prompt_is_recipes_rendering_in_sglang_settings(self): + """The prompt is recipe's own encoding of the request, with SGLang's + thinking and effort decisions spelled in its wire form.""" + import deepseek_recipe as recipe + + from sglang.srt.entrypoints.openai.protocol import ResponsesRequest + + encoding = recipe.DeepseekV41Encoding().with_tokenizer( + recipe.Tokenizer.from_file(_tokenizer_file()) + ) + + def reference(request_type, body): + request = request_type({"model": "DeepSeek-V4.1-Flash", **body}) + return encoding.encode( + request.convert(recipe.ConversionOptions()).conversation + ) + + anthropic, _ = self._messages() + body = { + "tools": [SEARCH_DOCS_TOOL, TICKET_TOOL], + "messages": [ASK_TICKET, ADD_TICKET], + } + # Thinking is off by default; an Anthropic `xhigh` effort is SGLang's `max`. + for fields, settings in ( + ({}, {"thinking": {"type": "disabled"}}), + ( + { + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "xhigh"}, + }, + {"output_config": {"effort": "max"}}, + ), + ): + with self.subTest(fields=fields): + chat_request = anthropic._convert_to_chat_completion_request( + self._messages_request(body["messages"], **fields) + ) + self.assertEqual( + chat_request.input_ids, + reference(recipe.MessagesRequest, {**body, **settings}), + ) + + serving, _ = self._responses() + body = {"tools": [SEARCH_TOOL], "input": "Weather in Paris?"} + request = ResponsesRequest(model="dummy", reasoning={"effort": "low"}, **body) + _, _, (prompt_ids,), _ = asyncio.run( + serving._make_request(request, None, self.tokenizer) + ) + self.assertEqual( + prompt_ids, + reference( + recipe.ResponsesRequest, {**body, "reasoning": {"effort": "low"}} + ), + ) + + if __name__ == "__main__": unittest.main()