vendor: DeepSeek-recipe namespace tools (#46370 + #49625)

Backport the upstream namespace-tool stack onto the B300 recipe base:
- #46370: Function.namespace / ChatMessage.action protocol fields.
- #49625: Responses/Messages native recipe rendering (convert_native),
  tool_search/defer_loading, namespace identity in/out/history replay.

Adapted to this base: skips serving_kimi_k3.py (not present), drops the
Minimax/Beam/wire-only dependencies added between 10-23 and the PR base,
and keeps the existing tool-parsing fallback where the newer
`parses_without_tools()` helper is unavailable.

Also includes the PR's Anthropic tool-reference plumbing and unit tests.
This commit is contained in:
2026-11-13 22:43:37 +08:00
parent f4de9f2141
commit 5fa8c23227
13 changed files with 2834 additions and 94 deletions
@@ -56,6 +56,7 @@ class AnthropicUsage(BaseModel):
class TextBlock(BaseModel):
type: Literal["text"] = "text"
text: str
cache_control: Optional[dict[str, Any]] = None
class ImageBlock(BaseModel):
@@ -91,6 +92,50 @@ class ToolReferenceBlock(BaseModel):
id: Optional[str] = None
class McpToolReference(BaseModel):
type: Literal["mcp_tool_reference"]
server_name: str
name: str
class McpToolsetReference(BaseModel):
type: Literal["mcp_toolset_reference"]
server_name: str
class ToolDefinition(BaseModel):
"""A tool given by value: `definition` is a `tools` entry."""
type: Literal["tool_definition"]
definition: dict[str, Any]
class ToolAdditionBlock(BaseModel):
"""Anthropic beta block: a ``system`` message offers a tool from its position
on. Rendered through deepseek-recipe, which reads only a ``tool_reference``
target."""
type: Literal["tool_addition"] = "tool_addition"
tool: Annotated[
Union[
ToolReferenceBlock, McpToolReference, McpToolsetReference, ToolDefinition
],
Field(discriminator="type"),
]
cache_control: Optional[dict[str, Any]] = None
class ToolRemovalBlock(BaseModel):
"""Anthropic beta block: a ``system`` message withdraws a tool."""
type: Literal["tool_removal"] = "tool_removal"
tool: Annotated[
Union[ToolReferenceBlock, McpToolReference, McpToolsetReference],
Field(discriminator="type"),
]
cache_control: Optional[dict[str, Any]] = None
class SearchResultBlock(BaseModel):
type: Literal["search_result"] = "search_result"
# ``source`` here is a URL/identifier string (unlike ImageBlock.source).
@@ -117,6 +162,8 @@ AnthropicContentBlock = Annotated[
ToolUseBlock,
ToolResultBlock,
ToolReferenceBlock,
ToolAdditionBlock,
ToolRemovalBlock,
SearchResultBlock,
ThinkingBlock,
RedactedThinkingBlock,
@@ -128,6 +175,8 @@ AnthropicContentBlock = Annotated[
class AnthropicMessage(BaseModel):
role: Literal["user", "assistant", "system"]
content: Union[str, list[AnthropicContentBlock]]
clear_at: Optional[Literal["next_user_message", "never"]] = None
output_config: Optional[dict[str, Any]] = None
# ---------- Tools (discriminated by ``type`` family) ----------
@@ -20,8 +20,10 @@ from sglang.srt.entrypoints.anthropic.protocol import (
AnthropicContentBlock,
AnthropicCountTokensRequest,
AnthropicCountTokensResponse,
AnthropicCustomTool,
AnthropicError,
AnthropicErrorResponse,
AnthropicMessage,
AnthropicMessageEndDelta,
AnthropicMessagesRequest,
AnthropicMessagesResponse,
@@ -32,6 +34,8 @@ from sglang.srt.entrypoints.anthropic.protocol import (
ContentBlockStopEvent,
ErrorEvent,
InputJsonDelta,
McpToolReference,
McpToolsetReference,
MessageDeltaEvent,
MessageStartEvent,
MessageStopEvent,
@@ -40,6 +44,8 @@ from sglang.srt.entrypoints.anthropic.protocol import (
TextDelta,
ThinkingBlock,
ThinkingDelta,
ToolDefinition,
ToolReferenceBlock,
ToolUseBlock,
is_server_tool,
)
@@ -152,6 +158,207 @@ def _extract_system_text(
return "\n".join(texts) if texts else None
def _block_types(request: AnthropicMessagesRequest) -> set[str]:
"""The block types of the request's system and messages, tool result contents
included."""
types = set()
for content in [request.system, *(message.content for message in request.messages)]:
for block in content if isinstance(content, list) else []:
types.add(block.type)
if block.type == "tool_result" and isinstance(block.content, list):
types.update(item.type for item in block.content)
return types
def _without_cleared_messages(
request: AnthropicMessagesRequest,
) -> AnthropicMessagesRequest:
"""`request` without the turn-scoped system messages (`clear_at:
"next_user_message"`) that a later user message has cleared."""
last_user = max(
(
index
for index, message in enumerate(request.messages)
if message.role == "user"
),
default=-1,
)
rendered = []
for index, message in enumerate(request.messages):
if message.clear_at is not None and message.role != "system":
raise ValueError(
f"messages.{index}.clear_at: clear_at is only permitted on role "
"'system' messages"
)
if message.clear_at == "next_user_message":
_check_turn_scoped(index=index, message=message)
_check_turn_scoped_placement(index=index, messages=request.messages)
if index < last_user:
continue
rendered.append(message)
return request.model_copy(update={"messages": rendered})
def _check_turn_scoped(index: int, message: AnthropicMessage) -> None:
scope = "a turn-scoped system message (clear_at: 'next_user_message')"
if message.output_config is not None:
raise ValueError(f"messages.{index}: output_config is not permitted on {scope}")
for position, block in enumerate(
message.content if isinstance(message.content, list) else []
):
if block.type != "text":
raise ValueError(
f"messages.{index}: a turn-scoped system message supports text "
"blocks only (clear_at: 'next_user_message')"
)
if block.cache_control is not None:
raise ValueError(
f"messages.{index}.content.{position}: cache_control is not "
f"permitted on {scope}"
)
def _check_turn_scoped_placement(index: int, messages: list[AnthropicMessage]) -> None:
# Consecutive system messages are placed as one group.
before = next(
(m.role for m in reversed(messages[:index]) if m.role != "system"), None
)
after = next((m.role for m in messages[index + 1 :] if m.role != "system"), None)
if before != "user" or after == "user":
raise ValueError(
f"messages.{index}: a turn-scoped system message (clear_at: "
"'next_user_message') must follow a user message and either end "
"messages or precede an assistant message"
)
def _declare_inline_tools(
request: AnthropicMessagesRequest,
) -> AnthropicMessagesRequest:
"""`request` with each custom tool a `tool_addition` block defines by value
declared in `tools` as deferred, and the block naming it by reference. A
definition of a tool already offered unchanged is dropped."""
tools = list(request.tools or [])
declared = {tool.name: tool for tool in tools}
offered = {tool.name for tool in tools if not tool.defer_loading}
unresolved: dict[str, str] = {}
messages = []
for index, message in enumerate(request.messages):
if isinstance(message.content, str):
messages.append(message)
continue
blocks = []
for position, block in enumerate(message.content):
where = f"messages.{index}.content.{position}"
for name in _referenced_tool_names(block):
if name not in declared:
unresolved.setdefault(name, where)
if block.type == "tool_addition" and block.tool.type != "tool_reference":
if message.role != "system":
raise ValueError(
f"{where}: tool_addition blocks can only be in system messages"
)
tool = _inline_custom_tool(where=where, target=block.tool)
# Declaring the tool makes it resolvable from the first message on.
if tool.name in unresolved:
raise ValueError(
f"{unresolved[tool.name]}: tool reference {tool.name!r} "
f"precedes the tool_definition that defines it at {where}"
)
earlier = declared.setdefault(tool.name, tool)
if earlier is tool:
tools.append(tool)
elif is_server_tool(earlier) or _definition(earlier) != _definition(
tool
):
raise ValueError(
f"{where}: tool_definition "
f"reuses the name {tool.name!r} with a different "
"definition; replacing a tool mid-conversation is not "
"supported"
)
if tool.name in offered:
continue
offered.add(tool.name)
block = block.model_copy(
update={"tool": ToolReferenceBlock(name=tool.name)}
)
blocks.append(block)
# recipe renders a system message left without blocks as an empty turn.
if blocks or not message.content:
messages.append(message.model_copy(update={"content": blocks}))
return request.model_copy(update={"tools": tools, "messages": messages})
def _referenced_tool_names(block: AnthropicContentBlock) -> list[Optional[str]]:
if block.type == "tool_addition" and block.tool.type == "tool_reference":
return [block.tool.name or block.tool.tool_name]
if block.type == "tool_reference":
return [block.tool_name or block.name]
if block.type == "tool_result" and isinstance(block.content, list):
return [name for item in block.content for name in _referenced_tool_names(item)]
return []
def _inline_custom_tool(
where: str, target: Union[McpToolReference, McpToolsetReference, ToolDefinition]
) -> AnthropicCustomTool:
if target.type != "tool_definition":
raise ValueError(
f"{where}: tool_addition with a {target.type} is not supported; this "
"server runs no MCP connector"
)
kind = target.definition.get("type")
if kind not in (None, "custom"):
raise ValueError(
f"{where}: tool_definition supports custom tools only, got a "
f"definition of type {kind!r}"
)
tool = AnthropicCustomTool.model_validate(target.definition)
if tool.defer_loading:
raise ValueError(
f"{where}: a tool_definition with defer_loading is not supported; "
"declare the tool in tools and add it by reference"
)
return tool.model_copy(update={"defer_loading": True})
def _definition(tool: AnthropicCustomTool) -> dict[str, Any]:
return tool.model_dump(exclude={"type", "defer_loading"})
def _native_block(block: dict[str, Any]) -> dict[str, Any]:
"""`block` with SGLang's accepted aliases spelled as deepseek-recipe reads them."""
kind = block["type"]
if kind == "tool_result":
content = block.get("content")
fields = {
"type": kind,
"tool_use_id": block.get("tool_use_id") or block.get("id"),
"content": (
[_native_block(item) for item in content]
if isinstance(content, list)
else content
),
"is_error": block.get("is_error"),
}
return {key: value for key, value in fields.items() if value is not None}
if kind == "tool_reference":
return {"type": kind, "tool_name": block.get("tool_name") or block.get("name")}
if kind == "tool_addition":
tool = block["tool"]
name = tool.get("name") or tool.get("tool_name")
return {"type": kind, "tool": {"type": "tool_reference", "name": name}}
return block
def _native_message(message: AnthropicMessage) -> dict[str, Any]:
data = message.model_dump(exclude_none=True)
if isinstance(data["content"], list):
data["content"] = [_native_block(block) for block in data["content"]]
return data
def _wrap_sse_event(data: str, event_type: str) -> str:
"""Format an Anthropic SSE event with event type and data lines."""
return f"event: {event_type}\ndata: {data}\n\n"
@@ -218,6 +425,24 @@ class AnthropicServing:
self, anthropic_request: AnthropicMessagesRequest
) -> ChatCompletionRequest:
"""Convert an Anthropic Messages request to an OpenAI ChatCompletion request."""
anthropic_request = _declare_inline_tools(
_without_cleared_messages(anthropic_request)
)
block_types = _block_types(anthropic_request)
if "tool_removal" in block_types:
raise ValueError(
"tool_removal blocks are not supported; a tool cannot be withdrawn "
"mid-conversation"
)
native = self.openai_serving_chat.native_tool_loading and (
bool(block_types & {"tool_reference", "tool_addition"})
or any(
not is_server_tool(tool) and tool.defer_loading
for tool in anthropic_request.tools or []
)
)
if not native and "tool_addition" in block_types:
raise ValueError("tool_addition blocks are not supported by this model")
openai_messages = []
def _convert_anthropic_image_source_to_openai_part(
@@ -719,8 +944,51 @@ class AnthropicServing:
elif chat_request.tools:
chat_request.tool_choice = "auto"
if native:
self._prepare_native_prompt(anthropic_request, chat_request)
return chat_request
def _prepare_native_prompt(
self,
anthropic_request: AnthropicMessagesRequest,
chat_request: ChatCompletionRequest,
) -> None:
"""Render the request's own wire form into `chat_request`, whose tools
become the ones the history has loaded."""
body = {
"messages": [
_native_message(message) for message in anthropic_request.messages
],
# recipe reads a tool with a `type` as a server tool.
"tools": [
tool.model_dump(exclude_none=True, exclude={"type"})
for tool in anthropic_request.tools or []
if not is_server_tool(tool)
],
}
system = anthropic_request.system
if system:
body["system"] = (
system
if isinstance(system, str)
else [block.model_dump(exclude_none=True) for block in system]
)
choice = anthropic_request.tool_choice
if choice is not None and choice.type == "none":
body["tool_choice"] = {"type": "none"}
tools = self.openai_serving_chat.prepare_native_prompt(
chat_request, protocol="messages", body=body
).tools
if (
choice is not None
and choice.type == "tool"
and not any(tool.function.name == choice.name for tool in tools)
):
raise ValueError(
f"tool_choice names {choice.name!r}, a deferred tool no "
"tool_reference or tool_addition block has loaded"
)
async def _handle_non_streaming(
self,
chat_request: ChatCompletionRequest,
@@ -3,7 +3,7 @@ concrete adapters load only when selected."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol, Union
import msgspec
@@ -15,8 +15,27 @@ if TYPE_CHECKING:
ToolCall,
)
from sglang.srt.function_call.core_types import ToolCallItem
from sglang.srt.multimodal.media_io import ImageData
EncodingBackend = Literal["builtin", "recipe"]
HistoryToolCallArguments = Literal["strict", "lenient", "as_sent"]
NativeProtocol = Literal["responses", "messages"]
class NativePrompt(msgspec.Struct, frozen=True, kw_only=True):
"""A Responses or Messages request rendered from its own wire form."""
prompt_ids: list[int]
tools: list[Tool]
"""The tools the model can call after the request's history: those loaded from
the start and those the history has loaded since."""
image_data: list[ImageData] = []
"""The images the prompt's placeholders stand for, in order."""
namespace_names: dict[str, tuple[str, str]] = {}
"""The call name of each namespace member, mapped to its namespace and the
name the request gives it there."""
continued_turn: Optional[dict[str, Any]] = None
"""The final assistant message the prompt continues, as a chat message."""
class ChatOutputHooks(msgspec.Struct, frozen=True, kw_only=True):
@@ -78,6 +97,25 @@ class ChatEncodingOutputAdapter(Protocol):
) -> ResponsesOutputStream: ...
class NativeRequestEncoder(Protocol):
def encode_native(
self,
*,
protocol: NativeProtocol,
body: dict[str, Any],
thinking: bool,
effort: Union[str, int, float, None],
response_schema: Optional[dict[str, Any]],
continue_final_message: bool,
) -> NativePrompt:
"""``body`` carries the request's conversation, tools and, for
``tool_choice="none"``, its choice; ``thinking`` and ``effort`` are the
handler's decisions, as for ``encode_messages``. ``response_schema`` is
the JSON schema the answer follows; a trailing assistant turn is continued
under ``continue_final_message`` and otherwise re-sent as a user turn."""
...
class ChatEncodingAdapter(Protocol):
collects_openai_media: bool
"""Whether media still has to be collected from the OpenAI messages after
@@ -92,6 +130,12 @@ class ChatEncodingAdapter(Protocol):
"""None when the configured parsers leave output to the text parsers."""
...
@property
def native_encoder(self) -> Optional[NativeRequestEncoder]:
"""None when Responses and Messages requests that load tools mid-conversation
have no rendering of their own wire form."""
...
def encode_messages(
self,
*,
@@ -17,6 +17,7 @@ from __future__ import annotations
import logging
import time
import msgspec
import unicodedata
import uuid
from dataclasses import dataclass
@@ -743,6 +744,10 @@ class ChatCompletionMessageGenericParam(BaseModel):
name: Optional[str] = None
phase: Optional[Literal["commentary", "final_answer"]] = None
reasoning_content: Optional[str] = None
action: Optional[str] = Field(
default=None,
description="Action tag for assistant messages (used by deepseek-recipe).",
)
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
# DeepSeek API prefix completion: continue this final assistant message.
@@ -798,12 +803,18 @@ class Function(BaseModel):
parameters: Optional[object] = None
strict: bool = False
defer_loading: Optional[bool] = None
namespace: Optional[str] = Field(
default=None,
description="Tool namespace for disambiguation (used by deepseek-recipe).",
)
@model_serializer(mode="wrap")
def _serialize(self, handler):
data = handler(self)
if self.defer_loading is None:
data.pop("defer_loading", None)
if self.namespace is None:
data.pop("namespace", None)
return data
@@ -913,6 +924,15 @@ class ChatCompletionRequest(PDRoutingFields):
# renders these, while ``tools`` (flattened to ``<namespace>.<function>``) still
# drives tool-call parsing.
_template_tools: Optional[List[Dict[str, Any]]] = PrivateAttr(default=None)
# Set with ``input_ids`` by a caller that rendered them itself: the images the
# prompt's placeholders stand for, in order.
_input_image_data: Optional[List[Any]] = PrivateAttr(default=None)
# Whether ``tools`` are the only tools the rendered prompt shows, so a call
# to any other tool is dropped rather than returned.
_drops_unlisted_tool_calls: bool = PrivateAttr(default=False)
# Set with ``input_ids`` under ``continue_final_message``: the final assistant
# message the prompt continues, where output parsing resumes.
_continued_turn: Optional[Dict[str, Any]] = PrivateAttr(default=None)
tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field(
default="auto", examples=["none"]
) # noqa
@@ -1252,6 +1272,7 @@ class ChatMessage(BaseModel):
role: Optional[str] = None
content: Optional[str] = None
reasoning_content: Optional[str] = None
action: Optional[str] = None
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
@model_serializer(mode="wrap")
@@ -1259,6 +1280,8 @@ class ChatMessage(BaseModel):
data = handler(self)
if self.tool_calls is None:
data.pop("tool_calls", None)
if self.action is None:
data.pop("action", None)
return data
@@ -1830,6 +1853,40 @@ class ResponseTool(BaseModel):
return self
class ResponseToolState(msgspec.Struct, frozen=True):
"""Public tool metadata and name mappings for one translated request."""
public_tools: list[ResponseTool] | None = None
public_choice: str | dict[str, Any] | None = None
names: dict[str, dict[str, str]] = {}
# The tools the model can call after the history, from a request rendered in
# its own wire form; None when `tools` alone declares them.
chat_tools: list[Tool] | None = None
# ``custom`` tools a ``tool_search_output`` loaded, beside the declared ones.
discovered_custom_names: frozenset[str] = frozenset()
class ResponseToolSearchCall(BaseModel):
"""A client-executed ``tool_search`` call; the pinned openai SDK has no type."""
id: str
type: Literal["tool_search_call"] = "tool_search_call"
call_id: str
execution: Literal["client"] = "client"
arguments: Dict[str, Any]
status: Literal["in_progress", "completed", "incomplete"]
class ResponseToolSearchCallEvent(BaseModel):
"""``response.output_item.added`` / ``.done`` for a tool search call, which the
SDK's event types reject."""
type: Literal["response.output_item.added", "response.output_item.done"]
sequence_number: int
output_index: int
item: ResponseToolSearchCall
class ResponseInputMessageParam(EasyInputMessageParam, total=False):
phase: Optional[Literal["commentary", "final_answer"]]
@@ -1856,6 +1913,10 @@ ResponseInputOutputItem: TypeAlias = Union[
class ResponsesRequest(PDRoutingFields):
"""Request body for v1/responses endpoint."""
_tool_state: ResponseToolState = PrivateAttr(default_factory=ResponseToolState)
# By call_id, the name the model called a tool by when the output item
# reports another; the stored history replays the call under it.
_called_tool_names: Dict[str, str] = PrivateAttr(default_factory=dict)
_response_parser_prefix: str = PrivateAttr(default="")
_response_parser_grammar_start: Optional[str] = PrivateAttr(default=None)
_response_parser_json_response_format: bool = PrivateAttr(default=False)
@@ -2176,6 +2237,7 @@ class ResponsesResponse(BaseModel):
ResponseOutputItem,
ResponseReasoningItem,
ResponseFunctionToolCall,
ResponseToolSearchCall,
]
] = Field(default_factory=list)
status: Literal[
@@ -2241,6 +2303,7 @@ class ResponsesResponse(BaseModel):
ResponseOutputItem,
ResponseReasoningItem,
ResponseFunctionToolCall,
ResponseToolSearchCall,
]
],
status: str,
@@ -957,12 +957,13 @@ class OpenAIServingChat(OpenAIServingBase):
The reasoning parser is told the same thing, so channel-framed formats
keep their framing intact exactly when a tool-call parser consumes it.
"""
return bool(
request.tool_choice != "none"
and self._effective_tools(request)
and self.tool_call_parser
and not request._response_parser_json_response_format
)
if not self.tool_call_parser or request._response_parser_json_response_format:
return False
if request._drops_unlisted_tool_calls and request.tool_choice == "none":
# The history showed the model tools, so it can still write a call;
# parsing drops it, where skipping the parser would return its markup.
return True
return bool(request.tool_choice != "none" and self._effective_tools(request))
def _validate_request(self, request: ChatCompletionRequest) -> str | None:
"""Validate that the input is valid."""
@@ -1367,18 +1368,7 @@ class OpenAIServingChat(OpenAIServingBase):
self, request: ChatCompletionRequest, is_multimodal: bool
) -> MessageProcessingResult:
"""Process chat messages and apply chat template"""
if self.default_chat_template_kwargs:
ctk = dict(request.chat_template_kwargs or {})
for k, v in self.default_chat_template_kwargs.items():
ctk.setdefault(k, v)
request.chat_template_kwargs = ctk
effort = ctk.get("reasoning_effort")
if effort is not None and request.reasoning_effort is None:
request.reasoning_effort = effort
normalize_hunyuan_reasoning_effort(
request, self.reasoning_parser, self.template_manager.reasoning_config
)
self._apply_request_defaults(request)
# GptOss model needs to keep special tokens for harmony parsing
if self.is_gpt_oss or self.is_gemma4:
@@ -1484,7 +1474,7 @@ class OpenAIServingChat(OpenAIServingBase):
result = MessageProcessingResult(
prompt="",
prompt_ids=request.input_ids,
image_data=None,
image_data=request._input_image_data,
audio_data=None,
video_data=None,
modalities=[],
@@ -1540,6 +1530,64 @@ class OpenAIServingChat(OpenAIServingBase):
result.reasoning_end_token_ids = list(token_ids)
return result
def _apply_request_defaults(self, request: ChatCompletionRequest) -> None:
"""Settle the thinking and effort that rendering reads; repeating it
changes nothing."""
if self.default_chat_template_kwargs:
ctk = dict(request.chat_template_kwargs or {})
for k, v in self.default_chat_template_kwargs.items():
ctk.setdefault(k, v)
request.chat_template_kwargs = ctk
effort = ctk.get("reasoning_effort")
if effort is not None and request.reasoning_effort is None:
request.reasoning_effort = effort
normalize_hunyuan_reasoning_effort(
request, self.reasoning_parser, self.template_manager.reasoning_config
)
@property
def native_tool_loading(self) -> bool:
"""Whether Responses and Messages requests that load tools mid-conversation
render from their own wire form (see `prepare_native_prompt`)."""
return (
self._encoding_adapter is not None
and self._encoding_adapter.native_encoder is not None
)
def prepare_native_prompt(
self,
request: ChatCompletionRequest,
*,
protocol: chat_encoding_adapters.NativeProtocol,
body: dict[str, Any],
) -> chat_encoding_adapters.NativePrompt:
"""Render `body`, the Responses or Messages form of `request`, into its
`input_ids` and images, and set its `tools` to the tools the model can call
after the history; rendering then keeps only grammar, stops and parsing."""
self._apply_request_defaults(request)
response_format = request.response_format
prompt = self._encoding_adapter.native_encoder.encode_native(
protocol=protocol,
body=body,
thinking=self._get_reasoning_from_request(request),
effort=request.reasoning_effort,
response_schema=(
response_format.json_schema.schema_
if response_format is not None and response_format.json_schema
else None
),
continue_final_message=request.continue_final_message,
)
if prompt.image_data and not self.tokenizer_manager.model_config.is_multimodal:
raise ValueError("image input is not supported for this model")
request.input_ids = prompt.prompt_ids
request.tools = prompt.tools or None
request._input_image_data = prompt.image_data or None
request._drops_unlisted_tool_calls = True
request._continued_turn = prompt.continued_turn
return prompt
def _apply_jinja_template(
self,
request: ChatCompletionRequest,
@@ -5,15 +5,20 @@ request semantics."""
from __future__ import annotations
from collections.abc import Sequence
import re
from typing import TYPE_CHECKING, Any, Optional, Union
from sglang.srt.entrypoints.openai.chat_encoding_adapters import (
ADAPTER_OUTPUT_PARSERS,
NativePrompt,
NativeProtocol,
select_new_output_ids,
token_output_sse_chunks,
)
from sglang.srt.entrypoints.openai.protocol import (
Function,
FunctionResponse,
Tool,
ToolCall,
)
from sglang.srt.environ import envs
@@ -26,7 +31,9 @@ from sglang.srt.parser.deepseek_recipe_output import (
whole_tool_calls,
)
from sglang.srt.parser.deepseek_recipe_renderer import (
BUDGET_TIER,
DeepSeekRecipe,
loaded_tools,
parse_effort_budget,
)
from sglang.srt.runtime_context import get_serving
@@ -37,7 +44,6 @@ if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
ResponsesRequest,
Tool,
)
DEFAULT_EFFORT_ENVS = {
@@ -45,6 +51,12 @@ DEFAULT_EFFORT_ENVS = {
"dsv41": envs.SGLANG_DSV41_REASONING_EFFORT,
}
JSON_RESPONSE_FORMATS = ("json_object", "json_schema")
# V4 renders deferred tools up front and never loads one mid-conversation.
NATIVE_TOOL_LOADING_SPEC = "dsv41"
NATIVE_EFFORT_FIELDS: dict[str, str] = {
"responses": "reasoning",
"messages": "output_config",
}
class DeepSeekRecipeChatEncodingAdapter:
@@ -96,6 +108,13 @@ class DeepSeekRecipeChatEncodingAdapter:
def output(self) -> Optional[DeepSeekRecipeChatEncodingAdapter]:
return self if self._token_output else None
def validate_request(self, request: ChatCompletionRequest) -> Optional[str]:
return None
@property
def native_encoder(self) -> Optional[DeepSeekRecipeChatEncodingAdapter]:
return self if self._spec == NATIVE_TOOL_LOADING_SPEC else None
def encode_messages(
self,
*,
@@ -149,6 +168,66 @@ class DeepSeekRecipeChatEncodingAdapter:
prompt_ids += self.deepseek_recipe.encode_text(task_prefix)
return prompt_ids
def encode_native(
self,
*,
protocol: NativeProtocol,
body: dict[str, Any],
thinking: bool,
effort: Union[str, int, float, None],
response_schema: Optional[dict[str, Any]],
continue_final_message: bool,
) -> NativePrompt:
tier, budget = self._effort(effort)
# recipe reads an effort as a request to think.
if thinking and tier is not None:
body = {**body, NATIVE_EFFORT_FIELDS[protocol]: {"effort": tier}}
conversation = self.deepseek_recipe.with_host_settings(
self.deepseek_recipe.convert_native(protocol, body, thinking=thinking),
response_schema=response_schema,
continue_final_message=continue_final_message,
)
last = conversation.messages[-1] if conversation.messages else None
# recipe's Messages conversion drops `strict`, which SGLang's grammar enforces.
strict = {tool.get("name") for tool in body["tools"] if tool.get("strict")}
tools = [
Tool(
type="function",
function=Function(
name=tool.name,
description=tool.description,
parameters=tool.parameters,
strict=True if tool.name in strict else tool.strict,
),
)
for tool in loaded_tools(conversation)
]
return NativePrompt(
prompt_ids=self.deepseek_recipe.encode_conversation(
conversation, effort_budget=budget
),
tools=tools,
image_data=[
ImageData(url=url)
for url in self.deepseek_recipe.image_urls(conversation)
],
namespace_names=(
self.deepseek_recipe.namespace_tool_names(_responses_namespaces(body))
if protocol == "responses"
else {}
),
continued_turn=(
{"role": "assistant", "content": last.content}
| (
{"reasoning_content": last.reasoning_content}
if last.reasoning_content is not None
else {}
)
if last is not None and last.prefix
else None
),
)
def _encode(
self,
body: dict[str, Any],
@@ -157,21 +236,29 @@ class DeepSeekRecipeChatEncodingAdapter:
task: Optional[str] = None,
image_urls: Optional[list[str]] = None,
) -> list[int]:
if effort is None or effort == "none":
effort = self._default_effort
budget = None
if effort is not None and not isinstance(effort, str):
if self._spec != "dsv41":
raise ValueError(
f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}"
)
budget = parse_effort_budget(effort)
elif effort is not None:
body["reasoning_effort"] = effort
tier, budget = self._effort(effort)
if budget is None and tier is not None:
body["reasoning_effort"] = tier
return self.deepseek_recipe.encode(
body, task=task, effort_budget=budget, image_urls=image_urls
)
def _effort(
self, effort: Union[str, int, float, None]
) -> tuple[Optional[str], Optional[int]]:
"""The named tier recipe renders and, for a V4.1 integer budget, the budget
that replaces the tier's line."""
effort = _numeric_effort(effort)
if effort is None or effort == "none":
effort = self._default_effort
if effort is None or isinstance(effort, str):
return effort, None
if self._spec != "dsv41":
raise ValueError(
f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}"
)
return BUDGET_TIER, parse_effort_budget(effort)
def parse_chat_response(
self,
*,
@@ -320,7 +407,7 @@ class DeepSeekRecipeChatEncodingAdapter:
self, request: ChatCompletionRequest
) -> DeepSeekRecipeOutputParser:
response_format = request.response_format
continued = (
continued = request._continued_turn or (
_trailing_assistant_turn(
[
message.model_dump(exclude_unset=True, by_alias=True, mode="json")
@@ -344,6 +431,11 @@ class DeepSeekRecipeChatEncodingAdapter:
separate_reasoning=self._separate_reasoning and request.separate_reasoning,
stream_reasoning=request.stream_reasoning,
stop=_stop_strings(request.stop),
callable_names=(
_native_callable_names(request.tools or [], request.tool_choice)
if request._drops_unlisted_tool_calls
else None
),
)
def _new_responses_parser(
@@ -353,12 +445,25 @@ class DeepSeekRecipeChatEncodingAdapter:
separate_reasoning: bool,
parse_tool_calls: bool,
) -> DeepSeekRecipeOutputParser:
native = request._tool_state.chat_tools is not None
return DeepSeekRecipeOutputParser(
self.deepseek_recipe,
thinking_stage=self.deepseek_recipe.parse_start(thinking=require_reasoning),
parse_tool_calls=parse_tool_calls,
# Under `none` the tool parser runs only to drop calls; a JSON answer
# must be read as JSON, or a call marker spelled in a string ends it.
parse_json_output=native
and request.tool_choice == "none"
and request.has_json_schema_constraint(),
separate_reasoning=separate_reasoning,
stop=_stop_strings(request.stop),
callable_names=(
_native_callable_names(
request._tool_state.chat_tools, request.tool_choice
)
if native
else None
),
)
def _tool_call_id(self, index: int, name: str, history_tool_calls_cnt: int) -> str:
@@ -390,6 +495,37 @@ class DeepSeekRecipeResponsesStream:
return delta.reasoning, delta.content, _tool_call_items(delta.tool_calls)
def _responses_namespaces(body: dict[str, Any]) -> list[dict[str, Any]]:
"""The namespace tools a Responses body declares and its history discovers, in
the order recipe's conversion names their members."""
discovered = [
tool
for item in body["input"]
if isinstance(item, dict) and item.get("type") == "tool_search_output"
for tool in item["tools"]
]
return [
tool for tool in [*body["tools"], *discovered] if tool["type"] == "namespace"
]
def _null_content_as_empty(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""As `_process_messages` renders them: a null content is ""."""
return [
message if message.get("content") is not None else {**message, "content": ""}
for message in messages
]
def _numeric_effort(
effort: Union[str, int, float, None],
) -> Union[str, int, float, None]:
"""A numeric string effort ("30", "0.3") as the number it spells."""
if isinstance(effort, str) and re.fullmatch(r"\d+(\.\d+)?", effort.strip()):
return float(effort) if "." in effort else int(effort)
return effort
def _trailing_assistant_turn(
messages: list[dict[str, Any]],
) -> Optional[dict[str, Any]]:
@@ -405,6 +541,18 @@ def _stop_strings(stop: Union[str, list[str], None]) -> list[str]:
return [stop] if isinstance(stop, str) else stop or []
def _native_callable_names(
tools: list[Tool], tool_choice: Any
) -> Optional[frozenset[str]]:
"""The names of `tools`, all that a natively rendered prompt shows the model;
None when calls to other tools are returned. `none` allows no call at all."""
if tool_choice == "none":
return frozenset()
if envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get():
return None
return frozenset(tool.function.name for tool in tools)
def _tool_call_items(
calls: Sequence[Union[RecipeToolCall, RecipeToolCallDelta]],
) -> list[ToolCallItem]:
@@ -36,7 +36,10 @@ if TYPE_CHECKING:
class InklingChatEncodingAdapter:
collects_openai_media = True
strict_tool_call_arguments = True
history_tool_call_arguments = "strict"
renders_response_format = False
uncounted_prompt_tokens = 0
native_encoder = None
def __init__(
self,
@@ -13,6 +13,7 @@ from http import HTTPStatus
from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterator, Optional, Union
import jinja2
import msgspec
import openai.types.responses as openai_responses_types
import orjson
from fastapi import Request
@@ -43,6 +44,7 @@ from openai.types.responses.response_reasoning_summary_part_done_event import (
Part as ResponseReasoningSummaryDonePart,
)
from openai_harmony import Message as OpenAIMessage
from pydantic import BaseModel, TypeAdapter
from sglang.srt.entrypoints.context import (
ConversationContext,
@@ -70,7 +72,12 @@ from sglang.srt.entrypoints.openai.protocol import (
ResponseOutputMessage,
ResponsesRequest,
ResponsesResponse,
ResponseTool,
ResponseToolSearchCall,
ResponseToolSearchCallEvent,
Tool,
ToolChoice,
ToolChoiceFuncName,
UsageInfo,
)
from sglang.srt.entrypoints.openai.responses_adapters import (
@@ -114,6 +121,100 @@ class _MediaInputValidationError(ValueError):
pass
class ResponseToolChoiceError(ValueError):
"""A response tool_choice that names a tool the model cannot call."""
TOOL_SEARCH = "tool_search"
TOOL_SEARCH_ITEMS = ("tool_search_call", "tool_search_output")
APPLY_PATCH = "apply_patch"
NAMESPACE_MEMBERS_ERROR = (
"namespace tools must list function tools, each with a type and a name"
)
def _holds_function_tools(members: Any) -> bool:
return isinstance(members, list) and all(
isinstance(member, dict)
and member.get("type") == "function"
and isinstance(member.get("name"), str)
for member in members
)
def _defers_loading(tool: ResponseTool) -> bool:
if tool.type == "namespace":
return bool(tool.defer_loading) or any(
isinstance(member, dict) and member.get("defer_loading")
for member in tool.tools or []
)
return tool.type in ("function", "custom") and bool(tool.defer_loading)
def _defer_tool_loading(
tools: list[dict[str, Any]], items: list
) -> tuple[list[dict[str, Any]], list]:
"""`tools` without those that defer loading, and `items` whose
`tool_search_output` entries naming a deferred tool take the fields they
leave out from its declaration, so a search result loads it where it returns.
recipe's Responses conversion renders a deferred tool up front and rejects a
search result that repeats a declared name."""
deferred: dict[tuple[Optional[str], str], dict[str, Any]] = {}
shown = []
for tool in tools:
if tool["type"] == "namespace":
members = []
for member in tool["tools"]:
if tool.get("defer_loading") or member.get("defer_loading"):
deferred[(tool["name"], member["name"])] = member
else:
members.append(member)
if members:
shown.append({**tool, "tools": members})
elif tool["type"] in ("function", "custom") and tool.get("defer_loading"):
deferred[(None, tool["name"])] = tool
else:
shown.append(tool)
def loaded(tool: dict[str, Any], namespace: Optional[str] = None) -> dict:
if tool["type"] == "namespace":
members = [loaded(member, tool["name"]) for member in tool["tools"]]
return {**tool, "tools": members}
declared = deferred.get((namespace, tool.get("name")))
if declared is None or declared["type"] != tool["type"]:
return tool
return declared | {k: v for k, v in tool.items() if v is not None}
if deferred:
items = [
(
{**item, "tools": [loaded(tool) for tool in item["tools"]]}
if isinstance(item, dict) and item.get("type") == "tool_search_output"
else item
)
for item in items
]
return shown, items
def _tool_search_call(
item_id: str, call_id: str, arguments: str
) -> ResponseToolSearchCall:
"""A search call reports object arguments; one whose arguments are not a JSON
object is incomplete."""
try:
parsed = json.loads(arguments)
except ValueError:
parsed = None
complete = isinstance(parsed, dict)
return ResponseToolSearchCall(
id=item_id,
call_id=call_id,
arguments=parsed if complete else {},
status="completed" if complete else "incomplete",
)
def _build_output_text_logprobs(meta_info: dict) -> list[Logprob]:
"""Reshape decoded ``meta_info`` logprobs into the Responses logprob type,
covering every generated token."""
@@ -319,9 +420,21 @@ class OpenAIServingResponses(OpenAIServingChat):
# FIXME: If the engine is dead, raise an error
# This is required for the streaming case
# Such a request's callable tools are known once its history renders,
# so `_make_request` checks its tool_choice instead.
native = self._renders_natively(request)
if native:
tools_error = self._native_tools_error(request)
if tools_error is not None:
return self.create_error_response(tools_error, param="tools")
# ``tool_choice="required"`` needs a tool the parser can actually force.
if request.tool_choice == "required" and not any(
tool.type in ("function", "custom") for tool in (request.tools or [])
if (
not native
and request.tool_choice == "required"
and not any(
tool.type in ("function", "custom") for tool in (request.tools or [])
)
):
return self.create_error_response(
'tool_choice="required" requires at least one tool with '
@@ -330,9 +443,13 @@ class OpenAIServingResponses(OpenAIServingChat):
)
tool_choice = request.effective_tool_choice()
if isinstance(tool_choice, dict) and not any(
tool.type in ("function", "custom") and tool.name == tool_choice["name"]
for tool in request.tools or []
if (
not native
and isinstance(tool_choice, dict)
and not any(
tool.type in ("function", "custom") and tool.name == tool_choice["name"]
for tool in request.tools or []
)
):
return self.create_error_response(
f"Tool {tool_choice['name']!r} is not declared in tools",
@@ -415,6 +532,8 @@ class OpenAIServingResponses(OpenAIServingChat):
except _MediaInputValidationError as e:
return self.create_error_response(str(e))
except ResponseToolChoiceError as e:
return self.create_error_response(str(e), param="tool_choice")
except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
logger.exception("Error in preprocessing prompt inputs")
return self.create_error_response(f"{e} {e.__cause__}")
@@ -684,7 +803,10 @@ class OpenAIServingResponses(OpenAIServingChat):
prev_response: Optional[ResponsesResponse],
tokenizer: Any,
):
messages = self._construct_input_messages(request, prev_response)
native = self._renders_natively(request)
messages = (
[] if native else self._construct_input_messages(request, prev_response)
)
chat_tools = self._response_tools_to_chat_tools(request)
chat_request = ChatCompletionRequest(
@@ -707,6 +829,8 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_template_kwargs=request.chat_template_kwargs,
)
chat_request._template_tools = self._response_tools_to_template_tools(request)
if native:
self._prepare_native_responses_prompt(request, chat_request)
media_error = self._validate_media_content(chat_request)
if media_error:
@@ -906,10 +1030,7 @@ class OpenAIServingResponses(OpenAIServingChat):
else:
self.msg_store[response.id] = self._response_input_history(
request
) + [
item.model_dump(exclude_none=True)
for item in response.output
]
) + self._replayed_output_items(request, response.output)
return response
@@ -952,12 +1073,18 @@ class OpenAIServingResponses(OpenAIServingChat):
name: str,
arguments: str,
custom_names: set[str],
namespace_names: Optional[dict[str, tuple[str, str]]] = None,
) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]:
tool_names: dict[str, dict[str, str]],
*,
client_tool_search: bool = False,
) -> Union[
ResponseFunctionToolCall, ResponseCustomToolCall, ResponseToolSearchCall
]:
"""A call against a ``custom`` tool reports its freeform payload rather
than the JSON arguments of the shim function tool. A call against a
namespace member reports ``namespace`` and the bare function ``name``."""
call_id = f"call_{random_uuid()[:24]}"
if client_tool_search and name == TOOL_SEARCH:
return _tool_search_call(f"tsc_{random_uuid()[:8]}", call_id, arguments)
if name in custom_names:
return ResponseCustomToolCall(
type="custom_tool_call",
@@ -966,7 +1093,13 @@ class OpenAIServingResponses(OpenAIServingChat):
name=name,
input=decode_custom_tool_input(arguments),
)
namespace, name = (namespace_names or {}).get(name, (None, name))
entry = (tool_names or {}).get(name)
if isinstance(entry, tuple):
namespace, name = entry
elif isinstance(entry, dict):
namespace, name = entry.get("namespace"), entry.get("name", name)
else:
namespace = None
return ResponseFunctionToolCall(
arguments=arguments,
call_id=call_id,
@@ -1063,12 +1196,17 @@ class OpenAIServingResponses(OpenAIServingChat):
finish_reason: Optional[dict[str, Any]] = None,
):
chat_tools = self._response_tools_to_chat_tools(request)
custom_names = custom_tool_names(request.tools)
custom_names = self._custom_tool_names(request)
namespace_names = (
namespace_function_names(request.tools)
if self.tool_call_parser == "muse"
else {}
else {
call_name: (entry.get("namespace"), entry.get("name", call_name))
for call_name, entry in request._tool_state.names.items()
if entry.get("namespace")
}
)
client_tool_search = self._client_tool_search(request)
encoding_output = self._encoding_output()
if encoding_output is not None:
required_json = self._uses_required_tool_json(request, chat_tools)
@@ -1088,12 +1226,17 @@ class OpenAIServingResponses(OpenAIServingChat):
content, custom_names, namespace_names
)
else:
tool_call_items = [
self._make_tool_call_item(
call.name, call.parameters, custom_names, namespace_names
tool_call_items = []
for call in calls:
item = self._make_tool_call_item(
call.name,
call.parameters,
custom_names,
namespace_names,
client_tool_search=client_tool_search,
)
for call in calls
]
self._record_called_name(request, item.call_id, call.name)
tool_call_items.append(item)
else:
reasoning_content, content, tool_call_items = self._parse_text_output(
request,
@@ -1194,14 +1337,15 @@ class OpenAIServingResponses(OpenAIServingChat):
try:
content, call_info_list = parser.parse_non_stream(content)
for call_info in call_info_list:
tool_call_items.append(
self._make_tool_call_item(
call_info.name,
call_info.parameters or "",
custom_names,
namespace_names,
)
item = self._make_tool_call_item(
call_info.name,
call_info.parameters or "",
custom_names,
namespace_names,
client_tool_search=self._client_tool_search(request),
)
self._record_called_name(request, item.call_id, call_info.name)
tool_call_items.append(item)
parsed_via_native = bool(call_info_list)
except Exception as e:
logger.error("Tool call parsing error: %s", e)
@@ -1268,6 +1412,10 @@ class OpenAIServingResponses(OpenAIServingChat):
def _tool_call_parsing_enabled(
self, request: ResponsesRequest, chat_tools: list[Tool]
) -> bool:
if request._tool_state.chat_tools is not None and request.tool_choice == "none":
# The history showed the model tools, so it can still write a call;
# parsing drops it, where skipping the parser would return its markup.
return bool(self.tool_call_parser)
return bool(
chat_tools and self.tool_call_parser and request.tool_choice != "none"
)
@@ -1326,7 +1474,213 @@ class OpenAIServingResponses(OpenAIServingChat):
return tool_choice
return {"type": "function", "function": {"name": tool_choice["name"]}}
def _renders_natively(self, request: ResponsesRequest) -> bool:
"""Whether the served encoding renders the request in its own wire form,
which tool search, namespace tools and deferred tools take."""
return self.native_tool_loading and (
any(
tool.type == "namespace" or _defers_loading(tool)
for tool in request.tools
)
or self._uses_tool_search(request)
)
def _uses_tool_search(self, request: ResponsesRequest) -> bool:
"""Whether the request searches for tools, or its history has."""
if any(tool.type == TOOL_SEARCH for tool in request.tools):
return True
return any(
isinstance(item, dict) and item.get("type") in TOOL_SEARCH_ITEMS
for item in self._history_items(request)
)
def _history_items(self, request: ResponsesRequest) -> list:
"""The input history, before `previous_response_id` is known to exist."""
stored = self.msg_store.get(request.previous_response_id) or []
return list(stored) + (request.input if isinstance(request.input, list) else [])
def _native_tools_error(self, request: ResponsesRequest) -> Optional[str]:
deferred_types = {
tool.name: tool.type
for tool in request.tools
if tool.type in ("function", "custom") and tool.defer_loading
}
for tool in request.tools:
if (
tool.type == TOOL_SEARCH
and tool.model_extra.get("execution") != "client"
):
return (
'tool_search requires execution "client": this server runs no '
"searches"
)
if tool.type == "namespace" and not _holds_function_tools(tool.tools):
return NAMESPACE_MEMBERS_ERROR
for item in self._history_items(request):
if not isinstance(item, dict) or item.get("type") != "tool_search_output":
continue
tools = item.get("tools")
if not isinstance(tools, list) or not all(
isinstance(tool, dict)
and tool.get("type") in ("function", "namespace", "custom")
for tool in tools
):
return (
"tool_search_output tools must be a list of function, namespace "
"or custom tools"
)
for tool in tools:
if tool["type"] == "namespace" and not _holds_function_tools(
tool.get("tools")
):
return NAMESPACE_MEMBERS_ERROR
# The served encoding renders no other custom tool.
if tool["type"] == "custom" and tool.get("name") != APPLY_PATCH:
return (
f"tool_search_output can load only the {APPLY_PATCH!r} "
"custom tool"
)
# A deferred declaration is loaded as the tool it declares.
declared = deferred_types.get(tool.get("name"), tool["type"])
if tool["type"] != "namespace" and declared != tool["type"]:
return (
f"tool_search_output returns {tool.get('name')!r} as a "
f"{tool['type']} tool, but tools declares it as a deferred "
f"{declared} tool"
)
return None
def _prepare_native_responses_prompt(
self, request: ResponsesRequest, chat_request: ChatCompletionRequest
) -> None:
"""Render the request's own wire form into `chat_request`, whose tools and
choice become the ones the history has loaded."""
choice = request.effective_tool_choice()
tools, items = _defer_tool_loading(
[tool.model_dump(exclude_unset=True) for tool in request.tools],
[
item
for item in map(
self._native_input_item, self._response_input_history(request)
)
if item is not None
],
)
body = {"input": items, "tools": tools}
if request.instructions:
body["instructions"] = request.instructions
if choice == "none":
body["tool_choice"] = "none"
prompt = self.prepare_native_prompt(
chat_request, protocol="responses", body=body
)
tools = prompt.tools
request._tool_state = msgspec.structs.replace(
request._tool_state,
chat_tools=tools,
names={
call_name: {"namespace": namespace, "name": name}
for call_name, (namespace, name) in prompt.namespace_names.items()
},
discovered_custom_names=frozenset(
tool["name"]
for item in body["input"]
if isinstance(item, dict) and item.get("type") == "tool_search_output"
for tool in item["tools"]
if tool["type"] == "custom"
),
)
if isinstance(choice, dict):
name = choice["name"]
if "namespace" in choice:
# `tools` keeps conversion order, declared before discovered, so
# the last match is the member's latest definition.
member = (choice["namespace"], name)
name = next(
(
tool.function.name
for tool in reversed(tools)
if prompt.namespace_names.get(tool.function.name) == member
),
None,
)
if not any(tool.function.name == name for tool in tools) or (
choice["type"] == "custom"
and name not in self._custom_tool_names(request)
):
raise ResponseToolChoiceError(
f"Tool {choice['name']!r} is not declared in tools without "
"defer_loading or loaded by a tool_search_output"
)
chat_request.tool_choice = ToolChoice(
function=ToolChoiceFuncName(name=name)
)
elif choice == "required" and not tools:
raise ResponseToolChoiceError(
'tool_choice="required" needs a tool the model can call'
)
else:
chat_request.tool_choice = choice if tools else "none"
@classmethod
def _native_input_item(cls, item: Any) -> Any:
if isinstance(item, BaseModel):
item = item.model_dump(exclude_none=True)
if isinstance(item, dict) and item.get("type") == "reasoning":
# recipe reads only `content`; keep the text SGLang replays from a
# summary or a store=false client's state.
message = cls._normalize_response_message_for_chat(item)
if message is None:
return None
return {
"type": "reasoning",
"content": [
{"type": "reasoning_text", "text": message["reasoning_content"]}
],
}
return item
@staticmethod
def _record_called_name(request: ResponsesRequest, call_id: str, name: str) -> None:
"""Keep the name the model called a namespace member by: the item reports
its namespace and own name, which several call names can share."""
if request._tool_state.chat_tools is not None and (
name in request._tool_state.names
):
request._called_tool_names[call_id] = name
@staticmethod
def _replayed_output_items(request: ResponsesRequest, output: list) -> list[dict]:
"""`output` as the history a later request replays: a namespace member's
call under the name the model called it by."""
items = []
for item in output:
item = item.model_dump(exclude_none=True)
called_name = request._called_tool_names.get(item.get("call_id"))
if called_name is not None:
item = {**item, "name": called_name}
del item["namespace"]
items.append(item)
return items
@staticmethod
def _custom_tool_names(request: ResponsesRequest) -> set[str]:
declared = custom_tool_names(request.tools)
if request._tool_state.chat_tools is not None:
# A deferred declaration counts once a search result has loaded it.
declared -= {tool.name for tool in request.tools if tool.defer_loading}
return declared | request._tool_state.discovered_custom_names
@staticmethod
def _client_tool_search(request: ResponsesRequest) -> bool:
"""Whether a `tool_search` call is the client-executed search tool."""
return request._tool_state.chat_tools is not None and any(
tool.type == TOOL_SEARCH for tool in request.tools
)
def _response_tools_to_chat_tools(self, request: ResponsesRequest) -> list[Tool]:
if request._tool_state.chat_tools is not None:
return list(request._tool_state.chat_tools)
# ``function`` and ``custom`` tools flow to chat; built-ins go through
# harmony. A custom tool is shimmed into a single-string function tool.
chat_tools = []
@@ -2133,6 +2487,7 @@ class OpenAIServingResponses(OpenAIServingChat):
tool_follows = opens_item and item_type in (
"function_call",
"custom_tool_call",
"tool_search_call",
)
terminal = event_type in (
"response.completed",
@@ -2215,14 +2570,19 @@ class OpenAIServingResponses(OpenAIServingChat):
)
chat_tools = self._response_tools_to_chat_tools(request)
custom_names = custom_tool_names(request.tools)
custom_names = self._custom_tool_names(request)
namespace_names = (
namespace_function_names(request.tools)
if self.tool_call_parser == "muse"
else {}
else {
call_name: (entry.get("namespace"), entry.get("name", call_name))
for call_name, entry in request._tool_state.names.items()
if entry.get("namespace")
}
)
tool_choice = request.effective_tool_choice()
required_json = self._uses_required_tool_json(request, chat_tools)
client_tool_search = self._client_tool_search(request)
encoding_output = self._encoding_output()
token_stream = (
encoding_output.new_responses_stream(
@@ -2453,7 +2813,13 @@ class OpenAIServingResponses(OpenAIServingChat):
return []
arguments = state["arguments"]
events: list = []
if state["custom"]:
done_event = openai_responses_types.ResponseOutputItemDoneEvent
if state["search"]:
completed_item = _tool_search_call(
state["item_id"], state["call_id"], arguments
)
done_event = ResponseToolSearchCallEvent
elif state["custom"]:
payload = decode_custom_tool_input(arguments)
# Deltas cannot be retracted, so a payload that no longer
# extends what was already streamed defers to the streamed text.
@@ -2520,7 +2886,7 @@ class OpenAIServingResponses(OpenAIServingChat):
)
events.append(
_send_event(
openai_responses_types.ResponseOutputItemDoneEvent(
done_event(
type="response.output_item.done",
sequence_number=-1,
output_index=state["output_index"],
@@ -2717,27 +3083,43 @@ class OpenAIServingResponses(OpenAIServingChat):
current_output_index += 1
name = call.name or ""
is_custom = name in custom_names
is_search = client_tool_search and name == TOOL_SEARCH
namespace, name = namespace_names.get(name, (None, name))
prefix = (
"ctc" if is_custom else "tsc" if is_search else "fc"
)
state = {
"item_id": (
f"ctc_{random_uuid()[:8]}"
if is_custom
else f"fc_{random_uuid()[:8]}"
),
"item_id": f"{prefix}_{random_uuid()[:8]}",
"call_id": f"call_{random_uuid()[:24]}",
"output_index": current_output_index,
"name": name,
"namespace": namespace,
"arguments": "",
"custom": is_custom,
"search": is_search,
"payload": "",
"added": False,
"done": False,
}
tool_call_states[tool_index] = state
self._record_called_name(
request, state["call_id"], call.name or ""
)
if not state["added"]:
state["added"] = True
if state["custom"]:
added_event = (
openai_responses_types.ResponseOutputItemAddedEvent
)
if state["search"]:
# Search arguments are reported whole when it completes.
added_item = ResponseToolSearchCall(
id=state["item_id"],
call_id=state["call_id"],
arguments={},
status="in_progress",
)
added_event = ResponseToolSearchCallEvent
elif state["custom"]:
added_item = ResponseCustomToolCall(
type="custom_tool_call",
id=state["item_id"],
@@ -2756,7 +3138,7 @@ class OpenAIServingResponses(OpenAIServingChat):
**_namespace_field(state),
)
yield _send_event(
openai_responses_types.ResponseOutputItemAddedEvent(
added_event(
type="response.output_item.added",
sequence_number=-1,
output_index=state["output_index"],
@@ -2786,7 +3168,7 @@ class OpenAIServingResponses(OpenAIServingChat):
delta=delta,
)
)
else:
elif not state["search"]:
yield _send_event(
openai_responses_types.ResponseFunctionCallArgumentsDeltaEvent(
type="response.function_call_arguments.delta",
@@ -2936,10 +3318,7 @@ class OpenAIServingResponses(OpenAIServingChat):
self.response_store[final_response.id] = final_response
self.msg_store[final_response.id] = self._response_input_history(
request
) + [
item.model_dump(exclude_none=True)
for item in final_response.output
]
) + self._replayed_output_items(request, final_response.output)
response_dict = _sanitize_response_dict(final_response.model_dump())
@@ -5,6 +5,7 @@ parses them, and tool-call arguments arrive as the API streams them."""
from __future__ import annotations
import json
import logging
from collections.abc import Sequence
from typing import Any, Optional
@@ -17,6 +18,11 @@ from sglang.srt.parser.deepseek_recipe_renderer import (
load_deepseek_recipe,
)
logger = logging.getLogger(__name__)
# The most tokens one UTF-8 character can span, a byte each.
_MAX_UTF8_BYTES = 4
class RecipeToolCallDelta(msgspec.Struct, frozen=True):
"""A piece of one call's JSON arguments; the first piece also names the call."""
@@ -74,6 +80,7 @@ class DeepSeekRecipeOutputParser:
separate_reasoning: bool = True,
stream_reasoning: bool = True,
stop: Sequence[str] = (),
callable_names: Optional[frozenset[str]] = None,
):
recipe = load_deepseek_recipe()
options = recipe.ParsingOptions(
@@ -101,6 +108,10 @@ class DeepSeekRecipeOutputParser:
# out, so a matched stop is cut before recipe has emitted any of it.
self._stop_hold_chars = max(map(len, stop), default=1) - 1
self._pending_ids: list[int] = []
self._callable_names = callable_names
# recipe's index of each call so far, mapped to the index it is reported
# at; None for a dropped call.
self._call_indices: dict[int, Optional[int]] = {}
self.num_consumed_tokens = 0
def feed(self, token_ids: Sequence[int]) -> RecipeOutputDelta:
@@ -192,6 +203,27 @@ class DeepSeekRecipeOutputParser:
+ self._hf_tokenizer.encode(text[:cut], add_special_tokens=False).ids
)
def _reported_call(
self, call: RecipeToolCallDelta
) -> Optional[RecipeToolCallDelta]:
"""`call` at the index it is reported at; None when it belongs to a call
outside `callable_names`."""
if self._callable_names is None:
return call
if call.name is not None:
index = None
if call.name in self._callable_names:
index = sum(kept is not None for kept in self._call_indices.values())
else:
logger.warning(
"Model attempted to call undefined function: %s", call.name
)
self._call_indices[call.index] = index
index = self._call_indices[call.index]
if index is None:
return None
return msgspec.structs.replace(call, index=index)
def _deltas(self, chunks: list[Any]) -> RecipeOutputDelta:
delta = RecipeOutputDelta()
for chunk in chunks:
@@ -202,12 +234,19 @@ class DeepSeekRecipeOutputParser:
reasoning = piece.get("reasoning_content") or ""
content = piece.get("content") or ""
tool_calls = tuple(
RecipeToolCallDelta(
index=call["index"],
arguments=call["function"]["arguments"],
name=call["function"].get("name"),
call
for call in map(
self._reported_call,
(
RecipeToolCallDelta(
index=call["index"],
arguments=call["function"]["arguments"],
name=call["function"].get("name"),
)
for call in piece.get("tool_calls") or ()
),
)
for call in piece.get("tool_calls") or ()
if call is not None
)
if self._reasoning_as_content:
content, reasoning = reasoning + content, ""
@@ -95,6 +95,23 @@ def pre_normalize(body: dict[str, Any]) -> dict[str, Any]:
return request
def loaded_tools(conversation: Any) -> list[Any]:
"""The tools the model can call after `conversation`'s history: those it does
not defer and the deferred ones a system message has loaded."""
recipe = load_deepseek_recipe()
loaded = {
tool.name
for message in conversation.messages
if isinstance(message, recipe.SystemMessage)
for tool in message.tools or []
}
return [
tool
for tool in conversation.tools
if not tool.defer_loading or tool.name in loaded
]
def _error_message(body: Any) -> str:
try:
return json.loads(body)["error"]["message"]
@@ -146,6 +163,19 @@ class DeepSeekRecipe:
) -> list[int]:
"""Prompt ids for `body` (see `pre_normalize`); a request recipe rejects
raises ValueError. `image_urls` receives the images in placeholder order."""
conversation = self.convert(body, effort_budget=effort_budget)
prompt_ids = self.encode_conversation(conversation, effort_budget=effort_budget)
if image_urls is not None:
image_urls.extend(self.image_urls(conversation))
if task is not None:
prompt_ids = self._attach_task(prompt_ids, task, body["thinking"])
return prompt_ids
def convert(
self, body: dict[str, Any], *, effort_budget: Optional[int] = None
) -> Any:
"""Convert `body` to recipe's `Conversation`; invalid requests raise ValueError.
Pass the same `effort_budget` to `encode_conversation`."""
recipe = load_deepseek_recipe()
normalized = pre_normalize(body)
normalized["model"] = self.model
@@ -155,16 +185,104 @@ class DeepSeekRecipe:
converted = recipe.ChatCompletionRequest(normalized).convert(self._options)
except recipe.ConversionError as error:
raise ValueError(_error_message(error.body)) from error
prompt_ids = self._encoding.encode(converted.conversation)
if image_urls is not None:
rendered = self._encoding.render_conversation(converted.conversation)
image_urls.extend(
source.url or source.data_url for source in rendered.image_sources
)
if effort_budget is not None and body["thinking"]:
return converted.conversation
def convert_native(
self, protocol: str, body: dict[str, Any], *, thinking: bool
) -> Any:
"""Convert a Responses or Messages request to recipe's `Conversation` in the
host's thinking mode; invalid requests raise ValueError."""
recipe = load_deepseek_recipe()
request_type = {
"responses": recipe.ResponsesRequest,
"messages": recipe.MessagesRequest,
}[protocol]
options = self._options.with_default_thinking_mode(thinking)
try:
converted = request_type({**body, "model": self.model}).convert(options)
except recipe.ConversionError as error:
raise ValueError(_error_message(error.body)) from error
return converted.conversation
def with_host_settings(
self,
conversation: Any,
*,
response_schema: Optional[dict[str, Any]],
continue_final_message: bool,
) -> Any:
"""`conversation` with what recipe's Responses and Messages conversions
leave out: a JSON response schema, and SGLang's reading of a trailing
assistant turn, continued or else re-sent as a user turn."""
recipe = load_deepseek_recipe()
messages = conversation.messages
last = messages[-1] if messages else None
if last is not None and last.role == "assistant":
# recipe closes the turn and opens another; SGLang continues it or,
# as `_handle_last_assistant_message`, re-sends it as a user turn.
if continue_final_message:
last = recipe.AssistantMessage(
last.content, last.reasoning_content, prefix=True
)
else:
last = recipe.UserMessage(last.content)
messages = [*messages[:-1], last]
elif response_schema is None:
return conversation
return recipe.Conversation(
messages,
thinking_mode=conversation.thinking_mode,
tools=conversation.tools,
tool_choice=conversation.tool_choice,
reasoning_effort=conversation.reasoning_effort,
response_schema=(
response_schema
if response_schema is not None
else conversation.response_schema
),
)
def namespace_tool_names(
self, namespaces: list[dict[str, Any]]
) -> dict[str, tuple[str, str]]:
"""The name recipe's Responses conversion calls each member of `namespaces`
by, mapped to its namespace and its name there. `namespaces` lists the
request's namespace tools in conversion order: declared, then discovered."""
recipe = load_deepseek_recipe()
body = {"model": self.model, "input": "", "tools": namespaces}
try:
return recipe.ResponsesRequest(body).namespace_tool_map(self._options)
except recipe.ConversionError as error:
raise ValueError(_error_message(error.body)) from error
def image_urls(self, conversation: Any) -> list[str]:
"""The images of `conversation` in the order its prompt places them."""
rendered = self._encoding.render_conversation(conversation)
return [source.url or source.data_url for source in rendered.image_sources]
def encode_conversation(
self, conversation: Any, *, effort_budget: Optional[int] = None
) -> list[int]:
"""Encode a recipe `Conversation`, including a prefix of its messages."""
if self._fast_prompt_tokenizer is None:
prompt_ids = self._encoding.encode(conversation)
else:
rendered = self._encoding.render_conversation(conversation)
prompt = rendered.prompt
segments = []
cursor = 0
# Recipe distinguishes control tokens from identical spellings in
# user text. Its Python ranges use code-point offsets, not bytes.
for start, end in rendered.special_ranges:
if start > cursor:
segments.append((prompt[cursor:start], False))
segments.append((prompt[start:end], True))
cursor = end
if cursor < len(prompt):
segments.append((prompt[cursor:], False))
prompt_ids = self._fast_prompt_tokenizer.encode_segments(segments).ids
if effort_budget is not None and conversation.thinking_mode:
prompt_ids = self._replace_effort_budget(prompt_ids, effort_budget)
if task is not None:
prompt_ids = self._attach_task(prompt_ids, task, body["thinking"])
return prompt_ids
def parse_start(
@@ -97,7 +97,9 @@ class AnthropicMessagesMixin:
)
serving = AnthropicServing(
openai_serving_chat=SimpleNamespace(supports_inline_system=True)
openai_serving_chat=SimpleNamespace(
supports_inline_system=True, native_tool_loading=False
)
)
chat_request = serving._convert_to_chat_completion_request(anthropic_request)
converted = chat_request.model_dump()
@@ -31,6 +31,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu")
class _FakeOpenAIServingChat:
native_reasoning_history = False
native_tool_loading = False
def __init__(self, stream_lines=None, chat_template=None):
self.stream_lines = stream_lines or []
@@ -61,6 +62,7 @@ class _FakeNonStreamingErrorOpenAI:
"""Returns an error response or raises an exception from the OpenAI handler."""
supports_inline_system = False
native_tool_loading = False
def __init__(self, status_code=400, body=None, content=None, exception=None):
self._status_code = status_code
@@ -496,6 +498,26 @@ class TestAnthropicServing(unittest.TestCase):
],
)
def test_tool_addition_fails_where_tools_cannot_load_mid_conversation(self):
"""A tool_addition block must not vanish on a model that cannot render it."""
request = self._anthropic_request(
messages=[
{"role": "user", "content": "hello"},
{
"role": "system",
"content": [
{
"type": "tool_addition",
"tool": {"type": "tool_reference", "name": "Bash"},
}
],
},
]
)
with self.assertRaisesRegex(ValueError, "tool_addition"):
self._serving()._convert_to_chat_completion_request(request)
def test_builtin_web_search_tool_without_schema_is_skipped(self):
request = AnthropicMessagesRequest.model_validate(
{
File diff suppressed because it is too large Load Diff