Backport the upstream namespace-tool stack onto the B300 recipe base: - #46370: Function.namespace / ChatMessage.action protocol fields. - #49625: Responses/Messages native recipe rendering (convert_native), tool_search/defer_loading, namespace identity in/out/history replay. Adapted to this base: skips serving_kimi_k3.py (not present), drops the Minimax/Beam/wire-only dependencies added between 10-23 and the PR base, and keeps the existing tool-parsing fallback where the newer `parses_without_tools()` helper is unavailable. Also includes the PR's Anthropic tool-reference plumbing and unit tests.
This commit is contained in:
@@ -56,6 +56,7 @@ class AnthropicUsage(BaseModel):
|
||||
class TextBlock(BaseModel):
|
||||
type: Literal["text"] = "text"
|
||||
text: str
|
||||
cache_control: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
class ImageBlock(BaseModel):
|
||||
@@ -91,6 +92,50 @@ class ToolReferenceBlock(BaseModel):
|
||||
id: Optional[str] = None
|
||||
|
||||
|
||||
class McpToolReference(BaseModel):
|
||||
type: Literal["mcp_tool_reference"]
|
||||
server_name: str
|
||||
name: str
|
||||
|
||||
|
||||
class McpToolsetReference(BaseModel):
|
||||
type: Literal["mcp_toolset_reference"]
|
||||
server_name: str
|
||||
|
||||
|
||||
class ToolDefinition(BaseModel):
|
||||
"""A tool given by value: `definition` is a `tools` entry."""
|
||||
|
||||
type: Literal["tool_definition"]
|
||||
definition: dict[str, Any]
|
||||
|
||||
|
||||
class ToolAdditionBlock(BaseModel):
|
||||
"""Anthropic beta block: a ``system`` message offers a tool from its position
|
||||
on. Rendered through deepseek-recipe, which reads only a ``tool_reference``
|
||||
target."""
|
||||
|
||||
type: Literal["tool_addition"] = "tool_addition"
|
||||
tool: Annotated[
|
||||
Union[
|
||||
ToolReferenceBlock, McpToolReference, McpToolsetReference, ToolDefinition
|
||||
],
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
cache_control: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
class ToolRemovalBlock(BaseModel):
|
||||
"""Anthropic beta block: a ``system`` message withdraws a tool."""
|
||||
|
||||
type: Literal["tool_removal"] = "tool_removal"
|
||||
tool: Annotated[
|
||||
Union[ToolReferenceBlock, McpToolReference, McpToolsetReference],
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
cache_control: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
class SearchResultBlock(BaseModel):
|
||||
type: Literal["search_result"] = "search_result"
|
||||
# ``source`` here is a URL/identifier string (unlike ImageBlock.source).
|
||||
@@ -117,6 +162,8 @@ AnthropicContentBlock = Annotated[
|
||||
ToolUseBlock,
|
||||
ToolResultBlock,
|
||||
ToolReferenceBlock,
|
||||
ToolAdditionBlock,
|
||||
ToolRemovalBlock,
|
||||
SearchResultBlock,
|
||||
ThinkingBlock,
|
||||
RedactedThinkingBlock,
|
||||
@@ -128,6 +175,8 @@ AnthropicContentBlock = Annotated[
|
||||
class AnthropicMessage(BaseModel):
|
||||
role: Literal["user", "assistant", "system"]
|
||||
content: Union[str, list[AnthropicContentBlock]]
|
||||
clear_at: Optional[Literal["next_user_message", "never"]] = None
|
||||
output_config: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
# ---------- Tools (discriminated by ``type`` family) ----------
|
||||
|
||||
@@ -20,8 +20,10 @@ from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
AnthropicContentBlock,
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicCountTokensResponse,
|
||||
AnthropicCustomTool,
|
||||
AnthropicError,
|
||||
AnthropicErrorResponse,
|
||||
AnthropicMessage,
|
||||
AnthropicMessageEndDelta,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
@@ -32,6 +34,8 @@ from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
ContentBlockStopEvent,
|
||||
ErrorEvent,
|
||||
InputJsonDelta,
|
||||
McpToolReference,
|
||||
McpToolsetReference,
|
||||
MessageDeltaEvent,
|
||||
MessageStartEvent,
|
||||
MessageStopEvent,
|
||||
@@ -40,6 +44,8 @@ from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
TextDelta,
|
||||
ThinkingBlock,
|
||||
ThinkingDelta,
|
||||
ToolDefinition,
|
||||
ToolReferenceBlock,
|
||||
ToolUseBlock,
|
||||
is_server_tool,
|
||||
)
|
||||
@@ -152,6 +158,207 @@ def _extract_system_text(
|
||||
return "\n".join(texts) if texts else None
|
||||
|
||||
|
||||
def _block_types(request: AnthropicMessagesRequest) -> set[str]:
|
||||
"""The block types of the request's system and messages, tool result contents
|
||||
included."""
|
||||
types = set()
|
||||
for content in [request.system, *(message.content for message in request.messages)]:
|
||||
for block in content if isinstance(content, list) else []:
|
||||
types.add(block.type)
|
||||
if block.type == "tool_result" and isinstance(block.content, list):
|
||||
types.update(item.type for item in block.content)
|
||||
return types
|
||||
|
||||
|
||||
def _without_cleared_messages(
|
||||
request: AnthropicMessagesRequest,
|
||||
) -> AnthropicMessagesRequest:
|
||||
"""`request` without the turn-scoped system messages (`clear_at:
|
||||
"next_user_message"`) that a later user message has cleared."""
|
||||
last_user = max(
|
||||
(
|
||||
index
|
||||
for index, message in enumerate(request.messages)
|
||||
if message.role == "user"
|
||||
),
|
||||
default=-1,
|
||||
)
|
||||
rendered = []
|
||||
for index, message in enumerate(request.messages):
|
||||
if message.clear_at is not None and message.role != "system":
|
||||
raise ValueError(
|
||||
f"messages.{index}.clear_at: clear_at is only permitted on role "
|
||||
"'system' messages"
|
||||
)
|
||||
if message.clear_at == "next_user_message":
|
||||
_check_turn_scoped(index=index, message=message)
|
||||
_check_turn_scoped_placement(index=index, messages=request.messages)
|
||||
if index < last_user:
|
||||
continue
|
||||
rendered.append(message)
|
||||
return request.model_copy(update={"messages": rendered})
|
||||
|
||||
|
||||
def _check_turn_scoped(index: int, message: AnthropicMessage) -> None:
|
||||
scope = "a turn-scoped system message (clear_at: 'next_user_message')"
|
||||
if message.output_config is not None:
|
||||
raise ValueError(f"messages.{index}: output_config is not permitted on {scope}")
|
||||
for position, block in enumerate(
|
||||
message.content if isinstance(message.content, list) else []
|
||||
):
|
||||
if block.type != "text":
|
||||
raise ValueError(
|
||||
f"messages.{index}: a turn-scoped system message supports text "
|
||||
"blocks only (clear_at: 'next_user_message')"
|
||||
)
|
||||
if block.cache_control is not None:
|
||||
raise ValueError(
|
||||
f"messages.{index}.content.{position}: cache_control is not "
|
||||
f"permitted on {scope}"
|
||||
)
|
||||
|
||||
|
||||
def _check_turn_scoped_placement(index: int, messages: list[AnthropicMessage]) -> None:
|
||||
# Consecutive system messages are placed as one group.
|
||||
before = next(
|
||||
(m.role for m in reversed(messages[:index]) if m.role != "system"), None
|
||||
)
|
||||
after = next((m.role for m in messages[index + 1 :] if m.role != "system"), None)
|
||||
if before != "user" or after == "user":
|
||||
raise ValueError(
|
||||
f"messages.{index}: a turn-scoped system message (clear_at: "
|
||||
"'next_user_message') must follow a user message and either end "
|
||||
"messages or precede an assistant message"
|
||||
)
|
||||
|
||||
|
||||
def _declare_inline_tools(
|
||||
request: AnthropicMessagesRequest,
|
||||
) -> AnthropicMessagesRequest:
|
||||
"""`request` with each custom tool a `tool_addition` block defines by value
|
||||
declared in `tools` as deferred, and the block naming it by reference. A
|
||||
definition of a tool already offered unchanged is dropped."""
|
||||
tools = list(request.tools or [])
|
||||
declared = {tool.name: tool for tool in tools}
|
||||
offered = {tool.name for tool in tools if not tool.defer_loading}
|
||||
unresolved: dict[str, str] = {}
|
||||
messages = []
|
||||
for index, message in enumerate(request.messages):
|
||||
if isinstance(message.content, str):
|
||||
messages.append(message)
|
||||
continue
|
||||
blocks = []
|
||||
for position, block in enumerate(message.content):
|
||||
where = f"messages.{index}.content.{position}"
|
||||
for name in _referenced_tool_names(block):
|
||||
if name not in declared:
|
||||
unresolved.setdefault(name, where)
|
||||
if block.type == "tool_addition" and block.tool.type != "tool_reference":
|
||||
if message.role != "system":
|
||||
raise ValueError(
|
||||
f"{where}: tool_addition blocks can only be in system messages"
|
||||
)
|
||||
tool = _inline_custom_tool(where=where, target=block.tool)
|
||||
# Declaring the tool makes it resolvable from the first message on.
|
||||
if tool.name in unresolved:
|
||||
raise ValueError(
|
||||
f"{unresolved[tool.name]}: tool reference {tool.name!r} "
|
||||
f"precedes the tool_definition that defines it at {where}"
|
||||
)
|
||||
earlier = declared.setdefault(tool.name, tool)
|
||||
if earlier is tool:
|
||||
tools.append(tool)
|
||||
elif is_server_tool(earlier) or _definition(earlier) != _definition(
|
||||
tool
|
||||
):
|
||||
raise ValueError(
|
||||
f"{where}: tool_definition "
|
||||
f"reuses the name {tool.name!r} with a different "
|
||||
"definition; replacing a tool mid-conversation is not "
|
||||
"supported"
|
||||
)
|
||||
if tool.name in offered:
|
||||
continue
|
||||
offered.add(tool.name)
|
||||
block = block.model_copy(
|
||||
update={"tool": ToolReferenceBlock(name=tool.name)}
|
||||
)
|
||||
blocks.append(block)
|
||||
# recipe renders a system message left without blocks as an empty turn.
|
||||
if blocks or not message.content:
|
||||
messages.append(message.model_copy(update={"content": blocks}))
|
||||
return request.model_copy(update={"tools": tools, "messages": messages})
|
||||
|
||||
|
||||
def _referenced_tool_names(block: AnthropicContentBlock) -> list[Optional[str]]:
|
||||
if block.type == "tool_addition" and block.tool.type == "tool_reference":
|
||||
return [block.tool.name or block.tool.tool_name]
|
||||
if block.type == "tool_reference":
|
||||
return [block.tool_name or block.name]
|
||||
if block.type == "tool_result" and isinstance(block.content, list):
|
||||
return [name for item in block.content for name in _referenced_tool_names(item)]
|
||||
return []
|
||||
|
||||
|
||||
def _inline_custom_tool(
|
||||
where: str, target: Union[McpToolReference, McpToolsetReference, ToolDefinition]
|
||||
) -> AnthropicCustomTool:
|
||||
if target.type != "tool_definition":
|
||||
raise ValueError(
|
||||
f"{where}: tool_addition with a {target.type} is not supported; this "
|
||||
"server runs no MCP connector"
|
||||
)
|
||||
kind = target.definition.get("type")
|
||||
if kind not in (None, "custom"):
|
||||
raise ValueError(
|
||||
f"{where}: tool_definition supports custom tools only, got a "
|
||||
f"definition of type {kind!r}"
|
||||
)
|
||||
tool = AnthropicCustomTool.model_validate(target.definition)
|
||||
if tool.defer_loading:
|
||||
raise ValueError(
|
||||
f"{where}: a tool_definition with defer_loading is not supported; "
|
||||
"declare the tool in tools and add it by reference"
|
||||
)
|
||||
return tool.model_copy(update={"defer_loading": True})
|
||||
|
||||
|
||||
def _definition(tool: AnthropicCustomTool) -> dict[str, Any]:
|
||||
return tool.model_dump(exclude={"type", "defer_loading"})
|
||||
|
||||
|
||||
def _native_block(block: dict[str, Any]) -> dict[str, Any]:
|
||||
"""`block` with SGLang's accepted aliases spelled as deepseek-recipe reads them."""
|
||||
kind = block["type"]
|
||||
if kind == "tool_result":
|
||||
content = block.get("content")
|
||||
fields = {
|
||||
"type": kind,
|
||||
"tool_use_id": block.get("tool_use_id") or block.get("id"),
|
||||
"content": (
|
||||
[_native_block(item) for item in content]
|
||||
if isinstance(content, list)
|
||||
else content
|
||||
),
|
||||
"is_error": block.get("is_error"),
|
||||
}
|
||||
return {key: value for key, value in fields.items() if value is not None}
|
||||
if kind == "tool_reference":
|
||||
return {"type": kind, "tool_name": block.get("tool_name") or block.get("name")}
|
||||
if kind == "tool_addition":
|
||||
tool = block["tool"]
|
||||
name = tool.get("name") or tool.get("tool_name")
|
||||
return {"type": kind, "tool": {"type": "tool_reference", "name": name}}
|
||||
return block
|
||||
|
||||
|
||||
def _native_message(message: AnthropicMessage) -> dict[str, Any]:
|
||||
data = message.model_dump(exclude_none=True)
|
||||
if isinstance(data["content"], list):
|
||||
data["content"] = [_native_block(block) for block in data["content"]]
|
||||
return data
|
||||
|
||||
|
||||
def _wrap_sse_event(data: str, event_type: str) -> str:
|
||||
"""Format an Anthropic SSE event with event type and data lines."""
|
||||
return f"event: {event_type}\ndata: {data}\n\n"
|
||||
@@ -218,6 +425,24 @@ class AnthropicServing:
|
||||
self, anthropic_request: AnthropicMessagesRequest
|
||||
) -> ChatCompletionRequest:
|
||||
"""Convert an Anthropic Messages request to an OpenAI ChatCompletion request."""
|
||||
anthropic_request = _declare_inline_tools(
|
||||
_without_cleared_messages(anthropic_request)
|
||||
)
|
||||
block_types = _block_types(anthropic_request)
|
||||
if "tool_removal" in block_types:
|
||||
raise ValueError(
|
||||
"tool_removal blocks are not supported; a tool cannot be withdrawn "
|
||||
"mid-conversation"
|
||||
)
|
||||
native = self.openai_serving_chat.native_tool_loading and (
|
||||
bool(block_types & {"tool_reference", "tool_addition"})
|
||||
or any(
|
||||
not is_server_tool(tool) and tool.defer_loading
|
||||
for tool in anthropic_request.tools or []
|
||||
)
|
||||
)
|
||||
if not native and "tool_addition" in block_types:
|
||||
raise ValueError("tool_addition blocks are not supported by this model")
|
||||
openai_messages = []
|
||||
|
||||
def _convert_anthropic_image_source_to_openai_part(
|
||||
@@ -719,8 +944,51 @@ class AnthropicServing:
|
||||
elif chat_request.tools:
|
||||
chat_request.tool_choice = "auto"
|
||||
|
||||
if native:
|
||||
self._prepare_native_prompt(anthropic_request, chat_request)
|
||||
return chat_request
|
||||
|
||||
def _prepare_native_prompt(
|
||||
self,
|
||||
anthropic_request: AnthropicMessagesRequest,
|
||||
chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Render the request's own wire form into `chat_request`, whose tools
|
||||
become the ones the history has loaded."""
|
||||
body = {
|
||||
"messages": [
|
||||
_native_message(message) for message in anthropic_request.messages
|
||||
],
|
||||
# recipe reads a tool with a `type` as a server tool.
|
||||
"tools": [
|
||||
tool.model_dump(exclude_none=True, exclude={"type"})
|
||||
for tool in anthropic_request.tools or []
|
||||
if not is_server_tool(tool)
|
||||
],
|
||||
}
|
||||
system = anthropic_request.system
|
||||
if system:
|
||||
body["system"] = (
|
||||
system
|
||||
if isinstance(system, str)
|
||||
else [block.model_dump(exclude_none=True) for block in system]
|
||||
)
|
||||
choice = anthropic_request.tool_choice
|
||||
if choice is not None and choice.type == "none":
|
||||
body["tool_choice"] = {"type": "none"}
|
||||
tools = self.openai_serving_chat.prepare_native_prompt(
|
||||
chat_request, protocol="messages", body=body
|
||||
).tools
|
||||
if (
|
||||
choice is not None
|
||||
and choice.type == "tool"
|
||||
and not any(tool.function.name == choice.name for tool in tools)
|
||||
):
|
||||
raise ValueError(
|
||||
f"tool_choice names {choice.name!r}, a deferred tool no "
|
||||
"tool_reference or tool_addition block has loaded"
|
||||
)
|
||||
|
||||
async def _handle_non_streaming(
|
||||
self,
|
||||
chat_request: ChatCompletionRequest,
|
||||
|
||||
@@ -3,7 +3,7 @@ concrete adapters load only when selected."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol
|
||||
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol, Union
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -15,8 +15,27 @@ if TYPE_CHECKING:
|
||||
ToolCall,
|
||||
)
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
from sglang.srt.multimodal.media_io import ImageData
|
||||
|
||||
EncodingBackend = Literal["builtin", "recipe"]
|
||||
HistoryToolCallArguments = Literal["strict", "lenient", "as_sent"]
|
||||
NativeProtocol = Literal["responses", "messages"]
|
||||
|
||||
|
||||
class NativePrompt(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""A Responses or Messages request rendered from its own wire form."""
|
||||
|
||||
prompt_ids: list[int]
|
||||
tools: list[Tool]
|
||||
"""The tools the model can call after the request's history: those loaded from
|
||||
the start and those the history has loaded since."""
|
||||
image_data: list[ImageData] = []
|
||||
"""The images the prompt's placeholders stand for, in order."""
|
||||
namespace_names: dict[str, tuple[str, str]] = {}
|
||||
"""The call name of each namespace member, mapped to its namespace and the
|
||||
name the request gives it there."""
|
||||
continued_turn: Optional[dict[str, Any]] = None
|
||||
"""The final assistant message the prompt continues, as a chat message."""
|
||||
|
||||
|
||||
class ChatOutputHooks(msgspec.Struct, frozen=True, kw_only=True):
|
||||
@@ -78,6 +97,25 @@ class ChatEncodingOutputAdapter(Protocol):
|
||||
) -> ResponsesOutputStream: ...
|
||||
|
||||
|
||||
class NativeRequestEncoder(Protocol):
|
||||
def encode_native(
|
||||
self,
|
||||
*,
|
||||
protocol: NativeProtocol,
|
||||
body: dict[str, Any],
|
||||
thinking: bool,
|
||||
effort: Union[str, int, float, None],
|
||||
response_schema: Optional[dict[str, Any]],
|
||||
continue_final_message: bool,
|
||||
) -> NativePrompt:
|
||||
"""``body`` carries the request's conversation, tools and, for
|
||||
``tool_choice="none"``, its choice; ``thinking`` and ``effort`` are the
|
||||
handler's decisions, as for ``encode_messages``. ``response_schema`` is
|
||||
the JSON schema the answer follows; a trailing assistant turn is continued
|
||||
under ``continue_final_message`` and otherwise re-sent as a user turn."""
|
||||
...
|
||||
|
||||
|
||||
class ChatEncodingAdapter(Protocol):
|
||||
collects_openai_media: bool
|
||||
"""Whether media still has to be collected from the OpenAI messages after
|
||||
@@ -92,6 +130,12 @@ class ChatEncodingAdapter(Protocol):
|
||||
"""None when the configured parsers leave output to the text parsers."""
|
||||
...
|
||||
|
||||
@property
|
||||
def native_encoder(self) -> Optional[NativeRequestEncoder]:
|
||||
"""None when Responses and Messages requests that load tools mid-conversation
|
||||
have no rendering of their own wire form."""
|
||||
...
|
||||
|
||||
def encode_messages(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -17,6 +17,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import msgspec
|
||||
import unicodedata
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
@@ -743,6 +744,10 @@ class ChatCompletionMessageGenericParam(BaseModel):
|
||||
name: Optional[str] = None
|
||||
phase: Optional[Literal["commentary", "final_answer"]] = None
|
||||
reasoning_content: Optional[str] = None
|
||||
action: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Action tag for assistant messages (used by deepseek-recipe).",
|
||||
)
|
||||
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
|
||||
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
|
||||
# DeepSeek API prefix completion: continue this final assistant message.
|
||||
@@ -798,12 +803,18 @@ class Function(BaseModel):
|
||||
parameters: Optional[object] = None
|
||||
strict: bool = False
|
||||
defer_loading: Optional[bool] = None
|
||||
namespace: Optional[str] = Field(
|
||||
default=None,
|
||||
description="Tool namespace for disambiguation (used by deepseek-recipe).",
|
||||
)
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
data = handler(self)
|
||||
if self.defer_loading is None:
|
||||
data.pop("defer_loading", None)
|
||||
if self.namespace is None:
|
||||
data.pop("namespace", None)
|
||||
return data
|
||||
|
||||
|
||||
@@ -913,6 +924,15 @@ class ChatCompletionRequest(PDRoutingFields):
|
||||
# renders these, while ``tools`` (flattened to ``<namespace>.<function>``) still
|
||||
# drives tool-call parsing.
|
||||
_template_tools: Optional[List[Dict[str, Any]]] = PrivateAttr(default=None)
|
||||
# Set with ``input_ids`` by a caller that rendered them itself: the images the
|
||||
# prompt's placeholders stand for, in order.
|
||||
_input_image_data: Optional[List[Any]] = PrivateAttr(default=None)
|
||||
# Whether ``tools`` are the only tools the rendered prompt shows, so a call
|
||||
# to any other tool is dropped rather than returned.
|
||||
_drops_unlisted_tool_calls: bool = PrivateAttr(default=False)
|
||||
# Set with ``input_ids`` under ``continue_final_message``: the final assistant
|
||||
# message the prompt continues, where output parsing resumes.
|
||||
_continued_turn: Optional[Dict[str, Any]] = PrivateAttr(default=None)
|
||||
tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field(
|
||||
default="auto", examples=["none"]
|
||||
) # noqa
|
||||
@@ -1252,6 +1272,7 @@ class ChatMessage(BaseModel):
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
reasoning_content: Optional[str] = None
|
||||
action: Optional[str] = None
|
||||
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
@@ -1259,6 +1280,8 @@ class ChatMessage(BaseModel):
|
||||
data = handler(self)
|
||||
if self.tool_calls is None:
|
||||
data.pop("tool_calls", None)
|
||||
if self.action is None:
|
||||
data.pop("action", None)
|
||||
return data
|
||||
|
||||
|
||||
@@ -1830,6 +1853,40 @@ class ResponseTool(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class ResponseToolState(msgspec.Struct, frozen=True):
|
||||
"""Public tool metadata and name mappings for one translated request."""
|
||||
|
||||
public_tools: list[ResponseTool] | None = None
|
||||
public_choice: str | dict[str, Any] | None = None
|
||||
names: dict[str, dict[str, str]] = {}
|
||||
# The tools the model can call after the history, from a request rendered in
|
||||
# its own wire form; None when `tools` alone declares them.
|
||||
chat_tools: list[Tool] | None = None
|
||||
# ``custom`` tools a ``tool_search_output`` loaded, beside the declared ones.
|
||||
discovered_custom_names: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
class ResponseToolSearchCall(BaseModel):
|
||||
"""A client-executed ``tool_search`` call; the pinned openai SDK has no type."""
|
||||
|
||||
id: str
|
||||
type: Literal["tool_search_call"] = "tool_search_call"
|
||||
call_id: str
|
||||
execution: Literal["client"] = "client"
|
||||
arguments: Dict[str, Any]
|
||||
status: Literal["in_progress", "completed", "incomplete"]
|
||||
|
||||
|
||||
class ResponseToolSearchCallEvent(BaseModel):
|
||||
"""``response.output_item.added`` / ``.done`` for a tool search call, which the
|
||||
SDK's event types reject."""
|
||||
|
||||
type: Literal["response.output_item.added", "response.output_item.done"]
|
||||
sequence_number: int
|
||||
output_index: int
|
||||
item: ResponseToolSearchCall
|
||||
|
||||
|
||||
class ResponseInputMessageParam(EasyInputMessageParam, total=False):
|
||||
phase: Optional[Literal["commentary", "final_answer"]]
|
||||
|
||||
@@ -1856,6 +1913,10 @@ ResponseInputOutputItem: TypeAlias = Union[
|
||||
class ResponsesRequest(PDRoutingFields):
|
||||
"""Request body for v1/responses endpoint."""
|
||||
|
||||
_tool_state: ResponseToolState = PrivateAttr(default_factory=ResponseToolState)
|
||||
# By call_id, the name the model called a tool by when the output item
|
||||
# reports another; the stored history replays the call under it.
|
||||
_called_tool_names: Dict[str, str] = PrivateAttr(default_factory=dict)
|
||||
_response_parser_prefix: str = PrivateAttr(default="")
|
||||
_response_parser_grammar_start: Optional[str] = PrivateAttr(default=None)
|
||||
_response_parser_json_response_format: bool = PrivateAttr(default=False)
|
||||
@@ -2176,6 +2237,7 @@ class ResponsesResponse(BaseModel):
|
||||
ResponseOutputItem,
|
||||
ResponseReasoningItem,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseToolSearchCall,
|
||||
]
|
||||
] = Field(default_factory=list)
|
||||
status: Literal[
|
||||
@@ -2241,6 +2303,7 @@ class ResponsesResponse(BaseModel):
|
||||
ResponseOutputItem,
|
||||
ResponseReasoningItem,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseToolSearchCall,
|
||||
]
|
||||
],
|
||||
status: str,
|
||||
|
||||
@@ -957,12 +957,13 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
The reasoning parser is told the same thing, so channel-framed formats
|
||||
keep their framing intact exactly when a tool-call parser consumes it.
|
||||
"""
|
||||
return bool(
|
||||
request.tool_choice != "none"
|
||||
and self._effective_tools(request)
|
||||
and self.tool_call_parser
|
||||
and not request._response_parser_json_response_format
|
||||
)
|
||||
if not self.tool_call_parser or request._response_parser_json_response_format:
|
||||
return False
|
||||
if request._drops_unlisted_tool_calls and request.tool_choice == "none":
|
||||
# The history showed the model tools, so it can still write a call;
|
||||
# parsing drops it, where skipping the parser would return its markup.
|
||||
return True
|
||||
return bool(request.tool_choice != "none" and self._effective_tools(request))
|
||||
|
||||
def _validate_request(self, request: ChatCompletionRequest) -> str | None:
|
||||
"""Validate that the input is valid."""
|
||||
@@ -1367,18 +1368,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
self, request: ChatCompletionRequest, is_multimodal: bool
|
||||
) -> MessageProcessingResult:
|
||||
"""Process chat messages and apply chat template"""
|
||||
if self.default_chat_template_kwargs:
|
||||
ctk = dict(request.chat_template_kwargs or {})
|
||||
for k, v in self.default_chat_template_kwargs.items():
|
||||
ctk.setdefault(k, v)
|
||||
request.chat_template_kwargs = ctk
|
||||
effort = ctk.get("reasoning_effort")
|
||||
if effort is not None and request.reasoning_effort is None:
|
||||
request.reasoning_effort = effort
|
||||
|
||||
normalize_hunyuan_reasoning_effort(
|
||||
request, self.reasoning_parser, self.template_manager.reasoning_config
|
||||
)
|
||||
self._apply_request_defaults(request)
|
||||
|
||||
# GptOss model needs to keep special tokens for harmony parsing
|
||||
if self.is_gpt_oss or self.is_gemma4:
|
||||
@@ -1484,7 +1474,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
result = MessageProcessingResult(
|
||||
prompt="",
|
||||
prompt_ids=request.input_ids,
|
||||
image_data=None,
|
||||
image_data=request._input_image_data,
|
||||
audio_data=None,
|
||||
video_data=None,
|
||||
modalities=[],
|
||||
@@ -1540,6 +1530,64 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
result.reasoning_end_token_ids = list(token_ids)
|
||||
return result
|
||||
|
||||
def _apply_request_defaults(self, request: ChatCompletionRequest) -> None:
|
||||
"""Settle the thinking and effort that rendering reads; repeating it
|
||||
changes nothing."""
|
||||
if self.default_chat_template_kwargs:
|
||||
ctk = dict(request.chat_template_kwargs or {})
|
||||
for k, v in self.default_chat_template_kwargs.items():
|
||||
ctk.setdefault(k, v)
|
||||
request.chat_template_kwargs = ctk
|
||||
effort = ctk.get("reasoning_effort")
|
||||
if effort is not None and request.reasoning_effort is None:
|
||||
request.reasoning_effort = effort
|
||||
|
||||
normalize_hunyuan_reasoning_effort(
|
||||
request, self.reasoning_parser, self.template_manager.reasoning_config
|
||||
)
|
||||
|
||||
@property
|
||||
def native_tool_loading(self) -> bool:
|
||||
"""Whether Responses and Messages requests that load tools mid-conversation
|
||||
render from their own wire form (see `prepare_native_prompt`)."""
|
||||
return (
|
||||
self._encoding_adapter is not None
|
||||
and self._encoding_adapter.native_encoder is not None
|
||||
)
|
||||
|
||||
def prepare_native_prompt(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
*,
|
||||
protocol: chat_encoding_adapters.NativeProtocol,
|
||||
body: dict[str, Any],
|
||||
) -> chat_encoding_adapters.NativePrompt:
|
||||
"""Render `body`, the Responses or Messages form of `request`, into its
|
||||
`input_ids` and images, and set its `tools` to the tools the model can call
|
||||
after the history; rendering then keeps only grammar, stops and parsing."""
|
||||
self._apply_request_defaults(request)
|
||||
response_format = request.response_format
|
||||
prompt = self._encoding_adapter.native_encoder.encode_native(
|
||||
protocol=protocol,
|
||||
body=body,
|
||||
thinking=self._get_reasoning_from_request(request),
|
||||
effort=request.reasoning_effort,
|
||||
response_schema=(
|
||||
response_format.json_schema.schema_
|
||||
if response_format is not None and response_format.json_schema
|
||||
else None
|
||||
),
|
||||
continue_final_message=request.continue_final_message,
|
||||
)
|
||||
if prompt.image_data and not self.tokenizer_manager.model_config.is_multimodal:
|
||||
raise ValueError("image input is not supported for this model")
|
||||
request.input_ids = prompt.prompt_ids
|
||||
request.tools = prompt.tools or None
|
||||
request._input_image_data = prompt.image_data or None
|
||||
request._drops_unlisted_tool_calls = True
|
||||
request._continued_turn = prompt.continued_turn
|
||||
return prompt
|
||||
|
||||
def _apply_jinja_template(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
@@ -5,15 +5,20 @@ request semantics."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from sglang.srt.entrypoints.openai.chat_encoding_adapters import (
|
||||
ADAPTER_OUTPUT_PARSERS,
|
||||
NativePrompt,
|
||||
NativeProtocol,
|
||||
select_new_output_ids,
|
||||
token_output_sse_chunks,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
Function,
|
||||
FunctionResponse,
|
||||
Tool,
|
||||
ToolCall,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
@@ -26,7 +31,9 @@ from sglang.srt.parser.deepseek_recipe_output import (
|
||||
whole_tool_calls,
|
||||
)
|
||||
from sglang.srt.parser.deepseek_recipe_renderer import (
|
||||
BUDGET_TIER,
|
||||
DeepSeekRecipe,
|
||||
loaded_tools,
|
||||
parse_effort_budget,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_serving
|
||||
@@ -37,7 +44,6 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ResponsesRequest,
|
||||
Tool,
|
||||
)
|
||||
|
||||
DEFAULT_EFFORT_ENVS = {
|
||||
@@ -45,6 +51,12 @@ DEFAULT_EFFORT_ENVS = {
|
||||
"dsv41": envs.SGLANG_DSV41_REASONING_EFFORT,
|
||||
}
|
||||
JSON_RESPONSE_FORMATS = ("json_object", "json_schema")
|
||||
# V4 renders deferred tools up front and never loads one mid-conversation.
|
||||
NATIVE_TOOL_LOADING_SPEC = "dsv41"
|
||||
NATIVE_EFFORT_FIELDS: dict[str, str] = {
|
||||
"responses": "reasoning",
|
||||
"messages": "output_config",
|
||||
}
|
||||
|
||||
|
||||
class DeepSeekRecipeChatEncodingAdapter:
|
||||
@@ -96,6 +108,13 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
def output(self) -> Optional[DeepSeekRecipeChatEncodingAdapter]:
|
||||
return self if self._token_output else None
|
||||
|
||||
def validate_request(self, request: ChatCompletionRequest) -> Optional[str]:
|
||||
return None
|
||||
|
||||
@property
|
||||
def native_encoder(self) -> Optional[DeepSeekRecipeChatEncodingAdapter]:
|
||||
return self if self._spec == NATIVE_TOOL_LOADING_SPEC else None
|
||||
|
||||
def encode_messages(
|
||||
self,
|
||||
*,
|
||||
@@ -149,6 +168,66 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
prompt_ids += self.deepseek_recipe.encode_text(task_prefix)
|
||||
return prompt_ids
|
||||
|
||||
def encode_native(
|
||||
self,
|
||||
*,
|
||||
protocol: NativeProtocol,
|
||||
body: dict[str, Any],
|
||||
thinking: bool,
|
||||
effort: Union[str, int, float, None],
|
||||
response_schema: Optional[dict[str, Any]],
|
||||
continue_final_message: bool,
|
||||
) -> NativePrompt:
|
||||
tier, budget = self._effort(effort)
|
||||
# recipe reads an effort as a request to think.
|
||||
if thinking and tier is not None:
|
||||
body = {**body, NATIVE_EFFORT_FIELDS[protocol]: {"effort": tier}}
|
||||
conversation = self.deepseek_recipe.with_host_settings(
|
||||
self.deepseek_recipe.convert_native(protocol, body, thinking=thinking),
|
||||
response_schema=response_schema,
|
||||
continue_final_message=continue_final_message,
|
||||
)
|
||||
last = conversation.messages[-1] if conversation.messages else None
|
||||
# recipe's Messages conversion drops `strict`, which SGLang's grammar enforces.
|
||||
strict = {tool.get("name") for tool in body["tools"] if tool.get("strict")}
|
||||
tools = [
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name=tool.name,
|
||||
description=tool.description,
|
||||
parameters=tool.parameters,
|
||||
strict=True if tool.name in strict else tool.strict,
|
||||
),
|
||||
)
|
||||
for tool in loaded_tools(conversation)
|
||||
]
|
||||
return NativePrompt(
|
||||
prompt_ids=self.deepseek_recipe.encode_conversation(
|
||||
conversation, effort_budget=budget
|
||||
),
|
||||
tools=tools,
|
||||
image_data=[
|
||||
ImageData(url=url)
|
||||
for url in self.deepseek_recipe.image_urls(conversation)
|
||||
],
|
||||
namespace_names=(
|
||||
self.deepseek_recipe.namespace_tool_names(_responses_namespaces(body))
|
||||
if protocol == "responses"
|
||||
else {}
|
||||
),
|
||||
continued_turn=(
|
||||
{"role": "assistant", "content": last.content}
|
||||
| (
|
||||
{"reasoning_content": last.reasoning_content}
|
||||
if last.reasoning_content is not None
|
||||
else {}
|
||||
)
|
||||
if last is not None and last.prefix
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _encode(
|
||||
self,
|
||||
body: dict[str, Any],
|
||||
@@ -157,21 +236,29 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
task: Optional[str] = None,
|
||||
image_urls: Optional[list[str]] = None,
|
||||
) -> list[int]:
|
||||
if effort is None or effort == "none":
|
||||
effort = self._default_effort
|
||||
budget = None
|
||||
if effort is not None and not isinstance(effort, str):
|
||||
if self._spec != "dsv41":
|
||||
raise ValueError(
|
||||
f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}"
|
||||
)
|
||||
budget = parse_effort_budget(effort)
|
||||
elif effort is not None:
|
||||
body["reasoning_effort"] = effort
|
||||
tier, budget = self._effort(effort)
|
||||
if budget is None and tier is not None:
|
||||
body["reasoning_effort"] = tier
|
||||
return self.deepseek_recipe.encode(
|
||||
body, task=task, effort_budget=budget, image_urls=image_urls
|
||||
)
|
||||
|
||||
def _effort(
|
||||
self, effort: Union[str, int, float, None]
|
||||
) -> tuple[Optional[str], Optional[int]]:
|
||||
"""The named tier recipe renders and, for a V4.1 integer budget, the budget
|
||||
that replaces the tier's line."""
|
||||
effort = _numeric_effort(effort)
|
||||
if effort is None or effort == "none":
|
||||
effort = self._default_effort
|
||||
if effort is None or isinstance(effort, str):
|
||||
return effort, None
|
||||
if self._spec != "dsv41":
|
||||
raise ValueError(
|
||||
f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}"
|
||||
)
|
||||
return BUDGET_TIER, parse_effort_budget(effort)
|
||||
|
||||
def parse_chat_response(
|
||||
self,
|
||||
*,
|
||||
@@ -320,7 +407,7 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
self, request: ChatCompletionRequest
|
||||
) -> DeepSeekRecipeOutputParser:
|
||||
response_format = request.response_format
|
||||
continued = (
|
||||
continued = request._continued_turn or (
|
||||
_trailing_assistant_turn(
|
||||
[
|
||||
message.model_dump(exclude_unset=True, by_alias=True, mode="json")
|
||||
@@ -344,6 +431,11 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
separate_reasoning=self._separate_reasoning and request.separate_reasoning,
|
||||
stream_reasoning=request.stream_reasoning,
|
||||
stop=_stop_strings(request.stop),
|
||||
callable_names=(
|
||||
_native_callable_names(request.tools or [], request.tool_choice)
|
||||
if request._drops_unlisted_tool_calls
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _new_responses_parser(
|
||||
@@ -353,12 +445,25 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
separate_reasoning: bool,
|
||||
parse_tool_calls: bool,
|
||||
) -> DeepSeekRecipeOutputParser:
|
||||
native = request._tool_state.chat_tools is not None
|
||||
return DeepSeekRecipeOutputParser(
|
||||
self.deepseek_recipe,
|
||||
thinking_stage=self.deepseek_recipe.parse_start(thinking=require_reasoning),
|
||||
parse_tool_calls=parse_tool_calls,
|
||||
# Under `none` the tool parser runs only to drop calls; a JSON answer
|
||||
# must be read as JSON, or a call marker spelled in a string ends it.
|
||||
parse_json_output=native
|
||||
and request.tool_choice == "none"
|
||||
and request.has_json_schema_constraint(),
|
||||
separate_reasoning=separate_reasoning,
|
||||
stop=_stop_strings(request.stop),
|
||||
callable_names=(
|
||||
_native_callable_names(
|
||||
request._tool_state.chat_tools, request.tool_choice
|
||||
)
|
||||
if native
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def _tool_call_id(self, index: int, name: str, history_tool_calls_cnt: int) -> str:
|
||||
@@ -390,6 +495,37 @@ class DeepSeekRecipeResponsesStream:
|
||||
return delta.reasoning, delta.content, _tool_call_items(delta.tool_calls)
|
||||
|
||||
|
||||
def _responses_namespaces(body: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""The namespace tools a Responses body declares and its history discovers, in
|
||||
the order recipe's conversion names their members."""
|
||||
discovered = [
|
||||
tool
|
||||
for item in body["input"]
|
||||
if isinstance(item, dict) and item.get("type") == "tool_search_output"
|
||||
for tool in item["tools"]
|
||||
]
|
||||
return [
|
||||
tool for tool in [*body["tools"], *discovered] if tool["type"] == "namespace"
|
||||
]
|
||||
|
||||
|
||||
def _null_content_as_empty(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""As `_process_messages` renders them: a null content is ""."""
|
||||
return [
|
||||
message if message.get("content") is not None else {**message, "content": ""}
|
||||
for message in messages
|
||||
]
|
||||
|
||||
|
||||
def _numeric_effort(
|
||||
effort: Union[str, int, float, None],
|
||||
) -> Union[str, int, float, None]:
|
||||
"""A numeric string effort ("30", "0.3") as the number it spells."""
|
||||
if isinstance(effort, str) and re.fullmatch(r"\d+(\.\d+)?", effort.strip()):
|
||||
return float(effort) if "." in effort else int(effort)
|
||||
return effort
|
||||
|
||||
|
||||
def _trailing_assistant_turn(
|
||||
messages: list[dict[str, Any]],
|
||||
) -> Optional[dict[str, Any]]:
|
||||
@@ -405,6 +541,18 @@ def _stop_strings(stop: Union[str, list[str], None]) -> list[str]:
|
||||
return [stop] if isinstance(stop, str) else stop or []
|
||||
|
||||
|
||||
def _native_callable_names(
|
||||
tools: list[Tool], tool_choice: Any
|
||||
) -> Optional[frozenset[str]]:
|
||||
"""The names of `tools`, all that a natively rendered prompt shows the model;
|
||||
None when calls to other tools are returned. `none` allows no call at all."""
|
||||
if tool_choice == "none":
|
||||
return frozenset()
|
||||
if envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get():
|
||||
return None
|
||||
return frozenset(tool.function.name for tool in tools)
|
||||
|
||||
|
||||
def _tool_call_items(
|
||||
calls: Sequence[Union[RecipeToolCall, RecipeToolCallDelta]],
|
||||
) -> list[ToolCallItem]:
|
||||
|
||||
@@ -36,7 +36,10 @@ if TYPE_CHECKING:
|
||||
|
||||
class InklingChatEncodingAdapter:
|
||||
collects_openai_media = True
|
||||
strict_tool_call_arguments = True
|
||||
history_tool_call_arguments = "strict"
|
||||
renders_response_format = False
|
||||
uncounted_prompt_tokens = 0
|
||||
native_encoder = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -13,6 +13,7 @@ from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterator, Optional, Union
|
||||
|
||||
import jinja2
|
||||
import msgspec
|
||||
import openai.types.responses as openai_responses_types
|
||||
import orjson
|
||||
from fastapi import Request
|
||||
@@ -43,6 +44,7 @@ from openai.types.responses.response_reasoning_summary_part_done_event import (
|
||||
Part as ResponseReasoningSummaryDonePart,
|
||||
)
|
||||
from openai_harmony import Message as OpenAIMessage
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from sglang.srt.entrypoints.context import (
|
||||
ConversationContext,
|
||||
@@ -70,7 +72,12 @@ from sglang.srt.entrypoints.openai.protocol import (
|
||||
ResponseOutputMessage,
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
ResponseTool,
|
||||
ResponseToolSearchCall,
|
||||
ResponseToolSearchCallEvent,
|
||||
Tool,
|
||||
ToolChoice,
|
||||
ToolChoiceFuncName,
|
||||
UsageInfo,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.responses_adapters import (
|
||||
@@ -114,6 +121,100 @@ class _MediaInputValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ResponseToolChoiceError(ValueError):
|
||||
"""A response tool_choice that names a tool the model cannot call."""
|
||||
|
||||
|
||||
TOOL_SEARCH = "tool_search"
|
||||
TOOL_SEARCH_ITEMS = ("tool_search_call", "tool_search_output")
|
||||
APPLY_PATCH = "apply_patch"
|
||||
NAMESPACE_MEMBERS_ERROR = (
|
||||
"namespace tools must list function tools, each with a type and a name"
|
||||
)
|
||||
|
||||
|
||||
def _holds_function_tools(members: Any) -> bool:
|
||||
return isinstance(members, list) and all(
|
||||
isinstance(member, dict)
|
||||
and member.get("type") == "function"
|
||||
and isinstance(member.get("name"), str)
|
||||
for member in members
|
||||
)
|
||||
|
||||
|
||||
def _defers_loading(tool: ResponseTool) -> bool:
|
||||
if tool.type == "namespace":
|
||||
return bool(tool.defer_loading) or any(
|
||||
isinstance(member, dict) and member.get("defer_loading")
|
||||
for member in tool.tools or []
|
||||
)
|
||||
return tool.type in ("function", "custom") and bool(tool.defer_loading)
|
||||
|
||||
|
||||
def _defer_tool_loading(
|
||||
tools: list[dict[str, Any]], items: list
|
||||
) -> tuple[list[dict[str, Any]], list]:
|
||||
"""`tools` without those that defer loading, and `items` whose
|
||||
`tool_search_output` entries naming a deferred tool take the fields they
|
||||
leave out from its declaration, so a search result loads it where it returns.
|
||||
recipe's Responses conversion renders a deferred tool up front and rejects a
|
||||
search result that repeats a declared name."""
|
||||
deferred: dict[tuple[Optional[str], str], dict[str, Any]] = {}
|
||||
shown = []
|
||||
for tool in tools:
|
||||
if tool["type"] == "namespace":
|
||||
members = []
|
||||
for member in tool["tools"]:
|
||||
if tool.get("defer_loading") or member.get("defer_loading"):
|
||||
deferred[(tool["name"], member["name"])] = member
|
||||
else:
|
||||
members.append(member)
|
||||
if members:
|
||||
shown.append({**tool, "tools": members})
|
||||
elif tool["type"] in ("function", "custom") and tool.get("defer_loading"):
|
||||
deferred[(None, tool["name"])] = tool
|
||||
else:
|
||||
shown.append(tool)
|
||||
|
||||
def loaded(tool: dict[str, Any], namespace: Optional[str] = None) -> dict:
|
||||
if tool["type"] == "namespace":
|
||||
members = [loaded(member, tool["name"]) for member in tool["tools"]]
|
||||
return {**tool, "tools": members}
|
||||
declared = deferred.get((namespace, tool.get("name")))
|
||||
if declared is None or declared["type"] != tool["type"]:
|
||||
return tool
|
||||
return declared | {k: v for k, v in tool.items() if v is not None}
|
||||
|
||||
if deferred:
|
||||
items = [
|
||||
(
|
||||
{**item, "tools": [loaded(tool) for tool in item["tools"]]}
|
||||
if isinstance(item, dict) and item.get("type") == "tool_search_output"
|
||||
else item
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
return shown, items
|
||||
|
||||
|
||||
def _tool_search_call(
|
||||
item_id: str, call_id: str, arguments: str
|
||||
) -> ResponseToolSearchCall:
|
||||
"""A search call reports object arguments; one whose arguments are not a JSON
|
||||
object is incomplete."""
|
||||
try:
|
||||
parsed = json.loads(arguments)
|
||||
except ValueError:
|
||||
parsed = None
|
||||
complete = isinstance(parsed, dict)
|
||||
return ResponseToolSearchCall(
|
||||
id=item_id,
|
||||
call_id=call_id,
|
||||
arguments=parsed if complete else {},
|
||||
status="completed" if complete else "incomplete",
|
||||
)
|
||||
|
||||
|
||||
def _build_output_text_logprobs(meta_info: dict) -> list[Logprob]:
|
||||
"""Reshape decoded ``meta_info`` logprobs into the Responses logprob type,
|
||||
covering every generated token."""
|
||||
@@ -319,9 +420,21 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
# FIXME: If the engine is dead, raise an error
|
||||
# This is required for the streaming case
|
||||
|
||||
# Such a request's callable tools are known once its history renders,
|
||||
# so `_make_request` checks its tool_choice instead.
|
||||
native = self._renders_natively(request)
|
||||
if native:
|
||||
tools_error = self._native_tools_error(request)
|
||||
if tools_error is not None:
|
||||
return self.create_error_response(tools_error, param="tools")
|
||||
|
||||
# ``tool_choice="required"`` needs a tool the parser can actually force.
|
||||
if request.tool_choice == "required" and not any(
|
||||
tool.type in ("function", "custom") for tool in (request.tools or [])
|
||||
if (
|
||||
not native
|
||||
and request.tool_choice == "required"
|
||||
and not any(
|
||||
tool.type in ("function", "custom") for tool in (request.tools or [])
|
||||
)
|
||||
):
|
||||
return self.create_error_response(
|
||||
'tool_choice="required" requires at least one tool with '
|
||||
@@ -330,9 +443,13 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
|
||||
tool_choice = request.effective_tool_choice()
|
||||
if isinstance(tool_choice, dict) and not any(
|
||||
tool.type in ("function", "custom") and tool.name == tool_choice["name"]
|
||||
for tool in request.tools or []
|
||||
if (
|
||||
not native
|
||||
and isinstance(tool_choice, dict)
|
||||
and not any(
|
||||
tool.type in ("function", "custom") and tool.name == tool_choice["name"]
|
||||
for tool in request.tools or []
|
||||
)
|
||||
):
|
||||
return self.create_error_response(
|
||||
f"Tool {tool_choice['name']!r} is not declared in tools",
|
||||
@@ -415,6 +532,8 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
|
||||
except _MediaInputValidationError as e:
|
||||
return self.create_error_response(str(e))
|
||||
except ResponseToolChoiceError as e:
|
||||
return self.create_error_response(str(e), param="tool_choice")
|
||||
except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
|
||||
logger.exception("Error in preprocessing prompt inputs")
|
||||
return self.create_error_response(f"{e} {e.__cause__}")
|
||||
@@ -684,7 +803,10 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
prev_response: Optional[ResponsesResponse],
|
||||
tokenizer: Any,
|
||||
):
|
||||
messages = self._construct_input_messages(request, prev_response)
|
||||
native = self._renders_natively(request)
|
||||
messages = (
|
||||
[] if native else self._construct_input_messages(request, prev_response)
|
||||
)
|
||||
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
chat_request = ChatCompletionRequest(
|
||||
@@ -707,6 +829,8 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
)
|
||||
chat_request._template_tools = self._response_tools_to_template_tools(request)
|
||||
if native:
|
||||
self._prepare_native_responses_prompt(request, chat_request)
|
||||
|
||||
media_error = self._validate_media_content(chat_request)
|
||||
if media_error:
|
||||
@@ -906,10 +1030,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
else:
|
||||
self.msg_store[response.id] = self._response_input_history(
|
||||
request
|
||||
) + [
|
||||
item.model_dump(exclude_none=True)
|
||||
for item in response.output
|
||||
]
|
||||
) + self._replayed_output_items(request, response.output)
|
||||
|
||||
return response
|
||||
|
||||
@@ -952,12 +1073,18 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
name: str,
|
||||
arguments: str,
|
||||
custom_names: set[str],
|
||||
namespace_names: Optional[dict[str, tuple[str, str]]] = None,
|
||||
) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]:
|
||||
tool_names: dict[str, dict[str, str]],
|
||||
*,
|
||||
client_tool_search: bool = False,
|
||||
) -> Union[
|
||||
ResponseFunctionToolCall, ResponseCustomToolCall, ResponseToolSearchCall
|
||||
]:
|
||||
"""A call against a ``custom`` tool reports its freeform payload rather
|
||||
than the JSON arguments of the shim function tool. A call against a
|
||||
namespace member reports ``namespace`` and the bare function ``name``."""
|
||||
call_id = f"call_{random_uuid()[:24]}"
|
||||
if client_tool_search and name == TOOL_SEARCH:
|
||||
return _tool_search_call(f"tsc_{random_uuid()[:8]}", call_id, arguments)
|
||||
if name in custom_names:
|
||||
return ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
@@ -966,7 +1093,13 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
name=name,
|
||||
input=decode_custom_tool_input(arguments),
|
||||
)
|
||||
namespace, name = (namespace_names or {}).get(name, (None, name))
|
||||
entry = (tool_names or {}).get(name)
|
||||
if isinstance(entry, tuple):
|
||||
namespace, name = entry
|
||||
elif isinstance(entry, dict):
|
||||
namespace, name = entry.get("namespace"), entry.get("name", name)
|
||||
else:
|
||||
namespace = None
|
||||
return ResponseFunctionToolCall(
|
||||
arguments=arguments,
|
||||
call_id=call_id,
|
||||
@@ -1063,12 +1196,17 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
finish_reason: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
custom_names = self._custom_tool_names(request)
|
||||
namespace_names = (
|
||||
namespace_function_names(request.tools)
|
||||
if self.tool_call_parser == "muse"
|
||||
else {}
|
||||
else {
|
||||
call_name: (entry.get("namespace"), entry.get("name", call_name))
|
||||
for call_name, entry in request._tool_state.names.items()
|
||||
if entry.get("namespace")
|
||||
}
|
||||
)
|
||||
client_tool_search = self._client_tool_search(request)
|
||||
encoding_output = self._encoding_output()
|
||||
if encoding_output is not None:
|
||||
required_json = self._uses_required_tool_json(request, chat_tools)
|
||||
@@ -1088,12 +1226,17 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
content, custom_names, namespace_names
|
||||
)
|
||||
else:
|
||||
tool_call_items = [
|
||||
self._make_tool_call_item(
|
||||
call.name, call.parameters, custom_names, namespace_names
|
||||
tool_call_items = []
|
||||
for call in calls:
|
||||
item = self._make_tool_call_item(
|
||||
call.name,
|
||||
call.parameters,
|
||||
custom_names,
|
||||
namespace_names,
|
||||
client_tool_search=client_tool_search,
|
||||
)
|
||||
for call in calls
|
||||
]
|
||||
self._record_called_name(request, item.call_id, call.name)
|
||||
tool_call_items.append(item)
|
||||
else:
|
||||
reasoning_content, content, tool_call_items = self._parse_text_output(
|
||||
request,
|
||||
@@ -1194,14 +1337,15 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
try:
|
||||
content, call_info_list = parser.parse_non_stream(content)
|
||||
for call_info in call_info_list:
|
||||
tool_call_items.append(
|
||||
self._make_tool_call_item(
|
||||
call_info.name,
|
||||
call_info.parameters or "",
|
||||
custom_names,
|
||||
namespace_names,
|
||||
)
|
||||
item = self._make_tool_call_item(
|
||||
call_info.name,
|
||||
call_info.parameters or "",
|
||||
custom_names,
|
||||
namespace_names,
|
||||
client_tool_search=self._client_tool_search(request),
|
||||
)
|
||||
self._record_called_name(request, item.call_id, call_info.name)
|
||||
tool_call_items.append(item)
|
||||
parsed_via_native = bool(call_info_list)
|
||||
except Exception as e:
|
||||
logger.error("Tool call parsing error: %s", e)
|
||||
@@ -1268,6 +1412,10 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
def _tool_call_parsing_enabled(
|
||||
self, request: ResponsesRequest, chat_tools: list[Tool]
|
||||
) -> bool:
|
||||
if request._tool_state.chat_tools is not None and request.tool_choice == "none":
|
||||
# The history showed the model tools, so it can still write a call;
|
||||
# parsing drops it, where skipping the parser would return its markup.
|
||||
return bool(self.tool_call_parser)
|
||||
return bool(
|
||||
chat_tools and self.tool_call_parser and request.tool_choice != "none"
|
||||
)
|
||||
@@ -1326,7 +1474,213 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
return tool_choice
|
||||
return {"type": "function", "function": {"name": tool_choice["name"]}}
|
||||
|
||||
def _renders_natively(self, request: ResponsesRequest) -> bool:
|
||||
"""Whether the served encoding renders the request in its own wire form,
|
||||
which tool search, namespace tools and deferred tools take."""
|
||||
return self.native_tool_loading and (
|
||||
any(
|
||||
tool.type == "namespace" or _defers_loading(tool)
|
||||
for tool in request.tools
|
||||
)
|
||||
or self._uses_tool_search(request)
|
||||
)
|
||||
|
||||
def _uses_tool_search(self, request: ResponsesRequest) -> bool:
|
||||
"""Whether the request searches for tools, or its history has."""
|
||||
if any(tool.type == TOOL_SEARCH for tool in request.tools):
|
||||
return True
|
||||
return any(
|
||||
isinstance(item, dict) and item.get("type") in TOOL_SEARCH_ITEMS
|
||||
for item in self._history_items(request)
|
||||
)
|
||||
|
||||
def _history_items(self, request: ResponsesRequest) -> list:
|
||||
"""The input history, before `previous_response_id` is known to exist."""
|
||||
stored = self.msg_store.get(request.previous_response_id) or []
|
||||
return list(stored) + (request.input if isinstance(request.input, list) else [])
|
||||
|
||||
def _native_tools_error(self, request: ResponsesRequest) -> Optional[str]:
|
||||
deferred_types = {
|
||||
tool.name: tool.type
|
||||
for tool in request.tools
|
||||
if tool.type in ("function", "custom") and tool.defer_loading
|
||||
}
|
||||
for tool in request.tools:
|
||||
if (
|
||||
tool.type == TOOL_SEARCH
|
||||
and tool.model_extra.get("execution") != "client"
|
||||
):
|
||||
return (
|
||||
'tool_search requires execution "client": this server runs no '
|
||||
"searches"
|
||||
)
|
||||
if tool.type == "namespace" and not _holds_function_tools(tool.tools):
|
||||
return NAMESPACE_MEMBERS_ERROR
|
||||
for item in self._history_items(request):
|
||||
if not isinstance(item, dict) or item.get("type") != "tool_search_output":
|
||||
continue
|
||||
tools = item.get("tools")
|
||||
if not isinstance(tools, list) or not all(
|
||||
isinstance(tool, dict)
|
||||
and tool.get("type") in ("function", "namespace", "custom")
|
||||
for tool in tools
|
||||
):
|
||||
return (
|
||||
"tool_search_output tools must be a list of function, namespace "
|
||||
"or custom tools"
|
||||
)
|
||||
for tool in tools:
|
||||
if tool["type"] == "namespace" and not _holds_function_tools(
|
||||
tool.get("tools")
|
||||
):
|
||||
return NAMESPACE_MEMBERS_ERROR
|
||||
# The served encoding renders no other custom tool.
|
||||
if tool["type"] == "custom" and tool.get("name") != APPLY_PATCH:
|
||||
return (
|
||||
f"tool_search_output can load only the {APPLY_PATCH!r} "
|
||||
"custom tool"
|
||||
)
|
||||
# A deferred declaration is loaded as the tool it declares.
|
||||
declared = deferred_types.get(tool.get("name"), tool["type"])
|
||||
if tool["type"] != "namespace" and declared != tool["type"]:
|
||||
return (
|
||||
f"tool_search_output returns {tool.get('name')!r} as a "
|
||||
f"{tool['type']} tool, but tools declares it as a deferred "
|
||||
f"{declared} tool"
|
||||
)
|
||||
return None
|
||||
|
||||
def _prepare_native_responses_prompt(
|
||||
self, request: ResponsesRequest, chat_request: ChatCompletionRequest
|
||||
) -> None:
|
||||
"""Render the request's own wire form into `chat_request`, whose tools and
|
||||
choice become the ones the history has loaded."""
|
||||
choice = request.effective_tool_choice()
|
||||
tools, items = _defer_tool_loading(
|
||||
[tool.model_dump(exclude_unset=True) for tool in request.tools],
|
||||
[
|
||||
item
|
||||
for item in map(
|
||||
self._native_input_item, self._response_input_history(request)
|
||||
)
|
||||
if item is not None
|
||||
],
|
||||
)
|
||||
body = {"input": items, "tools": tools}
|
||||
if request.instructions:
|
||||
body["instructions"] = request.instructions
|
||||
if choice == "none":
|
||||
body["tool_choice"] = "none"
|
||||
prompt = self.prepare_native_prompt(
|
||||
chat_request, protocol="responses", body=body
|
||||
)
|
||||
tools = prompt.tools
|
||||
request._tool_state = msgspec.structs.replace(
|
||||
request._tool_state,
|
||||
chat_tools=tools,
|
||||
names={
|
||||
call_name: {"namespace": namespace, "name": name}
|
||||
for call_name, (namespace, name) in prompt.namespace_names.items()
|
||||
},
|
||||
discovered_custom_names=frozenset(
|
||||
tool["name"]
|
||||
for item in body["input"]
|
||||
if isinstance(item, dict) and item.get("type") == "tool_search_output"
|
||||
for tool in item["tools"]
|
||||
if tool["type"] == "custom"
|
||||
),
|
||||
)
|
||||
if isinstance(choice, dict):
|
||||
name = choice["name"]
|
||||
if "namespace" in choice:
|
||||
# `tools` keeps conversion order, declared before discovered, so
|
||||
# the last match is the member's latest definition.
|
||||
member = (choice["namespace"], name)
|
||||
name = next(
|
||||
(
|
||||
tool.function.name
|
||||
for tool in reversed(tools)
|
||||
if prompt.namespace_names.get(tool.function.name) == member
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not any(tool.function.name == name for tool in tools) or (
|
||||
choice["type"] == "custom"
|
||||
and name not in self._custom_tool_names(request)
|
||||
):
|
||||
raise ResponseToolChoiceError(
|
||||
f"Tool {choice['name']!r} is not declared in tools without "
|
||||
"defer_loading or loaded by a tool_search_output"
|
||||
)
|
||||
chat_request.tool_choice = ToolChoice(
|
||||
function=ToolChoiceFuncName(name=name)
|
||||
)
|
||||
elif choice == "required" and not tools:
|
||||
raise ResponseToolChoiceError(
|
||||
'tool_choice="required" needs a tool the model can call'
|
||||
)
|
||||
else:
|
||||
chat_request.tool_choice = choice if tools else "none"
|
||||
|
||||
@classmethod
|
||||
def _native_input_item(cls, item: Any) -> Any:
|
||||
if isinstance(item, BaseModel):
|
||||
item = item.model_dump(exclude_none=True)
|
||||
if isinstance(item, dict) and item.get("type") == "reasoning":
|
||||
# recipe reads only `content`; keep the text SGLang replays from a
|
||||
# summary or a store=false client's state.
|
||||
message = cls._normalize_response_message_for_chat(item)
|
||||
if message is None:
|
||||
return None
|
||||
return {
|
||||
"type": "reasoning",
|
||||
"content": [
|
||||
{"type": "reasoning_text", "text": message["reasoning_content"]}
|
||||
],
|
||||
}
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
def _record_called_name(request: ResponsesRequest, call_id: str, name: str) -> None:
|
||||
"""Keep the name the model called a namespace member by: the item reports
|
||||
its namespace and own name, which several call names can share."""
|
||||
if request._tool_state.chat_tools is not None and (
|
||||
name in request._tool_state.names
|
||||
):
|
||||
request._called_tool_names[call_id] = name
|
||||
|
||||
@staticmethod
|
||||
def _replayed_output_items(request: ResponsesRequest, output: list) -> list[dict]:
|
||||
"""`output` as the history a later request replays: a namespace member's
|
||||
call under the name the model called it by."""
|
||||
items = []
|
||||
for item in output:
|
||||
item = item.model_dump(exclude_none=True)
|
||||
called_name = request._called_tool_names.get(item.get("call_id"))
|
||||
if called_name is not None:
|
||||
item = {**item, "name": called_name}
|
||||
del item["namespace"]
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def _custom_tool_names(request: ResponsesRequest) -> set[str]:
|
||||
declared = custom_tool_names(request.tools)
|
||||
if request._tool_state.chat_tools is not None:
|
||||
# A deferred declaration counts once a search result has loaded it.
|
||||
declared -= {tool.name for tool in request.tools if tool.defer_loading}
|
||||
return declared | request._tool_state.discovered_custom_names
|
||||
|
||||
@staticmethod
|
||||
def _client_tool_search(request: ResponsesRequest) -> bool:
|
||||
"""Whether a `tool_search` call is the client-executed search tool."""
|
||||
return request._tool_state.chat_tools is not None and any(
|
||||
tool.type == TOOL_SEARCH for tool in request.tools
|
||||
)
|
||||
|
||||
def _response_tools_to_chat_tools(self, request: ResponsesRequest) -> list[Tool]:
|
||||
if request._tool_state.chat_tools is not None:
|
||||
return list(request._tool_state.chat_tools)
|
||||
# ``function`` and ``custom`` tools flow to chat; built-ins go through
|
||||
# harmony. A custom tool is shimmed into a single-string function tool.
|
||||
chat_tools = []
|
||||
@@ -2133,6 +2487,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
tool_follows = opens_item and item_type in (
|
||||
"function_call",
|
||||
"custom_tool_call",
|
||||
"tool_search_call",
|
||||
)
|
||||
terminal = event_type in (
|
||||
"response.completed",
|
||||
@@ -2215,14 +2570,19 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
custom_names = self._custom_tool_names(request)
|
||||
namespace_names = (
|
||||
namespace_function_names(request.tools)
|
||||
if self.tool_call_parser == "muse"
|
||||
else {}
|
||||
else {
|
||||
call_name: (entry.get("namespace"), entry.get("name", call_name))
|
||||
for call_name, entry in request._tool_state.names.items()
|
||||
if entry.get("namespace")
|
||||
}
|
||||
)
|
||||
tool_choice = request.effective_tool_choice()
|
||||
required_json = self._uses_required_tool_json(request, chat_tools)
|
||||
client_tool_search = self._client_tool_search(request)
|
||||
encoding_output = self._encoding_output()
|
||||
token_stream = (
|
||||
encoding_output.new_responses_stream(
|
||||
@@ -2453,7 +2813,13 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
return []
|
||||
arguments = state["arguments"]
|
||||
events: list = []
|
||||
if state["custom"]:
|
||||
done_event = openai_responses_types.ResponseOutputItemDoneEvent
|
||||
if state["search"]:
|
||||
completed_item = _tool_search_call(
|
||||
state["item_id"], state["call_id"], arguments
|
||||
)
|
||||
done_event = ResponseToolSearchCallEvent
|
||||
elif state["custom"]:
|
||||
payload = decode_custom_tool_input(arguments)
|
||||
# Deltas cannot be retracted, so a payload that no longer
|
||||
# extends what was already streamed defers to the streamed text.
|
||||
@@ -2520,7 +2886,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
events.append(
|
||||
_send_event(
|
||||
openai_responses_types.ResponseOutputItemDoneEvent(
|
||||
done_event(
|
||||
type="response.output_item.done",
|
||||
sequence_number=-1,
|
||||
output_index=state["output_index"],
|
||||
@@ -2717,27 +3083,43 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
current_output_index += 1
|
||||
name = call.name or ""
|
||||
is_custom = name in custom_names
|
||||
is_search = client_tool_search and name == TOOL_SEARCH
|
||||
namespace, name = namespace_names.get(name, (None, name))
|
||||
prefix = (
|
||||
"ctc" if is_custom else "tsc" if is_search else "fc"
|
||||
)
|
||||
state = {
|
||||
"item_id": (
|
||||
f"ctc_{random_uuid()[:8]}"
|
||||
if is_custom
|
||||
else f"fc_{random_uuid()[:8]}"
|
||||
),
|
||||
"item_id": f"{prefix}_{random_uuid()[:8]}",
|
||||
"call_id": f"call_{random_uuid()[:24]}",
|
||||
"output_index": current_output_index,
|
||||
"name": name,
|
||||
"namespace": namespace,
|
||||
"arguments": "",
|
||||
"custom": is_custom,
|
||||
"search": is_search,
|
||||
"payload": "",
|
||||
"added": False,
|
||||
"done": False,
|
||||
}
|
||||
tool_call_states[tool_index] = state
|
||||
self._record_called_name(
|
||||
request, state["call_id"], call.name or ""
|
||||
)
|
||||
if not state["added"]:
|
||||
state["added"] = True
|
||||
if state["custom"]:
|
||||
added_event = (
|
||||
openai_responses_types.ResponseOutputItemAddedEvent
|
||||
)
|
||||
if state["search"]:
|
||||
# Search arguments are reported whole when it completes.
|
||||
added_item = ResponseToolSearchCall(
|
||||
id=state["item_id"],
|
||||
call_id=state["call_id"],
|
||||
arguments={},
|
||||
status="in_progress",
|
||||
)
|
||||
added_event = ResponseToolSearchCallEvent
|
||||
elif state["custom"]:
|
||||
added_item = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
id=state["item_id"],
|
||||
@@ -2756,7 +3138,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
**_namespace_field(state),
|
||||
)
|
||||
yield _send_event(
|
||||
openai_responses_types.ResponseOutputItemAddedEvent(
|
||||
added_event(
|
||||
type="response.output_item.added",
|
||||
sequence_number=-1,
|
||||
output_index=state["output_index"],
|
||||
@@ -2786,7 +3168,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
delta=delta,
|
||||
)
|
||||
)
|
||||
else:
|
||||
elif not state["search"]:
|
||||
yield _send_event(
|
||||
openai_responses_types.ResponseFunctionCallArgumentsDeltaEvent(
|
||||
type="response.function_call_arguments.delta",
|
||||
@@ -2936,10 +3318,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
self.response_store[final_response.id] = final_response
|
||||
self.msg_store[final_response.id] = self._response_input_history(
|
||||
request
|
||||
) + [
|
||||
item.model_dump(exclude_none=True)
|
||||
for item in final_response.output
|
||||
]
|
||||
) + self._replayed_output_items(request, final_response.output)
|
||||
|
||||
response_dict = _sanitize_response_dict(final_response.model_dump())
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ parses them, and tool-call arguments arrive as the API streams them."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -17,6 +18,11 @@ from sglang.srt.parser.deepseek_recipe_renderer import (
|
||||
load_deepseek_recipe,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The most tokens one UTF-8 character can span, a byte each.
|
||||
_MAX_UTF8_BYTES = 4
|
||||
|
||||
|
||||
class RecipeToolCallDelta(msgspec.Struct, frozen=True):
|
||||
"""A piece of one call's JSON arguments; the first piece also names the call."""
|
||||
@@ -74,6 +80,7 @@ class DeepSeekRecipeOutputParser:
|
||||
separate_reasoning: bool = True,
|
||||
stream_reasoning: bool = True,
|
||||
stop: Sequence[str] = (),
|
||||
callable_names: Optional[frozenset[str]] = None,
|
||||
):
|
||||
recipe = load_deepseek_recipe()
|
||||
options = recipe.ParsingOptions(
|
||||
@@ -101,6 +108,10 @@ class DeepSeekRecipeOutputParser:
|
||||
# out, so a matched stop is cut before recipe has emitted any of it.
|
||||
self._stop_hold_chars = max(map(len, stop), default=1) - 1
|
||||
self._pending_ids: list[int] = []
|
||||
self._callable_names = callable_names
|
||||
# recipe's index of each call so far, mapped to the index it is reported
|
||||
# at; None for a dropped call.
|
||||
self._call_indices: dict[int, Optional[int]] = {}
|
||||
self.num_consumed_tokens = 0
|
||||
|
||||
def feed(self, token_ids: Sequence[int]) -> RecipeOutputDelta:
|
||||
@@ -192,6 +203,27 @@ class DeepSeekRecipeOutputParser:
|
||||
+ self._hf_tokenizer.encode(text[:cut], add_special_tokens=False).ids
|
||||
)
|
||||
|
||||
def _reported_call(
|
||||
self, call: RecipeToolCallDelta
|
||||
) -> Optional[RecipeToolCallDelta]:
|
||||
"""`call` at the index it is reported at; None when it belongs to a call
|
||||
outside `callable_names`."""
|
||||
if self._callable_names is None:
|
||||
return call
|
||||
if call.name is not None:
|
||||
index = None
|
||||
if call.name in self._callable_names:
|
||||
index = sum(kept is not None for kept in self._call_indices.values())
|
||||
else:
|
||||
logger.warning(
|
||||
"Model attempted to call undefined function: %s", call.name
|
||||
)
|
||||
self._call_indices[call.index] = index
|
||||
index = self._call_indices[call.index]
|
||||
if index is None:
|
||||
return None
|
||||
return msgspec.structs.replace(call, index=index)
|
||||
|
||||
def _deltas(self, chunks: list[Any]) -> RecipeOutputDelta:
|
||||
delta = RecipeOutputDelta()
|
||||
for chunk in chunks:
|
||||
@@ -202,12 +234,19 @@ class DeepSeekRecipeOutputParser:
|
||||
reasoning = piece.get("reasoning_content") or ""
|
||||
content = piece.get("content") or ""
|
||||
tool_calls = tuple(
|
||||
RecipeToolCallDelta(
|
||||
index=call["index"],
|
||||
arguments=call["function"]["arguments"],
|
||||
name=call["function"].get("name"),
|
||||
call
|
||||
for call in map(
|
||||
self._reported_call,
|
||||
(
|
||||
RecipeToolCallDelta(
|
||||
index=call["index"],
|
||||
arguments=call["function"]["arguments"],
|
||||
name=call["function"].get("name"),
|
||||
)
|
||||
for call in piece.get("tool_calls") or ()
|
||||
),
|
||||
)
|
||||
for call in piece.get("tool_calls") or ()
|
||||
if call is not None
|
||||
)
|
||||
if self._reasoning_as_content:
|
||||
content, reasoning = reasoning + content, ""
|
||||
|
||||
@@ -95,6 +95,23 @@ def pre_normalize(body: dict[str, Any]) -> dict[str, Any]:
|
||||
return request
|
||||
|
||||
|
||||
def loaded_tools(conversation: Any) -> list[Any]:
|
||||
"""The tools the model can call after `conversation`'s history: those it does
|
||||
not defer and the deferred ones a system message has loaded."""
|
||||
recipe = load_deepseek_recipe()
|
||||
loaded = {
|
||||
tool.name
|
||||
for message in conversation.messages
|
||||
if isinstance(message, recipe.SystemMessage)
|
||||
for tool in message.tools or []
|
||||
}
|
||||
return [
|
||||
tool
|
||||
for tool in conversation.tools
|
||||
if not tool.defer_loading or tool.name in loaded
|
||||
]
|
||||
|
||||
|
||||
def _error_message(body: Any) -> str:
|
||||
try:
|
||||
return json.loads(body)["error"]["message"]
|
||||
@@ -146,6 +163,19 @@ class DeepSeekRecipe:
|
||||
) -> list[int]:
|
||||
"""Prompt ids for `body` (see `pre_normalize`); a request recipe rejects
|
||||
raises ValueError. `image_urls` receives the images in placeholder order."""
|
||||
conversation = self.convert(body, effort_budget=effort_budget)
|
||||
prompt_ids = self.encode_conversation(conversation, effort_budget=effort_budget)
|
||||
if image_urls is not None:
|
||||
image_urls.extend(self.image_urls(conversation))
|
||||
if task is not None:
|
||||
prompt_ids = self._attach_task(prompt_ids, task, body["thinking"])
|
||||
return prompt_ids
|
||||
|
||||
def convert(
|
||||
self, body: dict[str, Any], *, effort_budget: Optional[int] = None
|
||||
) -> Any:
|
||||
"""Convert `body` to recipe's `Conversation`; invalid requests raise ValueError.
|
||||
Pass the same `effort_budget` to `encode_conversation`."""
|
||||
recipe = load_deepseek_recipe()
|
||||
normalized = pre_normalize(body)
|
||||
normalized["model"] = self.model
|
||||
@@ -155,16 +185,104 @@ class DeepSeekRecipe:
|
||||
converted = recipe.ChatCompletionRequest(normalized).convert(self._options)
|
||||
except recipe.ConversionError as error:
|
||||
raise ValueError(_error_message(error.body)) from error
|
||||
prompt_ids = self._encoding.encode(converted.conversation)
|
||||
if image_urls is not None:
|
||||
rendered = self._encoding.render_conversation(converted.conversation)
|
||||
image_urls.extend(
|
||||
source.url or source.data_url for source in rendered.image_sources
|
||||
)
|
||||
if effort_budget is not None and body["thinking"]:
|
||||
return converted.conversation
|
||||
|
||||
def convert_native(
|
||||
self, protocol: str, body: dict[str, Any], *, thinking: bool
|
||||
) -> Any:
|
||||
"""Convert a Responses or Messages request to recipe's `Conversation` in the
|
||||
host's thinking mode; invalid requests raise ValueError."""
|
||||
recipe = load_deepseek_recipe()
|
||||
request_type = {
|
||||
"responses": recipe.ResponsesRequest,
|
||||
"messages": recipe.MessagesRequest,
|
||||
}[protocol]
|
||||
options = self._options.with_default_thinking_mode(thinking)
|
||||
try:
|
||||
converted = request_type({**body, "model": self.model}).convert(options)
|
||||
except recipe.ConversionError as error:
|
||||
raise ValueError(_error_message(error.body)) from error
|
||||
return converted.conversation
|
||||
|
||||
def with_host_settings(
|
||||
self,
|
||||
conversation: Any,
|
||||
*,
|
||||
response_schema: Optional[dict[str, Any]],
|
||||
continue_final_message: bool,
|
||||
) -> Any:
|
||||
"""`conversation` with what recipe's Responses and Messages conversions
|
||||
leave out: a JSON response schema, and SGLang's reading of a trailing
|
||||
assistant turn, continued or else re-sent as a user turn."""
|
||||
recipe = load_deepseek_recipe()
|
||||
messages = conversation.messages
|
||||
last = messages[-1] if messages else None
|
||||
if last is not None and last.role == "assistant":
|
||||
# recipe closes the turn and opens another; SGLang continues it or,
|
||||
# as `_handle_last_assistant_message`, re-sends it as a user turn.
|
||||
if continue_final_message:
|
||||
last = recipe.AssistantMessage(
|
||||
last.content, last.reasoning_content, prefix=True
|
||||
)
|
||||
else:
|
||||
last = recipe.UserMessage(last.content)
|
||||
messages = [*messages[:-1], last]
|
||||
elif response_schema is None:
|
||||
return conversation
|
||||
return recipe.Conversation(
|
||||
messages,
|
||||
thinking_mode=conversation.thinking_mode,
|
||||
tools=conversation.tools,
|
||||
tool_choice=conversation.tool_choice,
|
||||
reasoning_effort=conversation.reasoning_effort,
|
||||
response_schema=(
|
||||
response_schema
|
||||
if response_schema is not None
|
||||
else conversation.response_schema
|
||||
),
|
||||
)
|
||||
|
||||
def namespace_tool_names(
|
||||
self, namespaces: list[dict[str, Any]]
|
||||
) -> dict[str, tuple[str, str]]:
|
||||
"""The name recipe's Responses conversion calls each member of `namespaces`
|
||||
by, mapped to its namespace and its name there. `namespaces` lists the
|
||||
request's namespace tools in conversion order: declared, then discovered."""
|
||||
recipe = load_deepseek_recipe()
|
||||
body = {"model": self.model, "input": "", "tools": namespaces}
|
||||
try:
|
||||
return recipe.ResponsesRequest(body).namespace_tool_map(self._options)
|
||||
except recipe.ConversionError as error:
|
||||
raise ValueError(_error_message(error.body)) from error
|
||||
|
||||
def image_urls(self, conversation: Any) -> list[str]:
|
||||
"""The images of `conversation` in the order its prompt places them."""
|
||||
rendered = self._encoding.render_conversation(conversation)
|
||||
return [source.url or source.data_url for source in rendered.image_sources]
|
||||
|
||||
def encode_conversation(
|
||||
self, conversation: Any, *, effort_budget: Optional[int] = None
|
||||
) -> list[int]:
|
||||
"""Encode a recipe `Conversation`, including a prefix of its messages."""
|
||||
if self._fast_prompt_tokenizer is None:
|
||||
prompt_ids = self._encoding.encode(conversation)
|
||||
else:
|
||||
rendered = self._encoding.render_conversation(conversation)
|
||||
prompt = rendered.prompt
|
||||
segments = []
|
||||
cursor = 0
|
||||
# Recipe distinguishes control tokens from identical spellings in
|
||||
# user text. Its Python ranges use code-point offsets, not bytes.
|
||||
for start, end in rendered.special_ranges:
|
||||
if start > cursor:
|
||||
segments.append((prompt[cursor:start], False))
|
||||
segments.append((prompt[start:end], True))
|
||||
cursor = end
|
||||
if cursor < len(prompt):
|
||||
segments.append((prompt[cursor:], False))
|
||||
prompt_ids = self._fast_prompt_tokenizer.encode_segments(segments).ids
|
||||
if effort_budget is not None and conversation.thinking_mode:
|
||||
prompt_ids = self._replace_effort_budget(prompt_ids, effort_budget)
|
||||
if task is not None:
|
||||
prompt_ids = self._attach_task(prompt_ids, task, body["thinking"])
|
||||
return prompt_ids
|
||||
|
||||
def parse_start(
|
||||
|
||||
@@ -97,7 +97,9 @@ class AnthropicMessagesMixin:
|
||||
)
|
||||
|
||||
serving = AnthropicServing(
|
||||
openai_serving_chat=SimpleNamespace(supports_inline_system=True)
|
||||
openai_serving_chat=SimpleNamespace(
|
||||
supports_inline_system=True, native_tool_loading=False
|
||||
)
|
||||
)
|
||||
chat_request = serving._convert_to_chat_completion_request(anthropic_request)
|
||||
converted = chat_request.model_dump()
|
||||
|
||||
Reference in New Issue
Block a user