[Fix] muse: Support Responses namespace tools and defer_loading (#44685)
Co-authored-by: sid-rp <sidrp@meta.com>
This commit is contained in:
@@ -52,6 +52,9 @@ from openai.types.responses.response import ToolChoice
|
||||
from openai.types.responses.response_format_text_json_schema_config import (
|
||||
ResponseFormatTextJSONSchemaConfig,
|
||||
)
|
||||
from openai.types.responses.response_function_tool_call_param import (
|
||||
ResponseFunctionToolCallParam,
|
||||
)
|
||||
from openai.types.shared.response_format_json_object import ResponseFormatJSONObject
|
||||
from pydantic import (
|
||||
AfterValidator,
|
||||
@@ -903,6 +906,10 @@ class ChatCompletionRequest(PDRoutingFields):
|
||||
top_p: Optional[float] = None
|
||||
user: Optional[str] = None
|
||||
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
|
||||
# Set by /v1/responses when the request has namespace tools: the chat template
|
||||
# renders these, while ``tools`` (flattened to ``<namespace>.<function>``) still
|
||||
# drives tool-call parsing.
|
||||
_template_tools: Optional[List[Dict[str, Any]]] = PrivateAttr(default=None)
|
||||
tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field(
|
||||
default="auto", examples=["none"]
|
||||
) # noqa
|
||||
@@ -1807,6 +1814,7 @@ class ResponseTool(BaseModel):
|
||||
strict: bool = False
|
||||
# Inner schemas for ``namespace`` tools.
|
||||
tools: Optional[List[Dict[str, Any]]] = None
|
||||
defer_loading: Optional[bool] = None
|
||||
# Input format of a ``custom`` tool: {"type": "text"} or
|
||||
# {"type": "grammar", "syntax": ..., "definition": ...}.
|
||||
format: Optional[Dict[str, Any]] = None
|
||||
@@ -1822,12 +1830,18 @@ class ResponseInputMessageParam(EasyInputMessageParam, total=False):
|
||||
phase: Optional[Literal["commentary", "final_answer"]]
|
||||
|
||||
|
||||
class ResponseFunctionCallInputParam(ResponseFunctionToolCallParam, total=False):
|
||||
# Set when the call targets a member of a ``namespace`` tool.
|
||||
namespace: str
|
||||
|
||||
|
||||
class ResponseOutputMessage(OpenAIResponseOutputMessage):
|
||||
phase: Optional[Literal["commentary", "final_answer"]] = None
|
||||
|
||||
|
||||
ResponseInputOutputItem: TypeAlias = Union[
|
||||
ResponseInputMessageParam,
|
||||
ResponseFunctionCallInputParam,
|
||||
ResponseOutputMessage,
|
||||
ResponseInputItemParam,
|
||||
"ResponseReasoningItem",
|
||||
@@ -2351,6 +2365,7 @@ class ResponseReasoningTextContent(BaseModel):
|
||||
|
||||
ResponseInputOutputItem: TypeAlias = Union[
|
||||
ResponseInputMessageParam,
|
||||
ResponseFunctionCallInputParam,
|
||||
ResponseOutputMessage,
|
||||
ResponseInputItemParam,
|
||||
"ResponseReasoningItem",
|
||||
|
||||
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import base64
|
||||
import json
|
||||
import zlib
|
||||
from typing import Any, Dict, Optional, Set, Tuple
|
||||
from typing import Any, Dict, Iterator, Optional, Set, Tuple
|
||||
|
||||
CUSTOM_TOOL_INPUT_KEY = "input"
|
||||
|
||||
@@ -72,6 +72,27 @@ def custom_tool_names(tools: Any) -> Set[str]:
|
||||
return {tool.name for tool in tools or [] if tool.type == "custom" and tool.name}
|
||||
|
||||
|
||||
def namespace_functions(tool: Any) -> Iterator[Dict[str, Any]]:
|
||||
"""Unwrap the flat and OpenAI-wrapped members accepted by Muse's template."""
|
||||
for member in tool.tools or []:
|
||||
if member.get("type", "function") != "function":
|
||||
continue
|
||||
function = member.get("function", member)
|
||||
if isinstance(function, dict) and function.get("name"):
|
||||
yield function
|
||||
|
||||
|
||||
def namespace_function_names(tools: Any) -> Dict[str, Tuple[str, str]]:
|
||||
"""Map each namespace member's call name ``<namespace>.<function>`` to
|
||||
``(namespace, function)``."""
|
||||
return {
|
||||
f"{tool.name}.{member['name']}": (tool.name, member["name"])
|
||||
for tool in tools or []
|
||||
if tool.type == "namespace" and tool.name
|
||||
for member in namespace_functions(tool)
|
||||
}
|
||||
|
||||
|
||||
def encode_custom_tool_input(payload: str) -> str:
|
||||
return json.dumps({CUSTOM_TOOL_INPUT_KEY: payload}, ensure_ascii=False)
|
||||
|
||||
|
||||
@@ -1454,6 +1454,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
] or None
|
||||
elif request.tools:
|
||||
tools = [item.model_dump() for item in request.tools]
|
||||
if self.tool_call_parser == "muse" and request._template_tools:
|
||||
tools = request._template_tools
|
||||
if self.tool_call_parser and not glm_constraint:
|
||||
parser = FunctionCallParser(
|
||||
effective_tools,
|
||||
|
||||
@@ -83,6 +83,8 @@ from sglang.srt.entrypoints.openai.responses_adapters import (
|
||||
encode_custom_tool_input,
|
||||
encode_reasoning_state,
|
||||
label_developer_content,
|
||||
namespace_function_names,
|
||||
namespace_functions,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_chat import (
|
||||
OpenAIServingChat,
|
||||
@@ -704,6 +706,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
reasoning_effort=(request.reasoning.effort if request.reasoning else None),
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
)
|
||||
chat_request._template_tools = self._response_tools_to_template_tools(request)
|
||||
|
||||
media_error = self._validate_media_content(chat_request)
|
||||
if media_error:
|
||||
@@ -946,10 +949,14 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
|
||||
@staticmethod
|
||||
def _make_tool_call_item(
|
||||
name: str, arguments: str, custom_names: set[str]
|
||||
name: str,
|
||||
arguments: str,
|
||||
custom_names: set[str],
|
||||
namespace_names: Optional[dict[str, tuple[str, str]]] = None,
|
||||
) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]:
|
||||
"""A call against a ``custom`` tool reports its freeform payload rather
|
||||
than the JSON arguments of the shim function tool."""
|
||||
than the JSON arguments of the shim function tool. A call against a
|
||||
namespace member reports ``namespace`` and the bare function ``name``."""
|
||||
call_id = f"call_{random_uuid()[:24]}"
|
||||
if name in custom_names:
|
||||
return ResponseCustomToolCall(
|
||||
@@ -959,6 +966,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
name=name,
|
||||
input=decode_custom_tool_input(arguments),
|
||||
)
|
||||
namespace, name = (namespace_names or {}).get(name, (None, name))
|
||||
return ResponseFunctionToolCall(
|
||||
arguments=arguments,
|
||||
call_id=call_id,
|
||||
@@ -966,6 +974,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
name=name,
|
||||
id=f"fc_{random_uuid()[:8]}",
|
||||
status="completed",
|
||||
**({"namespace": namespace} if namespace else {}),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1055,6 +1064,11 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
):
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
namespace_names = (
|
||||
namespace_function_names(request.tools)
|
||||
if self.tool_call_parser == "muse"
|
||||
else {}
|
||||
)
|
||||
encoding_output = self._encoding_output()
|
||||
if encoding_output is not None:
|
||||
required_json = self._uses_required_tool_json(request, chat_tools)
|
||||
@@ -1069,11 +1083,13 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
if required_json and content:
|
||||
content, tool_call_items = self._parse_required_tool_json(
|
||||
content, custom_names
|
||||
content, custom_names, namespace_names
|
||||
)
|
||||
else:
|
||||
tool_call_items = [
|
||||
self._make_tool_call_item(call.name, call.parameters, custom_names)
|
||||
self._make_tool_call_item(
|
||||
call.name, call.parameters, custom_names, namespace_names
|
||||
)
|
||||
for call in calls
|
||||
]
|
||||
else:
|
||||
@@ -1082,6 +1098,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
final_output,
|
||||
chat_tools,
|
||||
custom_names,
|
||||
namespace_names=namespace_names,
|
||||
require_reasoning=require_reasoning,
|
||||
)
|
||||
|
||||
@@ -1122,6 +1139,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_tools: list[Tool],
|
||||
custom_names: set[str],
|
||||
*,
|
||||
namespace_names: dict[str, tuple[str, str]],
|
||||
require_reasoning: bool,
|
||||
) -> tuple[
|
||||
Optional[str],
|
||||
@@ -1179,6 +1197,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
call_info.name,
|
||||
call_info.parameters or "",
|
||||
custom_names,
|
||||
namespace_names,
|
||||
)
|
||||
)
|
||||
parsed_via_native = bool(call_info_list)
|
||||
@@ -1192,12 +1211,17 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
and not parsed_via_native
|
||||
and not detector_owns_format
|
||||
):
|
||||
content, json_items = self._parse_required_tool_json(content, custom_names)
|
||||
content, json_items = self._parse_required_tool_json(
|
||||
content, custom_names, namespace_names
|
||||
)
|
||||
tool_call_items.extend(json_items)
|
||||
return reasoning_content, content, tool_call_items
|
||||
|
||||
def _parse_required_tool_json(
|
||||
self, content: str, custom_names: set[str]
|
||||
self,
|
||||
content: str,
|
||||
custom_names: set[str],
|
||||
namespace_names: Optional[dict[str, tuple[str, str]]] = None,
|
||||
) -> tuple[str, list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]]]:
|
||||
tool_call_items = []
|
||||
try:
|
||||
@@ -1212,7 +1236,9 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
tool.get("parameters", {}), ensure_ascii=False
|
||||
)
|
||||
tool_call_items.append(
|
||||
self._make_tool_call_item(tool["name"], arguments, custom_names)
|
||||
self._make_tool_call_item(
|
||||
tool["name"], arguments, custom_names, namespace_names
|
||||
)
|
||||
)
|
||||
content = ""
|
||||
except Exception as e:
|
||||
@@ -1298,12 +1324,29 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
return tool_choice
|
||||
return {"type": "function", "function": {"name": tool_choice["name"]}}
|
||||
|
||||
@staticmethod
|
||||
def _response_tools_to_chat_tools(request: ResponsesRequest) -> list[Tool]:
|
||||
def _response_tools_to_chat_tools(self, request: ResponsesRequest) -> list[Tool]:
|
||||
# ``function`` and ``custom`` tools flow to chat; built-ins go through
|
||||
# harmony. A custom tool is shimmed into a single-string function tool.
|
||||
chat_tools = []
|
||||
is_muse = self.tool_call_parser == "muse"
|
||||
for tool in request.tools:
|
||||
if is_muse and tool.type == "namespace" and tool.name:
|
||||
# Members are called as ``<namespace>.<function>``; register those
|
||||
# names so the tool-call parser accepts the calls.
|
||||
for member in namespace_functions(tool):
|
||||
chat_tools.append(
|
||||
Tool(
|
||||
type="function",
|
||||
function=Function(
|
||||
name=f"{tool.name}.{member['name']}",
|
||||
description=member.get("description"),
|
||||
parameters=member.get("parameters"),
|
||||
strict=bool(member.get("strict", False)),
|
||||
defer_loading=member.get("defer_loading"),
|
||||
),
|
||||
)
|
||||
)
|
||||
continue
|
||||
if tool.type == "function":
|
||||
description, parameters = tool.description, tool.parameters
|
||||
elif tool.type == "custom" and tool.name:
|
||||
@@ -1319,11 +1362,51 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
description=description,
|
||||
parameters=parameters,
|
||||
strict=tool.strict,
|
||||
defer_loading=tool.defer_loading if is_muse else None,
|
||||
),
|
||||
)
|
||||
)
|
||||
return chat_tools
|
||||
|
||||
def _response_tools_to_template_tools(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
) -> Optional[list[dict]]:
|
||||
"""Tools for Muse's chat template when the request has namespace tools:
|
||||
namespaces stay nested, so the template can render namespace metadata and
|
||||
namespace-level ``defer_loading``. None keeps the flattened tool list."""
|
||||
if self.tool_call_parser != "muse" or not any(
|
||||
tool.type == "namespace" for tool in request.tools
|
||||
):
|
||||
return None
|
||||
template_tools: list[dict] = []
|
||||
for tool in request.tools:
|
||||
if tool.type == "namespace" and tool.name:
|
||||
namespace = {
|
||||
"type": "namespace",
|
||||
"name": tool.name,
|
||||
"description": tool.description or "",
|
||||
"tools": [dict(member) for member in namespace_functions(tool)],
|
||||
}
|
||||
if tool.defer_loading is not None:
|
||||
namespace["defer_loading"] = tool.defer_loading
|
||||
template_tools.append(namespace)
|
||||
elif tool.type in ("function", "custom") and tool.name:
|
||||
function = {
|
||||
"name": tool.name,
|
||||
"description": tool.description,
|
||||
"parameters": tool.parameters,
|
||||
}
|
||||
if tool.type == "custom":
|
||||
function["description"] = custom_tool_description(
|
||||
tool.description, tool.format
|
||||
)
|
||||
function["parameters"] = custom_tool_parameters()
|
||||
if tool.defer_loading is not None:
|
||||
function["defer_loading"] = tool.defer_loading
|
||||
template_tools.append({"type": "function", "function": function})
|
||||
return template_tools
|
||||
|
||||
@staticmethod
|
||||
def _normalize_response_content_part_for_chat(content_part: Any) -> Any:
|
||||
# Default detail=\"auto\" and lift flat min/max_dynamic_patch onto
|
||||
@@ -1390,7 +1473,9 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _normalize_response_message_for_chat(cls, message: Any) -> Any:
|
||||
def _normalize_response_message_for_chat(
|
||||
cls, message: Any, *, tool_call_parser: Optional[str] = None
|
||||
) -> Any:
|
||||
"""Convert one Responses-API input item to a chat-completions message."""
|
||||
if hasattr(message, "model_dump"):
|
||||
message = message.model_dump(exclude_none=True)
|
||||
@@ -1424,7 +1509,10 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
raw = orjson.dumps(raw).decode("utf-8")
|
||||
else:
|
||||
raw = "{}"
|
||||
return cls._chat_tool_call_message(message, message.get("name"), raw)
|
||||
name = message.get("name")
|
||||
if tool_call_parser == "muse" and message.get("namespace"):
|
||||
name = f"{message['namespace']}.{name}"
|
||||
return cls._chat_tool_call_message(message, name, raw)
|
||||
if msg_type == "custom_tool_call":
|
||||
# Replay through the same single-string shim the tool was offered as.
|
||||
return cls._chat_tool_call_message(
|
||||
@@ -1572,7 +1660,9 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
|
||||
for input_item in self._response_input_history(request):
|
||||
normalized = self._normalize_response_message_for_chat(input_item)
|
||||
normalized = self._normalize_response_message_for_chat(
|
||||
input_item, tool_call_parser=self.tool_call_parser
|
||||
)
|
||||
if normalized is not None:
|
||||
messages.append(normalized)
|
||||
|
||||
@@ -2124,6 +2214,11 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
namespace_names = (
|
||||
namespace_function_names(request.tools)
|
||||
if self.tool_call_parser == "muse"
|
||||
else {}
|
||||
)
|
||||
tool_choice = request.effective_tool_choice()
|
||||
required_json = self._uses_required_tool_json(request, chat_tools)
|
||||
encoding_output = self._encoding_output()
|
||||
@@ -2345,6 +2440,9 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
message_state["open"] = False
|
||||
return events
|
||||
|
||||
def _namespace_field(state: dict) -> dict:
|
||||
return {"namespace": state["namespace"]} if state["namespace"] else {}
|
||||
|
||||
def _close_tool_call_state(tool_index: int):
|
||||
state = tool_call_states.get(tool_index)
|
||||
if state is None or state.get("done"):
|
||||
@@ -2402,6 +2500,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
type="function_call",
|
||||
id=state["item_id"],
|
||||
status=item_status,
|
||||
**_namespace_field(state),
|
||||
)
|
||||
events.append(
|
||||
_send_event(
|
||||
@@ -2614,6 +2713,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
current_output_index += 1
|
||||
name = call.name or ""
|
||||
is_custom = name in custom_names
|
||||
namespace, name = namespace_names.get(name, (None, name))
|
||||
state = {
|
||||
"item_id": (
|
||||
f"ctc_{random_uuid()[:8]}"
|
||||
@@ -2623,6 +2723,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
"call_id": f"call_{random_uuid()[:24]}",
|
||||
"output_index": current_output_index,
|
||||
"name": name,
|
||||
"namespace": namespace,
|
||||
"arguments": "",
|
||||
"custom": is_custom,
|
||||
"payload": "",
|
||||
@@ -2648,6 +2749,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
type="function_call",
|
||||
id=state["item_id"],
|
||||
status="in_progress",
|
||||
**_namespace_field(state),
|
||||
)
|
||||
yield _send_event(
|
||||
openai_responses_types.ResponseOutputItemAddedEvent(
|
||||
|
||||
Reference in New Issue
Block a user