[Fix] muse: Support Responses namespace tools and defer_loading (#44685)

Co-authored-by: sid-rp <sidrp@meta.com>
This commit is contained in:
Baizhou Zhang
2026-10-23 16:48:08 -07:00
committed by GitHub
co-authored by sid-rp
parent 0d592c475b
commit 6a319e7b34
6 changed files with 515 additions and 18 deletions
@@ -52,6 +52,9 @@ from openai.types.responses.response import ToolChoice
from openai.types.responses.response_format_text_json_schema_config import (
ResponseFormatTextJSONSchemaConfig,
)
from openai.types.responses.response_function_tool_call_param import (
ResponseFunctionToolCallParam,
)
from openai.types.shared.response_format_json_object import ResponseFormatJSONObject
from pydantic import (
AfterValidator,
@@ -903,6 +906,10 @@ class ChatCompletionRequest(PDRoutingFields):
top_p: Optional[float] = None
user: Optional[str] = None
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
# Set by /v1/responses when the request has namespace tools: the chat template
# renders these, while ``tools`` (flattened to ``<namespace>.<function>``) still
# drives tool-call parsing.
_template_tools: Optional[List[Dict[str, Any]]] = PrivateAttr(default=None)
tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field(
default="auto", examples=["none"]
) # noqa
@@ -1807,6 +1814,7 @@ class ResponseTool(BaseModel):
strict: bool = False
# Inner schemas for ``namespace`` tools.
tools: Optional[List[Dict[str, Any]]] = None
defer_loading: Optional[bool] = None
# Input format of a ``custom`` tool: {"type": "text"} or
# {"type": "grammar", "syntax": ..., "definition": ...}.
format: Optional[Dict[str, Any]] = None
@@ -1822,12 +1830,18 @@ class ResponseInputMessageParam(EasyInputMessageParam, total=False):
phase: Optional[Literal["commentary", "final_answer"]]
class ResponseFunctionCallInputParam(ResponseFunctionToolCallParam, total=False):
# Set when the call targets a member of a ``namespace`` tool.
namespace: str
class ResponseOutputMessage(OpenAIResponseOutputMessage):
phase: Optional[Literal["commentary", "final_answer"]] = None
ResponseInputOutputItem: TypeAlias = Union[
ResponseInputMessageParam,
ResponseFunctionCallInputParam,
ResponseOutputMessage,
ResponseInputItemParam,
"ResponseReasoningItem",
@@ -2351,6 +2365,7 @@ class ResponseReasoningTextContent(BaseModel):
ResponseInputOutputItem: TypeAlias = Union[
ResponseInputMessageParam,
ResponseFunctionCallInputParam,
ResponseOutputMessage,
ResponseInputItemParam,
"ResponseReasoningItem",
@@ -17,7 +17,7 @@ from __future__ import annotations
import base64
import json
import zlib
from typing import Any, Dict, Optional, Set, Tuple
from typing import Any, Dict, Iterator, Optional, Set, Tuple
CUSTOM_TOOL_INPUT_KEY = "input"
@@ -72,6 +72,27 @@ def custom_tool_names(tools: Any) -> Set[str]:
return {tool.name for tool in tools or [] if tool.type == "custom" and tool.name}
def namespace_functions(tool: Any) -> Iterator[Dict[str, Any]]:
"""Unwrap the flat and OpenAI-wrapped members accepted by Muse's template."""
for member in tool.tools or []:
if member.get("type", "function") != "function":
continue
function = member.get("function", member)
if isinstance(function, dict) and function.get("name"):
yield function
def namespace_function_names(tools: Any) -> Dict[str, Tuple[str, str]]:
"""Map each namespace member's call name ``<namespace>.<function>`` to
``(namespace, function)``."""
return {
f"{tool.name}.{member['name']}": (tool.name, member["name"])
for tool in tools or []
if tool.type == "namespace" and tool.name
for member in namespace_functions(tool)
}
def encode_custom_tool_input(payload: str) -> str:
return json.dumps({CUSTOM_TOOL_INPUT_KEY: payload}, ensure_ascii=False)
@@ -1454,6 +1454,8 @@ class OpenAIServingChat(OpenAIServingBase):
] or None
elif request.tools:
tools = [item.model_dump() for item in request.tools]
if self.tool_call_parser == "muse" and request._template_tools:
tools = request._template_tools
if self.tool_call_parser and not glm_constraint:
parser = FunctionCallParser(
effective_tools,
@@ -83,6 +83,8 @@ from sglang.srt.entrypoints.openai.responses_adapters import (
encode_custom_tool_input,
encode_reasoning_state,
label_developer_content,
namespace_function_names,
namespace_functions,
)
from sglang.srt.entrypoints.openai.serving_chat import (
OpenAIServingChat,
@@ -704,6 +706,7 @@ class OpenAIServingResponses(OpenAIServingChat):
reasoning_effort=(request.reasoning.effort if request.reasoning else None),
chat_template_kwargs=request.chat_template_kwargs,
)
chat_request._template_tools = self._response_tools_to_template_tools(request)
media_error = self._validate_media_content(chat_request)
if media_error:
@@ -946,10 +949,14 @@ class OpenAIServingResponses(OpenAIServingChat):
@staticmethod
def _make_tool_call_item(
name: str, arguments: str, custom_names: set[str]
name: str,
arguments: str,
custom_names: set[str],
namespace_names: Optional[dict[str, tuple[str, str]]] = None,
) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]:
"""A call against a ``custom`` tool reports its freeform payload rather
than the JSON arguments of the shim function tool."""
than the JSON arguments of the shim function tool. A call against a
namespace member reports ``namespace`` and the bare function ``name``."""
call_id = f"call_{random_uuid()[:24]}"
if name in custom_names:
return ResponseCustomToolCall(
@@ -959,6 +966,7 @@ class OpenAIServingResponses(OpenAIServingChat):
name=name,
input=decode_custom_tool_input(arguments),
)
namespace, name = (namespace_names or {}).get(name, (None, name))
return ResponseFunctionToolCall(
arguments=arguments,
call_id=call_id,
@@ -966,6 +974,7 @@ class OpenAIServingResponses(OpenAIServingChat):
name=name,
id=f"fc_{random_uuid()[:8]}",
status="completed",
**({"namespace": namespace} if namespace else {}),
)
@staticmethod
@@ -1055,6 +1064,11 @@ class OpenAIServingResponses(OpenAIServingChat):
):
chat_tools = self._response_tools_to_chat_tools(request)
custom_names = custom_tool_names(request.tools)
namespace_names = (
namespace_function_names(request.tools)
if self.tool_call_parser == "muse"
else {}
)
encoding_output = self._encoding_output()
if encoding_output is not None:
required_json = self._uses_required_tool_json(request, chat_tools)
@@ -1069,11 +1083,13 @@ class OpenAIServingResponses(OpenAIServingChat):
)
if required_json and content:
content, tool_call_items = self._parse_required_tool_json(
content, custom_names
content, custom_names, namespace_names
)
else:
tool_call_items = [
self._make_tool_call_item(call.name, call.parameters, custom_names)
self._make_tool_call_item(
call.name, call.parameters, custom_names, namespace_names
)
for call in calls
]
else:
@@ -1082,6 +1098,7 @@ class OpenAIServingResponses(OpenAIServingChat):
final_output,
chat_tools,
custom_names,
namespace_names=namespace_names,
require_reasoning=require_reasoning,
)
@@ -1122,6 +1139,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools: list[Tool],
custom_names: set[str],
*,
namespace_names: dict[str, tuple[str, str]],
require_reasoning: bool,
) -> tuple[
Optional[str],
@@ -1179,6 +1197,7 @@ class OpenAIServingResponses(OpenAIServingChat):
call_info.name,
call_info.parameters or "",
custom_names,
namespace_names,
)
)
parsed_via_native = bool(call_info_list)
@@ -1192,12 +1211,17 @@ class OpenAIServingResponses(OpenAIServingChat):
and not parsed_via_native
and not detector_owns_format
):
content, json_items = self._parse_required_tool_json(content, custom_names)
content, json_items = self._parse_required_tool_json(
content, custom_names, namespace_names
)
tool_call_items.extend(json_items)
return reasoning_content, content, tool_call_items
def _parse_required_tool_json(
self, content: str, custom_names: set[str]
self,
content: str,
custom_names: set[str],
namespace_names: Optional[dict[str, tuple[str, str]]] = None,
) -> tuple[str, list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]]]:
tool_call_items = []
try:
@@ -1212,7 +1236,9 @@ class OpenAIServingResponses(OpenAIServingChat):
tool.get("parameters", {}), ensure_ascii=False
)
tool_call_items.append(
self._make_tool_call_item(tool["name"], arguments, custom_names)
self._make_tool_call_item(
tool["name"], arguments, custom_names, namespace_names
)
)
content = ""
except Exception as e:
@@ -1298,12 +1324,29 @@ class OpenAIServingResponses(OpenAIServingChat):
return tool_choice
return {"type": "function", "function": {"name": tool_choice["name"]}}
@staticmethod
def _response_tools_to_chat_tools(request: ResponsesRequest) -> list[Tool]:
def _response_tools_to_chat_tools(self, request: ResponsesRequest) -> list[Tool]:
# ``function`` and ``custom`` tools flow to chat; built-ins go through
# harmony. A custom tool is shimmed into a single-string function tool.
chat_tools = []
is_muse = self.tool_call_parser == "muse"
for tool in request.tools:
if is_muse and tool.type == "namespace" and tool.name:
# Members are called as ``<namespace>.<function>``; register those
# names so the tool-call parser accepts the calls.
for member in namespace_functions(tool):
chat_tools.append(
Tool(
type="function",
function=Function(
name=f"{tool.name}.{member['name']}",
description=member.get("description"),
parameters=member.get("parameters"),
strict=bool(member.get("strict", False)),
defer_loading=member.get("defer_loading"),
),
)
)
continue
if tool.type == "function":
description, parameters = tool.description, tool.parameters
elif tool.type == "custom" and tool.name:
@@ -1319,11 +1362,51 @@ class OpenAIServingResponses(OpenAIServingChat):
description=description,
parameters=parameters,
strict=tool.strict,
defer_loading=tool.defer_loading if is_muse else None,
),
)
)
return chat_tools
def _response_tools_to_template_tools(
self,
request: ResponsesRequest,
) -> Optional[list[dict]]:
"""Tools for Muse's chat template when the request has namespace tools:
namespaces stay nested, so the template can render namespace metadata and
namespace-level ``defer_loading``. None keeps the flattened tool list."""
if self.tool_call_parser != "muse" or not any(
tool.type == "namespace" for tool in request.tools
):
return None
template_tools: list[dict] = []
for tool in request.tools:
if tool.type == "namespace" and tool.name:
namespace = {
"type": "namespace",
"name": tool.name,
"description": tool.description or "",
"tools": [dict(member) for member in namespace_functions(tool)],
}
if tool.defer_loading is not None:
namespace["defer_loading"] = tool.defer_loading
template_tools.append(namespace)
elif tool.type in ("function", "custom") and tool.name:
function = {
"name": tool.name,
"description": tool.description,
"parameters": tool.parameters,
}
if tool.type == "custom":
function["description"] = custom_tool_description(
tool.description, tool.format
)
function["parameters"] = custom_tool_parameters()
if tool.defer_loading is not None:
function["defer_loading"] = tool.defer_loading
template_tools.append({"type": "function", "function": function})
return template_tools
@staticmethod
def _normalize_response_content_part_for_chat(content_part: Any) -> Any:
# Default detail=\"auto\" and lift flat min/max_dynamic_patch onto
@@ -1390,7 +1473,9 @@ class OpenAIServingResponses(OpenAIServingChat):
}
@classmethod
def _normalize_response_message_for_chat(cls, message: Any) -> Any:
def _normalize_response_message_for_chat(
cls, message: Any, *, tool_call_parser: Optional[str] = None
) -> Any:
"""Convert one Responses-API input item to a chat-completions message."""
if hasattr(message, "model_dump"):
message = message.model_dump(exclude_none=True)
@@ -1424,7 +1509,10 @@ class OpenAIServingResponses(OpenAIServingChat):
raw = orjson.dumps(raw).decode("utf-8")
else:
raw = "{}"
return cls._chat_tool_call_message(message, message.get("name"), raw)
name = message.get("name")
if tool_call_parser == "muse" and message.get("namespace"):
name = f"{message['namespace']}.{name}"
return cls._chat_tool_call_message(message, name, raw)
if msg_type == "custom_tool_call":
# Replay through the same single-string shim the tool was offered as.
return cls._chat_tool_call_message(
@@ -1572,7 +1660,9 @@ class OpenAIServingResponses(OpenAIServingChat):
)
for input_item in self._response_input_history(request):
normalized = self._normalize_response_message_for_chat(input_item)
normalized = self._normalize_response_message_for_chat(
input_item, tool_call_parser=self.tool_call_parser
)
if normalized is not None:
messages.append(normalized)
@@ -2124,6 +2214,11 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools = self._response_tools_to_chat_tools(request)
custom_names = custom_tool_names(request.tools)
namespace_names = (
namespace_function_names(request.tools)
if self.tool_call_parser == "muse"
else {}
)
tool_choice = request.effective_tool_choice()
required_json = self._uses_required_tool_json(request, chat_tools)
encoding_output = self._encoding_output()
@@ -2345,6 +2440,9 @@ class OpenAIServingResponses(OpenAIServingChat):
message_state["open"] = False
return events
def _namespace_field(state: dict) -> dict:
return {"namespace": state["namespace"]} if state["namespace"] else {}
def _close_tool_call_state(tool_index: int):
state = tool_call_states.get(tool_index)
if state is None or state.get("done"):
@@ -2402,6 +2500,7 @@ class OpenAIServingResponses(OpenAIServingChat):
type="function_call",
id=state["item_id"],
status=item_status,
**_namespace_field(state),
)
events.append(
_send_event(
@@ -2614,6 +2713,7 @@ class OpenAIServingResponses(OpenAIServingChat):
current_output_index += 1
name = call.name or ""
is_custom = name in custom_names
namespace, name = namespace_names.get(name, (None, name))
state = {
"item_id": (
f"ctc_{random_uuid()[:8]}"
@@ -2623,6 +2723,7 @@ class OpenAIServingResponses(OpenAIServingChat):
"call_id": f"call_{random_uuid()[:24]}",
"output_index": current_output_index,
"name": name,
"namespace": namespace,
"arguments": "",
"custom": is_custom,
"payload": "",
@@ -2648,6 +2749,7 @@ class OpenAIServingResponses(OpenAIServingChat):
type="function_call",
id=state["item_id"],
status="in_progress",
**_namespace_field(state),
)
yield _send_event(
openai_responses_types.ResponseOutputItemAddedEvent(