Support defer_loading field at function level for Chat Completions API (#22702)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

python/sglang/srt/entrypoints/anthropic/protocol.py

@@ -33,7 +33,13 @@ class AnthropicContentBlock(BaseModel):
     """Content block in message"""
 
     type: Literal[
-        "text", "image", "tool_use", "tool_result", "thinking", "redacted_thinking"
+        "text",
+        "image",
+        "tool_use",
+        "tool_result",
+        "tool_reference",
+        "thinking",
+        "redacted_thinking",
     ]
     text: Optional[str] = None
     # For image content
@@ -63,6 +69,7 @@ class AnthropicTool(BaseModel):
     name: str
     description: Optional[str] = None
     input_schema: dict[str, Any]
+    defer_loading: Optional[bool] = None
 
     @field_validator("input_schema")
     @classmethod

python/sglang/srt/entrypoints/anthropic/serving.py

@@ -146,6 +146,14 @@ class AnthropicServing:
                         )
                         if image_part is not None:
                             tool_content_parts.append(image_part)
+                    elif item_type == "tool_reference":
+                        # Anthropic uses `tool_name`; the SGLang chat template
+                        # matches on `name`. Translate at the boundary.
+                        ref_name = item.get("tool_name") or item.get("name")
+                        if ref_name:
+                            tool_content_parts.append(
+                                {"type": "tool_reference", "name": ref_name}
+                            )
 
                 tool_text = "\n".join(tool_text_parts)
                 if (
@@ -270,39 +278,41 @@ class AnthropicServing:
 
         chat_request = ChatCompletionRequest(**request_data)
 
-        # Convert tools
+        # Convert tools. Deferred tools stay in the list with defer_loading=True;
+        # the chat template hides them from the initial <tools> block and renders
+        # them on demand when a tool_reference block names them.
         if anthropic_request.tools:
-            tools = []
-            for tool in anthropic_request.tools:
-                tools.append(
-                    Tool(
-                        type="function",
-                        function={
-                            "name": tool.name,
-                            "description": tool.description or "",
-                            "parameters": tool.input_schema,
-                        },
-                    )
+            chat_request.tools = [
+                Tool(
+                    type="function",
+                    defer_loading=tool.defer_loading,
+                    function={
+                        "name": tool.name,
+                        "description": tool.description or "",
+                        "parameters": tool.input_schema,
+                    },
                 )
-            chat_request.tools = tools
+                for tool in anthropic_request.tools
+            ]
 
         # Convert tool choice
         if anthropic_request.tool_choice is not None:
-            if anthropic_request.tool_choice.type == "none":
+            tc_type = anthropic_request.tool_choice.type
+            if tc_type == "none":
                 chat_request.tool_choice = "none"
-            elif anthropic_request.tool_choice.type == "auto":
-                chat_request.tool_choice = "auto"
-            elif anthropic_request.tool_choice.type == "any":
-                chat_request.tool_choice = "required"
-            elif anthropic_request.tool_choice.type == "tool":
-                chat_request.tool_choice = ToolChoice(
-                    type="function",
-                    function=ToolChoiceFuncName(
-                        name=anthropic_request.tool_choice.name
-                    ),
-                )
-        elif anthropic_request.tools:
-            # Default to auto when tools are provided
+            elif chat_request.tools:
+                if tc_type == "auto":
+                    chat_request.tool_choice = "auto"
+                elif tc_type == "any":
+                    chat_request.tool_choice = "required"
+                elif tc_type == "tool":
+                    chat_request.tool_choice = ToolChoice(
+                        type="function",
+                        function=ToolChoiceFuncName(
+                            name=anthropic_request.tool_choice.name
+                        ),
+                    )
+        elif chat_request.tools:
             chat_request.tool_choice = "auto"
 
         return chat_request

python/sglang/srt/entrypoints/openai/protocol.py

@@ -454,11 +454,23 @@ class ChatCompletionMessageContentAudioPart(BaseModel):
     audio_url: ChatCompletionMessageContentAudioURL
 
 
+class ChatCompletionMessageContentToolReferenceBlock(BaseModel):
+    # GLM-specific extension used alongside `defer_loading` tools. The chat
+    # template looks up `tools[*].function.name == tr.name` and renders the
+    # referenced tool schemas inline for the current turn. Not part of any
+    # OpenAI API; included here so Pydantic accepts the content through the
+    # Chat Completions path (the Anthropic endpoint translates its
+    # `tool_name` field to `name` before forwarding).
+    type: Literal["tool_reference"]
+    name: str
+
+
 ChatCompletionMessageContentPart = Union[
     ChatCompletionMessageContentTextPart,
     ChatCompletionMessageContentImagePart,
     ChatCompletionMessageContentVideoPart,
     ChatCompletionMessageContentAudioPart,
+    ChatCompletionMessageContentToolReferenceBlock,
 ]
 
 # Rerank content types for multimodal reranking (e.g., Qwen3-VL-Reranker)
@@ -528,6 +540,14 @@ class Function(BaseModel):
     name: str
     parameters: Optional[object] = None
     strict: bool = False
+    defer_loading: Optional[bool] = None
+
+    @model_serializer(mode="wrap")
+    def _serialize(self, handler):
+        data = handler(self)
+        if self.defer_loading is None:
+            data.pop("defer_loading", None)
+        return data
 
 
 class Tool(BaseModel):
@@ -535,6 +555,13 @@ class Tool(BaseModel):
 
     type: str = Field(default="function", examples=["function"])
     function: Function
+    defer_loading: Optional[bool] = None
+
+    @model_validator(mode="after")
+    def _propagate_defer_loading(self) -> "Tool":
+        if self.defer_loading is not None and self.function.defer_loading is None:
+            self.function.defer_loading = self.defer_loading
+        return self
 
 
 class ToolChoiceFuncName(BaseModel):

python/sglang/srt/parser/jinja_template_utils.py

@@ -201,6 +201,11 @@ def process_content_for_template_format(
                     else:
                         # Keep text content as-is for openai format
                         processed_content_parts.append(chunk)
+                elif chunk_type == "tool_reference":
+                    # GLM-specific extension: pass through so the chat template
+                    # can match tool_reference.name against tools[*].function.name
+                    # and render the referenced tool schemas inline.
+                    processed_content_parts.append(chunk)
 
         new_msg = {
             k: v for k, v in msg_dict.items() if v is not None and k != "content"