From 09193bf36fbec930bd54649ac64a7a5ede76d46b Mon Sep 17 00:00:00 2001 From: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:57:52 +0800 Subject: [PATCH] [Fix]: render tool_reference schema regardless of tool_result part order (#32522) Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Xinyuan Tong --- .../srt/entrypoints/anthropic/serving.py | 49 +++++--- .../entrypoints/anthropic/test_serving.py | 105 ++++++++++++++++++ 2 files changed, 136 insertions(+), 18 deletions(-) diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index d84614210..96800e68a 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -297,7 +297,7 @@ class AnthropicServing: def _convert_tool_result_content( content: Any, - ) -> tuple[Union[str, list[dict]], str]: + ) -> tuple[list[Union[str, list[dict]]], str]: if isinstance(content, list): tool_content_parts = [] tool_text_parts = [] @@ -342,17 +342,29 @@ class AnthropicServing: ) tool_text = "\n".join(tool_text_parts) - if ( - len(tool_content_parts) == 1 - and tool_content_parts[0]["type"] == "text" - ): - return tool_content_parts[0]["text"], tool_text - if tool_content_parts: - return tool_content_parts, tool_text - return "", tool_text + # GLM templates expand references only at the start of a tool + # message, so isolate reference runs without changing part order. + tool_content_groups: list[list[dict]] = [] + for part in tool_content_parts: + is_reference = part["type"] == "tool_reference" + if ( + not tool_content_groups + or (tool_content_groups[-1][0]["type"] == "tool_reference") + != is_reference + ): + tool_content_groups.append([]) + tool_content_groups[-1].append(part) + + tool_contents: list[Union[str, list[dict]]] = [] + for group in tool_content_groups: + if len(group) == 1 and group[0]["type"] == "text": + tool_contents.append(group[0]["text"]) + else: + tool_contents.append(group) + return tool_contents or [""], tool_text tool_text = str(content) if content else "" - return tool_text, tool_text + return [tool_text], tool_text def _convert_assistant_thinking_blocks( blocks: list[AnthropicContentBlock], @@ -484,7 +496,7 @@ class AnthropicServing: tool_calls.append(tool_call) elif block.type == "tool_result": - tool_content, tool_text = _convert_tool_result_content( + tool_contents, tool_text = _convert_tool_result_content( block.content ) @@ -497,13 +509,14 @@ class AnthropicServing: # block must come AFTER that text in OpenAI form too). if msg.role == "user": _emit_user_message(content_parts) - openai_messages.append( - { - "role": "tool", - "tool_call_id": tool_call_id, - "content": tool_content, - } - ) + for tool_content in tool_contents: + openai_messages.append( + { + "role": "tool", + "tool_call_id": tool_call_id, + "content": tool_content, + } + ) else: content_parts.append( { diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index c1fa902de..aa3008e96 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -8,6 +8,7 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel from fastapi.responses import JSONResponse # noqa: E402 +from jinja2 import Environment # noqa: E402 from sglang.srt.entrypoints.anthropic.protocol import ( # noqa: E402 AnthropicMessage, @@ -140,6 +141,22 @@ class TestAnthropicServing(unittest.TestCase): "{{- message.role }}: {{ message.content }}\n" "{%- endfor %}" ) + GLM_TOOL_RESULT_TEMPLATE = """ +{%- for message in messages if message.role == "tool" -%} +{%- if loop.first -%}<|observation|>{%- endif -%} +{%- if message.content is string -%} +{{ message.content }} +{%- elif message.content.0.type == "tool_reference" -%} + +{%- for reference in message.content -%} +{%- for tool in tools if tool.function.name == reference.name -%} +{{ tool.function.name }} +{%- endfor -%} +{%- endfor -%} + +{%- endif -%} +{%- endfor -%} +""" def _serving(self, stream_lines=None, chat_template=None): return AnthropicServing(_FakeOpenAIServingChat(stream_lines, chat_template)) @@ -154,6 +171,26 @@ class TestAnthropicServing(unittest.TestCase): data.update(overrides) return AnthropicMessagesRequest.model_validate(data) + def _tool_result_request(self, content, tools=None): + overrides = { + "stream": False, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": content, + } + ], + } + ], + } + if tools is not None: + overrides["tools"] = tools + return self._anthropic_request(**overrides) + def test_stream_closes_tool_block_before_text_delta(self): serving = self._serving( [ @@ -373,6 +410,74 @@ class TestAnthropicServing(unittest.TestCase): self.assertIn("https://docs.sglang.ai", tool_message["content"]) self.assertIn("Anthropic API notes", tool_message["content"]) + def test_mixed_tool_reference_content_preserves_part_order(self): + request = self._tool_result_request( + [ + {"type": "text", "text": "Tool loaded: Bash"}, + {"type": "tool_reference", "tool_name": "Bash"}, + {"type": "text", "text": "Ready"}, + ] + ) + + chat_request = self._serving()._convert_to_chat_completion_request(request) + messages = chat_request.model_dump(exclude_none=True)["messages"] + + self.assertEqual([message["role"] for message in messages], ["tool"] * 3) + self.assertEqual( + [message["tool_call_id"] for message in messages], ["call_1"] * 3 + ) + self.assertEqual(messages[0]["content"], "Tool loaded: Bash") + self.assertEqual( + messages[1]["content"], + [{"type": "tool_reference", "name": "Bash"}], + ) + self.assertEqual(messages[2]["content"], "Ready") + + def test_mixed_tool_reference_content_renders_text_and_schema(self): + template = Environment().from_string(self.GLM_TOOL_RESULT_TEMPLATE) + request = self._tool_result_request( + [ + {"type": "text", "text": "Tool loaded: Bash"}, + {"type": "tool_reference", "tool_name": "Bash"}, + ], + tools=[ + { + "name": "Bash", + "description": "Run a shell command", + "input_schema": {"type": "object", "properties": {}}, + "defer_loading": True, + } + ], + ) + + chat_request = self._serving()._convert_to_chat_completion_request(request) + payload = chat_request.model_dump(exclude_none=True) + prompt = template.render(messages=payload["messages"], tools=payload["tools"]) + + self.assertIn("Tool loaded: Bash", prompt) + self.assertIn("Bash", prompt) + self.assertEqual(prompt.count("<|observation|>"), 1) + + def test_reference_only_tool_result_remains_one_message(self): + request = self._tool_result_request( + [ + {"type": "tool_reference", "tool_name": "Bash"}, + {"type": "tool_reference", "tool_name": "Read"}, + ] + ) + + chat_request = self._serving()._convert_to_chat_completion_request(request) + messages = chat_request.model_dump(exclude_none=True)["messages"] + + self.assertEqual(len(messages), 1) + self.assertEqual( + messages[0]["content"], + [ + {"type": "tool_reference", "name": "Bash"}, + {"type": "tool_reference", "name": "Read"}, + ], + ) + def test_builtin_web_search_tool_without_schema_is_skipped(self): request = AnthropicMessagesRequest.model_validate( {