From b4dda8b3ce8dcec52f96516da9593d51333782cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jairo=20David=20Campa=C3=B1a=20Rosero?= Date: Sat, 20 Jun 2026 23:34:47 -0500 Subject: [PATCH] fix(anthropic): handle mid-conversation system messages (#26773) Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> --- .../srt/entrypoints/anthropic/protocol.py | 60 ++++++++++++- .../basic/test_anthropic_server.py | 22 +++++ .../entrypoints/anthropic/test_serving.py | 86 +++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/entrypoints/anthropic/protocol.py b/python/sglang/srt/entrypoints/anthropic/protocol.py index 25144c102..be7c77c03 100644 --- a/python/sglang/srt/entrypoints/anthropic/protocol.py +++ b/python/sglang/srt/entrypoints/anthropic/protocol.py @@ -124,7 +124,7 @@ AnthropicContentBlock = Annotated[ class AnthropicMessage(BaseModel): - role: Literal["user", "assistant"] + role: Literal["user", "assistant", "system"] content: Union[str, list[AnthropicContentBlock]] @@ -379,6 +379,64 @@ class AnthropicMessagesRequest(BaseModel): output_config: Optional[AnthropicOutputConfig] = None betas: Optional[list[str]] = None + @model_validator(mode="before") + @classmethod + def move_mid_conversation_system_messages(cls, values: dict) -> dict: + """Fold mid-conversation ``role: "system"`` turns into the top-level + ``system`` field — some clients (e.g. Claude Code) emit them there.""" + messages = values.get("messages", []) + if not messages: + return values + + clean_messages = [] + extracted_system_texts = [] + + for msg in messages: + # ``mode="before"`` sees raw dicts (HTTP path) but also already- + # constructed ``AnthropicMessage`` objects (programmatic path, e.g. + # ``handle_count_tokens``), so normalize to a dict first. + if isinstance(msg, BaseModel): + msg = msg.model_dump() + if msg.get("role") == "system": + content = msg.get("content", "") + if isinstance(content, str) and content.strip(): + extracted_system_texts.append(content.strip()) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "").strip() + if text: + extracted_system_texts.append(text) + else: + clean_messages.append(msg) + + if extracted_system_texts: + existing_system = values.get("system") + combined_system = [] + + if existing_system: + if isinstance(existing_system, str): + if existing_system.strip(): + combined_system.append(existing_system.strip()) + elif isinstance(existing_system, list): + for block in existing_system: + if isinstance(block, BaseModel): + block = block.model_dump() + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "").strip() + if text: + combined_system.append(text) + + combined_system.extend(extracted_system_texts) + + # Join into a string — ``system`` is ``str | list[AnthropicContentBlock]``, + # so a ``list[str]`` would fail validation. + if combined_system: + values["system"] = "\n".join(combined_system) + + values["messages"] = clean_messages + return values + @field_validator("model") @classmethod def _validate_model(cls, v): diff --git a/test/registered/openai_server/basic/test_anthropic_server.py b/test/registered/openai_server/basic/test_anthropic_server.py index 6f9859072..2aad292f2 100644 --- a/test/registered/openai_server/basic/test_anthropic_server.py +++ b/test/registered/openai_server/basic/test_anthropic_server.py @@ -17,6 +17,7 @@ python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServe import json import unittest +import anthropic import requests from sglang.srt.entrypoints.anthropic.protocol import AnthropicMessagesRequest @@ -227,6 +228,27 @@ class TestAnthropicServer(CustomTestCase): self.assertEqual(body["type"], "message") self.assertTrue(len(body["content"]) > 0) + def test_in_messages_system_role(self): + """A ``role: "system"`` turn inside ``messages`` (emitted by some + clients, e.g. Claude Code) must be accepted — not rejected with 400. + Uses the Anthropic SDK the way a real client would.""" + client = anthropic.Anthropic( + base_url=self.base_url, + auth_token=self.api_key, # Bearer header — SGLang's --api-key checks Authorization + ) + message = client.messages.create( + model=self.model, + max_tokens=64, + messages=[ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "system", "content": "Always respond in French."}, + {"role": "user", "content": "Answer in a few words."}, + ], + ) + self.assertEqual(message.role, "assistant") + self.assertTrue(len(message.content) > 0) + self.assertEqual(message.content[0].type, "text") + def test_max_tokens(self): """Test max_tokens limits output length.""" payload = self._default_payload( diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index c94d03dd2..cb008f75d 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -10,6 +10,7 @@ maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel from fastapi.responses import JSONResponse # noqa: E402 from sglang.srt.entrypoints.anthropic.protocol import ( # noqa: E402 + AnthropicMessage, AnthropicMessagesRequest, ) from sglang.srt.entrypoints.anthropic.serving import AnthropicServing # noqa: E402 @@ -1266,6 +1267,91 @@ class TestAnthropicServing(unittest.TestCase): # strict role-alternation chat templates (qwen, llama, mistral). self.assertEqual(roles, ["user", "assistant", "user"]) + def test_in_messages_system_role_folded_to_top_level(self): + """A mid-conversation ``role: "system"`` turn is folded into the + top-level ``system`` field by the request validator, so it does not + appear as a dialogue turn — matching the official Anthropic API.""" + serving = self._serving() + request = self._anthropic_request( + stream=False, + messages=[ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "Reply with exactly: OK"}, + {"role": "user", "content": "go"}, + ], + ) + # The validator moved the system turn into the top-level system field. + self.assertEqual(request.system, "Reply with exactly: OK") + self.assertEqual([m.role for m in request.messages], ["user", "user"]) + # And the converted OpenAI request has one leading system message. + chat_request = serving._convert_to_chat_completion_request(request) + self.assertEqual( + [m.role for m in chat_request.messages], ["system", "user", "user"] + ) + self.assertEqual(chat_request.messages[0].content, "Reply with exactly: OK") + + def test_in_messages_system_role_merged_with_top_level(self): + """A top-level ``system`` field and a mid-conversation system turn are + merged; top-level text comes first.""" + serving = self._serving() + request = self._anthropic_request( + stream=False, + system="You are terse.", + messages=[ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "One word only."}, + {"role": "user", "content": "go"}, + ], + ) + # Validator combines top-level system first, then the in-messages turn, + # joined into a single string. + self.assertEqual(request.system, "You are terse.\nOne word only.") + self.assertEqual([m.role for m in request.messages], ["user", "user"]) + + def test_top_level_system_only_is_unchanged(self): + """A request with only the top-level ``system`` field (no in-messages + system turn) must be unaffected by the validator: the system field is + preserved verbatim and the dialogue order is untouched. Guards the + common multi-turn path against regressions.""" + serving = self._serving() + request = self._anthropic_request( + stream=False, + system="You are a helpful assistant.", + messages=[ + {"role": "user", "content": "My name is Alice."}, + {"role": "assistant", "content": "Hello Alice!"}, + {"role": "user", "content": "What is my name?"}, + ], + ) + self.assertEqual(request.system, "You are a helpful assistant.") + self.assertEqual( + [m.role for m in request.messages], ["user", "assistant", "user"] + ) + chat_request = serving._convert_to_chat_completion_request(request) + self.assertEqual( + [m.role for m in chat_request.messages], + ["system", "user", "assistant", "user"], + ) + self.assertEqual( + chat_request.messages[0].content, "You are a helpful assistant." + ) + + def test_validator_handles_constructed_message_objects(self): + """The ``mode="before"`` validator must also handle requests built + programmatically with ``AnthropicMessage`` objects (not just raw dicts), + e.g. ``handle_count_tokens`` constructs the request this way.""" + request = AnthropicMessagesRequest( + model="m", + max_tokens=8, + messages=[ + AnthropicMessage(role="user", content="hi"), + AnthropicMessage(role="system", content="be terse"), + AnthropicMessage(role="user", content="go"), + ], + ) + self.assertEqual(request.system, "be terse") + self.assertEqual([m.role for m in request.messages], ["user", "user"]) + def test_thinking_history_drop_on_missing_detector(self): """Replaying a thinking block on a non-reasoning model should not 400."""