fix(anthropic): detect-and-passthrough mid-conversation system messages (#28906)

python/sglang/srt/entrypoints/anthropic/protocol.py

@@ -379,64 +379,6 @@ class AnthropicMessagesRequest(BaseModel):
     output_config: Optional[AnthropicOutputConfig] = None
     betas: Optional[list[str]] = None
 
-    @model_validator(mode="before")
-    @classmethod
-    def move_mid_conversation_system_messages(cls, values: dict) -> dict:
-        """Fold mid-conversation ``role: "system"`` turns into the top-level
-        ``system`` field — some clients (e.g. Claude Code) emit them there."""
-        messages = values.get("messages", [])
-        if not messages:
-            return values
-
-        clean_messages = []
-        extracted_system_texts = []
-
-        for msg in messages:
-            # ``mode="before"`` sees raw dicts (HTTP path) but also already-
-            # constructed ``AnthropicMessage`` objects (programmatic path, e.g.
-            # ``handle_count_tokens``), so normalize to a dict first.
-            if isinstance(msg, BaseModel):
-                msg = msg.model_dump()
-            if msg.get("role") == "system":
-                content = msg.get("content", "")
-                if isinstance(content, str) and content.strip():
-                    extracted_system_texts.append(content.strip())
-                elif isinstance(content, list):
-                    for block in content:
-                        if isinstance(block, dict) and block.get("type") == "text":
-                            text = block.get("text", "").strip()
-                            if text:
-                                extracted_system_texts.append(text)
-            else:
-                clean_messages.append(msg)
-
-        if extracted_system_texts:
-            existing_system = values.get("system")
-            combined_system = []
-
-            if existing_system:
-                if isinstance(existing_system, str):
-                    if existing_system.strip():
-                        combined_system.append(existing_system.strip())
-                elif isinstance(existing_system, list):
-                    for block in existing_system:
-                        if isinstance(block, BaseModel):
-                            block = block.model_dump()
-                        if isinstance(block, dict) and block.get("type") == "text":
-                            text = block.get("text", "").strip()
-                            if text:
-                                combined_system.append(text)
-
-            combined_system.extend(extracted_system_texts)
-
-            # Join into a string — ``system`` is ``str | list[AnthropicContentBlock]``,
-            # so a ``list[str]`` would fail validation.
-            if combined_system:
-                values["system"] = "\n".join(combined_system)
-
-        values["messages"] = clean_messages
-        return values
-
     @field_validator("model")
     @classmethod
     def _validate_model(cls, v):

python/sglang/srt/entrypoints/anthropic/serving.py

@@ -52,6 +52,7 @@ from sglang.srt.entrypoints.openai.protocol import (
     ToolChoice,
     ToolChoiceFuncName,
 )
+from sglang.srt.managers.template_detection import detect_inline_system_support
 from sglang.srt.observability.req_time_stats import monotonic_time
 
 if TYPE_CHECKING:
@@ -132,6 +133,26 @@ def _anthropic_usage_from_openai(
     return AnthropicUsage(**usage_fields)
 
 
+def _extract_system_text(
+    content: Union[str, list[AnthropicContentBlock]],
+) -> Optional[str]:
+    """Flatten a system message's content to a trimmed string, or ``None``."""
+    if isinstance(content, str):
+        return content.strip() or None
+    texts = []
+    for block in content:
+        if isinstance(block, BaseModel) and getattr(block, "type", None) == "text":
+            text = getattr(block, "text", "")
+        elif isinstance(block, dict) and block.get("type") == "text":
+            text = block.get("text", "")
+        else:
+            continue
+        text = (text or "").strip()
+        if text:
+            texts.append(text)
+    return "\n".join(texts) if texts else None
+
+
 def _wrap_sse_event(data: str, event_type: str) -> str:
     """Format an Anthropic SSE event with event type and data lines."""
     return f"event: {event_type}\ndata: {data}\n\n"
@@ -169,6 +190,18 @@ class AnthropicServing:
 
     def __init__(self, openai_serving_chat: OpenAIServingChat):
         self.openai_serving_chat = openai_serving_chat
+        self._merge_inline_system = not detect_inline_system_support(
+            self._chat_template()
+        )
+
+    def _chat_template(self) -> Optional[str]:
+        tokenizer_manager = getattr(self.openai_serving_chat, "tokenizer_manager", None)
+        if tokenizer_manager is None:
+            return None
+        tokenizer = getattr(tokenizer_manager, "tokenizer", None)
+        if tokenizer is None:
+            return None
+        return getattr(tokenizer, "chat_template", None)
 
     async def handle_messages(
         self,
@@ -356,19 +389,28 @@ class AnthropicServing:
                 )
                 return None
 
-        # Add system message if provided
+        system_parts: list[str] = []
         if anthropic_request.system:
             if isinstance(anthropic_request.system, str):
-                openai_messages.append(
+                if anthropic_request.system.strip():
-                    {"role": "system", "content": anthropic_request.system}
+                    system_parts.append(anthropic_request.system)
-                )
             else:
-                system_parts = []
                 for block in anthropic_request.system:
                     if block.type == "text" and block.text:
                         system_parts.append(block.text)
-                system_text = "\n".join(system_parts)
+
-                openai_messages.append({"role": "system", "content": system_text})
+        if self._merge_inline_system:
+            for msg in anthropic_request.messages:
+                if msg.role != "system":
+                    continue
+                text = _extract_system_text(msg.content)
+                if text:
+                    system_parts.append(text)
+
+        if system_parts:
+            openai_messages.append(
+                {"role": "system", "content": "\n".join(system_parts)}
+            )
 
         def _emit_user_message(parts: list[dict]) -> None:
             """Append accumulated parts as a user message, then clear them.
@@ -388,6 +430,8 @@ class AnthropicServing:
 
         # Convert messages
         for msg in anthropic_request.messages:
+            if msg.role == "system" and self._merge_inline_system:
+                continue
             if isinstance(msg.content, str):
                 openai_messages.append({"role": msg.role, "content": msg.content})
                 continue

python/sglang/srt/managers/template_detection.py

@@ -24,6 +24,10 @@ import re
 from dataclasses import dataclass
 from typing import Callable, Optional, Tuple
 
+import jinja2
+import jinja2.ext
+import jinja2.sandbox
+
 logger = logging.getLogger(__name__)
 
 
@@ -483,6 +487,37 @@ def detect_tool_call_parser(
     return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser")
 
 
+def detect_inline_system_support(chat_template: Optional[str]) -> bool:
+    """True if mid-conversation ``role: "system"`` renders inline; False if the
+    template raises or silently drops it (then merge into the leading block).
+
+    The probe requires the second system's sentinel to appear in the output —
+    not raising isn't enough, since some templates ignore non-leading system."""
+    if not chat_template:
+        return False
+    sentinel = "__sglang_inline_system_sentinel__"
+    try:
+        env = jinja2.sandbox.ImmutableSandboxedEnvironment(
+            trim_blocks=True,
+            lstrip_blocks=True,
+            extensions=[jinja2.ext.loopcontrols],
+        )
+        rendered = env.from_string(chat_template).render(
+            messages=[
+                {"role": "system", "content": "t"},
+                {"role": "user", "content": "t"},
+                {"role": "system", "content": sentinel},
+                {"role": "user", "content": "t"},
+            ],
+            add_generation_prompt=False,
+        )
+        return sentinel in rendered
+    except jinja2.TemplateError:
+        return False
+    except Exception:
+        return False
+
+
 def _resolve_auto_parser(
     server_args,
     attr: str,

test/registered/unit/entrypoints/anthropic/test_serving.py

@@ -18,15 +18,21 @@ from sglang.srt.entrypoints.openai.protocol import (  # noqa: E402
     ChatCompletionRequest,
     ChatCompletionResponse,
 )
+from sglang.srt.managers.template_detection import (  # noqa: E402
+    detect_inline_system_support,
+)
 from sglang.test.ci.ci_register import register_cpu_ci  # noqa: E402
 
 register_cpu_ci(est_time=1, suite="base-a-test-cpu")
 
 
 class _FakeOpenAIServingChat:
-    def __init__(self, stream_lines=None):
+    def __init__(self, stream_lines=None, chat_template=None):
         self.stream_lines = stream_lines or []
         self.apply_reasoning_calls: list[bool] = []
+        self.tokenizer_manager = SimpleNamespace(
+            tokenizer=SimpleNamespace(chat_template=chat_template)
+        )
 
     def _generate_chat_stream(self, adapted_request, processed_request, raw_request):
         async def _gen():
@@ -128,8 +134,25 @@ async def _collect_anthropic_events(serving, anthropic_request):
 
 
 class TestAnthropicServing(unittest.TestCase):
-    def _serving(self, stream_lines=None):
+    # System-first guard (Qwen-style): rejects non-first system → must merge.
-        return AnthropicServing(_FakeOpenAIServingChat(stream_lines))
+    QWEN_SYSTEM_FIRST_TEMPLATE = (
+        "{%- for message in messages %}"
+        "{%- if message.role == 'system' and not loop.first %}"
+        "{{- raise_exception('system must be first') }}"
+        "{%- endif %}"
+        "{{- message.role }}: {{ message.content }}\n"
+        "{%- endfor %}"
+    )
+
+    # Renders system at any position (GLM/Kimi/Qwen3) → can pass through.
+    INLINE_SYSTEM_TEMPLATE = (
+        "{%- for message in messages %}"
+        "{{- message.role }}: {{ message.content }}\n"
+        "{%- endfor %}"
+    )
+
+    def _serving(self, stream_lines=None, chat_template=None):
+        return AnthropicServing(_FakeOpenAIServingChat(stream_lines, chat_template))
 
     def _anthropic_request(self, **overrides):
         data = {
@@ -1267,10 +1290,12 @@ class TestAnthropicServing(unittest.TestCase):
         # strict role-alternation chat templates (qwen, llama, mistral).
         self.assertEqual(roles, ["user", "assistant", "user"])
 
-    def test_in_messages_system_role_folded_to_top_level(self):
+    def test_in_messages_system_merged_when_template_requires_first(self):
-        """A mid-conversation ``role: "system"`` turn is folded into the
+        """When the chat template rejects mid-conversation ``role: "system"``
-        top-level ``system`` field by the request validator, so it does not
+        (e.g. Qwen's system-first guard), the converter folds the inline
-        appear as a dialogue turn — matching the official Anthropic API."""
+        system turn into the leading system block so the template doesn't
+        400. The request object itself is no longer mutated — detection runs
+        in the serving layer on conversion."""
         serving = self._serving()
         request = self._anthropic_request(
             stream=False,
@@ -1280,19 +1305,18 @@ class TestAnthropicServing(unittest.TestCase):
                 {"role": "user", "content": "go"},
             ],
         )
-        # The validator moved the system turn into the top-level system field.
+        self.assertIsNone(request.system)
-        self.assertEqual(request.system, "Reply with exactly: OK")
+        self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
-        self.assertEqual([m.role for m in request.messages], ["user", "user"])
-        # And the converted OpenAI request has one leading system message.
         chat_request = serving._convert_to_chat_completion_request(request)
         self.assertEqual(
             [m.role for m in chat_request.messages], ["system", "user", "user"]
         )
         self.assertEqual(chat_request.messages[0].content, "Reply with exactly: OK")
 
-    def test_in_messages_system_role_merged_with_top_level(self):
+    def test_in_messages_system_merged_with_top_level_when_merge(self):
-        """A top-level ``system`` field and a mid-conversation system turn are
+        """On the merge path, a top-level ``system`` field and a mid-conversation
-        merged; top-level text comes first."""
+        system turn are joined into the leading system block; top-level text
+        comes first."""
         serving = self._serving()
         request = self._anthropic_request(
             stream=False,
@@ -1303,43 +1327,73 @@ class TestAnthropicServing(unittest.TestCase):
                 {"role": "user", "content": "go"},
             ],
         )
-        # Validator combines top-level system first, then the in-messages turn,
+        self.assertEqual(request.system, "You are terse.")
-        # joined into a single string.
+        self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
-        self.assertEqual(request.system, "You are terse.\nOne word only.")
+        chat_request = serving._convert_to_chat_completion_request(request)
-        self.assertEqual([m.role for m in request.messages], ["user", "user"])
+        self.assertEqual(
+            [m.role for m in chat_request.messages], ["system", "user", "user"]
+        )
+        self.assertEqual(
+            chat_request.messages[0].content, "You are terse.\nOne word only."
+        )
 
-    def test_top_level_system_only_is_unchanged(self):
+    def test_in_messages_system_passed_through_when_template_allows_inline(self):
-        """A request with only the top-level ``system`` field (no in-messages
+        """When the chat template renders ``role: "system"`` at any position
-        system turn) must be unaffected by the validator: the system field is
+        (GLM / Kimi / Qwen3), the inline system turn stays at its original
-        preserved verbatim and the dialogue order is untouched. Guards the
+        position — preserving the prefix cache and the request's structure."""
-        common multi-turn path against regressions."""
+        serving = self._serving(chat_template=self.INLINE_SYSTEM_TEMPLATE)
-        serving = self._serving()
+        self.assertFalse(serving._merge_inline_system)
         request = self._anthropic_request(
             stream=False,
-            system="You are a helpful assistant.",
+            system="You are terse.",
             messages=[
-                {"role": "user", "content": "My name is Alice."},
+                {"role": "user", "content": "hi"},
-                {"role": "assistant", "content": "Hello Alice!"},
+                {"role": "system", "content": "Reply with exactly: OK"},
-                {"role": "user", "content": "What is my name?"},
+                {"role": "user", "content": "go"},
             ],
         )
-        self.assertEqual(request.system, "You are a helpful assistant.")
-        self.assertEqual(
-            [m.role for m in request.messages], ["user", "assistant", "user"]
-        )
         chat_request = serving._convert_to_chat_completion_request(request)
         self.assertEqual(
             [m.role for m in chat_request.messages],
-            ["system", "user", "assistant", "user"],
+            ["system", "user", "system", "user"],
-        )
-        self.assertEqual(
-            chat_request.messages[0].content, "You are a helpful assistant."
         )
+        self.assertEqual(chat_request.messages[0].content, "You are terse.")
+        self.assertEqual(chat_request.messages[2].content, "Reply with exactly: OK")
 
-    def test_validator_handles_constructed_message_objects(self):
+    def test_top_level_system_only_is_unchanged(self):
-        """The ``mode="before"`` validator must also handle requests built
+        """A request with only the top-level ``system`` field (no in-messages
-        programmatically with ``AnthropicMessage`` objects (not just raw dicts),
+        system turn) is unaffected on both detection paths: the system field is
-        e.g. ``handle_count_tokens`` constructs the request this way."""
+        preserved verbatim and the dialogue order is untouched. Guards the
+        common multi-turn path against regressions."""
+        for template in (None, self.INLINE_SYSTEM_TEMPLATE):
+            serving = self._serving(chat_template=template)
+            request = self._anthropic_request(
+                stream=False,
+                system="You are a helpful assistant.",
+                messages=[
+                    {"role": "user", "content": "My name is Alice."},
+                    {"role": "assistant", "content": "Hello Alice!"},
+                    {"role": "user", "content": "What is my name?"},
+                ],
+            )
+            self.assertEqual(request.system, "You are a helpful assistant.")
+            self.assertEqual(
+                [m.role for m in request.messages], ["user", "assistant", "user"]
+            )
+            chat_request = serving._convert_to_chat_completion_request(request)
+            self.assertEqual(
+                [m.role for m in chat_request.messages],
+                ["system", "user", "assistant", "user"],
+            )
+            self.assertEqual(
+                chat_request.messages[0].content, "You are a helpful assistant."
+            )
+
+    def test_constructed_message_objects_merged_on_merge_path(self):
+        """Requests built programmatically with ``AnthropicMessage`` objects
+        (e.g. ``handle_count_tokens``) also get inline system folded into the
+        leading block on the merge path."""
+        serving = self._serving()
         request = AnthropicMessagesRequest(
             model="m",
             max_tokens=8,
@@ -1349,8 +1403,12 @@ class TestAnthropicServing(unittest.TestCase):
                 AnthropicMessage(role="user", content="go"),
             ],
         )
-        self.assertEqual(request.system, "be terse")
+        self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
-        self.assertEqual([m.role for m in request.messages], ["user", "user"])
+        chat_request = serving._convert_to_chat_completion_request(request)
+        self.assertEqual(
+            [m.role for m in chat_request.messages], ["system", "user", "user"]
+        )
+        self.assertEqual(chat_request.messages[0].content, "be terse")
 
     def test_thinking_history_drop_on_missing_detector(self):
         """Replaying a thinking block on a non-reasoning model should not 400."""
@@ -1421,5 +1479,45 @@ class TestAnthropicServing(unittest.TestCase):
         )
 
 
+class TestDetectInlineSystemSupport(unittest.TestCase):
+    """Chat-template detection for mid-conversation system messages (#28883)."""
+
+    def test_guarded_template_not_supported(self):
+        guarded = (
+            "{%- for message in messages %}"
+            "{%- if message.role == 'system' and not loop.first %}"
+            "{{- raise_exception('system must be first') }}"
+            "{%- endif %}"
+            "{%- endfor %}"
+        )
+        self.assertFalse(detect_inline_system_support(guarded))
+
+    def test_inline_template_supported(self):
+        inline = (
+            "{%- for message in messages %}"
+            "{{- message.role }}: {{ message.content }}\n"
+            "{%- endfor %}"
+        )
+        self.assertTrue(detect_inline_system_support(inline))
+
+    def test_silent_drop_template_not_supported(self):
+        # Renders only the leading system; silently ignores later system turns.
+        silent_drop = (
+            "{%- if messages[0].role == 'system' %}"
+            "{{ messages[0].content }}\n"
+            "{%- endif %}"
+            "{%- for message in messages %}"
+            "{%- if message.role in ('user', 'assistant') %}"
+            "{{ message.role }}: {{ message.content }}\n"
+            "{%- endif %}"
+            "{%- endfor %}"
+        )
+        self.assertFalse(detect_inline_system_support(silent_drop))
+
+    def test_no_template_not_supported(self):
+        self.assertFalse(detect_inline_system_support(None))
+        self.assertFalse(detect_inline_system_support(""))
+
+
 if __name__ == "__main__":
     unittest.main()