fix(anthropic): detect-and-passthrough mid-conversation system messages (#28906)

test/registered/unit/entrypoints/anthropic/test_serving.py

@@ -18,15 +18,21 @@ from sglang.srt.entrypoints.openai.protocol import (  # noqa: E402
     ChatCompletionRequest,
     ChatCompletionResponse,
 )
+from sglang.srt.managers.template_detection import (  # noqa: E402
+    detect_inline_system_support,
+)
 from sglang.test.ci.ci_register import register_cpu_ci  # noqa: E402
 
 register_cpu_ci(est_time=1, suite="base-a-test-cpu")
 
 
 class _FakeOpenAIServingChat:
-    def __init__(self, stream_lines=None):
+    def __init__(self, stream_lines=None, chat_template=None):
         self.stream_lines = stream_lines or []
         self.apply_reasoning_calls: list[bool] = []
+        self.tokenizer_manager = SimpleNamespace(
+            tokenizer=SimpleNamespace(chat_template=chat_template)
+        )
 
     def _generate_chat_stream(self, adapted_request, processed_request, raw_request):
         async def _gen():
@@ -128,8 +134,25 @@ async def _collect_anthropic_events(serving, anthropic_request):
 
 
 class TestAnthropicServing(unittest.TestCase):
-    def _serving(self, stream_lines=None):
-        return AnthropicServing(_FakeOpenAIServingChat(stream_lines))
+    # System-first guard (Qwen-style): rejects non-first system → must merge.
+    QWEN_SYSTEM_FIRST_TEMPLATE = (
+        "{%- for message in messages %}"
+        "{%- if message.role == 'system' and not loop.first %}"
+        "{{- raise_exception('system must be first') }}"
+        "{%- endif %}"
+        "{{- message.role }}: {{ message.content }}\n"
+        "{%- endfor %}"
+    )
+
+    # Renders system at any position (GLM/Kimi/Qwen3) → can pass through.
+    INLINE_SYSTEM_TEMPLATE = (
+        "{%- for message in messages %}"
+        "{{- message.role }}: {{ message.content }}\n"
+        "{%- endfor %}"
+    )
+
+    def _serving(self, stream_lines=None, chat_template=None):
+        return AnthropicServing(_FakeOpenAIServingChat(stream_lines, chat_template))
 
     def _anthropic_request(self, **overrides):
         data = {
@@ -1267,10 +1290,12 @@ class TestAnthropicServing(unittest.TestCase):
         # strict role-alternation chat templates (qwen, llama, mistral).
         self.assertEqual(roles, ["user", "assistant", "user"])
 
-    def test_in_messages_system_role_folded_to_top_level(self):
-        """A mid-conversation ``role: "system"`` turn is folded into the
-        top-level ``system`` field by the request validator, so it does not
-        appear as a dialogue turn — matching the official Anthropic API."""
+    def test_in_messages_system_merged_when_template_requires_first(self):
+        """When the chat template rejects mid-conversation ``role: "system"``
+        (e.g. Qwen's system-first guard), the converter folds the inline
+        system turn into the leading system block so the template doesn't
+        400. The request object itself is no longer mutated — detection runs
+        in the serving layer on conversion."""
         serving = self._serving()
         request = self._anthropic_request(
             stream=False,
@@ -1280,19 +1305,18 @@ class TestAnthropicServing(unittest.TestCase):
                 {"role": "user", "content": "go"},
             ],
         )
-        # The validator moved the system turn into the top-level system field.
-        self.assertEqual(request.system, "Reply with exactly: OK")
-        self.assertEqual([m.role for m in request.messages], ["user", "user"])
-        # And the converted OpenAI request has one leading system message.
+        self.assertIsNone(request.system)
+        self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
         chat_request = serving._convert_to_chat_completion_request(request)
         self.assertEqual(
             [m.role for m in chat_request.messages], ["system", "user", "user"]
         )
         self.assertEqual(chat_request.messages[0].content, "Reply with exactly: OK")
 
-    def test_in_messages_system_role_merged_with_top_level(self):
-        """A top-level ``system`` field and a mid-conversation system turn are
-        merged; top-level text comes first."""
+    def test_in_messages_system_merged_with_top_level_when_merge(self):
+        """On the merge path, a top-level ``system`` field and a mid-conversation
+        system turn are joined into the leading system block; top-level text
+        comes first."""
         serving = self._serving()
         request = self._anthropic_request(
             stream=False,
@@ -1303,43 +1327,73 @@ class TestAnthropicServing(unittest.TestCase):
                 {"role": "user", "content": "go"},
             ],
         )
-        # Validator combines top-level system first, then the in-messages turn,
-        # joined into a single string.
-        self.assertEqual(request.system, "You are terse.\nOne word only.")
-        self.assertEqual([m.role for m in request.messages], ["user", "user"])
+        self.assertEqual(request.system, "You are terse.")
+        self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
+        chat_request = serving._convert_to_chat_completion_request(request)
+        self.assertEqual(
+            [m.role for m in chat_request.messages], ["system", "user", "user"]
+        )
+        self.assertEqual(
+            chat_request.messages[0].content, "You are terse.\nOne word only."
+        )
 
-    def test_top_level_system_only_is_unchanged(self):
-        """A request with only the top-level ``system`` field (no in-messages
-        system turn) must be unaffected by the validator: the system field is
-        preserved verbatim and the dialogue order is untouched. Guards the
-        common multi-turn path against regressions."""
-        serving = self._serving()
+    def test_in_messages_system_passed_through_when_template_allows_inline(self):
+        """When the chat template renders ``role: "system"`` at any position
+        (GLM / Kimi / Qwen3), the inline system turn stays at its original
+        position — preserving the prefix cache and the request's structure."""
+        serving = self._serving(chat_template=self.INLINE_SYSTEM_TEMPLATE)
+        self.assertFalse(serving._merge_inline_system)
         request = self._anthropic_request(
             stream=False,
-            system="You are a helpful assistant.",
+            system="You are terse.",
             messages=[
-                {"role": "user", "content": "My name is Alice."},
-                {"role": "assistant", "content": "Hello Alice!"},
-                {"role": "user", "content": "What is my name?"},
+                {"role": "user", "content": "hi"},
+                {"role": "system", "content": "Reply with exactly: OK"},
+                {"role": "user", "content": "go"},
             ],
         )
-        self.assertEqual(request.system, "You are a helpful assistant.")
-        self.assertEqual(
-            [m.role for m in request.messages], ["user", "assistant", "user"]
-        )
         chat_request = serving._convert_to_chat_completion_request(request)
         self.assertEqual(
             [m.role for m in chat_request.messages],
-            ["system", "user", "assistant", "user"],
-        )
-        self.assertEqual(
-            chat_request.messages[0].content, "You are a helpful assistant."
+            ["system", "user", "system", "user"],
         )
+        self.assertEqual(chat_request.messages[0].content, "You are terse.")
+        self.assertEqual(chat_request.messages[2].content, "Reply with exactly: OK")
 
-    def test_validator_handles_constructed_message_objects(self):
-        """The ``mode="before"`` validator must also handle requests built
-        programmatically with ``AnthropicMessage`` objects (not just raw dicts),
-        e.g. ``handle_count_tokens`` constructs the request this way."""
+    def test_top_level_system_only_is_unchanged(self):
+        """A request with only the top-level ``system`` field (no in-messages
+        system turn) is unaffected on both detection paths: the system field is
+        preserved verbatim and the dialogue order is untouched. Guards the
+        common multi-turn path against regressions."""
+        for template in (None, self.INLINE_SYSTEM_TEMPLATE):
+            serving = self._serving(chat_template=template)
+            request = self._anthropic_request(
+                stream=False,
+                system="You are a helpful assistant.",
+                messages=[
+                    {"role": "user", "content": "My name is Alice."},
+                    {"role": "assistant", "content": "Hello Alice!"},
+                    {"role": "user", "content": "What is my name?"},
+                ],
+            )
+            self.assertEqual(request.system, "You are a helpful assistant.")
+            self.assertEqual(
+                [m.role for m in request.messages], ["user", "assistant", "user"]
+            )
+            chat_request = serving._convert_to_chat_completion_request(request)
+            self.assertEqual(
+                [m.role for m in chat_request.messages],
+                ["system", "user", "assistant", "user"],
+            )
+            self.assertEqual(
+                chat_request.messages[0].content, "You are a helpful assistant."
+            )
+
+    def test_constructed_message_objects_merged_on_merge_path(self):
+        """Requests built programmatically with ``AnthropicMessage`` objects
+        (e.g. ``handle_count_tokens``) also get inline system folded into the
+        leading block on the merge path."""
+        serving = self._serving()
         request = AnthropicMessagesRequest(
             model="m",
             max_tokens=8,
@@ -1349,8 +1403,12 @@ class TestAnthropicServing(unittest.TestCase):
                 AnthropicMessage(role="user", content="go"),
             ],
         )
-        self.assertEqual(request.system, "be terse")
-        self.assertEqual([m.role for m in request.messages], ["user", "user"])
+        self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
+        chat_request = serving._convert_to_chat_completion_request(request)
+        self.assertEqual(
+            [m.role for m in chat_request.messages], ["system", "user", "user"]
+        )
+        self.assertEqual(chat_request.messages[0].content, "be terse")
 
     def test_thinking_history_drop_on_missing_detector(self):
         """Replaying a thinking block on a non-reasoning model should not 400."""
@@ -1421,5 +1479,45 @@ class TestAnthropicServing(unittest.TestCase):
         )
 
 
+class TestDetectInlineSystemSupport(unittest.TestCase):
+    """Chat-template detection for mid-conversation system messages (#28883)."""
+
+    def test_guarded_template_not_supported(self):
+        guarded = (
+            "{%- for message in messages %}"
+            "{%- if message.role == 'system' and not loop.first %}"
+            "{{- raise_exception('system must be first') }}"
+            "{%- endif %}"
+            "{%- endfor %}"
+        )
+        self.assertFalse(detect_inline_system_support(guarded))
+
+    def test_inline_template_supported(self):
+        inline = (
+            "{%- for message in messages %}"
+            "{{- message.role }}: {{ message.content }}\n"
+            "{%- endfor %}"
+        )
+        self.assertTrue(detect_inline_system_support(inline))
+
+    def test_silent_drop_template_not_supported(self):
+        # Renders only the leading system; silently ignores later system turns.
+        silent_drop = (
+            "{%- if messages[0].role == 'system' %}"
+            "{{ messages[0].content }}\n"
+            "{%- endif %}"
+            "{%- for message in messages %}"
+            "{%- if message.role in ('user', 'assistant') %}"
+            "{{ message.role }}: {{ message.content }}\n"
+            "{%- endif %}"
+            "{%- endfor %}"
+        )
+        self.assertFalse(detect_inline_system_support(silent_drop))
+
+    def test_no_template_not_supported(self):
+        self.assertFalse(detect_inline_system_support(None))
+        self.assertFalse(detect_inline_system_support(""))
+
+
 if __name__ == "__main__":
     unittest.main()