From ed71fb8f95ec2a15125e0d697237931c9fe9f13b Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Fri, 26 Jun 2026 01:14:12 +0100 Subject: [PATCH] fix(anthropic): detect-and-passthrough mid-conversation system messages (#28906) --- .../srt/entrypoints/anthropic/protocol.py | 58 ------ .../srt/entrypoints/anthropic/serving.py | 58 +++++- .../sglang/srt/managers/template_detection.py | 35 ++++ .../entrypoints/anthropic/test_serving.py | 182 ++++++++++++++---- 4 files changed, 226 insertions(+), 107 deletions(-) diff --git a/python/sglang/srt/entrypoints/anthropic/protocol.py b/python/sglang/srt/entrypoints/anthropic/protocol.py index be7c77c03..4d7d2a46e 100644 --- a/python/sglang/srt/entrypoints/anthropic/protocol.py +++ b/python/sglang/srt/entrypoints/anthropic/protocol.py @@ -379,64 +379,6 @@ class AnthropicMessagesRequest(BaseModel): output_config: Optional[AnthropicOutputConfig] = None betas: Optional[list[str]] = None - @model_validator(mode="before") - @classmethod - def move_mid_conversation_system_messages(cls, values: dict) -> dict: - """Fold mid-conversation ``role: "system"`` turns into the top-level - ``system`` field — some clients (e.g. Claude Code) emit them there.""" - messages = values.get("messages", []) - if not messages: - return values - - clean_messages = [] - extracted_system_texts = [] - - for msg in messages: - # ``mode="before"`` sees raw dicts (HTTP path) but also already- - # constructed ``AnthropicMessage`` objects (programmatic path, e.g. - # ``handle_count_tokens``), so normalize to a dict first. - if isinstance(msg, BaseModel): - msg = msg.model_dump() - if msg.get("role") == "system": - content = msg.get("content", "") - if isinstance(content, str) and content.strip(): - extracted_system_texts.append(content.strip()) - elif isinstance(content, list): - for block in content: - if isinstance(block, dict) and block.get("type") == "text": - text = block.get("text", "").strip() - if text: - extracted_system_texts.append(text) - else: - clean_messages.append(msg) - - if extracted_system_texts: - existing_system = values.get("system") - combined_system = [] - - if existing_system: - if isinstance(existing_system, str): - if existing_system.strip(): - combined_system.append(existing_system.strip()) - elif isinstance(existing_system, list): - for block in existing_system: - if isinstance(block, BaseModel): - block = block.model_dump() - if isinstance(block, dict) and block.get("type") == "text": - text = block.get("text", "").strip() - if text: - combined_system.append(text) - - combined_system.extend(extracted_system_texts) - - # Join into a string — ``system`` is ``str | list[AnthropicContentBlock]``, - # so a ``list[str]`` would fail validation. - if combined_system: - values["system"] = "\n".join(combined_system) - - values["messages"] = clean_messages - return values - @field_validator("model") @classmethod def _validate_model(cls, v): diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index 392e09ae2..ad59cc30a 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -52,6 +52,7 @@ from sglang.srt.entrypoints.openai.protocol import ( ToolChoice, ToolChoiceFuncName, ) +from sglang.srt.managers.template_detection import detect_inline_system_support from sglang.srt.observability.req_time_stats import monotonic_time if TYPE_CHECKING: @@ -132,6 +133,26 @@ def _anthropic_usage_from_openai( return AnthropicUsage(**usage_fields) +def _extract_system_text( + content: Union[str, list[AnthropicContentBlock]], +) -> Optional[str]: + """Flatten a system message's content to a trimmed string, or ``None``.""" + if isinstance(content, str): + return content.strip() or None + texts = [] + for block in content: + if isinstance(block, BaseModel) and getattr(block, "type", None) == "text": + text = getattr(block, "text", "") + elif isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + else: + continue + text = (text or "").strip() + if text: + texts.append(text) + return "\n".join(texts) if texts else None + + def _wrap_sse_event(data: str, event_type: str) -> str: """Format an Anthropic SSE event with event type and data lines.""" return f"event: {event_type}\ndata: {data}\n\n" @@ -169,6 +190,18 @@ class AnthropicServing: def __init__(self, openai_serving_chat: OpenAIServingChat): self.openai_serving_chat = openai_serving_chat + self._merge_inline_system = not detect_inline_system_support( + self._chat_template() + ) + + def _chat_template(self) -> Optional[str]: + tokenizer_manager = getattr(self.openai_serving_chat, "tokenizer_manager", None) + if tokenizer_manager is None: + return None + tokenizer = getattr(tokenizer_manager, "tokenizer", None) + if tokenizer is None: + return None + return getattr(tokenizer, "chat_template", None) async def handle_messages( self, @@ -356,19 +389,28 @@ class AnthropicServing: ) return None - # Add system message if provided + system_parts: list[str] = [] if anthropic_request.system: if isinstance(anthropic_request.system, str): - openai_messages.append( - {"role": "system", "content": anthropic_request.system} - ) + if anthropic_request.system.strip(): + system_parts.append(anthropic_request.system) else: - system_parts = [] for block in anthropic_request.system: if block.type == "text" and block.text: system_parts.append(block.text) - system_text = "\n".join(system_parts) - openai_messages.append({"role": "system", "content": system_text}) + + if self._merge_inline_system: + for msg in anthropic_request.messages: + if msg.role != "system": + continue + text = _extract_system_text(msg.content) + if text: + system_parts.append(text) + + if system_parts: + openai_messages.append( + {"role": "system", "content": "\n".join(system_parts)} + ) def _emit_user_message(parts: list[dict]) -> None: """Append accumulated parts as a user message, then clear them. @@ -388,6 +430,8 @@ class AnthropicServing: # Convert messages for msg in anthropic_request.messages: + if msg.role == "system" and self._merge_inline_system: + continue if isinstance(msg.content, str): openai_messages.append({"role": msg.role, "content": msg.content}) continue diff --git a/python/sglang/srt/managers/template_detection.py b/python/sglang/srt/managers/template_detection.py index ead57b0b8..4c6f367f0 100644 --- a/python/sglang/srt/managers/template_detection.py +++ b/python/sglang/srt/managers/template_detection.py @@ -24,6 +24,10 @@ import re from dataclasses import dataclass from typing import Callable, Optional, Tuple +import jinja2 +import jinja2.ext +import jinja2.sandbox + logger = logging.getLogger(__name__) @@ -483,6 +487,37 @@ def detect_tool_call_parser( return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser") +def detect_inline_system_support(chat_template: Optional[str]) -> bool: + """True if mid-conversation ``role: "system"`` renders inline; False if the + template raises or silently drops it (then merge into the leading block). + + The probe requires the second system's sentinel to appear in the output — + not raising isn't enough, since some templates ignore non-leading system.""" + if not chat_template: + return False + sentinel = "__sglang_inline_system_sentinel__" + try: + env = jinja2.sandbox.ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True, + extensions=[jinja2.ext.loopcontrols], + ) + rendered = env.from_string(chat_template).render( + messages=[ + {"role": "system", "content": "t"}, + {"role": "user", "content": "t"}, + {"role": "system", "content": sentinel}, + {"role": "user", "content": "t"}, + ], + add_generation_prompt=False, + ) + return sentinel in rendered + except jinja2.TemplateError: + return False + except Exception: + return False + + def _resolve_auto_parser( server_args, attr: str, diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index cb008f75d..57aefedb2 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -18,15 +18,21 @@ from sglang.srt.entrypoints.openai.protocol import ( # noqa: E402 ChatCompletionRequest, ChatCompletionResponse, ) +from sglang.srt.managers.template_detection import ( # noqa: E402 + detect_inline_system_support, +) from sglang.test.ci.ci_register import register_cpu_ci # noqa: E402 register_cpu_ci(est_time=1, suite="base-a-test-cpu") class _FakeOpenAIServingChat: - def __init__(self, stream_lines=None): + def __init__(self, stream_lines=None, chat_template=None): self.stream_lines = stream_lines or [] self.apply_reasoning_calls: list[bool] = [] + self.tokenizer_manager = SimpleNamespace( + tokenizer=SimpleNamespace(chat_template=chat_template) + ) def _generate_chat_stream(self, adapted_request, processed_request, raw_request): async def _gen(): @@ -128,8 +134,25 @@ async def _collect_anthropic_events(serving, anthropic_request): class TestAnthropicServing(unittest.TestCase): - def _serving(self, stream_lines=None): - return AnthropicServing(_FakeOpenAIServingChat(stream_lines)) + # System-first guard (Qwen-style): rejects non-first system → must merge. + QWEN_SYSTEM_FIRST_TEMPLATE = ( + "{%- for message in messages %}" + "{%- if message.role == 'system' and not loop.first %}" + "{{- raise_exception('system must be first') }}" + "{%- endif %}" + "{{- message.role }}: {{ message.content }}\n" + "{%- endfor %}" + ) + + # Renders system at any position (GLM/Kimi/Qwen3) → can pass through. + INLINE_SYSTEM_TEMPLATE = ( + "{%- for message in messages %}" + "{{- message.role }}: {{ message.content }}\n" + "{%- endfor %}" + ) + + def _serving(self, stream_lines=None, chat_template=None): + return AnthropicServing(_FakeOpenAIServingChat(stream_lines, chat_template)) def _anthropic_request(self, **overrides): data = { @@ -1267,10 +1290,12 @@ class TestAnthropicServing(unittest.TestCase): # strict role-alternation chat templates (qwen, llama, mistral). self.assertEqual(roles, ["user", "assistant", "user"]) - def test_in_messages_system_role_folded_to_top_level(self): - """A mid-conversation ``role: "system"`` turn is folded into the - top-level ``system`` field by the request validator, so it does not - appear as a dialogue turn — matching the official Anthropic API.""" + def test_in_messages_system_merged_when_template_requires_first(self): + """When the chat template rejects mid-conversation ``role: "system"`` + (e.g. Qwen's system-first guard), the converter folds the inline + system turn into the leading system block so the template doesn't + 400. The request object itself is no longer mutated — detection runs + in the serving layer on conversion.""" serving = self._serving() request = self._anthropic_request( stream=False, @@ -1280,19 +1305,18 @@ class TestAnthropicServing(unittest.TestCase): {"role": "user", "content": "go"}, ], ) - # The validator moved the system turn into the top-level system field. - self.assertEqual(request.system, "Reply with exactly: OK") - self.assertEqual([m.role for m in request.messages], ["user", "user"]) - # And the converted OpenAI request has one leading system message. + self.assertIsNone(request.system) + self.assertEqual([m.role for m in request.messages], ["user", "system", "user"]) chat_request = serving._convert_to_chat_completion_request(request) self.assertEqual( [m.role for m in chat_request.messages], ["system", "user", "user"] ) self.assertEqual(chat_request.messages[0].content, "Reply with exactly: OK") - def test_in_messages_system_role_merged_with_top_level(self): - """A top-level ``system`` field and a mid-conversation system turn are - merged; top-level text comes first.""" + def test_in_messages_system_merged_with_top_level_when_merge(self): + """On the merge path, a top-level ``system`` field and a mid-conversation + system turn are joined into the leading system block; top-level text + comes first.""" serving = self._serving() request = self._anthropic_request( stream=False, @@ -1303,43 +1327,73 @@ class TestAnthropicServing(unittest.TestCase): {"role": "user", "content": "go"}, ], ) - # Validator combines top-level system first, then the in-messages turn, - # joined into a single string. - self.assertEqual(request.system, "You are terse.\nOne word only.") - self.assertEqual([m.role for m in request.messages], ["user", "user"]) + self.assertEqual(request.system, "You are terse.") + self.assertEqual([m.role for m in request.messages], ["user", "system", "user"]) + chat_request = serving._convert_to_chat_completion_request(request) + self.assertEqual( + [m.role for m in chat_request.messages], ["system", "user", "user"] + ) + self.assertEqual( + chat_request.messages[0].content, "You are terse.\nOne word only." + ) - def test_top_level_system_only_is_unchanged(self): - """A request with only the top-level ``system`` field (no in-messages - system turn) must be unaffected by the validator: the system field is - preserved verbatim and the dialogue order is untouched. Guards the - common multi-turn path against regressions.""" - serving = self._serving() + def test_in_messages_system_passed_through_when_template_allows_inline(self): + """When the chat template renders ``role: "system"`` at any position + (GLM / Kimi / Qwen3), the inline system turn stays at its original + position — preserving the prefix cache and the request's structure.""" + serving = self._serving(chat_template=self.INLINE_SYSTEM_TEMPLATE) + self.assertFalse(serving._merge_inline_system) request = self._anthropic_request( stream=False, - system="You are a helpful assistant.", + system="You are terse.", messages=[ - {"role": "user", "content": "My name is Alice."}, - {"role": "assistant", "content": "Hello Alice!"}, - {"role": "user", "content": "What is my name?"}, + {"role": "user", "content": "hi"}, + {"role": "system", "content": "Reply with exactly: OK"}, + {"role": "user", "content": "go"}, ], ) - self.assertEqual(request.system, "You are a helpful assistant.") - self.assertEqual( - [m.role for m in request.messages], ["user", "assistant", "user"] - ) chat_request = serving._convert_to_chat_completion_request(request) self.assertEqual( [m.role for m in chat_request.messages], - ["system", "user", "assistant", "user"], - ) - self.assertEqual( - chat_request.messages[0].content, "You are a helpful assistant." + ["system", "user", "system", "user"], ) + self.assertEqual(chat_request.messages[0].content, "You are terse.") + self.assertEqual(chat_request.messages[2].content, "Reply with exactly: OK") - def test_validator_handles_constructed_message_objects(self): - """The ``mode="before"`` validator must also handle requests built - programmatically with ``AnthropicMessage`` objects (not just raw dicts), - e.g. ``handle_count_tokens`` constructs the request this way.""" + def test_top_level_system_only_is_unchanged(self): + """A request with only the top-level ``system`` field (no in-messages + system turn) is unaffected on both detection paths: the system field is + preserved verbatim and the dialogue order is untouched. Guards the + common multi-turn path against regressions.""" + for template in (None, self.INLINE_SYSTEM_TEMPLATE): + serving = self._serving(chat_template=template) + request = self._anthropic_request( + stream=False, + system="You are a helpful assistant.", + messages=[ + {"role": "user", "content": "My name is Alice."}, + {"role": "assistant", "content": "Hello Alice!"}, + {"role": "user", "content": "What is my name?"}, + ], + ) + self.assertEqual(request.system, "You are a helpful assistant.") + self.assertEqual( + [m.role for m in request.messages], ["user", "assistant", "user"] + ) + chat_request = serving._convert_to_chat_completion_request(request) + self.assertEqual( + [m.role for m in chat_request.messages], + ["system", "user", "assistant", "user"], + ) + self.assertEqual( + chat_request.messages[0].content, "You are a helpful assistant." + ) + + def test_constructed_message_objects_merged_on_merge_path(self): + """Requests built programmatically with ``AnthropicMessage`` objects + (e.g. ``handle_count_tokens``) also get inline system folded into the + leading block on the merge path.""" + serving = self._serving() request = AnthropicMessagesRequest( model="m", max_tokens=8, @@ -1349,8 +1403,12 @@ class TestAnthropicServing(unittest.TestCase): AnthropicMessage(role="user", content="go"), ], ) - self.assertEqual(request.system, "be terse") - self.assertEqual([m.role for m in request.messages], ["user", "user"]) + self.assertEqual([m.role for m in request.messages], ["user", "system", "user"]) + chat_request = serving._convert_to_chat_completion_request(request) + self.assertEqual( + [m.role for m in chat_request.messages], ["system", "user", "user"] + ) + self.assertEqual(chat_request.messages[0].content, "be terse") def test_thinking_history_drop_on_missing_detector(self): """Replaying a thinking block on a non-reasoning model should not 400.""" @@ -1421,5 +1479,45 @@ class TestAnthropicServing(unittest.TestCase): ) +class TestDetectInlineSystemSupport(unittest.TestCase): + """Chat-template detection for mid-conversation system messages (#28883).""" + + def test_guarded_template_not_supported(self): + guarded = ( + "{%- for message in messages %}" + "{%- if message.role == 'system' and not loop.first %}" + "{{- raise_exception('system must be first') }}" + "{%- endif %}" + "{%- endfor %}" + ) + self.assertFalse(detect_inline_system_support(guarded)) + + def test_inline_template_supported(self): + inline = ( + "{%- for message in messages %}" + "{{- message.role }}: {{ message.content }}\n" + "{%- endfor %}" + ) + self.assertTrue(detect_inline_system_support(inline)) + + def test_silent_drop_template_not_supported(self): + # Renders only the leading system; silently ignores later system turns. + silent_drop = ( + "{%- if messages[0].role == 'system' %}" + "{{ messages[0].content }}\n" + "{%- endif %}" + "{%- for message in messages %}" + "{%- if message.role in ('user', 'assistant') %}" + "{{ message.role }}: {{ message.content }}\n" + "{%- endif %}" + "{%- endfor %}" + ) + self.assertFalse(detect_inline_system_support(silent_drop)) + + def test_no_template_not_supported(self): + self.assertFalse(detect_inline_system_support(None)) + self.assertFalse(detect_inline_system_support("")) + + if __name__ == "__main__": unittest.main()