minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

fix(anthropic): detect-and-passthrough mid-conversation system messages (#28906)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-06-25 17:14:12 -07:00
committed by GitHub
parent 4ce1c180bd
commit ed71fb8f95
4 changed files with 226 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/anthropic/protocol.py
-58
View File
@@ -379,64 +379,6 @@ class AnthropicMessagesRequest(BaseModel):
output_config: Optional[AnthropicOutputConfig] = None
betas: Optional[list[str]] = None
@model_validator(mode="before")
@classmethod
def move_mid_conversation_system_messages(cls, values: dict) -> dict:
"""Fold mid-conversation ``role: "system"`` turns into the top-level
``system`` field — some clients (e.g. Claude Code) emit them there."""
messages = values.get("messages", [])
if not messages:
return values
clean_messages = []
extracted_system_texts = []
for msg in messages:
# ``mode="before"`` sees raw dicts (HTTP path) but also already-
# constructed ``AnthropicMessage`` objects (programmatic path, e.g.
# ``handle_count_tokens``), so normalize to a dict first.
if isinstance(msg, BaseModel):
msg = msg.model_dump()
if msg.get("role") == "system":
content = msg.get("content", "")
if isinstance(content, str) and content.strip():
extracted_system_texts.append(content.strip())
elif isinstance(content, list):
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text", "").strip()
if text:
extracted_system_texts.append(text)
else:
clean_messages.append(msg)
if extracted_system_texts:
existing_system = values.get("system")
combined_system = []
if existing_system:
if isinstance(existing_system, str):
if existing_system.strip():
combined_system.append(existing_system.strip())
elif isinstance(existing_system, list):
for block in existing_system:
if isinstance(block, BaseModel):
block = block.model_dump()
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text", "").strip()
if text:
combined_system.append(text)
combined_system.extend(extracted_system_texts)
# Join into a string — ``system`` is ``str | list[AnthropicContentBlock]``,
# so a ``list[str]`` would fail validation.
if combined_system:
values["system"] = "\n".join(combined_system)
values["messages"] = clean_messages
return values
@field_validator("model")
@classmethod
def _validate_model(cls, v):
python/sglang/srt/entrypoints/anthropic/serving.py
+51 -7
View File
@@ -52,6 +52,7 @@ from sglang.srt.entrypoints.openai.protocol import (
ToolChoice,
ToolChoiceFuncName,
)
from sglang.srt.managers.template_detection import detect_inline_system_support
from sglang.srt.observability.req_time_stats import monotonic_time
if TYPE_CHECKING:
@@ -132,6 +133,26 @@ def _anthropic_usage_from_openai(
return AnthropicUsage(**usage_fields)
def _extract_system_text(
content: Union[str, list[AnthropicContentBlock]],
) -> Optional[str]:
"""Flatten a system message's content to a trimmed string, or ``None``."""
if isinstance(content, str):
return content.strip() or None
texts = []
for block in content:
if isinstance(block, BaseModel) and getattr(block, "type", None) == "text":
text = getattr(block, "text", "")
elif isinstance(block, dict) and block.get("type") == "text":
text = block.get("text", "")
else:
continue
text = (text or "").strip()
if text:
texts.append(text)
return "\n".join(texts) if texts else None
def _wrap_sse_event(data: str, event_type: str) -> str:
"""Format an Anthropic SSE event with event type and data lines."""
return f"event: {event_type}\ndata: {data}\n\n"
@@ -169,6 +190,18 @@ class AnthropicServing:
def __init__(self, openai_serving_chat: OpenAIServingChat):
self.openai_serving_chat = openai_serving_chat
self._merge_inline_system = not detect_inline_system_support(
self._chat_template()
)
def _chat_template(self) -> Optional[str]:
tokenizer_manager = getattr(self.openai_serving_chat, "tokenizer_manager", None)
if tokenizer_manager is None:
return None
tokenizer = getattr(tokenizer_manager, "tokenizer", None)
if tokenizer is None:
return None
return getattr(tokenizer, "chat_template", None)
async def handle_messages(
self,
@@ -356,19 +389,28 @@ class AnthropicServing:
)
return None
# Add system message if provided
system_parts: list[str] = []
if anthropic_request.system:
if isinstance(anthropic_request.system, str):
openai_messages.append(
{"role": "system", "content": anthropic_request.system}
)
if anthropic_request.system.strip():
system_parts.append(anthropic_request.system)
else:
system_parts = []
for block in anthropic_request.system:
if block.type == "text" and block.text:
system_parts.append(block.text)
system_text = "\n".join(system_parts)
openai_messages.append({"role": "system", "content": system_text})
if self._merge_inline_system:
for msg in anthropic_request.messages:
if msg.role != "system":
continue
text = _extract_system_text(msg.content)
if text:
system_parts.append(text)
if system_parts:
openai_messages.append(
{"role": "system", "content": "\n".join(system_parts)}
)
def _emit_user_message(parts: list[dict]) -> None:
"""Append accumulated parts as a user message, then clear them.
@@ -388,6 +430,8 @@ class AnthropicServing:
# Convert messages
for msg in anthropic_request.messages:
if msg.role == "system" and self._merge_inline_system:
continue
if isinstance(msg.content, str):
openai_messages.append({"role": msg.role, "content": msg.content})
continue
python/sglang/srt/managers/template_detection.py
+35
View File
@@ -24,6 +24,10 @@ import re
from dataclasses import dataclass
from typing import Callable, Optional, Tuple
import jinja2
import jinja2.ext
import jinja2.sandbox
logger = logging.getLogger(__name__)
@@ -483,6 +487,37 @@ def detect_tool_call_parser(
return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser")
def detect_inline_system_support(chat_template: Optional[str]) -> bool:
"""True if mid-conversation ``role: "system"`` renders inline; False if the
template raises or silently drops it (then merge into the leading block).
The probe requires the second system's sentinel to appear in the output —
not raising isn't enough, since some templates ignore non-leading system."""
if not chat_template:
return False
sentinel = "__sglang_inline_system_sentinel__"
try:
env = jinja2.sandbox.ImmutableSandboxedEnvironment(
trim_blocks=True,
lstrip_blocks=True,
extensions=[jinja2.ext.loopcontrols],
)
rendered = env.from_string(chat_template).render(
messages=[
{"role": "system", "content": "t"},
{"role": "user", "content": "t"},
{"role": "system", "content": sentinel},
{"role": "user", "content": "t"},
],
add_generation_prompt=False,
)
return sentinel in rendered
except jinja2.TemplateError:
return False
except Exception:
return False
def _resolve_auto_parser(
server_args,
attr: str,
test/registered/unit/entrypoints/anthropic/test_serving.py
+140 -42
View File
@@ -18,15 +18,21 @@ from sglang.srt.entrypoints.openai.protocol import ( # noqa: E402
ChatCompletionRequest,
ChatCompletionResponse,
)
from sglang.srt.managers.template_detection import ( # noqa: E402
detect_inline_system_support,
)
from sglang.test.ci.ci_register import register_cpu_ci # noqa: E402
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
class _FakeOpenAIServingChat:
def __init__(self, stream_lines=None):
def __init__(self, stream_lines=None, chat_template=None):
self.stream_lines = stream_lines or []
self.apply_reasoning_calls: list[bool] = []
self.tokenizer_manager = SimpleNamespace(
tokenizer=SimpleNamespace(chat_template=chat_template)
)
def _generate_chat_stream(self, adapted_request, processed_request, raw_request):
async def _gen():
@@ -128,8 +134,25 @@ async def _collect_anthropic_events(serving, anthropic_request):
class TestAnthropicServing(unittest.TestCase):
def _serving(self, stream_lines=None):
return AnthropicServing(_FakeOpenAIServingChat(stream_lines))
# System-first guard (Qwen-style): rejects non-first system → must merge.
QWEN_SYSTEM_FIRST_TEMPLATE = (
"{%- for message in messages %}"
"{%- if message.role == 'system' and not loop.first %}"
"{{- raise_exception('system must be first') }}"
"{%- endif %}"
"{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
# Renders system at any position (GLM/Kimi/Qwen3) → can pass through.
INLINE_SYSTEM_TEMPLATE = (
"{%- for message in messages %}"
"{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
def _serving(self, stream_lines=None, chat_template=None):
return AnthropicServing(_FakeOpenAIServingChat(stream_lines, chat_template))
def _anthropic_request(self, **overrides):
data = {
@@ -1267,10 +1290,12 @@ class TestAnthropicServing(unittest.TestCase):
# strict role-alternation chat templates (qwen, llama, mistral).
self.assertEqual(roles, ["user", "assistant", "user"])
def test_in_messages_system_role_folded_to_top_level(self):
"""A mid-conversation ``role: "system"`` turn is folded into the
top-level ``system`` field by the request validator, so it does not
appear as a dialogue turn — matching the official Anthropic API."""
def test_in_messages_system_merged_when_template_requires_first(self):
"""When the chat template rejects mid-conversation ``role: "system"``
(e.g. Qwen's system-first guard), the converter folds the inline
system turn into the leading system block so the template doesn't
400. The request object itself is no longer mutated — detection runs
in the serving layer on conversion."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
@@ -1280,19 +1305,18 @@ class TestAnthropicServing(unittest.TestCase):
{"role": "user", "content": "go"},
],
)
# The validator moved the system turn into the top-level system field.
self.assertEqual(request.system, "Reply with exactly: OK")
self.assertEqual([m.role for m in request.messages], ["user", "user"])
# And the converted OpenAI request has one leading system message.
self.assertIsNone(request.system)
self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages], ["system", "user", "user"]
)
self.assertEqual(chat_request.messages[0].content, "Reply with exactly: OK")
def test_in_messages_system_role_merged_with_top_level(self):
"""A top-level ``system`` field and a mid-conversation system turn are
merged; top-level text comes first."""
def test_in_messages_system_merged_with_top_level_when_merge(self):
"""On the merge path, a top-level ``system`` field and a mid-conversation
system turn are joined into the leading system block; top-level text
comes first."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
@@ -1303,43 +1327,73 @@ class TestAnthropicServing(unittest.TestCase):
{"role": "user", "content": "go"},
],
)
# Validator combines top-level system first, then the in-messages turn,
# joined into a single string.
self.assertEqual(request.system, "You are terse.\nOne word only.")
self.assertEqual([m.role for m in request.messages], ["user", "user"])
self.assertEqual(request.system, "You are terse.")
self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages], ["system", "user", "user"]
)
self.assertEqual(
chat_request.messages[0].content, "You are terse.\nOne word only."
)
def test_top_level_system_only_is_unchanged(self):
"""A request with only the top-level ``system`` field (no in-messages
system turn) must be unaffected by the validator: the system field is
preserved verbatim and the dialogue order is untouched. Guards the
common multi-turn path against regressions."""
serving = self._serving()
def test_in_messages_system_passed_through_when_template_allows_inline(self):
"""When the chat template renders ``role: "system"`` at any position
(GLM / Kimi / Qwen3), the inline system turn stays at its original
position — preserving the prefix cache and the request's structure."""
serving = self._serving(chat_template=self.INLINE_SYSTEM_TEMPLATE)
self.assertFalse(serving._merge_inline_system)
request = self._anthropic_request(
stream=False,
system="You are a helpful assistant.",
system="You are terse.",
messages=[
{"role": "user", "content": "My name is Alice."},
{"role": "assistant", "content": "Hello Alice!"},
{"role": "user", "content": "What is my name?"},
{"role": "user", "content": "hi"},
{"role": "system", "content": "Reply with exactly: OK"},
{"role": "user", "content": "go"},
],
)
self.assertEqual(request.system, "You are a helpful assistant.")
self.assertEqual(
[m.role for m in request.messages], ["user", "assistant", "user"]
)
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages],
["system", "user", "assistant", "user"],
)
self.assertEqual(
chat_request.messages[0].content, "You are a helpful assistant."
["system", "user", "system", "user"],
)
self.assertEqual(chat_request.messages[0].content, "You are terse.")
self.assertEqual(chat_request.messages[2].content, "Reply with exactly: OK")
def test_validator_handles_constructed_message_objects(self):
"""The ``mode="before"`` validator must also handle requests built
programmatically with ``AnthropicMessage`` objects (not just raw dicts),
e.g. ``handle_count_tokens`` constructs the request this way."""
def test_top_level_system_only_is_unchanged(self):
"""A request with only the top-level ``system`` field (no in-messages
system turn) is unaffected on both detection paths: the system field is
preserved verbatim and the dialogue order is untouched. Guards the
common multi-turn path against regressions."""
for template in (None, self.INLINE_SYSTEM_TEMPLATE):
serving = self._serving(chat_template=template)
request = self._anthropic_request(
stream=False,
system="You are a helpful assistant.",
messages=[
{"role": "user", "content": "My name is Alice."},
{"role": "assistant", "content": "Hello Alice!"},
{"role": "user", "content": "What is my name?"},
],
)
self.assertEqual(request.system, "You are a helpful assistant.")
self.assertEqual(
[m.role for m in request.messages], ["user", "assistant", "user"]
)
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages],
["system", "user", "assistant", "user"],
)
self.assertEqual(
chat_request.messages[0].content, "You are a helpful assistant."
)
def test_constructed_message_objects_merged_on_merge_path(self):
"""Requests built programmatically with ``AnthropicMessage`` objects
(e.g. ``handle_count_tokens``) also get inline system folded into the
leading block on the merge path."""
serving = self._serving()
request = AnthropicMessagesRequest(
model="m",
max_tokens=8,
@@ -1349,8 +1403,12 @@ class TestAnthropicServing(unittest.TestCase):
AnthropicMessage(role="user", content="go"),
],
)
self.assertEqual(request.system, "be terse")
self.assertEqual([m.role for m in request.messages], ["user", "user"])
self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages], ["system", "user", "user"]
)
self.assertEqual(chat_request.messages[0].content, "be terse")
def test_thinking_history_drop_on_missing_detector(self):
"""Replaying a thinking block on a non-reasoning model should not 400."""
@@ -1421,5 +1479,45 @@ class TestAnthropicServing(unittest.TestCase):
)
class TestDetectInlineSystemSupport(unittest.TestCase):
"""Chat-template detection for mid-conversation system messages (#28883)."""
def test_guarded_template_not_supported(self):
guarded = (
"{%- for message in messages %}"
"{%- if message.role == 'system' and not loop.first %}"
"{{- raise_exception('system must be first') }}"
"{%- endif %}"
"{%- endfor %}"
)
self.assertFalse(detect_inline_system_support(guarded))
def test_inline_template_supported(self):
inline = (
"{%- for message in messages %}"
"{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
self.assertTrue(detect_inline_system_support(inline))
def test_silent_drop_template_not_supported(self):
# Renders only the leading system; silently ignores later system turns.
silent_drop = (
"{%- if messages[0].role == 'system' %}"
"{{ messages[0].content }}\n"
"{%- endif %}"
"{%- for message in messages %}"
"{%- if message.role in ('user', 'assistant') %}"
"{{ message.role }}: {{ message.content }}\n"
"{%- endif %}"
"{%- endfor %}"
)
self.assertFalse(detect_inline_system_support(silent_drop))
def test_no_template_not_supported(self):
self.assertFalse(detect_inline_system_support(None))
self.assertFalse(detect_inline_system_support(""))
if __name__ == "__main__":
unittest.main()