diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index 68f1c63305..f96595fb45 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -53,7 +53,6 @@ from sglang.srt.entrypoints.openai.protocol import ( ToolChoiceFuncName, ) from sglang.srt.observability.req_time_stats import monotonic_time -from sglang.srt.parser.template_detection import detect_inline_system_support if TYPE_CHECKING: from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat @@ -190,18 +189,7 @@ class AnthropicServing: def __init__(self, openai_serving_chat: OpenAIServingChat): self.openai_serving_chat = openai_serving_chat - self._merge_inline_system = not detect_inline_system_support( - self._chat_template() - ) - - def _chat_template(self) -> Optional[str]: - tokenizer_manager = getattr(self.openai_serving_chat, "tokenizer_manager", None) - if tokenizer_manager is None: - return None - tokenizer = getattr(tokenizer_manager, "tokenizer", None) - if tokenizer is None: - return None - return getattr(tokenizer, "chat_template", None) + self._merge_inline_system = not openai_serving_chat.supports_inline_system async def handle_messages( self, diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding.py b/python/sglang/srt/entrypoints/openai/chat_encoding.py index 6b983a8d94..c5c65d561d 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding.py @@ -204,6 +204,12 @@ def default_dsv41_reasoning_effort_from_env(raw: Optional[str]) -> Union[str, in return effort +def spec_supports_inline_system(spec: Optional[str]) -> bool: + """Native encoders verified to render mid-conversation system messages in place.""" + # K3's tokenizer renders in Python, so there is no Jinja source to probe. + return spec == "kimi_k3" + + def spec_owns_reasoning_history(spec: Optional[str]) -> bool: """Whether the encoder for ``spec`` renders assistant reasoning history itself. diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 6845aa4936..4e1bf33eca 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -110,6 +110,7 @@ from sglang.srt.parser.reasoning_parser import ( IQuestQ1ReasoningDetector, ReasoningParser, ) +from sglang.srt.parser.template_detection import detect_inline_system_support from sglang.srt.sampling.sampling_params import ( set_request_reasoning_end_token_ids, ) @@ -328,6 +329,7 @@ class OpenAIServingChat(OpenAIServingBase): # Which Python-based chat encoder (if any) bypasses apply_chat_template. # Values: "dsv32", "dsv4", or custom values set by subclass. None for default. self.chat_encoding_spec = self._resolve_chat_encoding_spec() + self.supports_inline_system = self._resolve_inline_system_support() self._dsv4_reasoning_effort_profile = ( chat_encoding.resolve_dsv4_reasoning_effort_profile( model_path=self.tokenizer_manager.model_path, @@ -463,6 +465,16 @@ class OpenAIServingChat(OpenAIServingBase): encoded = encoded[1:] return prompt_ids + encoded + def _resolve_inline_system_support(self) -> bool: + if self.chat_encoding_spec is not None: + return chat_encoding.spec_supports_inline_system(self.chat_encoding_spec) + if self.template_manager.chat_template_name is not None: + return False + tokenizer = self.tokenizer_manager.tokenizer + return tokenizer is not None and detect_inline_system_support( + tokenizer.chat_template + ) + def _resolve_chat_encoding_spec(self) -> str | None: """Determine which chat encoding spec to use. diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index 72ac87c09c..66ee1c2a67 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -1460,6 +1460,10 @@ class OpenAIServingResponses(OpenAIServingChat): # so chat templates render a single assistant block per turn. messages = self._merge_consecutive_assistant_messages(messages) + # Preserve the history prefix when a later instruction is appended. + if self.supports_inline_system: + return messages + # Most chat templates expect a single leading ``system`` message; # coalesce any ``instructions`` + interleaved ``developer`` entries. system_chunks: list[str] = [] diff --git a/python/sglang/test/kits/anthropic_messages_kit.py b/python/sglang/test/kits/anthropic_messages_kit.py index 51feebdd74..6795f05bf2 100644 --- a/python/sglang/test/kits/anthropic_messages_kit.py +++ b/python/sglang/test/kits/anthropic_messages_kit.py @@ -5,6 +5,7 @@ Host class must provide ``self.base_url`` (with or without a trailing /v1), """ import json +from types import SimpleNamespace import anthropic import requests @@ -95,7 +96,9 @@ class AnthropicMessagesMixin: ], ) - serving = AnthropicServing(openai_serving_chat=object()) + serving = AnthropicServing( + openai_serving_chat=SimpleNamespace(supports_inline_system=True) + ) chat_request = serving._convert_to_chat_completion_request(anthropic_request) converted = chat_request.model_dump() diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index e9a8411c9a..3e78bc76f7 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -36,6 +36,7 @@ class _FakeOpenAIServingChat: self.tokenizer_manager = SimpleNamespace( tokenizer=SimpleNamespace(chat_template=chat_template) ) + self.supports_inline_system = detect_inline_system_support(chat_template) def supports_native_reasoning_history(self): return self.native_reasoning_history @@ -57,6 +58,8 @@ class _FakeOpenAIServingChat: class _FakeNonStreamingErrorOpenAI: """Returns a configurable error response from the OpenAI handler.""" + supports_inline_system = False + def __init__(self, status_code=400, body=None, content=None): self._status_code = status_code self._body = body @@ -1492,6 +1495,28 @@ class TestAnthropicServing(unittest.TestCase): self.assertEqual(chat_request.messages[0].content, "You are terse.") self.assertEqual(chat_request.messages[2].content, "Reply with exactly: OK") + def test_inline_system_support_comes_from_the_prompt_renderer(self): + """Native encoders have no Jinja template to probe, so Messages must use + the serving chat's renderer decision instead of hoisting inline system.""" + chat = _FakeOpenAIServingChat(chat_template=None) + chat.supports_inline_system = True + serving = AnthropicServing(chat) + request = self._anthropic_request( + stream=False, + system="Stable instructions", + messages=[ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "New instruction"}, + {"role": "user", "content": "go"}, + ], + ) + converted = serving._convert_to_chat_completion_request(request) + self.assertEqual( + [m.role for m in converted.messages], + ["system", "user", "system", "user"], + ) + self.assertEqual(converted.messages[0].content, "Stable instructions") + def test_top_level_system_only_is_unchanged(self): """A request with only the top-level ``system`` field (no in-messages system turn) is unaffected on both detection paths: the system field is diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses.py b/test/registered/unit/entrypoints/openai/test_serving_responses.py index 9b7e425bd5..0e712d08e2 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_responses.py +++ b/test/registered/unit/entrypoints/openai/test_serving_responses.py @@ -14,7 +14,14 @@ from openai.types.responses import ( ) from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from openai_harmony import Conversation, Message, Role, ToolNamespaceConfig -from utils import StreamFixture, engine_chunk, event_payloads, make_serving +from utils import ( + MockTemplateManager, + MockTokenizerManager, + StreamFixture, + engine_chunk, + event_payloads, + make_serving, +) from sglang.srt.entrypoints.context import ( HarmonyContext, @@ -1603,6 +1610,62 @@ def test_active_stream_cancel_and_final_history(response_serving): asyncio.run(run()) +INLINE_SYSTEM_TEMPLATE = ( + "{% for m in messages %}{{ m['role'] }}: {{ m['content'] }}{% endfor %}" +) + + +@pytest.mark.parametrize( + "encoding_spec, chat_template_name, jinja_template, preserve", + [ + ("kimi_k3", None, None, True), + ("dsv32", None, INLINE_SYSTEM_TEMPLATE, False), + (None, None, INLINE_SYSTEM_TEMPLATE, True), + (None, "llama-3", INLINE_SYSTEM_TEMPLATE, False), + (None, None, "{{ messages[0]['content'] }}", False), + ], +) +def test_inline_instruction_keeps_history_prefix( + encoding_spec, chat_template_name, jinja_template, preserve +): + """An appended developer instruction stays in place when the renderer supports + inline system; hoisting it rewrites the prompt prefix and defeats KV reuse.""" + publish(ServerArgs(model_path="dummy"), role="tokenizer") + tokenizer_manager = MockTokenizerManager() + tokenizer_manager.tokenizer.chat_template = jinja_template + template_manager = MockTemplateManager() + template_manager.chat_template_name = chat_template_name + with patch.object( + OpenAIServingResponses, + "_resolve_chat_encoding_spec", + return_value=encoding_spec, + ): + serving = OpenAIServingResponses(tokenizer_manager, template_manager) + history = [ + {"role": "user", "content": "Existing conversation"}, + {"role": "assistant", "content": "Previous answer"}, + ] + appended = [ + {"role": "developer", "content": "New instruction"}, + {"role": "user", "content": "Next turn"}, + ] + before = serving._construct_input_messages( + ResponsesRequest(model="x", instructions="Stable instructions", input=history) + ) + after = serving._construct_input_messages( + ResponsesRequest( + model="x", instructions="Stable instructions", input=history + appended + ) + ) + if preserve: + assert after[: len(before)] == before + assert after[len(before)]["role"] == "system" + assert "New instruction" in after[len(before)]["content"] + else: + assert [m["role"] for m in after].count("system") == 1 + assert "New instruction" in after[0]["content"] + + @pytest.mark.parametrize("stream", [False, True]) def test_completion_preserves_cancelled_response(response_serving, stream): serving = response_serving(enabled=True, harmony=not stream)