[Fix] Preserve inline instructions for Responses and Kimi K3 Messages (#40077)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
James Liu
2026-09-30 15:20:52 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 hnyls2002 Liangsheng Yin
parent 8e4bcc5ca9
commit 826f15bf9c
7 changed files with 116 additions and 15 deletions
@@ -53,7 +53,6 @@ from sglang.srt.entrypoints.openai.protocol import (
ToolChoiceFuncName,
)
from sglang.srt.observability.req_time_stats import monotonic_time
from sglang.srt.parser.template_detection import detect_inline_system_support
if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
@@ -190,18 +189,7 @@ class AnthropicServing:
def __init__(self, openai_serving_chat: OpenAIServingChat):
self.openai_serving_chat = openai_serving_chat
self._merge_inline_system = not detect_inline_system_support(
self._chat_template()
)
def _chat_template(self) -> Optional[str]:
tokenizer_manager = getattr(self.openai_serving_chat, "tokenizer_manager", None)
if tokenizer_manager is None:
return None
tokenizer = getattr(tokenizer_manager, "tokenizer", None)
if tokenizer is None:
return None
return getattr(tokenizer, "chat_template", None)
self._merge_inline_system = not openai_serving_chat.supports_inline_system
async def handle_messages(
self,
@@ -204,6 +204,12 @@ def default_dsv41_reasoning_effort_from_env(raw: Optional[str]) -> Union[str, in
return effort
def spec_supports_inline_system(spec: Optional[str]) -> bool:
"""Native encoders verified to render mid-conversation system messages in place."""
# K3's tokenizer renders in Python, so there is no Jinja source to probe.
return spec == "kimi_k3"
def spec_owns_reasoning_history(spec: Optional[str]) -> bool:
"""Whether the encoder for ``spec`` renders assistant reasoning history itself.
@@ -110,6 +110,7 @@ from sglang.srt.parser.reasoning_parser import (
IQuestQ1ReasoningDetector,
ReasoningParser,
)
from sglang.srt.parser.template_detection import detect_inline_system_support
from sglang.srt.sampling.sampling_params import (
set_request_reasoning_end_token_ids,
)
@@ -328,6 +329,7 @@ class OpenAIServingChat(OpenAIServingBase):
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
self.supports_inline_system = self._resolve_inline_system_support()
self._dsv4_reasoning_effort_profile = (
chat_encoding.resolve_dsv4_reasoning_effort_profile(
model_path=self.tokenizer_manager.model_path,
@@ -463,6 +465,16 @@ class OpenAIServingChat(OpenAIServingBase):
encoded = encoded[1:]
return prompt_ids + encoded
def _resolve_inline_system_support(self) -> bool:
if self.chat_encoding_spec is not None:
return chat_encoding.spec_supports_inline_system(self.chat_encoding_spec)
if self.template_manager.chat_template_name is not None:
return False
tokenizer = self.tokenizer_manager.tokenizer
return tokenizer is not None and detect_inline_system_support(
tokenizer.chat_template
)
def _resolve_chat_encoding_spec(self) -> str | None:
"""Determine which chat encoding spec to use.
@@ -1460,6 +1460,10 @@ class OpenAIServingResponses(OpenAIServingChat):
# so chat templates render a single assistant block per turn.
messages = self._merge_consecutive_assistant_messages(messages)
# Preserve the history prefix when a later instruction is appended.
if self.supports_inline_system:
return messages
# Most chat templates expect a single leading ``system`` message;
# coalesce any ``instructions`` + interleaved ``developer`` entries.
system_chunks: list[str] = []
@@ -5,6 +5,7 @@ Host class must provide ``self.base_url`` (with or without a trailing /v1),
"""
import json
from types import SimpleNamespace
import anthropic
import requests
@@ -95,7 +96,9 @@ class AnthropicMessagesMixin:
],
)
serving = AnthropicServing(openai_serving_chat=object())
serving = AnthropicServing(
openai_serving_chat=SimpleNamespace(supports_inline_system=True)
)
chat_request = serving._convert_to_chat_completion_request(anthropic_request)
converted = chat_request.model_dump()