[Fix] Preserve inline instructions for Responses and Kimi K3 Messages (#40077)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
hnyls2002
Liangsheng Yin
parent
8e4bcc5ca9
commit
826f15bf9c
@@ -53,7 +53,6 @@ from sglang.srt.entrypoints.openai.protocol import (
|
||||
ToolChoiceFuncName,
|
||||
)
|
||||
from sglang.srt.observability.req_time_stats import monotonic_time
|
||||
from sglang.srt.parser.template_detection import detect_inline_system_support
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
@@ -190,18 +189,7 @@ class AnthropicServing:
|
||||
|
||||
def __init__(self, openai_serving_chat: OpenAIServingChat):
|
||||
self.openai_serving_chat = openai_serving_chat
|
||||
self._merge_inline_system = not detect_inline_system_support(
|
||||
self._chat_template()
|
||||
)
|
||||
|
||||
def _chat_template(self) -> Optional[str]:
|
||||
tokenizer_manager = getattr(self.openai_serving_chat, "tokenizer_manager", None)
|
||||
if tokenizer_manager is None:
|
||||
return None
|
||||
tokenizer = getattr(tokenizer_manager, "tokenizer", None)
|
||||
if tokenizer is None:
|
||||
return None
|
||||
return getattr(tokenizer, "chat_template", None)
|
||||
self._merge_inline_system = not openai_serving_chat.supports_inline_system
|
||||
|
||||
async def handle_messages(
|
||||
self,
|
||||
|
||||
@@ -204,6 +204,12 @@ def default_dsv41_reasoning_effort_from_env(raw: Optional[str]) -> Union[str, in
|
||||
return effort
|
||||
|
||||
|
||||
def spec_supports_inline_system(spec: Optional[str]) -> bool:
|
||||
"""Native encoders verified to render mid-conversation system messages in place."""
|
||||
# K3's tokenizer renders in Python, so there is no Jinja source to probe.
|
||||
return spec == "kimi_k3"
|
||||
|
||||
|
||||
def spec_owns_reasoning_history(spec: Optional[str]) -> bool:
|
||||
"""Whether the encoder for ``spec`` renders assistant reasoning history itself.
|
||||
|
||||
|
||||
@@ -110,6 +110,7 @@ from sglang.srt.parser.reasoning_parser import (
|
||||
IQuestQ1ReasoningDetector,
|
||||
ReasoningParser,
|
||||
)
|
||||
from sglang.srt.parser.template_detection import detect_inline_system_support
|
||||
from sglang.srt.sampling.sampling_params import (
|
||||
set_request_reasoning_end_token_ids,
|
||||
)
|
||||
@@ -328,6 +329,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
|
||||
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
||||
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
||||
self.supports_inline_system = self._resolve_inline_system_support()
|
||||
self._dsv4_reasoning_effort_profile = (
|
||||
chat_encoding.resolve_dsv4_reasoning_effort_profile(
|
||||
model_path=self.tokenizer_manager.model_path,
|
||||
@@ -463,6 +465,16 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
encoded = encoded[1:]
|
||||
return prompt_ids + encoded
|
||||
|
||||
def _resolve_inline_system_support(self) -> bool:
|
||||
if self.chat_encoding_spec is not None:
|
||||
return chat_encoding.spec_supports_inline_system(self.chat_encoding_spec)
|
||||
if self.template_manager.chat_template_name is not None:
|
||||
return False
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
return tokenizer is not None and detect_inline_system_support(
|
||||
tokenizer.chat_template
|
||||
)
|
||||
|
||||
def _resolve_chat_encoding_spec(self) -> str | None:
|
||||
"""Determine which chat encoding spec to use.
|
||||
|
||||
|
||||
@@ -1460,6 +1460,10 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
# so chat templates render a single assistant block per turn.
|
||||
messages = self._merge_consecutive_assistant_messages(messages)
|
||||
|
||||
# Preserve the history prefix when a later instruction is appended.
|
||||
if self.supports_inline_system:
|
||||
return messages
|
||||
|
||||
# Most chat templates expect a single leading ``system`` message;
|
||||
# coalesce any ``instructions`` + interleaved ``developer`` entries.
|
||||
system_chunks: list[str] = []
|
||||
|
||||
@@ -5,6 +5,7 @@ Host class must provide ``self.base_url`` (with or without a trailing /v1),
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import anthropic
|
||||
import requests
|
||||
@@ -95,7 +96,9 @@ class AnthropicMessagesMixin:
|
||||
],
|
||||
)
|
||||
|
||||
serving = AnthropicServing(openai_serving_chat=object())
|
||||
serving = AnthropicServing(
|
||||
openai_serving_chat=SimpleNamespace(supports_inline_system=True)
|
||||
)
|
||||
chat_request = serving._convert_to_chat_completion_request(anthropic_request)
|
||||
converted = chat_request.model_dump()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user