From 61c2da42bb656303425d8af79b71e0bcdb0e7d0c Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:22:11 +0800 Subject: [PATCH] [Fix] Pass Anthropic thinking history as reasoning_content for custom chat encoders (#35480) Co-authored-by: Mohammad Angkad --- .../srt/entrypoints/anthropic/serving.py | 26 +++++--- .../srt/entrypoints/openai/chat_encoding.py | 17 ++++- .../srt/entrypoints/openai/serving_chat.py | 7 +++ .../entrypoints/anthropic/test_serving.py | 62 +++++++++++++++++++ .../entrypoints/openai/test_serving_chat.py | 33 ++++++++++ 5 files changed, 137 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/entrypoints/anthropic/serving.py b/python/sglang/srt/entrypoints/anthropic/serving.py index 96800e68a..37bcdb43a 100644 --- a/python/sglang/srt/entrypoints/anthropic/serving.py +++ b/python/sglang/srt/entrypoints/anthropic/serving.py @@ -368,8 +368,12 @@ class AnthropicServing: def _convert_assistant_thinking_blocks( blocks: list[AnthropicContentBlock], - ) -> Optional[str]: - """Re-wrap prior-turn thinking blocks in the parser's own tokens. + ) -> tuple[Optional[str], Optional[str]]: + """Reconstruct prior-turn thinking as ``(reasoning_content, text)``. + + At most one is set: encoders that frame the reasoning channel take + it as ``reasoning_content``, everything else gets it re-wrapped and + spliced into content. ``redacted_thinking`` carries encrypted bytes that no local parser can interpret, so we raise rather than silently drop it. @@ -387,11 +391,15 @@ class AnthropicServing: if block.type == "thinking" and block.thinking ] if not thinking_parts: - return None + return None, None + + reasoning_text = "\n".join(thinking_parts) + if self.openai_serving_chat.supports_native_reasoning_history(): + return reasoning_text, None try: - return self.openai_serving_chat.wrap_reasoning_history( - "\n".join(thinking_parts) + return None, self.openai_serving_chat.wrap_reasoning_history( + reasoning_text ) except ValueError as e: logger.warning( @@ -399,7 +407,7 @@ class AnthropicServing: len(thinking_parts), e, ) - return None + return None, None system_parts: list[str] = [] if anthropic_request.system: @@ -454,7 +462,11 @@ class AnthropicServing: tool_calls: list[dict] = [] if msg.role == "assistant": - reasoning_history = _convert_assistant_thinking_blocks(msg.content) + reasoning_content, reasoning_history = ( + _convert_assistant_thinking_blocks(msg.content) + ) + if reasoning_content is not None: + openai_msg["reasoning_content"] = reasoning_content if reasoning_history is not None: content_parts.append({"type": "text", "text": reasoning_history}) diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding.py b/python/sglang/srt/entrypoints/openai/chat_encoding.py index b6b136ca9..f70c3e13b 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding.py @@ -113,7 +113,8 @@ def resolve_chat_encoding_spec( ) -> Optional[str]: """Return the chat encoding spec for a model. - None means the default path (HF chat template). + None means the default path (HF chat template); any non-None spec also owns + reasoning-history rendering (:func:`spec_owns_reasoning_history`). """ if tool_call_parser == "deepseekv4": return "dsv4" @@ -142,6 +143,20 @@ def resolve_chat_encoding_spec( return None +def spec_owns_reasoning_history(spec: Optional[str]) -> bool: + """Whether the encoder for ``spec`` renders assistant reasoning history itself. + + Custom encoders frame the reasoning and content channels, so history must be + passed as assistant ``reasoning_content``. Splicing a detector's markers into + content instead nests a reasoning block inside the content channel and leaves + the real one empty, teaching the model to emit raw markers as visible text. + + Answered for the whole family rather than a list of specs, so a new spec gets + the safe default: worst case is dropped history, not a leak. + """ + return spec is not None + + def encode_simple_chat( *, tokenizer: Any, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 53ad5a448..7bdc08b28 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -2272,6 +2272,13 @@ class OpenAIServingChat(OpenAIServingBase): elif self.reasoning_parser == "muse": request.skip_special_tokens = False + def supports_native_reasoning_history(self) -> bool: + """Whether the chat encoder takes history as ``reasoning_content`` rather + than via :meth:`wrap_reasoning_history`; see + :func:`chat_encoding.spec_owns_reasoning_history` for why. + """ + return chat_encoding.spec_owns_reasoning_history(self.chat_encoding_spec) + def wrap_reasoning_history(self, reasoning_text: str) -> str: """Wrap prior-turn reasoning in the detector's own start/end tokens. diff --git a/test/registered/unit/entrypoints/anthropic/test_serving.py b/test/registered/unit/entrypoints/anthropic/test_serving.py index aa3008e96..8f5c5b61a 100644 --- a/test/registered/unit/entrypoints/anthropic/test_serving.py +++ b/test/registered/unit/entrypoints/anthropic/test_serving.py @@ -28,6 +28,8 @@ register_cpu_ci(est_time=1, suite="base-a-test-cpu") class _FakeOpenAIServingChat: + native_reasoning_history = False + def __init__(self, stream_lines=None, chat_template=None): self.stream_lines = stream_lines or [] self.apply_reasoning_calls: list[bool] = [] @@ -35,6 +37,9 @@ class _FakeOpenAIServingChat: tokenizer=SimpleNamespace(chat_template=chat_template) ) + def supports_native_reasoning_history(self): + return self.native_reasoning_history + def _generate_chat_stream(self, adapted_request, processed_request, raw_request): async def _gen(): for line in self.stream_lines: @@ -960,6 +965,63 @@ class TestAnthropicServing(unittest.TestCase): self.assertIn("ponder", joined) self.assertNotIn("\nponder\n\nponder", joined) + def test_assistant_thinking_history_uses_native_reasoning_content(self): + """Channel-framing encoders take thinking history as ``reasoning_content``. + + Splicing the detector's markers into content would nest a reasoning block + inside the content channel and leave the real one empty, training the + model to emit raw markers as visible text. + """ + + class _NativeOpenAI(_FakeOpenAIServingChat): + native_reasoning_history = True + + def wrap_reasoning_history(self, text): + raise AssertionError("must not rewrap for a native-history encoder") + + serving = AnthropicServing(_NativeOpenAI()) + request = self._anthropic_request( + stream=False, + messages=[ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "ponder"}, + {"type": "text", "text": "hello"}, + ], + }, + {"role": "user", "content": "again"}, + ], + ) + chat_request = serving._convert_to_chat_completion_request(request) + assistant_msg = next(m for m in chat_request.messages if m.role == "assistant") + self.assertEqual(assistant_msg.reasoning_content, "ponder") + self.assertEqual(assistant_msg.content, "hello") + + def test_thinking_only_turn_keeps_native_reasoning_content(self): + """An assistant turn that is only thinking still carries its reasoning.""" + + class _NativeOpenAI(_FakeOpenAIServingChat): + native_reasoning_history = True + + serving = AnthropicServing(_NativeOpenAI()) + request = self._anthropic_request( + stream=False, + messages=[ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "ponder"}], + }, + {"role": "user", "content": "again"}, + ], + ) + chat_request = serving._convert_to_chat_completion_request(request) + assistant_msg = next(m for m in chat_request.messages if m.role == "assistant") + self.assertEqual(assistant_msg.reasoning_content, "ponder") + self.assertEqual(assistant_msg.content, "") + def test_redacted_thinking_history_is_rejected(self): """``redacted_thinking`` cannot be rendered by local parsers.""" serving = self._serving() diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 181936d8e..033a14761 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -11,6 +11,7 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel import json +import re import tempfile import unittest import uuid @@ -21,6 +22,7 @@ from unittest.mock import Mock, patch from fastapi import Request +from sglang.srt.entrypoints.openai import chat_encoding from sglang.srt.entrypoints.openai.chat_encoding import ( resolve_dsv4_reasoning_effort_profile, ) @@ -43,6 +45,9 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") +# Every spec resolve_chat_encoding_spec can return; pinned by the guard below. +_ALL_CHAT_ENCODING_SPECS = ("dsv4", "dsv32", "inkling", "kimi_k3") + def _spec_result(index): return { @@ -1924,6 +1929,34 @@ class ServingChatTestCase(unittest.TestCase): serving_chat = OpenAIServingChat(tm, TemplateManager()) self.assertEqual(serving_chat.chat_encoding_spec, "kimi_k3") + def test_custom_encoders_own_reasoning_history(self): + """Every custom encoding spec takes reasoning history natively. + + The alternative splices a detector's markers into content, which an + encoder that frames its own channels turns into visible raw markers. + A new spec must not silently default to that path. + """ + for spec in _ALL_CHAT_ENCODING_SPECS: + with self.subTest(chat_encoding_spec=spec): + self.chat.chat_encoding_spec = spec + self.assertTrue(self.chat.supports_native_reasoning_history()) + + # The HF chat-template path keeps the wrap-into-content behaviour. + self.chat.chat_encoding_spec = None + self.assertFalse(self.chat.supports_native_reasoning_history()) + + def test_all_chat_encoding_specs_are_enumerated(self): + """Guard the spec list this file asserts capabilities over.""" + source = Path(chat_encoding.__file__).read_text() + returned = set( + re.findall( + r'^\s+return "(\w+)"$', + source[source.index("def resolve_chat_encoding_spec") :], + re.MULTILINE, + ) + ) + self.assertEqual(returned, set(_ALL_CHAT_ENCODING_SPECS)) + # ------------- dsv4 task + latest_reminder ------------- def test_dsv4_task_field_schema(self): """Top-level `task` accepts the 6 DS task tokens and rejects others."""