diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index d4b5010e27..49f0dd5b6a 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -2087,6 +2087,7 @@ class OpenAIServingChat(OpenAIServingBase): # First chunk with role if is_firsts.get(index, True): is_firsts[index] = False + # K2 templates require preserved thinking, including empty reasoning. yield build_sse_content( chunk_id=content["meta_info"]["id"], created=int(time.time()), @@ -2094,6 +2095,12 @@ class OpenAIServingChat(OpenAIServingBase): index=index, role="assistant", content="", + reasoning_content=( + "" + if self.reasoning_parser == "k2_horizon" + and request.separate_reasoning + else None + ), ) stream_started = True @@ -2446,7 +2453,7 @@ class OpenAIServingChat(OpenAIServingBase): role="assistant", content=text if text else "", tool_calls=tool_calls, - reasoning_content=reasoning_text if reasoning_text else None, + reasoning_content=reasoning_text, ), logprobs=choice_logprobs, finish_reason=finish_reason["type"] if finish_reason else None, diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 5be3a11e29..c0bb2ec102 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -23,6 +23,7 @@ from typing import Optional from unittest.mock import Mock, patch from fastapi import Request +from transformers.utils.chat_template_utils import _compile_jinja_template from sglang.srt.entrypoints.openai import chat_encoding from sglang.srt.entrypoints.openai.chat_encoding import ( @@ -4281,6 +4282,140 @@ class ServingChatTestCase(CustomTestCase): kwargs = self._run_jinja_with_effort("high") self.assertNotIn("low_effort", kwargs) + def _use_preserved_reasoning_template(self): + template = _compile_jinja_template( + "{% for m in messages if m.role == 'assistant' %}" + "{% if m.reasoning_content is not string %}" + "{{ raise_exception('Assistant thinking fields must be strings') }}" + "{% endif %}{{ m.reasoning_content }}{% endfor %}" + ) + self.tm.tokenizer.apply_chat_template.side_effect = lambda messages, **kwargs: ( + template.render(messages=messages) + ) + self.template_manager.jinja_template_content_format = "string" + self.template_manager.force_reasoning = True + self.chat.reasoning_parser = self.chat.tool_call_parser = "k2_horizon" + + def _assert_empty_reasoning_replays(self, message): + self.assertEqual(message["reasoning_content"], "") + request = ChatCompletionRequest( + model="x", + messages=[*self.basic_req.messages, message, *self.basic_req.messages], + ) + self.chat._apply_jinja_template(request, None, False) + self.assertEqual(self.tm.tokenizer.encode.call_args.args[0], "") + + _K2_TOOL = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object"}}, + } + _K2_TOOL_TEXT = '{"name":"lookup","arguments":{}}' + + def test_empty_reasoning_response_replays(self): + self._use_preserved_reasoning_template() + for tool_only in (False, True): + with self.subTest(tool_only=tool_only): + request = ChatCompletionRequest( + model="x", + messages=self.basic_req.messages, + tools=[self._K2_TOOL] if tool_only else None, + ) + ret = _spec_result(0) + ret["text"] = "" + ( + self._K2_TOOL_TEXT if tool_only else "Hello" + ) + response = self.chat._build_chat_response(request, [ret], created=123) + message = response.model_dump(mode="json")["choices"][0]["message"] + self.assertEqual(message["content"], "" if tool_only else "Hello") + self.assertEqual(bool(message["tool_calls"]), tool_only) + self._assert_empty_reasoning_replays(message) + + def test_empty_stream_reasoning_each_choice(self): + self.template_manager.force_reasoning = True + self.chat.reasoning_parser = self.chat.tool_call_parser = "k2_horizon" + for tool_only in (False, True): + with self.subTest(tool_only=tool_only): + request = ChatCompletionRequest( + model="x", + messages=self.stream_req.messages, + stream=True, + n=2, + tools=[self._K2_TOOL] if tool_only else None, + ) + + async def generate(): + for text in ( + "" + (self._K2_TOOL_TEXT if tool_only else "Hello"), + ): + for index in range(request.n): + ret = _spec_result(index) + ret["text"] = text + if text == "