minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Kimi K3] Add reasoning, tool-call, and OpenAI serving support (#33025)

Browse Source 
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
Co-authored-by: A-transformer <cl5743590921@gmail.com>
This commit is contained in:
Xinyuan Tong
2026-08-01 14:57:23 -07:00
committed by GitHub
co-authored by hnyls2002 Liangsheng Yin A-transformer
parent f1b41a5b3d
commit e2cf21b9e5
34 changed files with 3439 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/entrypoints/openai/test_protocol.py
+62
View File
@@ -438,6 +438,68 @@ class TestChatCompletionRequest(unittest.TestCase):
self.assertEqual(name, "VoiceNote")
self.assertEqual(strict, True)
def test_schema_derived_strict_false_constraint_gated_on_renderer(self):
"""A `strict` field on the user's model doubles as the protocol switch.
set_json_schema pops `strict` out of the schema's properties and feeds
its default into response_format. strict=False drops the sampling
constraint only when the renderer forwards response_format to the
model; otherwise the schema would be silently ignored, so the
constraint stays installed.
"""
class Note(BaseModel):
title: str
strict: bool = False
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Return JSON"}],
response_format={
"type": "json_schema",
"schema": Note.model_json_schema(),
},
)
self.assertIs(request.response_format.json_schema.strict, False)
self.assertNotIn(
"strict", request.response_format.json_schema.schema_["properties"]
)
sampling_params = request.to_sampling_params(
stop=[], model_generation_config={}
)
self.assertIn("json_schema", sampling_params)
sampling_params = request.to_sampling_params(
stop=[],
model_generation_config={},
renderer_handles_response_format=True,
)
self.assertNotIn("json_schema", sampling_params)
def test_non_strict_response_format_constraint_gated_on_renderer(self):
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Return JSON"}],
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {"type": "object"},
"strict": False,
},
},
)
sampling_params = request.to_sampling_params(
stop=[], model_generation_config={}
)
self.assertIn("json_schema", sampling_params)
sampling_params = request.to_sampling_params(
stop=[],
model_generation_config={},
renderer_handles_response_format=True,
)
self.assertNotIn("json_schema", sampling_params)
class TestModelSerialization(unittest.TestCase):
"""Test model serialization with hidden states"""
test/registered/unit/entrypoints/openai/test_serving_chat.py
+296
View File
@@ -27,6 +27,7 @@ from sglang.srt.entrypoints.openai.serving_chat import (
OpenAIServingChat,
normalize_tool_content,
)
from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.parser.template_detection import ReasoningToggleConfig
from sglang.srt.utils import get_or_create_event_loop
@@ -301,12 +302,40 @@ class ServingChatTestCase(unittest.TestCase):
[],
[],
None,
require_reasoning=True,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertTrue(adapted.require_reasoning)
def test_process_messages_records_template_reasoning_state(self):
self.chat.default_chat_template_kwargs = {"thinking": True}
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=False
)
self.chat.reasoning_parser = "deepseek-v3"
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
rendered = MessageProcessingResult(
prompt="prompt",
prompt_ids=[1, 2, 3],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
with patch.object(
self.chat, "_apply_conversation_template", return_value=rendered
):
processed = self.chat._process_messages(request, is_multimodal=False)
self.assertTrue(processed.require_reasoning)
def test_kimi_tool_call_respects_explicit_reasoning_disable(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
@@ -632,6 +661,260 @@ class ServingChatTestCase(unittest.TestCase):
parser.get_structure_constraint.call_args.kwargs["thinking_mode"]
)
def test_kimi_k3_constraint_failure_keeps_native_stop_format(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.chat_encoding_spec = "kimi_k3"
self.chat.tool_call_parser = "kimi_k3"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
tool = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
"required": ["location"],
},
"strict": True,
},
}
cases = (
(None, [TOOLS_CLOSE]),
("USER_STOP", ["USER_STOP", TOOLS_CLOSE]),
(["USER_STOP"], ["USER_STOP", TOOLS_CLOSE]),
([TOOLS_CLOSE], [TOOLS_CLOSE]),
)
for request_stop, expected in cases:
with (
self.subTest(request_stop=request_stop),
patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as parser_cls,
):
parser = parser_cls.return_value
parser.detector.eot_token = TOOLS_CLOSE
parser.detector.parses_required_natively.return_value = False
parser.get_structure_constraint.return_value = None
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather in Paris?"}],
tools=[tool],
tool_choice="required",
stop=request_stop,
)
original_stop = (
list(request.stop)
if isinstance(request.stop, list)
else request.stop
)
result = self.chat._process_messages(request, is_multimodal=False)
self.assertEqual(result.stop, expected)
self.assertEqual(request.stop, original_stop)
self.assertIsNone(result.tool_call_constraint)
def test_kimi_k3_tool_call_stop_is_scoped_to_active_tools(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.chat_encoding_spec = "kimi_k3"
self.chat.tool_call_parser = "kimi_k3"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather in Paris?"}],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object"},
},
}
],
tool_choice="none",
)
result = self.chat._process_messages(request, is_multimodal=False)
self.assertIsNone(result.stop)
def test_kimi_k3_encoder_receives_wire_request_fields(self):
self.template_manager.chat_template_name = None
self.chat.chat_encoding_spec = "kimi_k3"
self.tm.model_config.is_multimodal = True
self.tm.tokenizer.apply_chat_template.return_value = [7, 8, 9]
tool = {
"type": "function",
"function": {
"name": "weather",
"parameters": {"type": "object"},
},
}
request = ChatCompletionRequest(
model="x",
messages=[
{
"role": "developer",
"content": "<|kimi_image_placeholder|>",
"tools": [tool],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "Explain <|kimi_image_placeholder|>",
},
{"type": "image_url", "image_url": {"url": "image-1"}},
],
},
{
"role": "assistant",
"content": None,
"reasoning_content": "Inspect <|kimi_image_placeholder|>",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {
"name": "inspect",
"arguments": {
"source": "<|kimi_image_placeholder|>",
"nested": ["<|kimi_image_placeholder|>"],
},
},
}
],
},
],
tools=[tool],
tool_choice="required",
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {"type": "object"},
"strict": False,
},
},
)
result = self.chat._process_messages(request, is_multimodal=True)
call = self.tm.tokenizer.apply_chat_template.call_args
rendered_messages = call.args[0]
self.assertEqual(rendered_messages[0]["role"], "system")
self.assertEqual(
rendered_messages[0]["content"], "<| kimi_image_placeholder |>"
)
self.assertNotIn("strict", rendered_messages[0]["tools"][0]["function"])
self.assertEqual(
rendered_messages[1]["content"][0]["text"],
"Explain <| kimi_image_placeholder |>",
)
self.assertEqual(
rendered_messages[2]["reasoning_content"],
"Inspect <| kimi_image_placeholder |>",
)
self.assertEqual(
rendered_messages[2]["tool_calls"][0]["function"]["arguments"],
{
"source": "<| kimi_image_placeholder |>",
"nested": ["<| kimi_image_placeholder |>"],
},
)
self.assertEqual(call.kwargs["image_prompts"], ["<|media_pad|>"])
self.assertEqual(call.kwargs["tool_choice"], "required")
self.assertNotIn("strict", call.kwargs["tools"][0]["function"])
self.assertEqual(
call.kwargs["response_format"]["json_schema"]["schema"],
{"type": "object"},
)
self.assertNotIn("schema_", call.kwargs["response_format"]["json_schema"])
self.assertEqual(result.prompt_ids, [7, 8, 9])
self.assertEqual(result.image_data[0].url, "image-1")
def test_kimi_k3_neutralizes_text_only_assistant_history(self):
self.template_manager.chat_template_name = None
self.chat.chat_encoding_spec = "kimi_k3"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
request = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Run it"},
{
"role": "assistant",
"content": None,
"reasoning_content": "Read <|kimi_image_placeholder|>",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {
"name": "shell",
"arguments": "not-json <|kimi_image_placeholder|>",
},
}
],
},
],
)
self.chat._process_messages(request, is_multimodal=False)
messages = self.tm.tokenizer.apply_chat_template.call_args.args[0]
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertEqual(messages[-1]["role"], "assistant")
self.assertEqual(
messages[-1]["reasoning_content"],
"Read <| kimi_image_placeholder |>",
)
self.assertEqual(
messages[-1]["tool_calls"][0]["function"]["arguments"],
"not-json <| kimi_image_placeholder |>",
)
self.assertNotIn("image_prompts", kwargs)
def test_message_tools_participate_in_validation_across_encodings(self):
tool = {
"type": "function",
"function": {
"name": "weather",
"parameters": {"type": "object"},
},
}
messages = [
{"role": "system", "content": "", "tools": [tool]},
{"role": "user", "content": "Weather?"},
]
for chat_encoding_spec in (None, "dsv4", "dsv32", "kimi_k3"):
with self.subTest(chat_encoding_spec=chat_encoding_spec):
self.chat.chat_encoding_spec = chat_encoding_spec
automatic = ChatCompletionRequest(
model="x", messages=messages, tool_choice=None
)
self.assertEqual(automatic.tool_choice, "auto")
self.assertIsNone(self.chat._validate_request(automatic))
required = ChatCompletionRequest(
model="x", messages=messages, tool_choice="required"
)
self.assertIsNone(self.chat._validate_request(required))
duplicate = ChatCompletionRequest(
model="x",
messages=messages,
tools=[tool],
tool_choice="required",
)
self.assertEqual(
self.chat._validate_request(duplicate),
"Tool names must be unique across request and message tools.",
)
def test_jinja_rejects_non_object_tool_call_arguments(self):
"""History tool call arguments must parse to a JSON object."""
self.template_manager.chat_template_name = None
@@ -1256,6 +1539,19 @@ class ServingChatTestCase(unittest.TestCase):
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
def test_kimi_k3_encoding_detection(self):
from sglang.srt.parser.template_manager import TemplateManager
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["KimiK3ForConditionalGeneration"]
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "kimi_k3")
tm.model_config.hf_config.architectures = ["LlamaForCausalLM"]
tm.server_args.tool_call_parser = "kimi_k3"
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "kimi_k3")
# ------------- dsv4 task + latest_reminder -------------
def test_dsv4_task_field_schema(self):
"""Top-level `task` accepts the 6 DS task tokens and rejects others."""
test/registered/unit/entrypoints/openai/test_serving_responses.py
+111 -3
View File
@@ -18,6 +18,7 @@ from sglang.srt.entrypoints.openai.protocol import (
)
from sglang.srt.entrypoints.openai.serving_responses import OpenAIServingResponses
from sglang.srt.function_call.core_types import ToolCallItem
from sglang.srt.parser.template_detection import ReasoningToggleConfig
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
@@ -205,6 +206,105 @@ class ChatToolForwardingTestCase(unittest.TestCase):
result = asyncio.run(serving.create_responses(request, raw_request=None))
self.assertEqual(getattr(result, "status_code", None), 400)
def test_kimi_k3_request_uses_chat_encoder_fields(self):
serving = make_serving()
serving.chat_encoding_spec = "kimi_k3"
serving.default_chat_template_kwargs = {}
serving.template_manager.chat_template_name = None
serving.tokenizer_manager.tokenizer.apply_chat_template.return_value = [4, 5, 6]
request = ResponsesRequest(
model="x",
input="Explain <|kimi_image_placeholder|>",
tools=[
{
"type": "function",
"name": "lookup",
"parameters": {"type": "object"},
}
],
tool_choice="required",
reasoning={"effort": "high"},
store=False,
)
_, request_prompts, engine_prompts, _ = asyncio.run(
serving._make_request(request, None, serving.tokenizer_manager.tokenizer)
)
call = serving.tokenizer_manager.tokenizer.apply_chat_template.call_args
self.assertEqual(
call.args[0][0]["content"], "Explain <| kimi_image_placeholder |>"
)
self.assertEqual(call.kwargs["thinking_effort"], "high")
self.assertEqual(call.kwargs["tool_choice"], "required")
self.assertEqual(call.kwargs["tools"][0]["function"]["name"], "lookup")
self.assertEqual(request_prompts, [[4, 5, 6]])
self.assertEqual(engine_prompts, [[4, 5, 6]])
class ReasoningRequestForwardingTestCase(unittest.TestCase):
def test_create_responses_uses_processed_reasoning_state(self):
serving = make_serving()
serving.reasoning_parser = "deepseek-r1"
serving.default_chat_template_kwargs = {"thinking": False}
serving.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
rendered = MessageProcessingResult(
prompt="prompt",
prompt_ids=[1, 2, 3],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
captured = {}
async def fake_generate(
request_id,
request_prompt,
adapted_request,
sampling_params,
context,
**kwargs,
):
captured["adapted_request"] = adapted_request
context.append_output(
{
"text": "done",
"meta_info": {
"prompt_tokens": 3,
"completion_tokens": 1,
"cached_tokens": 0,
},
}
)
yield context
serving._generate_with_builtin_tools = fake_generate
request = ResponsesRequest(
model="x",
input="answer",
request_id="resp_reasoning",
store=False,
)
with (
patch.object(
serving, "_apply_conversation_template", return_value=rendered
),
patch(
"sglang.srt.entrypoints.openai.serving_responses.ReasoningParser"
) as parser_cls,
):
parser_cls.return_value.parse_non_stream.return_value = (None, "done")
response = asyncio.run(serving.create_responses(request))
self.assertEqual(response.status, "completed")
self.assertFalse(captured["adapted_request"].require_reasoning)
self.assertFalse(parser_cls.call_args.kwargs["force_reasoning"])
class InputItemNormalizationTestCase(unittest.TestCase):
def test_function_call_becomes_assistant_tool_call(self):
@@ -293,6 +393,7 @@ class FullResponseUsageTestCase(unittest.TestCase):
tokenizer=serving.tokenizer_manager.tokenizer,
request_metadata=metadata,
created_time=123,
require_reasoning=False,
)
)
@@ -432,6 +533,7 @@ class OutputItemsTestCase(unittest.TestCase):
self._function_tool_request(),
"raw model output with <tool_call>",
tokenizer=Mock(),
require_reasoning=False,
)
tool_calls = [
@@ -464,7 +566,10 @@ class OutputItemsTestCase(unittest.TestCase):
[fake_call],
)
output_items = serving._make_response_output_items(
self._function_tool_request(), "raw model output", tokenizer=Mock()
self._function_tool_request(),
"raw model output",
tokenizer=Mock(),
require_reasoning=False,
)
types = [type(item).__name__ for item in output_items]
@@ -489,7 +594,7 @@ class OutputItemsTestCase(unittest.TestCase):
raw = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]'
output_items = serving._make_response_output_items(
request, raw, tokenizer=Mock()
request, raw, tokenizer=Mock(), require_reasoning=False
)
tool_calls = [
@@ -524,7 +629,10 @@ class OutputItemsTestCase(unittest.TestCase):
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
) as parser_cls:
output_items = serving._make_response_output_items(
request, "just a plain answer", tokenizer=Mock()
request,
"just a plain answer",
tokenizer=Mock(),
require_reasoning=False,
)
parser_cls.assert_not_called()
test/registered/unit/entrypoints/openai/test_serving_responses_stream.py
+17 -1
View File
@@ -20,9 +20,10 @@ register_cpu_ci(est_time=4, suite="base-a-test-cpu")
class _StreamFixture:
def __init__(self, serving, request):
def __init__(self, serving, request, *, require_reasoning=False):
self.serving = serving
self.request = request
self.require_reasoning = require_reasoning
self.request_metadata = RequestResponseMetadata(request_id=request.request_id)
def run(self, chunks):
@@ -39,6 +40,7 @@ class _StreamFixture:
model_name="x",
tokenizer=Mock(),
request_metadata=self.request_metadata,
require_reasoning=self.require_reasoning,
)
)
@@ -60,6 +62,20 @@ def _engine_chunk(text, completion_tokens, *, finish=False):
class NonHarmonyStreamTestCase(unittest.TestCase):
def test_reasoning_parser_uses_processed_reasoning_state(self):
serving = make_serving()
serving.reasoning_parser = "deepseek-r1"
request = ResponsesRequest(model="x", input="hi", stream=True, store=False)
with patch(
"sglang.srt.entrypoints.openai.serving_responses.ReasoningParser"
) as parser_cls:
parser_cls.return_value.parse_stream_chunk.return_value = (None, "done")
fixture = _StreamFixture(serving, request, require_reasoning=True)
fixture.run([_engine_chunk("done", 1, finish=True)])
self.assertTrue(parser_cls.call_args.kwargs["force_reasoning"])
def test_emits_typed_sse_events_in_order(self):
serving = make_serving()
serving.reasoning_parser = None