minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Stop losing Kimi-K3 tool calls to reasoning, constraint conflicts, and truncation (#34881)

Browse Source 
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-08-18 12:42:56 -07:00
committed by GitHub
co-authored by Claude Opus 5 Xinyuan Tong Xinyuan Tong
parent 8bb106cee9
commit 307a90f6d3
9 changed files with 374 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/protocol.py
+8
View File
@@ -1081,6 +1081,14 @@ class ChatCompletionRequest(BaseModel):
)
if tool_call_constraint and has_existing_constraints:
if self.tool_choice == "required" or isinstance(
self.tool_choice, ToolChoice
):
raise ValueError(
"tool_choice 'required' or a named tool cannot be combined with "
"response_format, regex, or ebnf: the tool-call constraint and the "
"output constraint cannot both be honored."
)
logger.warning("Constrained decoding is not compatible with tool calls.")
elif tool_call_constraint:
constraint_type, constraint_value = tool_call_constraint
python/sglang/srt/entrypoints/openai/serving_chat.py
+41 -7
View File
@@ -2007,15 +2007,25 @@ class OpenAIServingChat(OpenAIServingBase):
parser = FunctionCallParser(
tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer
)
should_try_parser = (
not is_required
or parser.detector.supports_structural_tag()
detector_owns_format = (
parser.detector.supports_structural_tag()
or parser.detector.parses_required_natively()
)
should_try_parser = not is_required or detector_owns_format
if should_try_parser and parser.has_tool_call(text):
try:
text, call_info_list = parser.parse_non_stream(text)
if not call_info_list:
logger.warning(
"Tool call marker present but no complete call parsed "
"from %s output; dropping the incomplete call",
self.tool_call_parser,
)
logger.debug(
"Unparsed tool call output (%d chars): %r",
len(text),
text[:2000],
)
return ToolCallProcessingResult(None, text, finish_reason)
tool_calls = []
@@ -2041,6 +2051,15 @@ class OpenAIServingChat(OpenAIServingBase):
logger.error(f"Tool call parsing error: {e}")
return ToolCallProcessingResult(None, text, finish_reason)
if is_required and detector_owns_format:
logger.warning(
"Required tool call missing from %s output (%d chars)",
self.tool_call_parser,
len(text),
)
logger.debug("Unparsed required tool call output: %r", text[:2000])
return ToolCallProcessingResult(None, text, finish_reason)
# json_schema constraint → JSON array output for required/named
if is_required:
original_finish_type = finish_reason["type"]
@@ -2049,12 +2068,28 @@ class OpenAIServingChat(OpenAIServingBase):
finish_reason["matched"] = None
try:
tool_call_data = orjson.loads(text)
if isinstance(tool_call_data, dict):
tool_call_data = [tool_call_data]
if not isinstance(tool_call_data, list):
raise ValueError(
"expected a JSON array of tool calls, got "
f"{type(tool_call_data).__name__}"
)
if not all(
isinstance(tool, dict) and "name" in tool for tool in tool_call_data
):
raise ValueError(
"every tool call must be a JSON object with a 'name'"
)
tool_calls = []
for i, tool in enumerate(tool_call_data):
parameters = json.dumps(
tool.get("parameters", {}), ensure_ascii=False
)
call_info = ToolCallItem(
tool_index=i,
name=tool["name"],
parameters=json.dumps(tool["parameters"], ensure_ascii=False),
parameters=parameters,
)
tool_id = self._process_tool_call_id(
call_info, history_tool_calls_cnt
@@ -2065,15 +2100,14 @@ class OpenAIServingChat(OpenAIServingBase):
index=i,
function=FunctionResponse(
name=tool["name"],
arguments=json.dumps(
tool["parameters"], ensure_ascii=False
),
arguments=parameters,
),
)
)
return ToolCallProcessingResult(tool_calls, "", finish_reason)
except Exception as e:
logger.error(f"Tool call parsing error: {e}")
logger.debug("Unparsed required tool call output: %r", text[:2000])
finish_reason["type"] = original_finish_type
return ToolCallProcessingResult(None, text, finish_reason)
python/sglang/srt/entrypoints/openai/serving_responses.py
+12 -3
View File
@@ -859,6 +859,7 @@ class OpenAIServingResponses(OpenAIServingChat):
is_required = request.tool_choice == "required"
tool_call_items: list[ResponseFunctionToolCall] = []
parsed_via_native = False
detector_owns_format = False
if (
content
and chat_tools
@@ -870,9 +871,11 @@ class OpenAIServingResponses(OpenAIServingChat):
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
should_try_native = (
not is_required or parser.detector.supports_structural_tag()
detector_owns_format = (
parser.detector.supports_structural_tag()
or parser.detector.parses_required_natively()
)
should_try_native = not is_required or detector_owns_format
if should_try_native and parser.has_tool_call(content):
try:
content, call_info_list = parser.parse_non_stream(content)
@@ -891,7 +894,13 @@ class OpenAIServingResponses(OpenAIServingChat):
except Exception as e:
logger.error("Tool call parsing error: %s", e)
if content and chat_tools and is_required and not parsed_via_native:
if (
content
and chat_tools
and is_required
and not parsed_via_native
and not detector_owns_format
):
try:
tool_call_data = orjson.loads(content)
if isinstance(tool_call_data, dict):
python/sglang/srt/function_call/kimik3_detector.py
+15
View File
@@ -19,6 +19,7 @@ from sglang.srt.function_call.kimik3_format import (
TOOLS_CLOSE,
TOOLS_OPEN,
partial_suffix_len,
strip_partial_marker_suffix,
strip_response_wrappers,
)
from sglang.srt.function_call.kimik3_structural_tag import (
@@ -217,6 +218,20 @@ class KimiK3Detector(BaseFormatDetector):
self._sent_normal_idx = 0
return StreamingParseResult()
def finish(self, tools: List[Tool]) -> StreamingParseResult:
open_idx = self._buffer.find(self.bot_token)
if open_idx != -1:
section = self._buffer[open_idx + len(self.bot_token) :]
if not self._parse_calls(section):
logger.warning(
"Kimi K3 tools section ended with no complete tool call; "
"dropping %d buffered chars",
len(section),
)
return StreamingParseResult()
pending = self._emit_normal_text(limit=len(self._buffer))
return StreamingParseResult(normal_text=strip_partial_marker_suffix(pending))
def _emit_normal_text(self, limit: int | None = None) -> str:
if limit is None:
holdback = partial_suffix_len(
python/sglang/srt/parser/reasoning_parser.py
+8 -1
View File
@@ -542,6 +542,12 @@ class KimiK3Detector(BaseReasoningFormatDetector):
open_idx = text.find(self.think_start_token)
start = open_idx + len(self.think_start_token) if open_idx != -1 else 0
close_idx = text.find(self.think_end_token, start)
tools_idx = text.find(self.tool_start_token, start)
if close_idx != -1 and tools_idx != -1 and tools_idx < close_idx:
return StreamingParseResult(
reasoning_text=strip_partial_marker_suffix(text[start:tools_idx]),
normal_text=self._clean_content(text[tools_idx:]),
)
if close_idx == -1:
channel_idx = self._next_channel_idx(text, start)
if channel_idx != -1:
@@ -583,7 +589,8 @@ class KimiK3Detector(BaseReasoningFormatDetector):
self.stripped_think_start = True
close_idx = buf.find(self.think_end_token)
if close_idx != -1:
tools_idx = buf.find(self.tool_start_token)
if close_idx != -1 and not (tools_idx != -1 and tools_idx < close_idx):
reasoning_text = buf[:close_idx]
self._buffer = buf[close_idx + len(self.think_end_token) :]
self._in_reasoning = False
test/registered/function_call/test_kimik3_detector.py
+32
View File
@@ -208,6 +208,38 @@ def test_streaming_bookkeeping_for_serving_layer() -> None:
assert json.loads(detector.streamed_args_for_tool[0]) == {"code": "a"}
def test_stream_end_reports_truncated_tools_section(caplog) -> None:
"""A tools section cut off before its closing tag used to vanish at
end-of-stream: no call, no text, no log. It must at least be reported."""
detector = KimiK3Detector()
tools = [_make_tool("python")]
truncated = TOOLS_OPEN + '<|open|>call tool="python" index="1"<|sep|>'
text, calls = _stream(detector, _chunks(truncated, 7), tools)
assert calls == []
with caplog.at_level("WARNING", logger="sglang.srt.function_call.kimik3_detector"):
result = detector.finish(tools)
assert result.calls == []
assert TOOLS_OPEN not in (result.normal_text or "")
assert "no complete tool call" in caplog.text
def test_stream_end_releases_held_back_text() -> None:
detector = KimiK3Detector()
tools = [_make_tool("python")]
text, _ = _stream(detector, ["all done", "<"], tools)
assert text == "all done"
result = detector.finish(tools)
assert text + (result.normal_text or "") == "all done<"
def test_stream_end_drops_truncated_marker() -> None:
detector = KimiK3Detector()
tools = [_make_tool("python")]
text, _ = _stream(detector, ["all done", "<|open|>"], tools)
result = detector.finish(tools)
assert text + (result.normal_text or "") == "all done"
def test_detector_capabilities_and_registration() -> None:
detector = KimiK3Detector()
assert detector.supports_structural_tag()
test/registered/unit/entrypoints/openai/test_serving_chat.py
+188 -1
View File
@@ -27,13 +27,15 @@ from sglang.srt.entrypoints.openai.chat_encoding import (
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
MessageProcessingResult,
ToolChoice,
ToolChoiceFuncName,
)
from sglang.srt.entrypoints.openai.serving_chat import (
OpenAIServingChat,
normalize_tool_content,
)
from sglang.srt.environ import envs
from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE
from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.parser.template_detection import ReasoningToggleConfig
from sglang.srt.utils import get_or_create_event_loop
@@ -1569,6 +1571,191 @@ class ServingChatTestCase(unittest.TestCase):
self.assertEqual(tool_calls[1].id, "functions.get_weather:2")
self.assertEqual(tool_calls[1].function.name, "get_weather")
def test_required_tool_choice_skips_json_fallback_for_native_parser(self):
"""A structural-tag parser owns the output format, so a missing tool
call must not be pushed through the json_schema array fallback."""
self.chat.tool_call_parser = "kimi_k3"
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
texts = {
"prose": "<|open|>response<|sep|>I'll check that.<|close|>response<|sep|>",
"empty": "",
"json_object": '{"name": "get_weather", "parameters": {"city": "Paris"}}',
}
for choice in (
"required",
ToolChoice(function=ToolChoiceFuncName(name="get_weather")),
):
for label, text in texts.items():
with self.subTest(tool_choice=choice, payload=label):
finish_reason = {"type": "stop", "matched": None}
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="WARNING"
) as logs:
tool_calls, remaining, finish_reason = (
self.chat._process_tool_calls(
text=text,
tools=tools,
finish_reason=finish_reason,
tool_choice=choice,
)
)
self.assertIsNone(tool_calls)
self.assertEqual(remaining, text)
self.assertEqual(finish_reason["type"], "stop")
self.assertNotIn("Tool call parsing error", "\n".join(logs.output))
def test_truncated_native_tool_call_logs_and_drops(self):
"""A tools section cut off before its closing tag parses to zero calls
without raising; the sync path used to drop it with no log at all."""
self.chat.tool_call_parser = "kimi_k3"
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
truncated = TOOLS_OPEN + '<|open|>call tool="get_weather" index="1"<|sep|>'
for choice in ("auto", "required"):
with self.subTest(tool_choice=choice):
finish_reason = {"type": "stop", "matched": None}
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="WARNING"
) as logs:
tool_calls, remaining, finish_reason = (
self.chat._process_tool_calls(
text=truncated,
tools=tools,
finish_reason=finish_reason,
tool_choice=choice,
)
)
self.assertIsNone(tool_calls)
self.assertEqual(remaining, "")
self.assertEqual(finish_reason["type"], "stop")
self.assertIn("no complete call", "\n".join(logs.output))
def test_required_tool_choice_json_fallback_tolerates_odd_shapes(self):
"""Parsers without a structural tag keep the JSON array fallback, but a
non-array payload degrades instead of raising an opaque TypeError."""
self.chat.tool_call_parser = "glm45"
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
cases = [
(
'[{"name": "get_weather", "parameters": {"city": "Paris"}}]',
'{"city": "Paris"}',
),
(
'{"name": "get_weather", "parameters": {"city": "Paris"}}',
'{"city": "Paris"}',
),
('{"name": "get_weather"}', "{}"),
]
for text, expected_args in cases:
with self.subTest(text=text):
tool_calls, _, finish_reason = self.chat._process_tool_calls(
text=text,
tools=tools,
finish_reason={"type": "stop", "matched": None},
tool_choice="required",
)
self.assertEqual(len(tool_calls), 1)
self.assertEqual(tool_calls[0].function.name, "get_weather")
self.assertEqual(tool_calls[0].function.arguments, expected_args)
self.assertEqual(finish_reason["type"], "tool_calls")
finish_reason = {"type": "stop", "matched": None}
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="ERROR"
):
tool_calls, remaining, finish_reason = self.chat._process_tool_calls(
text='["get_weather"]',
tools=tools,
finish_reason=finish_reason,
tool_choice="required",
)
self.assertIsNone(tool_calls)
self.assertEqual(remaining, '["get_weather"]')
self.assertEqual(finish_reason["type"], "stop")
def test_required_tool_choice_rejects_conflicting_output_constraint(self):
"""response_format and a forced tool call cannot both be honored: the
tool-call constraint was dropped with only a warning, so the model was
constrained to a shape that can never contain a tool call."""
tools = [{"type": "function", "function": {"name": "get_weather"}}]
constraint = ("structural_tag", None)
conflicting = [
{"type": "json_object"},
{
"type": "json_schema",
"json_schema": {"name": "a", "schema": {"type": "object"}},
},
]
for tool_choice in (
"required",
ToolChoice(function=ToolChoiceFuncName(name="get_weather")),
):
for response_format in conflicting:
with self.subTest(tool_choice=tool_choice, rf=response_format["type"]):
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
tools=tools,
tool_choice=tool_choice,
response_format=response_format,
)
with self.assertRaises(ValueError) as ctx:
request.to_sampling_params(
stop=[],
model_generation_config={},
tool_call_constraint=constraint,
)
self.assertIn("cannot be combined", str(ctx.exception))
def test_auto_tool_choice_keeps_response_format_without_raising(self):
""" "auto" means the model need not call a tool, so dropping the
tool-call constraint still leaves a satisfiable request."""
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
tool_choice="auto",
response_format={"type": "json_object"},
)
sampling_params = request.to_sampling_params(
stop=[],
model_generation_config={},
tool_call_constraint=("structural_tag", None),
)
self.assertEqual(sampling_params["json_schema"], '{"type": "object"}')
def test_kimi_k2_streaming_tool_call_id_with_history(self):
"""Ensure streaming first chunk tool_call.id increase with tool calls history for kimi_k2 parser."""
test/registered/unit/entrypoints/openai/test_serving_responses.py
+38
View File
@@ -691,6 +691,44 @@ class OutputItemsTestCase(CustomTestCase):
[],
)
def test_required_tool_choice_skips_json_fallback_for_native_parser(self):
"""muse reports parses_required_natively, so required output must not
be pushed through the orjson JSON-array fallback (mirrors chat)."""
serving = self.serving
serving.tool_call_parser = "muse"
request = ResponsesRequest(
model="x",
input="hi",
tool_choice="required",
tools=[
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object"},
}
],
store=False,
)
raw = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]'
output_items = serving._make_response_output_items(
request, raw, tokenizer=Mock(), require_reasoning=False
)
self.assertEqual(
[
item
for item in output_items
if isinstance(item, ResponseFunctionToolCall)
],
[],
)
message_items = [
item for item in output_items if isinstance(item, ResponseOutputMessage)
]
self.assertEqual(len(message_items), 1)
self.assertEqual(message_items[0].content[0].text, raw)
def test_no_tool_call_extraction_when_tool_choice_none(self):
serving = self.serving
request = ResponsesRequest(
test/registered/unit/parser/test_kimik3_reasoning_parser.py
+32
View File
@@ -154,6 +154,38 @@ def test_streaming_recovers_missing_think_separator() -> None:
assert content == "reply"
_TOOLS_CHANNEL = (
f'{TOOLS_OPEN}<|open|>call tool="python" index="1"<|sep|>'
"<|close|>call<|sep|>"
f"{TOOLS_CLOSE}"
)
def test_non_stream_tools_channel_before_think_close_is_not_reasoning() -> None:
"""A tools channel emitted inside the think block, with think closing after
it, must still reach the tool-call parser.
The reasoning grammar is deferred until think closes, so nothing stops the
model from opening the tools channel first. Splitting on think_end put the
whole call in reasoning_text, dropping it with no log line.
"""
detector = KimiK3Detector(force_reasoning=True)
result = detector.detect_and_parse(
f"thought{_TOOLS_CHANNEL}{THINK_CLOSE}{MESSAGE_CLOSE}"
)
assert result.reasoning_text == "thought"
assert TOOLS_OPEN in result.normal_text
@pytest.mark.parametrize("chunk_size", [1, 3, 7, 1000])
def test_streaming_tools_channel_before_think_close(chunk_size: int) -> None:
detector = KimiK3Detector(force_reasoning=True)
text = f"thought{_TOOLS_CHANNEL}{THINK_CLOSE}{MESSAGE_CLOSE}"
reasoning, content = _stream(detector, _chunks(text, chunk_size))
assert reasoning == "thought"
assert TOOLS_OPEN in content
def test_reasoning_parser_registration() -> None:
assert isinstance(ReasoningParser("kimi_k3").detector, KimiK3Detector)