minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Stop losing Kimi-K3 tool calls to reasoning, constraint conflicts, and truncation (#34881)

Browse Source 
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Khoa Pham
2026-08-18 12:42:56 -07:00
committed by GitHub
co-authored by Claude Opus 5 Xinyuan Tong Xinyuan Tong
parent 8bb106cee9
commit 307a90f6d3
9 changed files with 374 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/protocol.py
+8
View File
@@ -1081,6 +1081,14 @@ class ChatCompletionRequest(BaseModel):
)
if tool_call_constraint and has_existing_constraints:
if self.tool_choice == "required" or isinstance(
self.tool_choice, ToolChoice
):
raise ValueError(
"tool_choice 'required' or a named tool cannot be combined with "
"response_format, regex, or ebnf: the tool-call constraint and the "
"output constraint cannot both be honored."
)
logger.warning("Constrained decoding is not compatible with tool calls.")
elif tool_call_constraint:
constraint_type, constraint_value = tool_call_constraint
python/sglang/srt/entrypoints/openai/serving_chat.py
+41 -7
View File
@@ -2007,15 +2007,25 @@ class OpenAIServingChat(OpenAIServingBase):
parser = FunctionCallParser(
tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer
)
should_try_parser = (
not is_required
or parser.detector.supports_structural_tag()
detector_owns_format = (
parser.detector.supports_structural_tag()
or parser.detector.parses_required_natively()
)
should_try_parser = not is_required or detector_owns_format
if should_try_parser and parser.has_tool_call(text):
try:
text, call_info_list = parser.parse_non_stream(text)
if not call_info_list:
logger.warning(
"Tool call marker present but no complete call parsed "
"from %s output; dropping the incomplete call",
self.tool_call_parser,
)
logger.debug(
"Unparsed tool call output (%d chars): %r",
len(text),
text[:2000],
)
return ToolCallProcessingResult(None, text, finish_reason)
tool_calls = []
@@ -2041,6 +2051,15 @@ class OpenAIServingChat(OpenAIServingBase):
logger.error(f"Tool call parsing error: {e}")
return ToolCallProcessingResult(None, text, finish_reason)
if is_required and detector_owns_format:
logger.warning(
"Required tool call missing from %s output (%d chars)",
self.tool_call_parser,
len(text),
)
logger.debug("Unparsed required tool call output: %r", text[:2000])
return ToolCallProcessingResult(None, text, finish_reason)
# json_schema constraint → JSON array output for required/named
if is_required:
original_finish_type = finish_reason["type"]
@@ -2049,12 +2068,28 @@ class OpenAIServingChat(OpenAIServingBase):
finish_reason["matched"] = None
try:
tool_call_data = orjson.loads(text)
if isinstance(tool_call_data, dict):
tool_call_data = [tool_call_data]
if not isinstance(tool_call_data, list):
raise ValueError(
"expected a JSON array of tool calls, got "
f"{type(tool_call_data).__name__}"
)
if not all(
isinstance(tool, dict) and "name" in tool for tool in tool_call_data
):
raise ValueError(
"every tool call must be a JSON object with a 'name'"
)
tool_calls = []
for i, tool in enumerate(tool_call_data):
parameters = json.dumps(
tool.get("parameters", {}), ensure_ascii=False
)
call_info = ToolCallItem(
tool_index=i,
name=tool["name"],
parameters=json.dumps(tool["parameters"], ensure_ascii=False),
parameters=parameters,
)
tool_id = self._process_tool_call_id(
call_info, history_tool_calls_cnt
@@ -2065,15 +2100,14 @@ class OpenAIServingChat(OpenAIServingBase):
index=i,
function=FunctionResponse(
name=tool["name"],
arguments=json.dumps(
tool["parameters"], ensure_ascii=False
),
arguments=parameters,
),
)
)
return ToolCallProcessingResult(tool_calls, "", finish_reason)
except Exception as e:
logger.error(f"Tool call parsing error: {e}")
logger.debug("Unparsed required tool call output: %r", text[:2000])
finish_reason["type"] = original_finish_type
return ToolCallProcessingResult(None, text, finish_reason)
python/sglang/srt/entrypoints/openai/serving_responses.py
+12 -3
View File
@@ -859,6 +859,7 @@ class OpenAIServingResponses(OpenAIServingChat):
is_required = request.tool_choice == "required"
tool_call_items: list[ResponseFunctionToolCall] = []
parsed_via_native = False
detector_owns_format = False
if (
content
and chat_tools
@@ -870,9 +871,11 @@ class OpenAIServingResponses(OpenAIServingChat):
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
should_try_native = (
not is_required or parser.detector.supports_structural_tag()
detector_owns_format = (
parser.detector.supports_structural_tag()
or parser.detector.parses_required_natively()
)
should_try_native = not is_required or detector_owns_format
if should_try_native and parser.has_tool_call(content):
try:
content, call_info_list = parser.parse_non_stream(content)
@@ -891,7 +894,13 @@ class OpenAIServingResponses(OpenAIServingChat):
except Exception as e:
logger.error("Tool call parsing error: %s", e)
if content and chat_tools and is_required and not parsed_via_native:
if (
content
and chat_tools
and is_required
and not parsed_via_native
and not detector_owns_format
):
try:
tool_call_data = orjson.loads(content)
if isinstance(tool_call_data, dict):
python/sglang/srt/function_call/kimik3_detector.py
+15
View File
@@ -19,6 +19,7 @@ from sglang.srt.function_call.kimik3_format import (
TOOLS_CLOSE,
TOOLS_OPEN,
partial_suffix_len,
strip_partial_marker_suffix,
strip_response_wrappers,
)
from sglang.srt.function_call.kimik3_structural_tag import (
@@ -217,6 +218,20 @@ class KimiK3Detector(BaseFormatDetector):
self._sent_normal_idx = 0
return StreamingParseResult()
def finish(self, tools: List[Tool]) -> StreamingParseResult:
open_idx = self._buffer.find(self.bot_token)
if open_idx != -1:
section = self._buffer[open_idx + len(self.bot_token) :]
if not self._parse_calls(section):
logger.warning(
"Kimi K3 tools section ended with no complete tool call; "
"dropping %d buffered chars",
len(section),
)
return StreamingParseResult()
pending = self._emit_normal_text(limit=len(self._buffer))
return StreamingParseResult(normal_text=strip_partial_marker_suffix(pending))
def _emit_normal_text(self, limit: int | None = None) -> str:
if limit is None:
holdback = partial_suffix_len(
python/sglang/srt/parser/reasoning_parser.py
+8 -1
View File
@@ -542,6 +542,12 @@ class KimiK3Detector(BaseReasoningFormatDetector):
open_idx = text.find(self.think_start_token)
start = open_idx + len(self.think_start_token) if open_idx != -1 else 0
close_idx = text.find(self.think_end_token, start)
tools_idx = text.find(self.tool_start_token, start)
if close_idx != -1 and tools_idx != -1 and tools_idx < close_idx:
return StreamingParseResult(
reasoning_text=strip_partial_marker_suffix(text[start:tools_idx]),
normal_text=self._clean_content(text[tools_idx:]),
)
if close_idx == -1:
channel_idx = self._next_channel_idx(text, start)
if channel_idx != -1:
@@ -583,7 +589,8 @@ class KimiK3Detector(BaseReasoningFormatDetector):
self.stripped_think_start = True
close_idx = buf.find(self.think_end_token)
if close_idx != -1:
tools_idx = buf.find(self.tool_start_token)
if close_idx != -1 and not (tools_idx != -1 and tools_idx < close_idx):
reasoning_text = buf[:close_idx]
self._buffer = buf[close_idx + len(self.think_end_token) :]
self._in_reasoning = False