diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 33904acc3..da62e3b0f 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -1081,6 +1081,14 @@ class ChatCompletionRequest(BaseModel): ) if tool_call_constraint and has_existing_constraints: + if self.tool_choice == "required" or isinstance( + self.tool_choice, ToolChoice + ): + raise ValueError( + "tool_choice 'required' or a named tool cannot be combined with " + "response_format, regex, or ebnf: the tool-call constraint and the " + "output constraint cannot both be honored." + ) logger.warning("Constrained decoding is not compatible with tool calls.") elif tool_call_constraint: constraint_type, constraint_value = tool_call_constraint diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 28e5d0a2c..f572e6360 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -2007,15 +2007,25 @@ class OpenAIServingChat(OpenAIServingBase): parser = FunctionCallParser( tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer ) - should_try_parser = ( - not is_required - or parser.detector.supports_structural_tag() + detector_owns_format = ( + parser.detector.supports_structural_tag() or parser.detector.parses_required_natively() ) + should_try_parser = not is_required or detector_owns_format if should_try_parser and parser.has_tool_call(text): try: text, call_info_list = parser.parse_non_stream(text) if not call_info_list: + logger.warning( + "Tool call marker present but no complete call parsed " + "from %s output; dropping the incomplete call", + self.tool_call_parser, + ) + logger.debug( + "Unparsed tool call output (%d chars): %r", + len(text), + text[:2000], + ) return ToolCallProcessingResult(None, text, finish_reason) tool_calls = [] @@ -2041,6 +2051,15 @@ class OpenAIServingChat(OpenAIServingBase): logger.error(f"Tool call parsing error: {e}") return ToolCallProcessingResult(None, text, finish_reason) + if is_required and detector_owns_format: + logger.warning( + "Required tool call missing from %s output (%d chars)", + self.tool_call_parser, + len(text), + ) + logger.debug("Unparsed required tool call output: %r", text[:2000]) + return ToolCallProcessingResult(None, text, finish_reason) + # json_schema constraint → JSON array output for required/named if is_required: original_finish_type = finish_reason["type"] @@ -2049,12 +2068,28 @@ class OpenAIServingChat(OpenAIServingBase): finish_reason["matched"] = None try: tool_call_data = orjson.loads(text) + if isinstance(tool_call_data, dict): + tool_call_data = [tool_call_data] + if not isinstance(tool_call_data, list): + raise ValueError( + "expected a JSON array of tool calls, got " + f"{type(tool_call_data).__name__}" + ) + if not all( + isinstance(tool, dict) and "name" in tool for tool in tool_call_data + ): + raise ValueError( + "every tool call must be a JSON object with a 'name'" + ) tool_calls = [] for i, tool in enumerate(tool_call_data): + parameters = json.dumps( + tool.get("parameters", {}), ensure_ascii=False + ) call_info = ToolCallItem( tool_index=i, name=tool["name"], - parameters=json.dumps(tool["parameters"], ensure_ascii=False), + parameters=parameters, ) tool_id = self._process_tool_call_id( call_info, history_tool_calls_cnt @@ -2065,15 +2100,14 @@ class OpenAIServingChat(OpenAIServingBase): index=i, function=FunctionResponse( name=tool["name"], - arguments=json.dumps( - tool["parameters"], ensure_ascii=False - ), + arguments=parameters, ), ) ) return ToolCallProcessingResult(tool_calls, "", finish_reason) except Exception as e: logger.error(f"Tool call parsing error: {e}") + logger.debug("Unparsed required tool call output: %r", text[:2000]) finish_reason["type"] = original_finish_type return ToolCallProcessingResult(None, text, finish_reason) diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index 41ae749a3..392f6533a 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -859,6 +859,7 @@ class OpenAIServingResponses(OpenAIServingChat): is_required = request.tool_choice == "required" tool_call_items: list[ResponseFunctionToolCall] = [] parsed_via_native = False + detector_owns_format = False if ( content and chat_tools @@ -870,9 +871,11 @@ class OpenAIServingResponses(OpenAIServingChat): self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer, ) - should_try_native = ( - not is_required or parser.detector.supports_structural_tag() + detector_owns_format = ( + parser.detector.supports_structural_tag() + or parser.detector.parses_required_natively() ) + should_try_native = not is_required or detector_owns_format if should_try_native and parser.has_tool_call(content): try: content, call_info_list = parser.parse_non_stream(content) @@ -891,7 +894,13 @@ class OpenAIServingResponses(OpenAIServingChat): except Exception as e: logger.error("Tool call parsing error: %s", e) - if content and chat_tools and is_required and not parsed_via_native: + if ( + content + and chat_tools + and is_required + and not parsed_via_native + and not detector_owns_format + ): try: tool_call_data = orjson.loads(content) if isinstance(tool_call_data, dict): diff --git a/python/sglang/srt/function_call/kimik3_detector.py b/python/sglang/srt/function_call/kimik3_detector.py index fe3b53f72..385e02f96 100644 --- a/python/sglang/srt/function_call/kimik3_detector.py +++ b/python/sglang/srt/function_call/kimik3_detector.py @@ -19,6 +19,7 @@ from sglang.srt.function_call.kimik3_format import ( TOOLS_CLOSE, TOOLS_OPEN, partial_suffix_len, + strip_partial_marker_suffix, strip_response_wrappers, ) from sglang.srt.function_call.kimik3_structural_tag import ( @@ -217,6 +218,20 @@ class KimiK3Detector(BaseFormatDetector): self._sent_normal_idx = 0 return StreamingParseResult() + def finish(self, tools: List[Tool]) -> StreamingParseResult: + open_idx = self._buffer.find(self.bot_token) + if open_idx != -1: + section = self._buffer[open_idx + len(self.bot_token) :] + if not self._parse_calls(section): + logger.warning( + "Kimi K3 tools section ended with no complete tool call; " + "dropping %d buffered chars", + len(section), + ) + return StreamingParseResult() + pending = self._emit_normal_text(limit=len(self._buffer)) + return StreamingParseResult(normal_text=strip_partial_marker_suffix(pending)) + def _emit_normal_text(self, limit: int | None = None) -> str: if limit is None: holdback = partial_suffix_len( diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index af57e94e8..4e386385a 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -542,6 +542,12 @@ class KimiK3Detector(BaseReasoningFormatDetector): open_idx = text.find(self.think_start_token) start = open_idx + len(self.think_start_token) if open_idx != -1 else 0 close_idx = text.find(self.think_end_token, start) + tools_idx = text.find(self.tool_start_token, start) + if close_idx != -1 and tools_idx != -1 and tools_idx < close_idx: + return StreamingParseResult( + reasoning_text=strip_partial_marker_suffix(text[start:tools_idx]), + normal_text=self._clean_content(text[tools_idx:]), + ) if close_idx == -1: channel_idx = self._next_channel_idx(text, start) if channel_idx != -1: @@ -583,7 +589,8 @@ class KimiK3Detector(BaseReasoningFormatDetector): self.stripped_think_start = True close_idx = buf.find(self.think_end_token) - if close_idx != -1: + tools_idx = buf.find(self.tool_start_token) + if close_idx != -1 and not (tools_idx != -1 and tools_idx < close_idx): reasoning_text = buf[:close_idx] self._buffer = buf[close_idx + len(self.think_end_token) :] self._in_reasoning = False diff --git a/test/registered/function_call/test_kimik3_detector.py b/test/registered/function_call/test_kimik3_detector.py index 20a0bf2e3..704237b2d 100644 --- a/test/registered/function_call/test_kimik3_detector.py +++ b/test/registered/function_call/test_kimik3_detector.py @@ -208,6 +208,38 @@ def test_streaming_bookkeeping_for_serving_layer() -> None: assert json.loads(detector.streamed_args_for_tool[0]) == {"code": "a"} +def test_stream_end_reports_truncated_tools_section(caplog) -> None: + """A tools section cut off before its closing tag used to vanish at + end-of-stream: no call, no text, no log. It must at least be reported.""" + detector = KimiK3Detector() + tools = [_make_tool("python")] + truncated = TOOLS_OPEN + '<|open|>call tool="python" index="1"<|sep|>' + text, calls = _stream(detector, _chunks(truncated, 7), tools) + assert calls == [] + with caplog.at_level("WARNING", logger="sglang.srt.function_call.kimik3_detector"): + result = detector.finish(tools) + assert result.calls == [] + assert TOOLS_OPEN not in (result.normal_text or "") + assert "no complete tool call" in caplog.text + + +def test_stream_end_releases_held_back_text() -> None: + detector = KimiK3Detector() + tools = [_make_tool("python")] + text, _ = _stream(detector, ["all done", "<"], tools) + assert text == "all done" + result = detector.finish(tools) + assert text + (result.normal_text or "") == "all done<" + + +def test_stream_end_drops_truncated_marker() -> None: + detector = KimiK3Detector() + tools = [_make_tool("python")] + text, _ = _stream(detector, ["all done", "<|open|>"], tools) + result = detector.finish(tools) + assert text + (result.normal_text or "") == "all done" + + def test_detector_capabilities_and_registration() -> None: detector = KimiK3Detector() assert detector.supports_structural_tag() diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index fbac3fa80..0e7861a13 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -27,13 +27,15 @@ from sglang.srt.entrypoints.openai.chat_encoding import ( from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, MessageProcessingResult, + ToolChoice, + ToolChoiceFuncName, ) from sglang.srt.entrypoints.openai.serving_chat import ( OpenAIServingChat, normalize_tool_content, ) from sglang.srt.environ import envs -from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE +from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN from sglang.srt.managers.io_struct import GenerateReqInput from sglang.srt.parser.template_detection import ReasoningToggleConfig from sglang.srt.utils import get_or_create_event_loop @@ -1569,6 +1571,191 @@ class ServingChatTestCase(unittest.TestCase): self.assertEqual(tool_calls[1].id, "functions.get_weather:2") self.assertEqual(tool_calls[1].function.name, "get_weather") + def test_required_tool_choice_skips_json_fallback_for_native_parser(self): + """A structural-tag parser owns the output format, so a missing tool + call must not be pushed through the json_schema array fallback.""" + self.chat.tool_call_parser = "kimi_k3" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + texts = { + "prose": "<|open|>response<|sep|>I'll check that.<|close|>response<|sep|>", + "empty": "", + "json_object": '{"name": "get_weather", "parameters": {"city": "Paris"}}', + } + for choice in ( + "required", + ToolChoice(function=ToolChoiceFuncName(name="get_weather")), + ): + for label, text in texts.items(): + with self.subTest(tool_choice=choice, payload=label): + finish_reason = {"type": "stop", "matched": None} + with self.assertLogs( + "sglang.srt.entrypoints.openai.serving_chat", level="WARNING" + ) as logs: + tool_calls, remaining, finish_reason = ( + self.chat._process_tool_calls( + text=text, + tools=tools, + finish_reason=finish_reason, + tool_choice=choice, + ) + ) + self.assertIsNone(tool_calls) + self.assertEqual(remaining, text) + self.assertEqual(finish_reason["type"], "stop") + self.assertNotIn("Tool call parsing error", "\n".join(logs.output)) + + def test_truncated_native_tool_call_logs_and_drops(self): + """A tools section cut off before its closing tag parses to zero calls + without raising; the sync path used to drop it with no log at all.""" + self.chat.tool_call_parser = "kimi_k3" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + truncated = TOOLS_OPEN + '<|open|>call tool="get_weather" index="1"<|sep|>' + for choice in ("auto", "required"): + with self.subTest(tool_choice=choice): + finish_reason = {"type": "stop", "matched": None} + with self.assertLogs( + "sglang.srt.entrypoints.openai.serving_chat", level="WARNING" + ) as logs: + tool_calls, remaining, finish_reason = ( + self.chat._process_tool_calls( + text=truncated, + tools=tools, + finish_reason=finish_reason, + tool_choice=choice, + ) + ) + self.assertIsNone(tool_calls) + self.assertEqual(remaining, "") + self.assertEqual(finish_reason["type"], "stop") + self.assertIn("no complete call", "\n".join(logs.output)) + + def test_required_tool_choice_json_fallback_tolerates_odd_shapes(self): + """Parsers without a structural tag keep the JSON array fallback, but a + non-array payload degrades instead of raising an opaque TypeError.""" + self.chat.tool_call_parser = "glm45" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + cases = [ + ( + '[{"name": "get_weather", "parameters": {"city": "Paris"}}]', + '{"city": "Paris"}', + ), + ( + '{"name": "get_weather", "parameters": {"city": "Paris"}}', + '{"city": "Paris"}', + ), + ('{"name": "get_weather"}', "{}"), + ] + for text, expected_args in cases: + with self.subTest(text=text): + tool_calls, _, finish_reason = self.chat._process_tool_calls( + text=text, + tools=tools, + finish_reason={"type": "stop", "matched": None}, + tool_choice="required", + ) + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0].function.name, "get_weather") + self.assertEqual(tool_calls[0].function.arguments, expected_args) + self.assertEqual(finish_reason["type"], "tool_calls") + + finish_reason = {"type": "stop", "matched": None} + with self.assertLogs( + "sglang.srt.entrypoints.openai.serving_chat", level="ERROR" + ): + tool_calls, remaining, finish_reason = self.chat._process_tool_calls( + text='["get_weather"]', + tools=tools, + finish_reason=finish_reason, + tool_choice="required", + ) + self.assertIsNone(tool_calls) + self.assertEqual(remaining, '["get_weather"]') + self.assertEqual(finish_reason["type"], "stop") + + def test_required_tool_choice_rejects_conflicting_output_constraint(self): + """response_format and a forced tool call cannot both be honored: the + tool-call constraint was dropped with only a warning, so the model was + constrained to a shape that can never contain a tool call.""" + tools = [{"type": "function", "function": {"name": "get_weather"}}] + constraint = ("structural_tag", None) + conflicting = [ + {"type": "json_object"}, + { + "type": "json_schema", + "json_schema": {"name": "a", "schema": {"type": "object"}}, + }, + ] + for tool_choice in ( + "required", + ToolChoice(function=ToolChoiceFuncName(name="get_weather")), + ): + for response_format in conflicting: + with self.subTest(tool_choice=tool_choice, rf=response_format["type"]): + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "hi"}], + tools=tools, + tool_choice=tool_choice, + response_format=response_format, + ) + with self.assertRaises(ValueError) as ctx: + request.to_sampling_params( + stop=[], + model_generation_config={}, + tool_call_constraint=constraint, + ) + self.assertIn("cannot be combined", str(ctx.exception)) + + def test_auto_tool_choice_keeps_response_format_without_raising(self): + """ "auto" means the model need not call a tool, so dropping the + tool-call constraint still leaves a satisfiable request.""" + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "hi"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + tool_choice="auto", + response_format={"type": "json_object"}, + ) + sampling_params = request.to_sampling_params( + stop=[], + model_generation_config={}, + tool_call_constraint=("structural_tag", None), + ) + self.assertEqual(sampling_params["json_schema"], '{"type": "object"}') + def test_kimi_k2_streaming_tool_call_id_with_history(self): """Ensure streaming first chunk tool_call.id increase with tool calls history for kimi_k2 parser.""" diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses.py b/test/registered/unit/entrypoints/openai/test_serving_responses.py index ab6c561ab..458128764 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_responses.py +++ b/test/registered/unit/entrypoints/openai/test_serving_responses.py @@ -691,6 +691,44 @@ class OutputItemsTestCase(CustomTestCase): [], ) + def test_required_tool_choice_skips_json_fallback_for_native_parser(self): + """muse reports parses_required_natively, so required output must not + be pushed through the orjson JSON-array fallback (mirrors chat).""" + serving = self.serving + serving.tool_call_parser = "muse" + request = ResponsesRequest( + model="x", + input="hi", + tool_choice="required", + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + } + ], + store=False, + ) + raw = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]' + + output_items = serving._make_response_output_items( + request, raw, tokenizer=Mock(), require_reasoning=False + ) + + self.assertEqual( + [ + item + for item in output_items + if isinstance(item, ResponseFunctionToolCall) + ], + [], + ) + message_items = [ + item for item in output_items if isinstance(item, ResponseOutputMessage) + ] + self.assertEqual(len(message_items), 1) + self.assertEqual(message_items[0].content[0].text, raw) + def test_no_tool_call_extraction_when_tool_choice_none(self): serving = self.serving request = ResponsesRequest( diff --git a/test/registered/unit/parser/test_kimik3_reasoning_parser.py b/test/registered/unit/parser/test_kimik3_reasoning_parser.py index cc0ecdb6e..db54703cc 100644 --- a/test/registered/unit/parser/test_kimik3_reasoning_parser.py +++ b/test/registered/unit/parser/test_kimik3_reasoning_parser.py @@ -154,6 +154,38 @@ def test_streaming_recovers_missing_think_separator() -> None: assert content == "reply" +_TOOLS_CHANNEL = ( + f'{TOOLS_OPEN}<|open|>call tool="python" index="1"<|sep|>' + "<|close|>call<|sep|>" + f"{TOOLS_CLOSE}" +) + + +def test_non_stream_tools_channel_before_think_close_is_not_reasoning() -> None: + """A tools channel emitted inside the think block, with think closing after + it, must still reach the tool-call parser. + + The reasoning grammar is deferred until think closes, so nothing stops the + model from opening the tools channel first. Splitting on think_end put the + whole call in reasoning_text, dropping it with no log line. + """ + detector = KimiK3Detector(force_reasoning=True) + result = detector.detect_and_parse( + f"thought{_TOOLS_CHANNEL}{THINK_CLOSE}{MESSAGE_CLOSE}" + ) + assert result.reasoning_text == "thought" + assert TOOLS_OPEN in result.normal_text + + +@pytest.mark.parametrize("chunk_size", [1, 3, 7, 1000]) +def test_streaming_tools_channel_before_think_close(chunk_size: int) -> None: + detector = KimiK3Detector(force_reasoning=True) + text = f"thought{_TOOLS_CHANNEL}{THINK_CLOSE}{MESSAGE_CLOSE}" + reasoning, content = _stream(detector, _chunks(text, chunk_size)) + assert reasoning == "thought" + assert TOOLS_OPEN in content + + def test_reasoning_parser_registration() -> None: assert isinstance(ReasoningParser("kimi_k3").detector, KimiK3Detector)