Fix Responses API request handling (#25881)
Co-authored-by: Kai-Hsun Chen <kaihsun@apache.org> Co-authored-by: Kristin Cowalcijk <kristincowalcijk@gmail.com> Co-authored-by: aerosta <63026763+aerosta@users.noreply.github.com> Co-authored-by: glaziermag <glaziermag@users.noreply.github.com> Co-authored-by: Blake Ledden <blake.ledden@gmail.com> Co-authored-by: PanJason <pyyjason@gmail.com> Co-authored-by: Leoyzen <leoyzen@gmail.com> Co-authored-by: kennyu <966806+kennyu@users.noreply.github.com>
This commit is contained in:
Xinyuan Tong
co-authored by
Kai-Hsun Chen
Kristin Cowalcijk
aerosta
glaziermag
Blake Ledden
PanJason
Leoyzen
kennyu
Kai-Hsun Chen
Kristin Cowalcijk
aerosta
glaziermag
Blake Ledden
PanJason
Leoyzen
kennyu
parent
b3270264e4
commit
85712fa5b0
@@ -0,0 +1,146 @@
|
||||
import unittest
|
||||
|
||||
from utils import make_serving # noqa: F401 — bootstrap import
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import ResponsesRequest, UsageInfo
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class ResponsesRequestTestCase(unittest.TestCase):
|
||||
def test_function_tool_accepted(self):
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="call the tool",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"description": "Look up a value.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"key": {"type": "string"}},
|
||||
"required": ["key"],
|
||||
},
|
||||
"strict": True,
|
||||
}
|
||||
],
|
||||
store=False,
|
||||
)
|
||||
|
||||
self.assertEqual(request.tools[0].type, "function")
|
||||
self.assertEqual(request.tools[0].name, "lookup")
|
||||
self.assertTrue(request.tools[0].strict)
|
||||
|
||||
def test_function_tool_requires_name(self):
|
||||
with self.assertRaises(ValueError):
|
||||
ResponsesRequest(
|
||||
model="x", input="hi", tools=[{"type": "function"}], store=False
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
tools=[{"type": "function", "name": ""}],
|
||||
store=False,
|
||||
)
|
||||
|
||||
def test_extended_tool_types_accepted(self):
|
||||
for tool_type in (
|
||||
"web_search",
|
||||
"web_search_preview",
|
||||
"code_interpreter",
|
||||
"file_search",
|
||||
"image_generation",
|
||||
"computer_use_preview",
|
||||
"local_shell",
|
||||
"mcp",
|
||||
"custom",
|
||||
"namespace",
|
||||
):
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
tools=[{"type": tool_type}],
|
||||
store=False,
|
||||
)
|
||||
self.assertEqual(request.tools[0].type, tool_type)
|
||||
|
||||
def test_namespace_tool_carries_inner_tools_list(self):
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
tools=[
|
||||
{
|
||||
"type": "namespace",
|
||||
"name": "codex",
|
||||
"tools": [
|
||||
{"type": "function", "name": "apply_patch"},
|
||||
{"type": "function", "name": "shell"},
|
||||
],
|
||||
}
|
||||
],
|
||||
store=False,
|
||||
)
|
||||
self.assertEqual(request.tools[0].type, "namespace")
|
||||
self.assertEqual(len(request.tools[0].tools), 2)
|
||||
self.assertEqual(request.tools[0].tools[0]["name"], "apply_patch")
|
||||
|
||||
|
||||
class ResponsesSamplingParamsTestCase(unittest.TestCase):
|
||||
def test_processed_stop_and_tool_constraint_propagate(self):
|
||||
request = ResponsesRequest(model="x", input="call the tool", store=False)
|
||||
params = request.to_sampling_params(
|
||||
default_max_tokens=128,
|
||||
default_params={},
|
||||
stop=["</s>"],
|
||||
tool_call_constraint=("json_schema", {"type": "object"}),
|
||||
)
|
||||
self.assertEqual(params["stop"], ["</s>"])
|
||||
self.assertEqual(params["json_schema"], '{"type": "object"}')
|
||||
|
||||
def test_constraint_conflict_raises(self):
|
||||
request = ResponsesRequest(model="x", input="hi", store=False)
|
||||
with self.assertRaises(ValueError):
|
||||
request.to_sampling_params(
|
||||
default_max_tokens=128,
|
||||
default_params={"json_schema": '{"type": "object"}'},
|
||||
tool_call_constraint=("json_schema", {"type": "object"}),
|
||||
)
|
||||
|
||||
def test_structural_tag_with_model_dump(self):
|
||||
class _FakeStructuralTag:
|
||||
def model_dump(self, by_alias=False):
|
||||
return {"type": "structural_tag"}
|
||||
|
||||
request = ResponsesRequest(model="x", input="hi", store=False)
|
||||
params = request.to_sampling_params(
|
||||
default_max_tokens=128,
|
||||
default_params={},
|
||||
tool_call_constraint=("structural_tag", _FakeStructuralTag()),
|
||||
)
|
||||
self.assertEqual(params["structural_tag"], '{"type": "structural_tag"}')
|
||||
|
||||
|
||||
class ResponsesResponseFromRequestTestCase(unittest.TestCase):
|
||||
def test_parallel_tool_calls_false_preserved(self):
|
||||
from sglang.srt.entrypoints.openai.protocol import ResponsesResponse
|
||||
|
||||
request = ResponsesRequest(
|
||||
model="x", input="hi", parallel_tool_calls=False, store=False
|
||||
)
|
||||
response = ResponsesResponse.from_request(
|
||||
request,
|
||||
sampling_params={},
|
||||
model_name="x",
|
||||
created_time=0,
|
||||
output=[],
|
||||
status="completed",
|
||||
usage=UsageInfo(prompt_tokens=1, completion_tokens=1, total_tokens=2),
|
||||
)
|
||||
self.assertFalse(response.parallel_tool_calls)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,529 @@
|
||||
import asyncio
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from openai.types.responses import (
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
||||
from utils import make_serving
|
||||
|
||||
from sglang.srt.entrypoints.context import SimpleContext
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
MessageProcessingResult,
|
||||
RequestResponseMetadata,
|
||||
ResponsesRequest,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.serving_responses import OpenAIServingResponses
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class InputMessageConstructionTestCase(unittest.TestCase):
|
||||
def test_previous_response_replays_assistant_text_not_instructions(self):
|
||||
serving = make_serving()
|
||||
prev_response = Mock(id="resp_prev")
|
||||
prev_response.output = [
|
||||
ResponseReasoningItem(
|
||||
id="rs_prev", summary=[], type="reasoning", content=None, status=None
|
||||
),
|
||||
ResponseOutputMessage(
|
||||
id="msg_prev",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
text="first answer part",
|
||||
annotations=[],
|
||||
type="output_text",
|
||||
logprobs=None,
|
||||
),
|
||||
ResponseOutputText(
|
||||
text="second answer part",
|
||||
annotations=[],
|
||||
type="output_text",
|
||||
logprobs=None,
|
||||
),
|
||||
],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
),
|
||||
]
|
||||
serving.msg_store["resp_prev"] = [{"role": "user", "content": "old input"}]
|
||||
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
instructions="Be brief",
|
||||
previous_response_id="resp_prev",
|
||||
input="new input",
|
||||
store=False,
|
||||
)
|
||||
|
||||
messages = serving._construct_input_messages(request, prev_response)
|
||||
|
||||
self.assertEqual(
|
||||
messages,
|
||||
[
|
||||
{"role": "system", "content": "Be brief"},
|
||||
{"role": "user", "content": "old input"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "first answer part\nsecond answer part",
|
||||
},
|
||||
{"role": "user", "content": "new input"},
|
||||
],
|
||||
)
|
||||
|
||||
def test_input_parts_normalized_for_chat_templates(self):
|
||||
serving = make_serving()
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "what is this?"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "http://example.com/cat.png",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
store=False,
|
||||
)
|
||||
|
||||
messages = serving._construct_input_messages(request)
|
||||
|
||||
self.assertEqual(
|
||||
messages,
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is this?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "http://example.com/cat.png",
|
||||
"detail": "auto",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_previous_response_id_input_list_does_not_call_copy_module(self):
|
||||
serving = make_serving()
|
||||
serving.use_harmony = True
|
||||
prev = Mock(id="resp_prev")
|
||||
prev.output = [
|
||||
ResponseFunctionToolCall(
|
||||
arguments="{}",
|
||||
call_id="call_x",
|
||||
name="t",
|
||||
type="function_call",
|
||||
id="fc_x",
|
||||
status="completed",
|
||||
)
|
||||
]
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input=[{"role": "user", "content": "hi"}],
|
||||
previous_response_id="resp_prev",
|
||||
store=False,
|
||||
)
|
||||
try:
|
||||
serving._construct_input_messages_with_harmony(request, prev)
|
||||
except TypeError as exc:
|
||||
self.fail(f"copy() module-call regression: {exc}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class ChatToolForwardingTestCase(unittest.TestCase):
|
||||
def test_make_request_passes_function_tools_to_chat_processing(self):
|
||||
serving = make_serving()
|
||||
seen = {}
|
||||
|
||||
def fake_process(chat_request, is_multimodal):
|
||||
seen["tools"] = chat_request.tools
|
||||
seen["tool_choice"] = chat_request.tool_choice
|
||||
seen["parallel_tool_calls"] = chat_request.parallel_tool_calls
|
||||
return MessageProcessingResult(
|
||||
prompt="prompt",
|
||||
prompt_ids=[1, 2, 3],
|
||||
image_data=None,
|
||||
audio_data=None,
|
||||
video_data=None,
|
||||
modalities=[],
|
||||
stop=["</s>"],
|
||||
tool_call_constraint=("json_schema", {"type": "object"}),
|
||||
)
|
||||
|
||||
serving._process_messages = Mock(side_effect=fake_process)
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="call the tool",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
tool_choice="required",
|
||||
parallel_tool_calls=False,
|
||||
store=False,
|
||||
)
|
||||
|
||||
messages, request_prompts, engine_prompts, processed = asyncio.run(
|
||||
serving._make_request(request, None, serving.tokenizer_manager.tokenizer)
|
||||
)
|
||||
|
||||
self.assertEqual(messages, [{"role": "user", "content": "call the tool"}])
|
||||
self.assertEqual(request_prompts, [[1, 2, 3]])
|
||||
self.assertEqual(engine_prompts, [[1, 2, 3]])
|
||||
self.assertEqual(seen["tools"][0].function.name, "lookup")
|
||||
self.assertEqual(seen["tool_choice"], "required")
|
||||
self.assertFalse(seen["parallel_tool_calls"])
|
||||
self.assertEqual(processed.tool_call_constraint[0], "json_schema")
|
||||
|
||||
def test_required_tool_choice_without_function_tool_returns_400(self):
|
||||
serving = make_serving()
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
tool_choice="required",
|
||||
tools=[{"type": "web_search"}, {"type": "mcp"}],
|
||||
store=False,
|
||||
)
|
||||
result = asyncio.run(serving.create_responses(request, raw_request=None))
|
||||
self.assertEqual(getattr(result, "status_code", None), 400)
|
||||
|
||||
|
||||
class InputItemNormalizationTestCase(unittest.TestCase):
|
||||
def test_function_call_becomes_assistant_tool_call(self):
|
||||
normalized = OpenAIServingResponses._normalize_response_message_for_chat(
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_1",
|
||||
"call_id": "call_abc",
|
||||
"name": "lookup",
|
||||
"arguments": '{"key": "val"}',
|
||||
"status": "completed",
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
normalized,
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_abc",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"arguments": '{"key": "val"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def test_developer_role_becomes_system(self):
|
||||
normalized = OpenAIServingResponses._normalize_response_message_for_chat(
|
||||
{"role": "developer", "content": "Be terse."}
|
||||
)
|
||||
self.assertEqual(normalized, {"role": "system", "content": "Be terse."})
|
||||
|
||||
def test_function_call_output_becomes_tool_message(self):
|
||||
normalized = OpenAIServingResponses._normalize_response_message_for_chat(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_abc",
|
||||
"output": "42",
|
||||
}
|
||||
)
|
||||
self.assertEqual(
|
||||
normalized,
|
||||
{"role": "tool", "tool_call_id": "call_abc", "content": "42"},
|
||||
)
|
||||
|
||||
def test_unknown_input_item_type_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
OpenAIServingResponses._normalize_response_message_for_chat(
|
||||
{"type": "web_search_call", "id": "ws_1"}
|
||||
)
|
||||
|
||||
|
||||
class FullResponseUsageTestCase(unittest.TestCase):
|
||||
def test_full_response_uses_dict_meta_info_for_usage(self):
|
||||
serving = make_serving()
|
||||
context = SimpleContext()
|
||||
context.last_output = {
|
||||
"text": "done",
|
||||
"meta_info": {
|
||||
"prompt_tokens": 11,
|
||||
"completion_tokens": 7,
|
||||
"cached_tokens": 3,
|
||||
"reasoning_tokens": 2,
|
||||
},
|
||||
}
|
||||
request = ResponsesRequest(
|
||||
model="x", input="hello", request_id="resp_usage", store=False
|
||||
)
|
||||
metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||
|
||||
async def empty_generator():
|
||||
if False:
|
||||
yield None
|
||||
|
||||
response = asyncio.run(
|
||||
serving.responses_full_generator(
|
||||
request,
|
||||
sampling_params={},
|
||||
result_generator=empty_generator(),
|
||||
context=context,
|
||||
model_name="x",
|
||||
tokenizer=serving.tokenizer_manager.tokenizer,
|
||||
request_metadata=metadata,
|
||||
created_time=123,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(response.usage.prompt_tokens, 11)
|
||||
self.assertEqual(response.usage.completion_tokens, 7)
|
||||
self.assertEqual(response.usage.reasoning_tokens, 2)
|
||||
self.assertEqual(metadata.final_usage_info, response.usage)
|
||||
|
||||
|
||||
class MultimodalRequestTestCase(unittest.TestCase):
|
||||
def test_multimodal_create_responses_sends_text_and_media_to_engine(self):
|
||||
serving = make_serving(is_multimodal=True)
|
||||
captured = {}
|
||||
|
||||
serving._process_messages = Mock(
|
||||
return_value=MessageProcessingResult(
|
||||
prompt="rendered multimodal prompt",
|
||||
prompt_ids=[9, 9, 9],
|
||||
image_data=["http://example.com/cat.png"],
|
||||
audio_data=None,
|
||||
video_data=None,
|
||||
modalities=["image"],
|
||||
stop=[],
|
||||
)
|
||||
)
|
||||
|
||||
async def fake_generate(
|
||||
request_id,
|
||||
request_prompt,
|
||||
adapted_request,
|
||||
sampling_params,
|
||||
context,
|
||||
**kwargs,
|
||||
):
|
||||
captured["request_prompt"] = request_prompt
|
||||
captured["adapted_request"] = adapted_request
|
||||
context.append_output(
|
||||
{
|
||||
"text": "looks like a cat",
|
||||
"meta_info": {
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 4,
|
||||
"cached_tokens": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
yield context
|
||||
|
||||
serving._generate_with_builtin_tools = fake_generate
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "describe it"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "http://example.com/cat.png",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
request_id="resp_mm",
|
||||
store=False,
|
||||
)
|
||||
|
||||
response = asyncio.run(serving.create_responses(request))
|
||||
|
||||
self.assertEqual(response.status, "completed")
|
||||
self.assertEqual(captured["request_prompt"], "rendered multimodal prompt")
|
||||
self.assertEqual(captured["adapted_request"].text, "rendered multimodal prompt")
|
||||
self.assertIsNone(captured["adapted_request"].input_ids)
|
||||
self.assertEqual(
|
||||
captured["adapted_request"].image_data, ["http://example.com/cat.png"]
|
||||
)
|
||||
self.assertEqual(captured["adapted_request"].modalities, ["image"])
|
||||
|
||||
|
||||
class OutputItemsTestCase(unittest.TestCase):
|
||||
def _function_tool_request(self):
|
||||
return ResponsesRequest(
|
||||
model="x",
|
||||
input="weather?",
|
||||
store=False,
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_function_tool_call_extracted_via_parser(self):
|
||||
serving = make_serving()
|
||||
serving.tool_call_parser = "qwen3_coder"
|
||||
fake_call = ToolCallItem(
|
||||
tool_index=0, name="get_weather", parameters='{"city": "Beijing"}'
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
||||
) as parser_cls:
|
||||
instance = parser_cls.return_value
|
||||
instance.has_tool_call.return_value = True
|
||||
instance.parse_non_stream.return_value = ("trailing text", [fake_call])
|
||||
output_items = serving._make_response_output_items(
|
||||
self._function_tool_request(),
|
||||
"raw model output with <tool_call>",
|
||||
tokenizer=Mock(),
|
||||
)
|
||||
|
||||
tool_calls = [
|
||||
item for item in output_items if isinstance(item, ResponseFunctionToolCall)
|
||||
]
|
||||
self.assertEqual(len(tool_calls), 1)
|
||||
self.assertEqual(tool_calls[0].name, "get_weather")
|
||||
self.assertEqual(tool_calls[0].arguments, '{"city": "Beijing"}')
|
||||
|
||||
message_items = [
|
||||
item for item in output_items if isinstance(item, ResponseOutputMessage)
|
||||
]
|
||||
self.assertEqual(len(message_items), 1)
|
||||
self.assertEqual(message_items[0].content[0].text, "trailing text")
|
||||
|
||||
def test_prose_emitted_before_tool_call_item(self):
|
||||
serving = make_serving()
|
||||
serving.tool_call_parser = "qwen3_coder"
|
||||
fake_call = ToolCallItem(
|
||||
tool_index=0, name="get_weather", parameters='{"city": "Beijing"}'
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
||||
) as parser_cls:
|
||||
instance = parser_cls.return_value
|
||||
instance.has_tool_call.return_value = True
|
||||
instance.parse_non_stream.return_value = (
|
||||
"I'll check the weather.",
|
||||
[fake_call],
|
||||
)
|
||||
output_items = serving._make_response_output_items(
|
||||
self._function_tool_request(), "raw model output", tokenizer=Mock()
|
||||
)
|
||||
|
||||
types = [type(item).__name__ for item in output_items]
|
||||
self.assertEqual(types, ["ResponseOutputMessage", "ResponseFunctionToolCall"])
|
||||
|
||||
def test_required_tool_choice_parses_json_array_without_native_parser(self):
|
||||
serving = make_serving()
|
||||
serving.tool_call_parser = None
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
tool_choice="required",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
store=False,
|
||||
)
|
||||
raw = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]'
|
||||
|
||||
output_items = serving._make_response_output_items(
|
||||
request, raw, tokenizer=Mock()
|
||||
)
|
||||
|
||||
tool_calls = [
|
||||
item for item in output_items if isinstance(item, ResponseFunctionToolCall)
|
||||
]
|
||||
self.assertEqual(len(tool_calls), 1)
|
||||
self.assertEqual(tool_calls[0].name, "get_weather")
|
||||
self.assertEqual(tool_calls[0].arguments, '{"city": "Beijing"}')
|
||||
self.assertEqual(
|
||||
[item for item in output_items if isinstance(item, ResponseOutputMessage)],
|
||||
[],
|
||||
)
|
||||
|
||||
def test_no_tool_call_extraction_when_tool_choice_none(self):
|
||||
serving = make_serving()
|
||||
serving.tool_call_parser = "qwen3_coder"
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
store=False,
|
||||
tool_choice="none",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
||||
) as parser_cls:
|
||||
output_items = serving._make_response_output_items(
|
||||
request, "just a plain answer", tokenizer=Mock()
|
||||
)
|
||||
parser_cls.assert_not_called()
|
||||
|
||||
self.assertEqual(len(output_items), 1)
|
||||
self.assertIsInstance(output_items[0], ResponseOutputMessage)
|
||||
|
||||
|
||||
class HarmonyResponsesTestCase(unittest.TestCase):
|
||||
def test_developer_message_skips_unsupported_tool_types(self):
|
||||
from sglang.srt.entrypoints.harmony_utils import get_developer_message
|
||||
from sglang.srt.entrypoints.openai.protocol import ResponseTool
|
||||
|
||||
tools = [
|
||||
ResponseTool(
|
||||
type="function",
|
||||
name="get_weather",
|
||||
description="Look up weather.",
|
||||
parameters={"type": "object"},
|
||||
),
|
||||
ResponseTool(type="web_search"),
|
||||
ResponseTool(type="namespace", name="codex"),
|
||||
ResponseTool(type="mcp"),
|
||||
]
|
||||
msg = get_developer_message(instructions="be helpful", tools=tools)
|
||||
self.assertIsNotNone(msg)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,211 @@
|
||||
import asyncio
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from utils import (
|
||||
collect_stream_events,
|
||||
event_payloads,
|
||||
event_types,
|
||||
find_completed_event,
|
||||
make_serving,
|
||||
)
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
RequestResponseMetadata,
|
||||
ResponsesRequest,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _StreamFixture:
|
||||
def __init__(self, serving, request):
|
||||
self.serving = serving
|
||||
self.request = request
|
||||
self.request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||
|
||||
def run(self, chunks):
|
||||
async def gen():
|
||||
for ch in chunks:
|
||||
yield ch
|
||||
|
||||
async def collect():
|
||||
return await collect_stream_events(
|
||||
self.serving.responses_stream_generator_non_harmony(
|
||||
self.request,
|
||||
sampling_params={},
|
||||
result_generator=gen(),
|
||||
model_name="x",
|
||||
tokenizer=Mock(),
|
||||
request_metadata=self.request_metadata,
|
||||
)
|
||||
)
|
||||
|
||||
return asyncio.run(collect())
|
||||
|
||||
|
||||
def _engine_chunk(text, completion_tokens, *, finish=False):
|
||||
return {
|
||||
"text": text,
|
||||
"meta_info": {
|
||||
"id": "rid",
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": completion_tokens,
|
||||
"cached_tokens": 0,
|
||||
"reasoning_tokens": 0,
|
||||
"finish_reason": {"type": "stop"} if finish else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class NonHarmonyStreamTestCase(unittest.TestCase):
|
||||
def test_emits_typed_sse_events_in_order(self):
|
||||
serving = make_serving()
|
||||
serving.reasoning_parser = None
|
||||
serving.tool_call_parser = None
|
||||
|
||||
request = ResponsesRequest(model="x", input="hi", stream=True, store=False)
|
||||
fixture = _StreamFixture(serving, request)
|
||||
events = fixture.run(
|
||||
[
|
||||
_engine_chunk("Hel", 1),
|
||||
_engine_chunk("Hello", 2),
|
||||
_engine_chunk("Hello world", 4, finish=True),
|
||||
]
|
||||
)
|
||||
|
||||
types = event_types(events)
|
||||
self.assertEqual(types[0], "response.created")
|
||||
self.assertEqual(types[1], "response.in_progress")
|
||||
for ev in (
|
||||
"response.output_item.added",
|
||||
"response.content_part.added",
|
||||
"response.output_text.delta",
|
||||
"response.output_text.done",
|
||||
"response.content_part.done",
|
||||
"response.output_item.done",
|
||||
):
|
||||
self.assertIn(ev, types)
|
||||
self.assertEqual(types[-1], "response.completed")
|
||||
|
||||
seqs = [p["sequence_number"] for p in event_payloads(events)]
|
||||
self.assertEqual(seqs, list(range(len(seqs))))
|
||||
|
||||
def test_required_tool_choice_emits_function_call_events(self):
|
||||
serving = make_serving()
|
||||
serving.reasoning_parser = None
|
||||
serving.tool_call_parser = None
|
||||
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
stream=True,
|
||||
store=False,
|
||||
tool_choice="required",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
)
|
||||
payload = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]'
|
||||
|
||||
chunks = []
|
||||
sent = 0
|
||||
while sent < len(payload):
|
||||
sent += min(8, len(payload) - sent)
|
||||
chunks.append(
|
||||
_engine_chunk(payload[:sent], sent, finish=sent == len(payload))
|
||||
)
|
||||
|
||||
fixture = _StreamFixture(serving, request)
|
||||
events = fixture.run(chunks)
|
||||
types = event_types(events)
|
||||
|
||||
self.assertIn("response.function_call_arguments.delta", types)
|
||||
self.assertIn("response.function_call_arguments.done", types)
|
||||
self.assertIn("response.output_item.added", types)
|
||||
self.assertIn("response.output_item.done", types)
|
||||
self.assertNotIn("response.output_text.delta", types)
|
||||
|
||||
added_kinds = [
|
||||
payload["item"]["type"]
|
||||
for payload in event_payloads(events)
|
||||
if payload.get("type") == "response.output_item.added"
|
||||
]
|
||||
self.assertIn("function_call", added_kinds)
|
||||
|
||||
def test_final_output_preserves_text_tool_text_order(self):
|
||||
from sglang.srt.function_call.core_types import (
|
||||
StreamingParseResult,
|
||||
ToolCallItem,
|
||||
)
|
||||
|
||||
serving = make_serving()
|
||||
serving.reasoning_parser = None
|
||||
serving.tool_call_parser = "qwen3_coder"
|
||||
|
||||
request = ResponsesRequest(
|
||||
model="x",
|
||||
input="hi",
|
||||
stream=True,
|
||||
store=False,
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
scripted = [
|
||||
StreamingParseResult(normal_text="I'll check.", calls=[]),
|
||||
StreamingParseResult(
|
||||
normal_text="",
|
||||
calls=[
|
||||
ToolCallItem(
|
||||
tool_index=0,
|
||||
name="get_weather",
|
||||
parameters='{"city": "Beijing"}',
|
||||
)
|
||||
],
|
||||
),
|
||||
StreamingParseResult(normal_text="It's sunny.", calls=[]),
|
||||
]
|
||||
chunks = [
|
||||
_engine_chunk(" " * 3, 3),
|
||||
_engine_chunk(" " * 10, 10),
|
||||
_engine_chunk(" " * 14, 14, finish=True),
|
||||
]
|
||||
|
||||
script_iter = iter(scripted)
|
||||
|
||||
def fake_parse_stream_chunk(delta):
|
||||
sp = next(script_iter)
|
||||
return sp.normal_text, sp.calls
|
||||
|
||||
with patch(
|
||||
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
||||
) as parser_cls:
|
||||
parser_cls.return_value.detector.supports_structural_tag.return_value = True
|
||||
parser_cls.return_value.parse_stream_chunk.side_effect = (
|
||||
fake_parse_stream_chunk
|
||||
)
|
||||
fixture = _StreamFixture(serving, request)
|
||||
events = fixture.run(chunks)
|
||||
|
||||
completed = find_completed_event(events)
|
||||
output = completed["response"]["output"]
|
||||
kinds = [item["type"] for item in output]
|
||||
self.assertEqual(kinds, ["message", "function_call", "message"])
|
||||
self.assertEqual(output[0]["content"][0]["text"], "I'll check.")
|
||||
self.assertEqual(output[1]["name"], "get_weather")
|
||||
self.assertEqual(output[2]["content"][0]["text"], "It's sunny.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Stub CUDA-only deps before importing sglang.srt serving modules. Must
|
||||
be imported first by every /v1/responses test that runs on CPU."""
|
||||
|
||||
try:
|
||||
import torch
|
||||
|
||||
_ORIGINAL_TORCH_COMPILE = torch.compile
|
||||
|
||||
def _identity_compile(fn=None, **kwargs):
|
||||
if fn is None:
|
||||
return lambda inner_fn: inner_fn
|
||||
return fn
|
||||
|
||||
torch.compile = _identity_compile
|
||||
except ImportError:
|
||||
torch = None
|
||||
_ORIGINAL_TORCH_COMPILE = None
|
||||
|
||||
from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
import json
|
||||
from typing import AsyncIterator
|
||||
from unittest.mock import Mock
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_responses import OpenAIServingResponses
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(
|
||||
est_time=0,
|
||||
suite="base-a-test-cpu",
|
||||
disabled="helper module — exported fixtures, not a test",
|
||||
)
|
||||
|
||||
if torch is not None:
|
||||
torch.compile = _ORIGINAL_TORCH_COMPILE
|
||||
|
||||
|
||||
class MockTokenizerManager:
|
||||
def __init__(self, *, is_multimodal: bool = False):
|
||||
self.model_config = Mock(is_multimodal=is_multimodal, context_len=4096)
|
||||
self.model_config.get_default_sampling_params.return_value = {}
|
||||
self.model_config.hf_config = Mock(
|
||||
model_type="llama", architectures=["LlamaForCausalLM"]
|
||||
)
|
||||
self.server_args = Mock(
|
||||
enable_cache_report=False,
|
||||
reasoning_parser=None,
|
||||
stream_response_default_include_usage=False,
|
||||
tokenizer_metrics_allowed_custom_labels=None,
|
||||
tool_call_parser=None,
|
||||
incremental_streaming_output=False,
|
||||
)
|
||||
self.tokenizer = Mock()
|
||||
self.tokenizer.encode.return_value = [1, 2, 3]
|
||||
self.tokenizer.chat_template = None
|
||||
self.tokenizer.bos_token_id = 1
|
||||
self.num_reserved_tokens = 0
|
||||
self.generate_request = Mock()
|
||||
self.create_abort_task = Mock()
|
||||
|
||||
|
||||
class MockTemplateManager:
|
||||
def __init__(self):
|
||||
self.chat_template_name = "llama-3"
|
||||
self.jinja_template_content_format = None
|
||||
self.completion_template_name = None
|
||||
self.reasoning_config = None
|
||||
self.force_reasoning = False
|
||||
|
||||
|
||||
def make_serving(*, is_multimodal: bool = False) -> OpenAIServingResponses:
|
||||
return OpenAIServingResponses(
|
||||
MockTokenizerManager(is_multimodal=is_multimodal), MockTemplateManager()
|
||||
)
|
||||
|
||||
|
||||
async def collect_stream_events(stream: AsyncIterator[str]) -> list[str]:
|
||||
events = []
|
||||
async for chunk in stream:
|
||||
events.append(chunk)
|
||||
return events
|
||||
|
||||
|
||||
def event_types(events: list[str]) -> list[str]:
|
||||
return [
|
||||
line[len("event: ") :].strip()
|
||||
for chunk in events
|
||||
for line in chunk.splitlines()
|
||||
if line.startswith("event: ")
|
||||
]
|
||||
|
||||
|
||||
def event_payloads(events: list[str]) -> list[dict]:
|
||||
return [
|
||||
json.loads(line[len("data: ") :])
|
||||
for chunk in events
|
||||
for line in chunk.splitlines()
|
||||
if line.startswith("data: ")
|
||||
]
|
||||
|
||||
|
||||
def find_completed_event(events: list[str]) -> dict:
|
||||
for chunk in events:
|
||||
lines = chunk.splitlines()
|
||||
if lines and lines[0] == "event: response.completed":
|
||||
return json.loads(lines[1][len("data: ") :])
|
||||
raise AssertionError("response.completed event missing from stream")
|
||||
Reference in New Issue
Block a user