2260 lines
84 KiB
Python
2260 lines
84 KiB
Python
import argparse
|
|
import asyncio
|
|
import functools
|
|
import sys
|
|
import unittest
|
|
from contextlib import asynccontextmanager
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock, patch
|
|
|
|
import orjson
|
|
import pytest
|
|
from openai.types.responses import (
|
|
ResponseOutputMessage,
|
|
ResponseOutputText,
|
|
ResponseReasoningItem,
|
|
)
|
|
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
|
from openai_harmony import Conversation, Message, Role, ToolNamespaceConfig
|
|
from utils import (
|
|
MockTemplateManager,
|
|
MockTokenizerManager,
|
|
StreamFixture,
|
|
engine_chunk,
|
|
event_payloads,
|
|
make_serving,
|
|
)
|
|
|
|
from sglang.srt.entrypoints.context import (
|
|
HarmonyContext,
|
|
SimpleContext,
|
|
)
|
|
from sglang.srt.entrypoints.harmony_utils import get_encoding
|
|
from sglang.srt.entrypoints.openai.protocol import (
|
|
MessageProcessingResult,
|
|
RequestResponseMetadata,
|
|
ResponsesRequest,
|
|
ResponsesResponse,
|
|
)
|
|
from sglang.srt.entrypoints.openai.serving_responses import (
|
|
OpenAIServingResponses,
|
|
_build_output_text_logprobs,
|
|
)
|
|
from sglang.srt.function_call.core_types import ToolCallItem
|
|
from sglang.srt.parser.response_template import ResponseTemplateReasoningDetector
|
|
from sglang.srt.parser.template_detection import ReasoningToggleConfig
|
|
from sglang.srt.runtime_context import get_serving, publish, reset_context
|
|
from sglang.srt.sampling.sampling_params import (
|
|
REQUEST_REASONING_END_TOKEN_IDS_KEY,
|
|
)
|
|
from sglang.srt.server_args import ServerArgs
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
|
|
|
|
|
class InputMessageConstructionTestCase(CustomTestCase):
|
|
def test_previous_response_replays_assistant_text_not_instructions(self):
|
|
serving = make_serving()
|
|
prev_response = Mock(id="resp_prev")
|
|
prev_response.output = [
|
|
ResponseReasoningItem(
|
|
id="rs_prev", summary=[], type="reasoning", content=None, status=None
|
|
),
|
|
ResponseOutputMessage(
|
|
id="msg_prev",
|
|
content=[
|
|
ResponseOutputText(
|
|
text="first answer part",
|
|
annotations=[],
|
|
type="output_text",
|
|
logprobs=None,
|
|
),
|
|
ResponseOutputText(
|
|
text="second answer part",
|
|
annotations=[],
|
|
type="output_text",
|
|
logprobs=None,
|
|
),
|
|
],
|
|
role="assistant",
|
|
status="completed",
|
|
type="message",
|
|
),
|
|
]
|
|
serving.msg_store["resp_prev"] = [
|
|
{"role": "user", "content": "old input"},
|
|
*[item.model_dump(exclude_none=True) for item in prev_response.output],
|
|
]
|
|
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
instructions="Be brief",
|
|
previous_response_id="resp_prev",
|
|
input="new input",
|
|
store=False,
|
|
)
|
|
|
|
messages = serving._construct_input_messages(request, prev_response)
|
|
|
|
self.assertEqual(
|
|
messages,
|
|
[
|
|
{"role": "system", "content": "Be brief"},
|
|
{"role": "user", "content": "old input"},
|
|
{
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "text", "text": "first answer part"},
|
|
{"type": "text", "text": "second answer part"},
|
|
],
|
|
},
|
|
{"role": "user", "content": "new input"},
|
|
],
|
|
)
|
|
|
|
def test_stored_tool_turn_matches_client_replay_without_old_instructions(self):
|
|
publish(
|
|
ServerArgs(model_path="dummy", enable_response_store=True), role="tokenizer"
|
|
)
|
|
for stream in (False, True):
|
|
with self.subTest(stream=stream):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "deepseek-r1"
|
|
serving.tool_call_parser = None
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="old input",
|
|
instructions="OLD INSTRUCTION",
|
|
tools=[{"type": "function", "name": "lookup"}],
|
|
tool_choice="required",
|
|
store=True,
|
|
stream=stream,
|
|
)
|
|
chunk = engine_chunk(
|
|
'<think>secret plan</think>[{"name":"lookup","parameters":{}}]',
|
|
finish=True,
|
|
)
|
|
if stream:
|
|
StreamFixture(serving, request).run([chunk])
|
|
response = serving.response_store[request.request_id]
|
|
else:
|
|
context = SimpleContext()
|
|
context.append_output(chunk)
|
|
|
|
async def empty():
|
|
if False:
|
|
yield
|
|
|
|
response = asyncio.run(
|
|
serving.responses_full_generator(
|
|
request,
|
|
{},
|
|
empty(),
|
|
context,
|
|
"x",
|
|
Mock(),
|
|
RequestResponseMetadata(request_id=request.request_id),
|
|
require_reasoning=False,
|
|
)
|
|
)
|
|
call = next(
|
|
item for item in response.output if item.type == "function_call"
|
|
)
|
|
result = {
|
|
"type": "function_call_output",
|
|
"call_id": call.call_id,
|
|
"output": "answer 42",
|
|
}
|
|
for instructions in ("NEW INSTRUCTION", None):
|
|
followup = ResponsesRequest(
|
|
model="x",
|
|
previous_response_id=response.id,
|
|
input=[result],
|
|
instructions=instructions,
|
|
store=False,
|
|
)
|
|
explicit = ResponsesRequest(
|
|
model="x",
|
|
input=[{"role": "user", "content": "old input"}]
|
|
+ [item.model_dump() for item in response.output]
|
|
+ [result],
|
|
instructions=instructions,
|
|
store=False,
|
|
)
|
|
messages = serving._construct_input_messages(followup, response)
|
|
self.assertEqual(
|
|
messages, serving._construct_input_messages(explicit)
|
|
)
|
|
self.assertNotIn("OLD INSTRUCTION", str(messages))
|
|
self.assertIn("secret plan", str(messages))
|
|
self.assertIn(call.call_id, str(messages))
|
|
self.assertEqual(
|
|
[item["type"] for item in serving.msg_store[response.id][1:]],
|
|
["reasoning", "function_call"],
|
|
)
|
|
|
|
def test_harmony_instructions_are_rebuilt_for_each_request(self):
|
|
serving = make_serving()
|
|
serving.use_harmony = True
|
|
previous = Mock(id="resp_previous", output=[])
|
|
first = ResponsesRequest(
|
|
model="x", input="old input", instructions="OLD INSTRUCTION"
|
|
)
|
|
messages = serving._construct_input_messages_with_harmony(first, None)
|
|
serving.msg_store[previous.id] = messages[2:]
|
|
for instructions in ("NEW INSTRUCTION", None):
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="next",
|
|
instructions=instructions,
|
|
previous_response_id=previous.id,
|
|
)
|
|
actual = serving._construct_input_messages_with_harmony(request, previous)
|
|
expected = serving._construct_input_messages_with_harmony(request, None)
|
|
self.assertEqual(actual[:2], expected[:2])
|
|
self.assertEqual(actual[2:], messages[2:] + expected[2:])
|
|
|
|
def test_harmony_replays_output_text_and_encoded_reasoning(self):
|
|
from sglang.srt.entrypoints.harmony_utils import parse_response_input
|
|
from sglang.srt.entrypoints.openai.responses_adapters import (
|
|
encode_reasoning_state,
|
|
)
|
|
|
|
message = parse_response_input(
|
|
{
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": [
|
|
{"type": "output_text", "text": "assistant-only secret 42"},
|
|
],
|
|
"phase": "final_answer",
|
|
},
|
|
[],
|
|
)
|
|
self.assertEqual(message.content[0].text, "assistant-only secret 42")
|
|
reasoning = parse_response_input(
|
|
{
|
|
"type": "reasoning",
|
|
"encrypted_content": encode_reasoning_state("private plan"),
|
|
},
|
|
[],
|
|
)
|
|
self.assertEqual(reasoning.content[0].text, "private plan")
|
|
self.assertEqual(reasoning.channel, "analysis")
|
|
|
|
def test_harmony_replays_dict_tool_calls_and_results_in_one_input(self):
|
|
serving = make_serving()
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input=[
|
|
{
|
|
"type": "function_call",
|
|
"name": "lookup",
|
|
"call_id": "call_1",
|
|
"arguments": "{}",
|
|
},
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_1",
|
|
"output": [{"type": "output_text", "text": "result"}],
|
|
},
|
|
],
|
|
store=False,
|
|
)
|
|
messages = serving._construct_input_messages_with_harmony(request, None)
|
|
self.assertEqual(messages[-1].author.name, "functions.lookup")
|
|
self.assertEqual(messages[-1].content[0].text, "result")
|
|
|
|
def test_harmony_message_channels_map_to_phases(self):
|
|
from sglang.srt.entrypoints.harmony_utils import (
|
|
parse_output_message,
|
|
parse_response_input,
|
|
)
|
|
|
|
for phase in ("commentary", "final_answer"):
|
|
message = parse_response_input(
|
|
{
|
|
"role": "assistant",
|
|
"content": "answer",
|
|
"phase": phase,
|
|
},
|
|
[],
|
|
)
|
|
(item,) = parse_output_message(message)
|
|
self.assertEqual(item.phase, phase)
|
|
self.assertEqual(item.content[0].text, "answer")
|
|
|
|
def test_replay_preserves_different_assistant_phases(self):
|
|
serving = make_serving()
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input=[
|
|
{"role": "assistant", "content": "working", "phase": "commentary"},
|
|
{"role": "assistant", "content": "answer", "phase": "final_answer"},
|
|
],
|
|
store=False,
|
|
)
|
|
messages = serving._construct_input_messages(request)
|
|
self.assertEqual([m["phase"] for m in messages], ["commentary", "final_answer"])
|
|
self.assertEqual([m["content"] for m in messages], ["working", "answer"])
|
|
|
|
def test_input_parts_normalized_for_chat_templates(self):
|
|
serving = make_serving()
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input=[
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "input_text", "text": "what is this?"},
|
|
{
|
|
"type": "input_image",
|
|
"image_url": "http://example.com/cat.png",
|
|
},
|
|
],
|
|
}
|
|
],
|
|
store=False,
|
|
)
|
|
|
|
messages = serving._construct_input_messages(request)
|
|
|
|
self.assertEqual(
|
|
messages,
|
|
[
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": "what is this?"},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url": "http://example.com/cat.png",
|
|
"detail": "auto",
|
|
},
|
|
},
|
|
],
|
|
}
|
|
],
|
|
)
|
|
|
|
def test_previous_response_id_input_list_does_not_call_copy_module(self):
|
|
serving = make_serving()
|
|
serving.use_harmony = True
|
|
prev = Mock(id="resp_prev")
|
|
prev.output = [
|
|
ResponseFunctionToolCall(
|
|
arguments="{}",
|
|
call_id="call_x",
|
|
name="t",
|
|
type="function_call",
|
|
id="fc_x",
|
|
status="completed",
|
|
)
|
|
]
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input=[{"role": "user", "content": "hi"}],
|
|
previous_response_id="resp_prev",
|
|
store=False,
|
|
)
|
|
try:
|
|
serving._construct_input_messages_with_harmony(request, prev)
|
|
except TypeError as exc:
|
|
self.fail(f"copy() module-call regression: {exc}")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
class ChatToolForwardingTestCase(CustomTestCase):
|
|
def test_make_request_passes_function_tools_to_chat_processing(self):
|
|
serving = make_serving()
|
|
seen = {}
|
|
|
|
def fake_process(chat_request, is_multimodal):
|
|
seen["tools"] = chat_request.tools
|
|
seen["tool_choice"] = chat_request.tool_choice
|
|
seen["parallel_tool_calls"] = chat_request.parallel_tool_calls
|
|
return MessageProcessingResult(
|
|
prompt="prompt",
|
|
prompt_ids=[1, 2, 3],
|
|
image_data=None,
|
|
audio_data=None,
|
|
video_data=None,
|
|
modalities=[],
|
|
stop=["</s>"],
|
|
tool_call_constraint=("json_schema", {"type": "object"}),
|
|
)
|
|
|
|
serving._process_messages = Mock(side_effect=fake_process)
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="call the tool",
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "lookup",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
tool_choice="required",
|
|
parallel_tool_calls=False,
|
|
store=False,
|
|
)
|
|
|
|
messages, request_prompts, engine_prompts, processed = asyncio.run(
|
|
serving._make_request(request, None, serving.tokenizer_manager.tokenizer)
|
|
)
|
|
|
|
self.assertEqual(messages, [{"role": "user", "content": "call the tool"}])
|
|
self.assertEqual(request_prompts, [[1, 2, 3]])
|
|
self.assertEqual(engine_prompts, [[1, 2, 3]])
|
|
self.assertEqual(seen["tools"][0].function.name, "lookup")
|
|
self.assertEqual(seen["tool_choice"], "required")
|
|
self.assertFalse(seen["parallel_tool_calls"])
|
|
self.assertEqual(processed.tool_call_constraint[0], "json_schema")
|
|
|
|
def test_harmony_forced_choices_explain_missing_routing_constraints(self):
|
|
serving = make_serving()
|
|
serving.use_harmony = True
|
|
for choice in ("none", "required", {"type": "function", "name": "lookup"}):
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
tool_choice=choice,
|
|
tools=[{"type": "function", "name": "lookup"}],
|
|
store=False,
|
|
)
|
|
response = asyncio.run(serving.create_responses(request))
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertIn(b"recipient", response.body)
|
|
self.assertIn(b"tool_choice", response.body)
|
|
serving.tokenizer_manager.generate_request.assert_not_called()
|
|
|
|
def test_required_tool_choice_without_function_tool_returns_400(self):
|
|
serving = make_serving()
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
tool_choice="required",
|
|
tools=[{"type": "web_search"}, {"type": "mcp"}],
|
|
store=False,
|
|
)
|
|
result = asyncio.run(serving.create_responses(request, raw_request=None))
|
|
self.assertEqual(getattr(result, "status_code", None), 400)
|
|
|
|
def test_kimi_k3_request_uses_chat_encoder_fields(self):
|
|
serving = make_serving()
|
|
serving.chat_encoding_spec = "kimi_k3"
|
|
serving.default_chat_template_kwargs = {}
|
|
serving.template_manager.chat_template_name = None
|
|
serving.tokenizer_manager.tokenizer.apply_chat_template.return_value = [4, 5, 6]
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="Explain <|kimi_image_placeholder|>",
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "lookup",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
tool_choice="required",
|
|
reasoning={"effort": "high"},
|
|
store=False,
|
|
)
|
|
|
|
_, request_prompts, engine_prompts, _ = asyncio.run(
|
|
serving._make_request(request, None, serving.tokenizer_manager.tokenizer)
|
|
)
|
|
|
|
call = serving.tokenizer_manager.tokenizer.apply_chat_template.call_args
|
|
self.assertEqual(
|
|
call.args[0][0]["content"], "Explain <| kimi_image_placeholder |>"
|
|
)
|
|
self.assertEqual(call.kwargs["thinking_effort"], "high")
|
|
self.assertEqual(call.kwargs["tool_choice"], "required")
|
|
self.assertEqual(call.kwargs["tools"][0]["function"]["name"], "lookup")
|
|
self.assertEqual(request_prompts, [[4, 5, 6]])
|
|
self.assertEqual(engine_prompts, [[4, 5, 6]])
|
|
|
|
def test_k2_output_parser_reuses_effective_template_default(self):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "k2_horizon"
|
|
serving.default_chat_template_kwargs = {"reasoning_effort": "low"}
|
|
serving.template_manager.chat_template_name = None
|
|
serving.tokenizer_manager.tokenizer.apply_chat_template.return_value = [4, 5, 6]
|
|
request = ResponsesRequest(
|
|
model="IFM/K2-Horizon-7B",
|
|
input="hi",
|
|
# Template kwargs are the final render inputs, so the server default
|
|
# below takes precedence over this API convenience field.
|
|
reasoning={"effort": "medium"},
|
|
store=False,
|
|
)
|
|
|
|
asyncio.run(
|
|
serving._make_request(request, None, serving.tokenizer_manager.tokenizer)
|
|
)
|
|
|
|
render_call = serving.tokenizer_manager.tokenizer.apply_chat_template.call_args
|
|
self.assertEqual(render_call.kwargs["reasoning_effort"], "low")
|
|
self.assertEqual(request.chat_template_kwargs["reasoning_effort"], "low")
|
|
|
|
output_items = serving._make_response_output_items(
|
|
request,
|
|
"work</ifm|think_faster>\nanswer",
|
|
tokenizer=Mock(),
|
|
require_reasoning=True,
|
|
)
|
|
self.assertEqual(output_items[0].content[0].text, "work")
|
|
self.assertEqual(output_items[1].content[0].text, "\nanswer")
|
|
|
|
|
|
class ReasoningRequestForwardingTestCase(unittest.TestCase):
|
|
def test_create_responses_uses_processed_reasoning_state(self):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "deepseek-r1"
|
|
serving.default_chat_template_kwargs = {"thinking": False}
|
|
serving.template_manager.reasoning_config = ReasoningToggleConfig(
|
|
toggle_param="thinking", default_enabled=True
|
|
)
|
|
rendered = MessageProcessingResult(
|
|
prompt="prompt",
|
|
prompt_ids=[1, 2, 3],
|
|
image_data=None,
|
|
audio_data=None,
|
|
video_data=None,
|
|
modalities=[],
|
|
stop=[],
|
|
reasoning_end_token_ids=[41, 42],
|
|
)
|
|
captured = {}
|
|
|
|
async def fake_generate(
|
|
request_id,
|
|
request_prompt,
|
|
adapted_request,
|
|
sampling_params,
|
|
context,
|
|
**kwargs,
|
|
):
|
|
captured["adapted_request"] = adapted_request
|
|
context.append_output(
|
|
{
|
|
"text": "done",
|
|
"meta_info": {
|
|
"prompt_tokens": 3,
|
|
"completion_tokens": 1,
|
|
"cached_tokens": 0,
|
|
},
|
|
}
|
|
)
|
|
yield context
|
|
|
|
serving._generate_with_builtin_tools = fake_generate
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="answer",
|
|
request_id="resp_reasoning",
|
|
store=False,
|
|
)
|
|
|
|
with (
|
|
patch.object(
|
|
serving, "_apply_conversation_template", return_value=rendered
|
|
),
|
|
patch(
|
|
"sglang.srt.entrypoints.openai.serving_responses.ReasoningParser"
|
|
) as parser_cls,
|
|
):
|
|
parser_cls.return_value.parse_non_stream.return_value = (None, "done")
|
|
response = asyncio.run(serving.create_responses(request))
|
|
|
|
self.assertEqual(response.status, "completed")
|
|
self.assertFalse(captured["adapted_request"].require_reasoning)
|
|
self.assertEqual(
|
|
captured["adapted_request"].sampling_params["custom_params"][
|
|
REQUEST_REASONING_END_TOKEN_IDS_KEY
|
|
],
|
|
[41, 42],
|
|
)
|
|
self.assertFalse(parser_cls.call_args.kwargs["force_reasoning"])
|
|
|
|
|
|
class SkipSpecialTokensForwardingTestCase(CustomTestCase):
|
|
"""The skip_special_tokens override from _process_messages must reach the
|
|
engine sampling params; muse's channel markers die in detok otherwise."""
|
|
|
|
def _create_responses_sampling_params(self, serving):
|
|
serving.default_chat_template_kwargs = None
|
|
rendered = MessageProcessingResult(
|
|
prompt="prompt",
|
|
prompt_ids=[1, 2, 3],
|
|
image_data=None,
|
|
audio_data=None,
|
|
video_data=None,
|
|
modalities=[],
|
|
stop=[],
|
|
)
|
|
captured = {}
|
|
|
|
async def fake_generate(
|
|
request_id,
|
|
request_prompt,
|
|
adapted_request,
|
|
sampling_params,
|
|
context,
|
|
**kwargs,
|
|
):
|
|
captured["sampling_params"] = sampling_params
|
|
context.append_output(
|
|
{
|
|
"text": "done",
|
|
"meta_info": {
|
|
"prompt_tokens": 3,
|
|
"completion_tokens": 1,
|
|
"cached_tokens": 0,
|
|
},
|
|
}
|
|
)
|
|
yield context
|
|
|
|
serving._generate_with_builtin_tools = fake_generate
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="answer",
|
|
request_id="resp_skip_special",
|
|
store=False,
|
|
)
|
|
|
|
with (
|
|
patch.object(
|
|
serving, "_apply_conversation_template", return_value=rendered
|
|
),
|
|
patch(
|
|
"sglang.srt.entrypoints.openai.serving_responses.ReasoningParser"
|
|
) as parser_cls,
|
|
):
|
|
parser_cls.return_value.parse_non_stream.return_value = (None, "done")
|
|
response = asyncio.run(serving.create_responses(request))
|
|
|
|
self.assertEqual(response.status, "completed")
|
|
return captured["sampling_params"]
|
|
|
|
def test_marker_preserving_parser_disables_skip_special_tokens(self):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "muse"
|
|
params = self._create_responses_sampling_params(serving)
|
|
self.assertFalse(params["skip_special_tokens"])
|
|
|
|
def test_default_parser_keeps_skip_special_tokens(self):
|
|
serving = make_serving()
|
|
params = self._create_responses_sampling_params(serving)
|
|
# The chat request's True is a synthesized default (ResponsesRequest has
|
|
# no such field), so leave it unset for --preferred-sampling-params.
|
|
self.assertNotIn("skip_special_tokens", params)
|
|
|
|
def test_response_template_parser_keeps_special_tokens_unspaced(self):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "response_template"
|
|
serving._reasoning_detector = ResponseTemplateReasoningDetector(
|
|
response_template={
|
|
"start_anchor": "<assistant>",
|
|
"fields": {"content": {"content": "text"}},
|
|
},
|
|
)
|
|
params = self._create_responses_sampling_params(serving)
|
|
self.assertFalse(params["skip_special_tokens"])
|
|
self.assertFalse(params["spaces_between_special_tokens"])
|
|
|
|
|
|
class ResponseTemplateRequiredToolTestCase(CustomTestCase):
|
|
# Every field is delimited, so the template gives no field to the JSON.
|
|
TEMPLATE = {
|
|
"start_anchor": "<assistant>",
|
|
"fields": {
|
|
"thinking": {"open": "<think>", "close": "</think>"},
|
|
"content": {"open": "<answer>", "close": "</answer>"},
|
|
"tool_calls": {
|
|
"open_pattern": r"<call:(?P<name>\w+)>",
|
|
"close": "</call>",
|
|
"content": "json",
|
|
"repeats": True,
|
|
"transform": {
|
|
"type": "function",
|
|
"function": {"name": "{name}", "arguments": "{content}"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def _output(text, matched):
|
|
return {
|
|
"text": text,
|
|
"meta_info": {
|
|
"prompt_tokens": 3,
|
|
"completion_tokens": 9,
|
|
"cached_tokens": 0,
|
|
"finish_reason": {"type": "stop", "matched": matched},
|
|
},
|
|
}
|
|
|
|
def _respond(
|
|
self, stream, *, output, tool_call_parser="response_template", **request_fields
|
|
):
|
|
serving = make_serving()
|
|
serving.default_chat_template_kwargs = None
|
|
serving.reasoning_parser = "response_template"
|
|
serving.tool_call_parser = tool_call_parser
|
|
serving._reasoning_detector = ResponseTemplateReasoningDetector(
|
|
response_template=self.TEMPLATE
|
|
)
|
|
# Always-on reasoning: the output grammar starts after the reasoning.
|
|
serving.template_manager.reasoning_config = ReasoningToggleConfig(
|
|
special_case="always"
|
|
)
|
|
tokenizer = serving.tokenizer_manager.tokenizer
|
|
tokenizer.response_template = self.TEMPLATE
|
|
tokenizer.encode.side_effect = lambda literal, **_: (
|
|
[7] if literal == "</call>" else [1, 2, 3]
|
|
)
|
|
tokenizer.decode.return_value = "<assistant>"
|
|
|
|
async def fake_generate(
|
|
request_id, request_prompt, adapted_request, sampling_params, context, **_
|
|
):
|
|
context.append_output(output)
|
|
yield context
|
|
|
|
serving._generate_with_builtin_tools = fake_generate
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="Weather?",
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"city": {"type": "string"}},
|
|
},
|
|
}
|
|
],
|
|
stream=stream,
|
|
store=False,
|
|
**request_fields,
|
|
)
|
|
rendered = MessageProcessingResult(
|
|
prompt="<assistant>",
|
|
prompt_ids=[1, 2, 3],
|
|
image_data=None,
|
|
audio_data=None,
|
|
video_data=None,
|
|
modalities=[],
|
|
stop=[],
|
|
)
|
|
|
|
async def respond():
|
|
response = await serving.create_responses(request)
|
|
if stream:
|
|
events = [event async for event in response]
|
|
return event_payloads(events)[-1]["response"]
|
|
return response.model_dump()
|
|
|
|
with patch.object(
|
|
serving, "_apply_conversation_template", return_value=rendered
|
|
):
|
|
return asyncio.run(respond())
|
|
|
|
def test_required_tool_json_is_parsed_under_delimited_template(self):
|
|
"""Required-tool JSON after the reasoning becomes a completed function
|
|
call, also when it ends on the tool closer that detokenization keeps."""
|
|
output = self._output(
|
|
"<think>Check the forecast.</think>"
|
|
'[{"name": "get_weather", "parameters": {"city": "Paris"}}]</call>',
|
|
matched=7,
|
|
)
|
|
for stream in (False, True):
|
|
with self.subTest(stream=stream):
|
|
response = self._respond(stream, output=output, tool_choice="required")
|
|
|
|
self.assertEqual(response["status"], "completed")
|
|
self.assertEqual(
|
|
[item["type"] for item in response["output"]],
|
|
["reasoning", "function_call"],
|
|
)
|
|
call = response["output"][1]
|
|
self.assertEqual(call["name"], "get_weather")
|
|
self.assertEqual(orjson.loads(call["arguments"]), {"city": "Paris"})
|
|
|
|
def test_json_text_format_with_auto_tools_stays_message(self):
|
|
"""A JSON text format with automatic tools must return the JSON intact;
|
|
template tool syntax inside its strings is never a function call."""
|
|
payload = '{"example": "<call:get_weather>{}</call>"}'
|
|
output = self._output(f"<think>Plan</think>{payload}", matched=2)
|
|
for stream in (False, True):
|
|
with self.subTest(stream=stream):
|
|
response = self._respond(
|
|
stream,
|
|
output=output,
|
|
tool_choice="auto",
|
|
text={"format": {"type": "json_object"}},
|
|
)
|
|
|
|
self.assertEqual(response["status"], "completed")
|
|
self.assertEqual(
|
|
[item["type"] for item in response["output"]],
|
|
["reasoning", "message"],
|
|
)
|
|
self.assertEqual(response["output"][1]["content"][0]["text"], payload)
|
|
|
|
def test_json_text_format_with_natively_required_tools_stays_message(self):
|
|
"""A required or named choice that a tool parser owns natively accepts a
|
|
JSON text format; the JSON answer stays a message, never a tool call."""
|
|
payload = '{"name": "get_weather", "parameters": {"city": "Paris"}}'
|
|
output = self._output(f"<think>Plan</think>{payload}", matched=2)
|
|
for tool_choice in ("required", {"type": "function", "name": "get_weather"}):
|
|
with self.subTest(tool_choice=tool_choice):
|
|
response = self._respond(
|
|
False,
|
|
output=output,
|
|
tool_call_parser="glm47",
|
|
tool_choice=tool_choice,
|
|
text={"format": {"type": "json_object"}},
|
|
)
|
|
|
|
self.assertEqual(response["status"], "completed")
|
|
self.assertEqual(
|
|
[item["type"] for item in response["output"]],
|
|
["reasoning", "message"],
|
|
)
|
|
self.assertEqual(response["output"][1]["content"][0]["text"], payload)
|
|
|
|
|
|
class InputItemNormalizationTestCase(CustomTestCase):
|
|
def test_function_call_becomes_assistant_tool_call(self):
|
|
normalized = OpenAIServingResponses._normalize_response_message_for_chat(
|
|
{
|
|
"type": "function_call",
|
|
"id": "fc_1",
|
|
"call_id": "call_abc",
|
|
"name": "lookup",
|
|
"arguments": '{"key": "val"}',
|
|
"status": "completed",
|
|
}
|
|
)
|
|
self.assertEqual(
|
|
normalized,
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_abc",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "lookup",
|
|
"arguments": '{"key": "val"}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
)
|
|
|
|
def test_developer_role_becomes_labelled_system(self):
|
|
normalized = OpenAIServingResponses._normalize_response_message_for_chat(
|
|
{"role": "developer", "content": "Be terse."}
|
|
)
|
|
self.assertEqual(
|
|
normalized,
|
|
{"role": "system", "content": "Developer instructions:\nBe terse."},
|
|
)
|
|
|
|
def test_function_call_output_becomes_tool_message(self):
|
|
normalized = OpenAIServingResponses._normalize_response_message_for_chat(
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_abc",
|
|
"output": "42",
|
|
}
|
|
)
|
|
self.assertEqual(
|
|
normalized,
|
|
{"role": "tool", "tool_call_id": "call_abc", "content": "42"},
|
|
)
|
|
|
|
def test_unknown_input_item_type_raises(self):
|
|
with self.assertRaises(ValueError):
|
|
OpenAIServingResponses._normalize_response_message_for_chat(
|
|
{"type": "web_search_call", "id": "ws_1"}
|
|
)
|
|
|
|
|
|
class FullResponseUsageTestCase(CustomTestCase):
|
|
def test_full_response_uses_dict_meta_info_for_usage(self):
|
|
serving = make_serving()
|
|
context = SimpleContext()
|
|
context.last_output = {
|
|
"text": "done",
|
|
"meta_info": {
|
|
"prompt_tokens": 11,
|
|
"completion_tokens": 7,
|
|
"cached_tokens": 3,
|
|
"reasoning_tokens": 2,
|
|
},
|
|
}
|
|
request = ResponsesRequest(
|
|
model="x", input="hello", request_id="resp_usage", store=False
|
|
)
|
|
metadata = RequestResponseMetadata(request_id=request.request_id)
|
|
|
|
async def empty_generator():
|
|
for _ in ():
|
|
yield None
|
|
|
|
response = asyncio.run(
|
|
serving.responses_full_generator(
|
|
request,
|
|
sampling_params={},
|
|
result_generator=empty_generator(),
|
|
context=context,
|
|
model_name="x",
|
|
tokenizer=serving.tokenizer_manager.tokenizer,
|
|
request_metadata=metadata,
|
|
created_time=123,
|
|
require_reasoning=False,
|
|
)
|
|
)
|
|
|
|
self.assertEqual(response.usage.prompt_tokens, 11)
|
|
self.assertEqual(response.usage.completion_tokens, 7)
|
|
self.assertEqual(response.usage.reasoning_tokens, 2)
|
|
self.assertEqual(metadata.final_usage_info, response.usage)
|
|
|
|
|
|
class MultimodalRequestTestCase(CustomTestCase):
|
|
def test_text_only_create_responses_rejects_media_before_generation(self):
|
|
serving = make_serving()
|
|
serving._process_messages = Mock()
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input=[
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "input_text", "text": "describe it"},
|
|
{
|
|
"type": "input_image",
|
|
"image_url": "http://example.com/cat.png",
|
|
},
|
|
],
|
|
}
|
|
],
|
|
store=False,
|
|
)
|
|
|
|
response = asyncio.run(serving.create_responses(request))
|
|
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertIn(b"received unsupported content type 'image_url'", response.body)
|
|
serving._process_messages.assert_not_called()
|
|
serving.tokenizer_manager.generate_request.assert_not_called()
|
|
|
|
def test_multimodal_create_responses_sends_text_and_media_to_engine(self):
|
|
serving = make_serving(is_multimodal=True)
|
|
captured = {}
|
|
|
|
serving._process_messages = Mock(
|
|
return_value=MessageProcessingResult(
|
|
prompt="rendered multimodal prompt",
|
|
prompt_ids=[9, 9, 9],
|
|
image_data=["http://example.com/cat.png"],
|
|
audio_data=None,
|
|
video_data=None,
|
|
modalities=["image"],
|
|
stop=[],
|
|
)
|
|
)
|
|
|
|
async def fake_generate(
|
|
request_id,
|
|
request_prompt,
|
|
adapted_request,
|
|
sampling_params,
|
|
context,
|
|
**kwargs,
|
|
):
|
|
captured["request_prompt"] = request_prompt
|
|
captured["adapted_request"] = adapted_request
|
|
context.append_output(
|
|
{
|
|
"text": "looks like a cat",
|
|
"meta_info": {
|
|
"prompt_tokens": 5,
|
|
"completion_tokens": 4,
|
|
"cached_tokens": 0,
|
|
},
|
|
}
|
|
)
|
|
yield context
|
|
|
|
serving._generate_with_builtin_tools = fake_generate
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input=[
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "input_text", "text": "describe it"},
|
|
{
|
|
"type": "input_image",
|
|
"image_url": "http://example.com/cat.png",
|
|
},
|
|
],
|
|
}
|
|
],
|
|
request_id="resp_mm",
|
|
store=False,
|
|
)
|
|
|
|
response = asyncio.run(serving.create_responses(request))
|
|
|
|
self.assertEqual(response.status, "completed")
|
|
self.assertEqual(captured["request_prompt"], "rendered multimodal prompt")
|
|
self.assertEqual(captured["adapted_request"].text, "rendered multimodal prompt")
|
|
self.assertIsNone(captured["adapted_request"].input_ids)
|
|
self.assertEqual(
|
|
captured["adapted_request"].image_data, ["http://example.com/cat.png"]
|
|
)
|
|
self.assertEqual(captured["adapted_request"].modalities, ["image"])
|
|
|
|
def test_multimodal_token_first_specs_route_through_prompt_ids(self):
|
|
"""Bug regression: token-first encoders leave prompt == "" with
|
|
non-empty prompt_ids; forwarding the empty text 400s in
|
|
_tokenize_texts, so the multimodal branch must forward prompt_ids."""
|
|
for spec in ("inkling", "kimi_k3"):
|
|
with self.subTest(spec=spec):
|
|
serving = make_serving(is_multimodal=True)
|
|
serving.chat_encoding_spec = spec
|
|
serving._process_messages = Mock(
|
|
return_value=MessageProcessingResult(
|
|
prompt="",
|
|
prompt_ids=[4, 5, 6],
|
|
image_data=None,
|
|
audio_data=None,
|
|
video_data=None,
|
|
modalities=[],
|
|
stop=[],
|
|
)
|
|
)
|
|
request = ResponsesRequest(model="x", input="hi", store=False)
|
|
|
|
_, request_prompts, engine_prompts, _ = asyncio.run(
|
|
serving._make_request(
|
|
request, None, serving.tokenizer_manager.tokenizer
|
|
)
|
|
)
|
|
|
|
self.assertEqual(engine_prompts, [[4, 5, 6]])
|
|
self.assertEqual(request_prompts, [[4, 5, 6]])
|
|
|
|
|
|
class OutputItemsTestCase(CustomTestCase):
|
|
def setUp(self):
|
|
# qwen3_coder is the default for this class; the one no-native-parser
|
|
# case overrides it.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
self.serving = make_serving()
|
|
self.serving.tool_call_parser = "qwen3_coder"
|
|
|
|
def _function_tool_request(self):
|
|
return ResponsesRequest(
|
|
model="x",
|
|
input="weather?",
|
|
store=False,
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"description": "Get weather",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
)
|
|
|
|
def test_function_tool_call_extracted_via_parser(self):
|
|
serving = self.serving
|
|
fake_call = ToolCallItem(
|
|
tool_index=0, name="get_weather", parameters='{"city": "Beijing"}'
|
|
)
|
|
|
|
with patch(
|
|
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
|
) as parser_cls:
|
|
instance = parser_cls.return_value
|
|
instance.has_tool_call.return_value = True
|
|
instance.parse_non_stream.return_value = ("trailing text", [fake_call])
|
|
output_items = serving._make_response_output_items(
|
|
self._function_tool_request(),
|
|
"raw model output with <tool_call>",
|
|
tokenizer=Mock(),
|
|
require_reasoning=False,
|
|
)
|
|
|
|
tool_calls = [
|
|
item for item in output_items if isinstance(item, ResponseFunctionToolCall)
|
|
]
|
|
self.assertEqual(len(tool_calls), 1)
|
|
self.assertEqual(tool_calls[0].name, "get_weather")
|
|
self.assertEqual(tool_calls[0].arguments, '{"city": "Beijing"}')
|
|
|
|
message_items = [
|
|
item for item in output_items if isinstance(item, ResponseOutputMessage)
|
|
]
|
|
self.assertEqual(len(message_items), 1)
|
|
self.assertEqual(message_items[0].content[0].text, "trailing text")
|
|
|
|
def test_prose_emitted_before_tool_call_item(self):
|
|
serving = self.serving
|
|
fake_call = ToolCallItem(
|
|
tool_index=0, name="get_weather", parameters='{"city": "Beijing"}'
|
|
)
|
|
|
|
with patch(
|
|
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
|
) as parser_cls:
|
|
instance = parser_cls.return_value
|
|
instance.has_tool_call.return_value = True
|
|
instance.parse_non_stream.return_value = (
|
|
"I'll check the weather.",
|
|
[fake_call],
|
|
)
|
|
output_items = serving._make_response_output_items(
|
|
self._function_tool_request(),
|
|
"raw model output",
|
|
tokenizer=Mock(),
|
|
require_reasoning=False,
|
|
)
|
|
|
|
types = [type(item).__name__ for item in output_items]
|
|
self.assertEqual(types, ["ResponseOutputMessage", "ResponseFunctionToolCall"])
|
|
self.assertEqual(output_items[0].phase, "commentary")
|
|
|
|
def test_required_tool_choice_parses_json_array_without_native_parser(self):
|
|
serving = self.serving
|
|
serving.tool_call_parser = None
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
tool_choice="required",
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
store=False,
|
|
)
|
|
raw = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]'
|
|
|
|
output_items = serving._make_response_output_items(
|
|
request, raw, tokenizer=Mock(), require_reasoning=False
|
|
)
|
|
|
|
tool_calls = [
|
|
item for item in output_items if isinstance(item, ResponseFunctionToolCall)
|
|
]
|
|
self.assertEqual(len(tool_calls), 1)
|
|
self.assertEqual(tool_calls[0].name, "get_weather")
|
|
self.assertEqual(tool_calls[0].arguments, '{"city": "Beijing"}')
|
|
self.assertEqual(
|
|
[item for item in output_items if isinstance(item, ResponseOutputMessage)],
|
|
[],
|
|
)
|
|
|
|
def test_required_tool_choice_skips_json_fallback_for_native_parser(self):
|
|
"""muse reports parses_required_natively, so required output must not
|
|
be pushed through the orjson JSON-array fallback (mirrors chat)."""
|
|
serving = self.serving
|
|
serving.tool_call_parser = "muse"
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
tool_choice="required",
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
store=False,
|
|
)
|
|
raw = '[{"name": "get_weather", "parameters": {"city": "Beijing"}}]'
|
|
|
|
output_items = serving._make_response_output_items(
|
|
request, raw, tokenizer=Mock(), require_reasoning=False
|
|
)
|
|
|
|
self.assertEqual(
|
|
[
|
|
item
|
|
for item in output_items
|
|
if isinstance(item, ResponseFunctionToolCall)
|
|
],
|
|
[],
|
|
)
|
|
message_items = [
|
|
item for item in output_items if isinstance(item, ResponseOutputMessage)
|
|
]
|
|
self.assertEqual(len(message_items), 1)
|
|
self.assertEqual(message_items[0].content[0].text, raw)
|
|
|
|
def test_no_tool_call_extraction_when_tool_choice_none(self):
|
|
serving = self.serving
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
store=False,
|
|
tool_choice="none",
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
)
|
|
|
|
with patch(
|
|
"sglang.srt.entrypoints.openai.serving_responses.FunctionCallParser"
|
|
) as parser_cls:
|
|
output_items = serving._make_response_output_items(
|
|
request,
|
|
"just a plain answer",
|
|
tokenizer=Mock(),
|
|
require_reasoning=False,
|
|
)
|
|
parser_cls.assert_not_called()
|
|
|
|
self.assertEqual(len(output_items), 1)
|
|
self.assertIsInstance(output_items[0], ResponseOutputMessage)
|
|
|
|
|
|
class InklingOutputItemsTestCase(CustomTestCase):
|
|
"""Inkling output items come from sampled token IDs in both response modes."""
|
|
|
|
def setUp(self):
|
|
self.serving = make_serving()
|
|
self.serving.tokenizer_manager.tokenizer = None
|
|
self.serving.chat_encoding_spec = "inkling"
|
|
self.serving.reasoning_parser = self.serving.tool_call_parser = "inkling"
|
|
self.serving._encoding_adapter = self.serving._build_chat_encoding_adapter()
|
|
self.request = ResponsesRequest(
|
|
model="x",
|
|
input="weather?",
|
|
stream=True,
|
|
store=False,
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "weather",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
)
|
|
|
|
@staticmethod
|
|
def _output_ids() -> list[int]:
|
|
from sglang.srt.parser.inkling_renderer import load_tml_renderers
|
|
|
|
tokenizer = load_tml_renderers().tokenizer
|
|
special, text = tokenizer.encode_special, tokenizer.encode_ordinary
|
|
return [
|
|
special("message_model"),
|
|
special("content_thinking"),
|
|
*text("plan"),
|
|
special("end_message"),
|
|
special("message_model"),
|
|
special("content_text"),
|
|
*text("see <|end_message|> here"),
|
|
special("end_message"),
|
|
special("message_model"),
|
|
*text("weather"),
|
|
special("content_invoke_tool_json"),
|
|
*text('{"name":"weather","args":{"city":"SF"}}'),
|
|
special("end_message"),
|
|
special("content_model_end_sampling"),
|
|
]
|
|
|
|
@staticmethod
|
|
def _summary(items: list[dict]) -> list[tuple]:
|
|
summary = []
|
|
for item in items:
|
|
if item["type"] == "function_call":
|
|
summary.append((item["name"], orjson.loads(item["arguments"])))
|
|
else:
|
|
summary.append((item["type"], item["content"][0]["text"]))
|
|
return summary
|
|
|
|
def test_stream_and_non_stream_items_match(self):
|
|
output_ids = self._output_ids()
|
|
finish = {"type": "stop", "matched": output_ids[-1]}
|
|
expected = [
|
|
("reasoning", "plan"),
|
|
("message", "see <|end_message|> here"),
|
|
("weather", {"city": "SF"}),
|
|
]
|
|
items = self.serving._make_response_output_items(
|
|
self.request,
|
|
"",
|
|
Mock(),
|
|
require_reasoning=False,
|
|
output_ids=output_ids,
|
|
finish_reason=finish,
|
|
)
|
|
self.assertEqual(self._summary([i.model_dump() for i in items]), expected)
|
|
|
|
for split in (3, len(output_ids) // 2, len(output_ids) - 2):
|
|
with self.subTest(split=split):
|
|
chunks = []
|
|
for ids, finish_reason in (
|
|
(output_ids[:split], None),
|
|
(output_ids, finish),
|
|
):
|
|
chunk = engine_chunk("", len(ids))
|
|
chunk["output_ids"] = ids
|
|
chunk["meta_info"]["finish_reason"] = finish_reason
|
|
chunks.append(chunk)
|
|
events = StreamFixture(self.serving, self.request).run(chunks)
|
|
output = event_payloads(events)[-1]["response"]["output"]
|
|
self.assertEqual(self._summary(output), expected)
|
|
|
|
def test_required_tool_choice_parses_json_array_without_native_parser(self):
|
|
"""With only the Inkling reasoning parser, required/named tool output is
|
|
the generic JSON array and must still become function_call items."""
|
|
from sglang.srt.parser.inkling_renderer import load_tml_renderers
|
|
|
|
self.serving.tool_call_parser = None
|
|
self.serving._encoding_adapter = self.serving._build_chat_encoding_adapter()
|
|
tokenizer = load_tml_renderers().tokenizer
|
|
special, text = tokenizer.encode_special, tokenizer.encode_ordinary
|
|
thinking = [
|
|
special("message_model"),
|
|
special("content_thinking"),
|
|
*text("plan"),
|
|
special("end_message"),
|
|
]
|
|
calls = text(
|
|
'[{"name": "weather", "parameters": {"city": "SF"}}, '
|
|
'{"name": "weather", "parameters": {"city": "NYC"}}]'
|
|
)
|
|
eos = special("content_model_end_sampling")
|
|
# Headerless after thinking, or framed as a text message as the Inkling
|
|
# constraint frame samples it.
|
|
shapes = {
|
|
"headerless": [*thinking, *calls, eos],
|
|
"framed": [
|
|
*thinking,
|
|
special("message_model"),
|
|
special("content_text"),
|
|
*calls,
|
|
special("end_message"),
|
|
eos,
|
|
],
|
|
}
|
|
expected = [
|
|
("reasoning", "plan"),
|
|
("weather", {"city": "SF"}),
|
|
("weather", {"city": "NYC"}),
|
|
]
|
|
|
|
for shape, output_ids in shapes.items():
|
|
finish = {"type": "stop", "matched": eos}
|
|
# Engine chunks may batch tokens; arguments must not depend on where
|
|
# the JSON payload is split, including a final stop-only chunk.
|
|
chunkings = {
|
|
"per_token": list(range(1, len(output_ids) + 1)),
|
|
"stop_only_last_chunk": [len(output_ids) - 1, len(output_ids)],
|
|
"single_terminal_chunk": [len(output_ids)],
|
|
}
|
|
for tool_choice in ("required", {"type": "function", "name": "weather"}):
|
|
request = self.request.model_copy(update={"tool_choice": tool_choice})
|
|
with self.subTest(shape=shape, tool_choice=tool_choice, stream=False):
|
|
items = self.serving._make_response_output_items(
|
|
request,
|
|
"",
|
|
Mock(),
|
|
require_reasoning=False,
|
|
output_ids=output_ids,
|
|
finish_reason=finish,
|
|
)
|
|
self.assertEqual(
|
|
self._summary([i.model_dump() for i in items]), expected
|
|
)
|
|
for chunking, ends in chunkings.items():
|
|
with self.subTest(
|
|
shape=shape, tool_choice=tool_choice, stream=chunking
|
|
):
|
|
chunks = []
|
|
for end in ends:
|
|
chunk = engine_chunk("", end)
|
|
chunk["output_ids"] = output_ids[:end]
|
|
chunk["meta_info"]["finish_reason"] = (
|
|
finish if end == len(output_ids) else None
|
|
)
|
|
chunks.append(chunk)
|
|
events = StreamFixture(self.serving, request).run(chunks)
|
|
output = event_payloads(events)[-1]["response"]["output"]
|
|
self.assertEqual(self._summary(output), expected)
|
|
|
|
|
|
class ChatEncodingAdapterDispatchTestCase(CustomTestCase):
|
|
@staticmethod
|
|
def _hooks():
|
|
from sglang.srt.entrypoints.openai import chat_encoding_adapters
|
|
|
|
return chat_encoding_adapters.ChatOutputHooks(
|
|
tool_call_parsing_active=Mock(),
|
|
history_tool_calls_count=Mock(),
|
|
tool_call_id=Mock(),
|
|
)
|
|
|
|
def test_only_a_selected_inkling_encoding_requires_tml_renderers(self):
|
|
"""Serving another model must not need tml-renderers, while selecting
|
|
Inkling must still fail at construction when it is missing."""
|
|
from sglang.srt.entrypoints.openai import chat_encoding_adapters
|
|
from sglang.srt.parser.inkling_renderer import load_tml_renderers
|
|
|
|
load_tml_renderers.cache_clear()
|
|
self.addCleanup(load_tml_renderers.cache_clear)
|
|
|
|
def ctx(encoding):
|
|
return chat_encoding_adapters.ChatEncodingAdapterContext(
|
|
chat_encoding_spec=encoding,
|
|
tokenizer=None,
|
|
reasoning_parser="inkling",
|
|
tool_call_parser="inkling",
|
|
chat_hooks=self._hooks(),
|
|
hf_config=None,
|
|
model_path="model",
|
|
revision=None,
|
|
)
|
|
|
|
with patch.dict(sys.modules, {"tml_renderers": None}):
|
|
for encoding in ("kimi_k3", "dsv32"):
|
|
with self.subTest(encoding=encoding):
|
|
self.assertIsNone(
|
|
chat_encoding_adapters.build_chat_encoding_adapter(
|
|
ctx(encoding)
|
|
)
|
|
)
|
|
with self.assertRaises(ImportError):
|
|
chat_encoding_adapters.build_chat_encoding_adapter(ctx("inkling"))
|
|
|
|
def test_inkling_parser_cannot_be_mixed_with_another_parser(self):
|
|
from sglang.srt.entrypoints.openai.serving_inkling import (
|
|
InklingChatEncodingAdapter,
|
|
)
|
|
|
|
for reasoning_parser, tool_call_parser in (
|
|
("inkling", "qwen25"),
|
|
("qwen3", "inkling"),
|
|
):
|
|
with self.subTest(reasoning=reasoning_parser, tool=tool_call_parser):
|
|
with self.assertRaises(ValueError):
|
|
InklingChatEncodingAdapter(
|
|
tokenizer=None,
|
|
reasoning_parser=reasoning_parser,
|
|
tool_call_parser=tool_call_parser,
|
|
chat_hooks=self._hooks(),
|
|
)
|
|
for reasoning_parser, tool_call_parser in (("inkling", None), (None, None)):
|
|
with self.subTest(reasoning=reasoning_parser, tool=tool_call_parser):
|
|
InklingChatEncodingAdapter(
|
|
tokenizer=None,
|
|
reasoning_parser=reasoning_parser,
|
|
tool_call_parser=tool_call_parser,
|
|
chat_hooks=self._hooks(),
|
|
)
|
|
|
|
def test_inkling_adapter_rejects_a_served_tokenizer_with_other_ids(self):
|
|
"""The served tokenizer detokenizes and matches stops, while prompts and
|
|
outputs go through tml-renderers; a vocabulary that differs must fail boot."""
|
|
from sglang.srt.entrypoints.openai.serving_inkling import (
|
|
InklingChatEncodingAdapter,
|
|
)
|
|
from sglang.srt.parser.inkling_renderer import load_tml_renderers
|
|
|
|
tml = load_tml_renderers().tokenizer
|
|
|
|
def served(shift_token=None, ordinary=tml.encode_ordinary):
|
|
# Only encode(), as on the tiktoken tokenizer wrapper.
|
|
def encode(text, add_special_tokens):
|
|
if text.startswith("<|") and text.endswith("|>"):
|
|
return [tml.encode_special(text[2:-2]) + (text == shift_token)]
|
|
return list(ordinary(text))
|
|
|
|
return SimpleNamespace(encode=encode)
|
|
|
|
build = functools.partial(
|
|
InklingChatEncodingAdapter,
|
|
reasoning_parser="inkling",
|
|
tool_call_parser="inkling",
|
|
chat_hooks=self._hooks(),
|
|
)
|
|
build(tokenizer=served())
|
|
with self.assertRaisesRegex(ValueError, "<\\|end_message\\|>"):
|
|
build(tokenizer=served(shift_token="<|end_message|>"))
|
|
with self.assertRaisesRegex(ValueError, "ordinary text"):
|
|
build(tokenizer=served(ordinary=lambda text: tml.encode_ordinary(text)[1:]))
|
|
|
|
def test_responses_adapter_hooks_belong_to_the_handler(self):
|
|
"""The Responses handler reuses the adapter from its base init; its
|
|
chat hooks must belong to that handler, not to another instance."""
|
|
from sglang.srt.entrypoints.openai import (
|
|
chat_encoding_adapters,
|
|
serving_inkling,
|
|
)
|
|
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
|
|
|
with (
|
|
patch.object(
|
|
OpenAIServingChat, "_resolve_chat_encoding_spec", return_value="inkling"
|
|
),
|
|
patch.object(serving_inkling, "_check_served_tokenizer"),
|
|
patch.object(
|
|
chat_encoding_adapters,
|
|
"build_chat_encoding_adapter",
|
|
wraps=chat_encoding_adapters.build_chat_encoding_adapter,
|
|
) as build,
|
|
):
|
|
serving = make_serving()
|
|
|
|
hooks = build.call_args.args[0].chat_hooks
|
|
self.assertIs(hooks.tool_call_id.__self__, serving)
|
|
self.assertIsNotNone(serving._encoding_adapter)
|
|
|
|
|
|
class HarmonyResponsesTestCase(CustomTestCase):
|
|
def test_developer_message_skips_unsupported_tool_types(self):
|
|
from sglang.srt.entrypoints.harmony_utils import get_developer_message
|
|
from sglang.srt.entrypoints.openai.protocol import ResponseTool
|
|
|
|
tools = [
|
|
ResponseTool(
|
|
type="function",
|
|
name="get_weather",
|
|
description="Look up weather.",
|
|
parameters={"type": "object"},
|
|
),
|
|
ResponseTool(type="web_search"),
|
|
ResponseTool(type="namespace", name="codex"),
|
|
ResponseTool(type="mcp"),
|
|
]
|
|
msg = get_developer_message(instructions="be helpful", tools=tools)
|
|
self.assertIsNotNone(msg)
|
|
|
|
|
|
class StatusFromFinishReasonTestCase(CustomTestCase):
|
|
def test_only_length_maps_to_incomplete(self):
|
|
fn = OpenAIServingResponses._status_from_finish_reason
|
|
self.assertEqual(fn({"type": "length"}), "incomplete")
|
|
self.assertEqual(fn("length"), "incomplete")
|
|
for other in ({"type": "stop"}, {"type": "tool_calls"}, "stop", None):
|
|
self.assertEqual(fn(other), "completed", other)
|
|
|
|
|
|
class BuildOutputTextLogprobsTestCase(CustomTestCase):
|
|
def test_tokens_and_top_logprobs_are_converted(self):
|
|
meta_info = {
|
|
"output_token_logprobs": [(-0.1, 10, "Hello"), (-0.2, 11, " world")],
|
|
"output_top_logprobs": [
|
|
[(-0.1, 10, "Hello"), (-2.0, 12, "Hi")],
|
|
[(-0.2, 11, " world"), (-3.0, 13, " earth")],
|
|
],
|
|
}
|
|
out = _build_output_text_logprobs(meta_info)
|
|
self.assertEqual(len(out), 2)
|
|
self.assertEqual(out[0].token, "Hello")
|
|
self.assertEqual(out[0].logprob, -0.1)
|
|
self.assertEqual(out[0].bytes, list("Hello".encode("utf-8")))
|
|
self.assertEqual(len(out[0].top_logprobs), 2)
|
|
self.assertEqual(out[0].top_logprobs[0].token, "Hello")
|
|
self.assertEqual(out[1].token, " world")
|
|
|
|
def test_no_top_logprobs_yields_empty_lists(self):
|
|
meta_info = {
|
|
"output_token_logprobs": [(-0.5, 7, "hi")],
|
|
"output_top_logprobs": None,
|
|
}
|
|
out = _build_output_text_logprobs(meta_info)
|
|
self.assertEqual(len(out), 1)
|
|
self.assertEqual(out[0].top_logprobs, [])
|
|
|
|
|
|
class ChatToolChoiceConversionTestCase(CustomTestCase):
|
|
def test_conversion(self):
|
|
fn = OpenAIServingResponses._chat_tool_choice
|
|
for s in ("auto", "required", "none"):
|
|
self.assertEqual(fn(s), s)
|
|
# Input is an effective_tool_choice() result, so the only object form
|
|
# reaching here is a named function; degrading the rest to "auto"
|
|
# happens there, once, so the echoed and the honored value agree.
|
|
self.assertEqual(
|
|
fn({"type": "function", "name": "get_weather"}),
|
|
{"type": "function", "function": {"name": "get_weather"}},
|
|
)
|
|
|
|
|
|
class EnginePassthroughTestCase(CustomTestCase):
|
|
"""Both flags cross hops with no type contract, and dropping either fails
|
|
silently."""
|
|
|
|
def _capture(self, serving, request, raw_request=None):
|
|
# Let the real _process_messages run: it is the hop that turns
|
|
# skip_special_tokens off, so mocking it would make that assertion vacuous.
|
|
# chat_template_name=None routes it through the tokenizer's template
|
|
# (mocked) instead of the conversation registry, which has no fixture entry.
|
|
serving.default_chat_template_kwargs = {}
|
|
serving.template_manager.chat_template_name = None
|
|
captured = {}
|
|
|
|
async def fake_generate(
|
|
request_id,
|
|
request_prompt,
|
|
adapted_request,
|
|
sampling_params,
|
|
context,
|
|
**kwargs,
|
|
):
|
|
captured["adapted_request"] = adapted_request
|
|
captured["sampling_params"] = sampling_params
|
|
context.append_output(
|
|
{
|
|
"text": "ok",
|
|
"meta_info": {
|
|
"prompt_tokens": 1,
|
|
"completion_tokens": 1,
|
|
"cached_tokens": 0,
|
|
},
|
|
}
|
|
)
|
|
yield context
|
|
|
|
serving._generate_with_builtin_tools = fake_generate
|
|
asyncio.run(serving.create_responses(request, raw_request=raw_request))
|
|
return captured
|
|
|
|
def test_pd_routing_fields_forwarded_to_engine(self):
|
|
serving = make_serving()
|
|
raw_request = Mock(headers={"x-data-parallel-rank": "2"}, state=Mock())
|
|
|
|
captured = self._capture(
|
|
serving,
|
|
ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
bootstrap_host="10.0.0.1",
|
|
bootstrap_port=8998,
|
|
bootstrap_room=42,
|
|
routed_dp_rank=1,
|
|
disagg_prefill_dp_rank=0,
|
|
store=False,
|
|
),
|
|
raw_request=raw_request,
|
|
)
|
|
|
|
adapted_request = captured["adapted_request"]
|
|
self.assertEqual(adapted_request.bootstrap_host, "10.0.0.1")
|
|
self.assertEqual(adapted_request.bootstrap_port, 8998)
|
|
self.assertEqual(adapted_request.bootstrap_room, 42)
|
|
self.assertEqual(adapted_request.routed_dp_rank, 2)
|
|
self.assertEqual(adapted_request.disagg_prefill_dp_rank, 0)
|
|
|
|
def test_require_reasoning_forwarded_when_reasoning_parser_configured(self):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "deepseek-r1"
|
|
serving.template_manager.reasoning_config = ReasoningToggleConfig(
|
|
toggle_param="thinking", default_enabled=True
|
|
)
|
|
|
|
captured = self._capture(
|
|
serving, ResponsesRequest(model="x", input="hi", store=False)
|
|
)
|
|
|
|
self.assertTrue(captured["adapted_request"].require_reasoning)
|
|
|
|
def test_prefilled_think_template_opens_the_parser(self):
|
|
"""``force_reasoning`` is a template property, not a request one, so it
|
|
drives the parser but never the engine flag -- as on the chat path."""
|
|
serving = make_serving()
|
|
serving.reasoning_parser = "deepseek-r1"
|
|
serving.template_manager.force_reasoning = True
|
|
|
|
with patch(
|
|
"sglang.srt.entrypoints.openai.serving_responses.ReasoningParser"
|
|
) as parser_cls:
|
|
parser_cls.return_value.parse_non_stream.return_value = (None, "hi")
|
|
serving._make_response_output_items(
|
|
ResponsesRequest(model="x", input="hi", store=False),
|
|
"hi",
|
|
tokenizer=Mock(),
|
|
require_reasoning=False,
|
|
)
|
|
|
|
self.assertTrue(parser_cls.call_args.kwargs["force_reasoning"])
|
|
|
|
def test_require_reasoning_false_without_reasoning_parser(self):
|
|
serving = make_serving()
|
|
serving.reasoning_parser = None
|
|
|
|
captured = self._capture(
|
|
serving, ResponsesRequest(model="x", input="hi", store=False)
|
|
)
|
|
|
|
self.assertFalse(captured["adapted_request"].require_reasoning)
|
|
|
|
def test_skip_special_tokens_disabled_for_tool_requests(self):
|
|
# _process_messages turns it off so tool-call markers survive detokenize;
|
|
# create_responses must re-apply it to the engine sampling dict.
|
|
serving = make_serving()
|
|
serving.tool_call_parser = "qwen25"
|
|
|
|
captured = self._capture(
|
|
serving,
|
|
ResponsesRequest(
|
|
model="x",
|
|
input="weather",
|
|
store=False,
|
|
tools=[
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"parameters": {"type": "object"},
|
|
}
|
|
],
|
|
),
|
|
)
|
|
|
|
self.assertFalse(captured["sampling_params"]["skip_special_tokens"])
|
|
|
|
|
|
class CancelIdempotencyTestCase(CustomTestCase):
|
|
def test_cancelling_a_terminal_response_returns_it_not_an_error(self):
|
|
publish(
|
|
ServerArgs(model_path="dummy", enable_response_store=True), role="tokenizer"
|
|
)
|
|
from sglang.srt.entrypoints.openai.protocol import ResponsesResponse
|
|
|
|
for status in ("cancelled", "completed"):
|
|
serving = make_serving()
|
|
resp = ResponsesResponse.from_request(
|
|
ResponsesRequest(model="x", input="hi", store=False),
|
|
sampling_params={},
|
|
model_name="x",
|
|
created_time=0,
|
|
output=[],
|
|
status=status,
|
|
usage=None,
|
|
)
|
|
serving.response_store[resp.id] = resp
|
|
|
|
out = asyncio.run(serving.cancel_responses(resp.id))
|
|
|
|
self.assertIs(out, resp, status)
|
|
self.assertEqual(out.status, status)
|
|
|
|
|
|
class StreamingLogprobsRejectionTestCase(CustomTestCase):
|
|
def test_stream_with_logprobs_include_rejected(self):
|
|
import orjson
|
|
|
|
serving = make_serving()
|
|
request = ResponsesRequest(
|
|
model="x",
|
|
input="hi",
|
|
store=False,
|
|
stream=True,
|
|
include=["message.output_text.logprobs"],
|
|
)
|
|
result = asyncio.run(serving.create_responses(request))
|
|
self.assertEqual(result.status_code, 400)
|
|
body = orjson.loads(result.body)
|
|
self.assertIn("streaming mode", body["error"]["message"])
|
|
|
|
|
|
STORE_DISABLED_MESSAGE = (
|
|
"Response store is disabled. Stateful Responses require "
|
|
"--enable-response-store on a standalone server; response storage "
|
|
"is unavailable in PD mode."
|
|
)
|
|
STORE_PD_MESSAGE = (
|
|
"--enable-response-store is not supported with "
|
|
"--disaggregation-mode=prefill or decode; response storage must "
|
|
"remain disabled in PD mode."
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def isolated_response_config():
|
|
reset_context()
|
|
yield
|
|
reset_context()
|
|
|
|
|
|
@pytest.fixture
|
|
def response_serving():
|
|
def build(enabled=False, mode="null", harmony=False):
|
|
reset_context()
|
|
publish(
|
|
ServerArgs(
|
|
model_path="dummy",
|
|
enable_response_store=enabled,
|
|
disaggregation_mode=mode,
|
|
),
|
|
role="tokenizer",
|
|
)
|
|
serving = make_serving()
|
|
serving.use_harmony = harmony
|
|
serving.default_chat_template_kwargs = {}
|
|
serving.template_manager.chat_template_name = None
|
|
serving.template_manager.jinja_template_content_format = "string"
|
|
serving.tokenizer_manager.tokenizer.apply_chat_template.return_value = [1, 2, 3]
|
|
serving.tokenizer_manager.abort_request = Mock()
|
|
serving.reasoning_parser = None
|
|
serving.tool_call_parser = None
|
|
|
|
async def generate(*args, **kwargs):
|
|
chunk = engine_chunk("ok", finish=True)
|
|
if harmony:
|
|
chunk["output_ids"] = get_encoding().render_conversation(
|
|
Conversation.from_messages(
|
|
[
|
|
Message.from_role_and_content(
|
|
Role.ASSISTANT, "ok"
|
|
).with_channel("final")
|
|
]
|
|
)
|
|
)
|
|
chunk["meta_info"]["completion_tokens"] = len(chunk["output_ids"])
|
|
yield chunk
|
|
|
|
serving.tokenizer_manager.generate_request = Mock(side_effect=generate)
|
|
return serving
|
|
|
|
return build
|
|
|
|
|
|
async def create_response_result(serving, request):
|
|
result = await serving.create_responses(request)
|
|
if request.stream:
|
|
payloads = event_payloads([event async for event in result])
|
|
assert payloads[0]["type"] == "response.created"
|
|
assert payloads[-1]["type"] == "response.completed"
|
|
return ResponsesResponse.model_validate(payloads[-1]["response"])
|
|
return result
|
|
|
|
|
|
def assert_response_error(response, param, message=STORE_DISABLED_MESSAGE, status=400):
|
|
assert response.status_code == status
|
|
assert orjson.loads(response.body) == {
|
|
"error": {
|
|
"message": message,
|
|
"type": "invalid_request_error",
|
|
"param": param,
|
|
"code": status,
|
|
}
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mode,enabled", [("null", False), ("null", True), ("decode", True)]
|
|
)
|
|
def test_response_store_configuration(mode, enabled):
|
|
parser = argparse.ArgumentParser()
|
|
ServerArgs.add_cli_args(parser)
|
|
argv = ["--model-path", "dummy", "--disaggregation-mode", mode]
|
|
if enabled:
|
|
argv.append("--enable-response-store")
|
|
args = ServerArgs.from_cli_args(parser.parse_args(argv))
|
|
if mode != "null":
|
|
with pytest.raises(ValueError) as error:
|
|
publish(args, role="tokenizer")
|
|
assert str(error.value) == STORE_PD_MESSAGE
|
|
else:
|
|
publish(args, role="tokenizer")
|
|
serving = make_serving()
|
|
assert get_serving().enable_response_store is enabled
|
|
assert serving.enable_response_store is enabled
|
|
assert not serving.is_disaggregated
|
|
|
|
|
|
@pytest.mark.parametrize("stream", [False, True])
|
|
@pytest.mark.parametrize("enabled", [False, True])
|
|
def test_response_store_persistence_and_continuation(response_serving, stream, enabled):
|
|
serving = response_serving(enabled=enabled, harmony=not stream)
|
|
|
|
async def run():
|
|
first = await create_response_result(
|
|
serving, ResponsesRequest(model="x", input="first", stream=stream)
|
|
)
|
|
assert first.status == "completed"
|
|
assert first.output[0].content[0].text == "ok"
|
|
assert first.store is True
|
|
assert not serving.background_tasks
|
|
if not enabled:
|
|
assert not serving.response_store and not serving.msg_store
|
|
return
|
|
assert serving.response_store[first.id].output == first.output
|
|
history = list(serving.msg_store[first.id])
|
|
assert len(history) == 2
|
|
second = await create_response_result(
|
|
serving,
|
|
ResponsesRequest(
|
|
model="x",
|
|
input="next",
|
|
previous_response_id=first.id,
|
|
store=False,
|
|
stream=stream,
|
|
),
|
|
)
|
|
if serving.use_harmony:
|
|
generated_request = (
|
|
serving.tokenizer_manager.generate_request.call_args.args[0]
|
|
)
|
|
prompt = get_encoding().decode(generated_request.input_ids)
|
|
else:
|
|
prompt = str(
|
|
serving.tokenizer_manager.tokenizer.apply_chat_template.call_args.args[
|
|
0
|
|
]
|
|
)
|
|
assert all(text in prompt for text in ("first", "ok", "next"))
|
|
assert second.status == "completed"
|
|
assert second.output[0].content[0].text == "ok"
|
|
assert second.store is False
|
|
assert set(serving.response_store) == {first.id}
|
|
assert set(serving.msg_store) == {first.id}
|
|
assert serving.msg_store[first.id] == history
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"enabled,fields,param,message",
|
|
[
|
|
(
|
|
False,
|
|
{"previous_response_id": ""},
|
|
"previous_response_id",
|
|
STORE_DISABLED_MESSAGE,
|
|
),
|
|
(False, {}, "background", STORE_DISABLED_MESSAGE),
|
|
(True, {}, "store", "background=true requires store=true."),
|
|
],
|
|
ids=["empty-predecessor", "disabled-background", "background-without-store"],
|
|
)
|
|
def test_response_store_admission(response_serving, enabled, fields, param, message):
|
|
serving = response_serving(enabled=enabled, mode="null" if enabled else "prefill")
|
|
serving._make_request = AsyncMock(side_effect=AssertionError("prompt reached"))
|
|
serving.response_store = Mock()
|
|
serving.msg_store = Mock()
|
|
request = ResponsesRequest(
|
|
model="unknown", input="hi", background=True, stream=True, store=None, **fields
|
|
)
|
|
assert_response_error(
|
|
asyncio.run(serving.create_responses(request)), param, message
|
|
)
|
|
serving._make_request.assert_not_called()
|
|
serving.tokenizer_manager.generate_request.assert_not_called()
|
|
assert not serving.response_store.mock_calls
|
|
assert not serving.msg_store.mock_calls
|
|
assert not serving.background_tasks
|
|
|
|
|
|
@pytest.mark.parametrize("enabled", [False, True])
|
|
def test_response_store_read_endpoints(response_serving, enabled):
|
|
serving = response_serving(enabled=enabled)
|
|
|
|
async def run():
|
|
if not enabled:
|
|
serving.response_store = Mock()
|
|
serving.background_tasks = Mock()
|
|
for method in (serving.retrieve_responses, serving.cancel_responses):
|
|
assert_response_error(await method("bad"), "response_id")
|
|
assert not serving.response_store.mock_calls
|
|
assert not serving.background_tasks.mock_calls
|
|
else:
|
|
response = await create_response_result(
|
|
serving, ResponsesRequest(model="x", input="hi")
|
|
)
|
|
assert await serving.retrieve_responses(response.id) is response
|
|
for response_id, status in (("bad", 400), ("resp_missing", 404)):
|
|
for method in (serving.retrieve_responses, serving.cancel_responses):
|
|
assert (await method(response_id)).status_code == status
|
|
request = ResponsesRequest(
|
|
model="x", input="hi", previous_response_id=response_id
|
|
)
|
|
assert (await serving.create_responses(request)).status_code == status
|
|
serving.tokenizer_manager.abort_request.assert_not_called()
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"outcome", ["success", "failure", "cancel_queued", "cancel_running"]
|
|
)
|
|
def test_background_task_lifecycle(response_serving, outcome):
|
|
serving = response_serving(enabled=True)
|
|
original_generate = serving.tokenizer_manager.generate_request
|
|
|
|
async def run():
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
|
|
async def generate(*args, **kwargs):
|
|
entered.set()
|
|
await release.wait()
|
|
if outcome == "failure":
|
|
raise ValueError("generation failed")
|
|
async for chunk in original_generate(*args, **kwargs):
|
|
yield chunk
|
|
|
|
serving.tokenizer_manager.generate_request = generate
|
|
request = ResponsesRequest(model="x", input="hi", background=True)
|
|
queued = await serving.create_responses(request)
|
|
assert queued.status == "queued"
|
|
assert serving.response_store[queued.id] is queued
|
|
assert request.request_id in serving.msg_store
|
|
task = serving.background_tasks[queued.id]
|
|
if outcome != "cancel_queued":
|
|
await entered.wait()
|
|
assert (await serving.retrieve_responses(queued.id)).status == "in_progress"
|
|
if outcome.startswith("cancel"):
|
|
cancelled = await serving.cancel_responses(queued.id)
|
|
assert cancelled.status == "cancelled"
|
|
serving.tokenizer_manager.abort_request.assert_called_once_with(
|
|
rid=queued.id
|
|
)
|
|
assert task.done()
|
|
if outcome == "cancel_queued":
|
|
assert task.cancelled()
|
|
else:
|
|
release.set()
|
|
await task
|
|
assert serving.response_store[queued.id].status == (
|
|
"failed" if outcome == "failure" else "completed"
|
|
)
|
|
assert not serving.background_tasks
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_active_stream_cancel_and_final_history(response_serving):
|
|
serving = response_serving(enabled=True, harmony=True)
|
|
|
|
async def run():
|
|
request = ResponsesRequest(model="x", input="hi", background=True, stream=True)
|
|
stream = await serving.create_responses(request)
|
|
assert "response.created" in await anext(stream)
|
|
assert not serving.background_tasks
|
|
assert (await serving.cancel_responses(request.request_id)).status_code == 404
|
|
serving.tokenizer_manager.abort_request.assert_not_called()
|
|
events = [event async for event in stream]
|
|
assert event_payloads(events)[-1]["type"] == "response.completed"
|
|
assert serving.response_store[request.request_id].status == "completed"
|
|
assert len(serving.msg_store[request.request_id]) == 2
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
INLINE_SYSTEM_TEMPLATE = (
|
|
"{% for m in messages %}{{ m['role'] }}: {{ m['content'] }}{% endfor %}"
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"encoding_spec, chat_template_name, jinja_template, preserve",
|
|
[
|
|
("kimi_k3", None, None, True),
|
|
("dsv32", None, INLINE_SYSTEM_TEMPLATE, False),
|
|
(None, None, INLINE_SYSTEM_TEMPLATE, True),
|
|
(None, "llama-3", INLINE_SYSTEM_TEMPLATE, False),
|
|
(None, None, "{{ messages[0]['content'] }}", False),
|
|
],
|
|
)
|
|
def test_inline_instruction_keeps_history_prefix(
|
|
encoding_spec, chat_template_name, jinja_template, preserve
|
|
):
|
|
"""An appended developer instruction stays in place when the renderer supports
|
|
inline system; hoisting it rewrites the prompt prefix and defeats KV reuse."""
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
tokenizer_manager = MockTokenizerManager()
|
|
tokenizer_manager.tokenizer.chat_template = jinja_template
|
|
template_manager = MockTemplateManager()
|
|
template_manager.chat_template_name = chat_template_name
|
|
with patch.object(
|
|
OpenAIServingResponses,
|
|
"_resolve_chat_encoding_spec",
|
|
return_value=encoding_spec,
|
|
):
|
|
serving = OpenAIServingResponses(tokenizer_manager, template_manager)
|
|
history = [
|
|
{"role": "user", "content": "Existing conversation"},
|
|
{"role": "assistant", "content": "Previous answer"},
|
|
]
|
|
appended = [
|
|
{"role": "developer", "content": "New instruction"},
|
|
{"role": "user", "content": "Next turn"},
|
|
]
|
|
before = serving._construct_input_messages(
|
|
ResponsesRequest(model="x", instructions="Stable instructions", input=history)
|
|
)
|
|
after = serving._construct_input_messages(
|
|
ResponsesRequest(
|
|
model="x", instructions="Stable instructions", input=history + appended
|
|
)
|
|
)
|
|
if preserve:
|
|
assert after[: len(before)] == before
|
|
assert after[len(before)]["role"] == "system"
|
|
assert "New instruction" in after[len(before)]["content"]
|
|
else:
|
|
assert [m["role"] for m in after].count("system") == 1
|
|
assert "New instruction" in after[0]["content"]
|
|
|
|
|
|
@pytest.mark.parametrize("stream", [False, True])
|
|
def test_completion_preserves_cancelled_response(response_serving, stream):
|
|
serving = response_serving(enabled=True, harmony=not stream)
|
|
request = ResponsesRequest(model="x", input="hi", stream=stream)
|
|
cancelled = ResponsesResponse.from_request(
|
|
request,
|
|
{},
|
|
model_name="x",
|
|
created_time=0,
|
|
output=[],
|
|
status="cancelled",
|
|
usage=None,
|
|
)
|
|
serving.response_store[request.request_id] = cancelled
|
|
serving.msg_store[request.request_id] = ["original"]
|
|
if stream:
|
|
StreamFixture(serving, request).run([engine_chunk("ok", finish=True)])
|
|
else:
|
|
messages = serving._construct_input_messages_with_harmony(request, None)
|
|
context = HarmonyContext(messages, {})
|
|
|
|
async def generate():
|
|
async for chunk in serving.tokenizer_manager.generate_request():
|
|
context.append_output(chunk)
|
|
yield context
|
|
|
|
response = asyncio.run(
|
|
serving.responses_full_generator(
|
|
request,
|
|
{},
|
|
generate(),
|
|
context,
|
|
"x",
|
|
Mock(),
|
|
RequestResponseMetadata(request_id=request.request_id),
|
|
require_reasoning=False,
|
|
)
|
|
)
|
|
assert response.status == "completed"
|
|
assert serving.response_store[request.request_id] is cancelled
|
|
assert serving.msg_store[request.request_id] == ["original"]
|
|
|
|
|
|
def test_pd_builtin_tool_admission(response_serving):
|
|
serving = response_serving(mode="prefill", harmony=True)
|
|
serving.tool_server = Mock()
|
|
serving.supports_browsing = True
|
|
request = ResponsesRequest(model="x", input="hi", tools=[{"type": "web_search"}])
|
|
result = asyncio.run(serving.create_responses(request))
|
|
assert result.status_code == 400
|
|
assert orjson.loads(result.body)["error"]["param"] == "tools"
|
|
serving.tool_server.get_tool_session.assert_not_called()
|
|
serving.tokenizer_manager.generate_request.assert_not_called()
|
|
|
|
|
|
def test_standalone_builtin_tools_without_storage(response_serving):
|
|
serving = response_serving(harmony=True)
|
|
tool_session = Mock()
|
|
tool_session.call_tool = AsyncMock(
|
|
return_value=Mock(content=[Mock(text="Search result: 42")])
|
|
)
|
|
|
|
@asynccontextmanager
|
|
async def session(name):
|
|
yield tool_session
|
|
|
|
turns = iter(
|
|
[
|
|
Message.from_role_and_content(Role.ASSISTANT, '{"query":"answer"}')
|
|
.with_channel("commentary")
|
|
.with_recipient("browser.search"),
|
|
Message.from_role_and_content(
|
|
Role.ASSISTANT, "The answer is 42."
|
|
).with_channel("final"),
|
|
]
|
|
)
|
|
|
|
async def generate(*args, **kwargs):
|
|
chunk = engine_chunk("", finish=True)
|
|
chunk["output_ids"] = get_encoding().render_conversation(
|
|
Conversation.from_messages([next(turns)])
|
|
)
|
|
if serving.tokenizer_manager.generate_request.call_count == 1:
|
|
# The parser starts inside the prompt's open assistant header.
|
|
chunk["output_ids"] = chunk["output_ids"][2:]
|
|
chunk["meta_info"]["completion_tokens"] = len(chunk["output_ids"])
|
|
yield chunk
|
|
|
|
serving.tokenizer_manager.generate_request = Mock(side_effect=generate)
|
|
serving.tool_server = Mock()
|
|
serving.tool_server.get_tool_session = session
|
|
serving.tool_server.get_tool_description.return_value = ToolNamespaceConfig(
|
|
name="browser", description="Browser", tools=[]
|
|
)
|
|
serving.supports_browsing = True
|
|
request = ResponsesRequest(model="x", input="hi", tools=[{"type": "web_search"}])
|
|
response = asyncio.run(create_response_result(serving, request))
|
|
assert isinstance(response, ResponsesResponse), response.body
|
|
tool_session.call_tool.assert_awaited_once_with("search", {"query": "answer"})
|
|
assert response.status == "completed"
|
|
assert response.output[-1].content[0].text == "The answer is 42."
|
|
assert serving.tokenizer_manager.generate_request.call_count == 2
|
|
continuation = serving.tokenizer_manager.generate_request.call_args_list[1].args[0]
|
|
assert "Search result: 42" in get_encoding().decode(continuation.input_ids)
|
|
assert not serving.msg_store and not serving.response_store
|
|
|
|
|
|
def test_pd_tool_continuation_stops_before_side_effect(response_serving):
|
|
serving = response_serving(mode="decode")
|
|
context = Mock()
|
|
context.need_builtin_tool_call.return_value = True
|
|
context.call_tool = AsyncMock()
|
|
|
|
async def run():
|
|
async for _ in serving._generate_with_builtin_tools(
|
|
"resp_tool", "hi", Mock(), {}, context
|
|
):
|
|
pass
|
|
|
|
with pytest.raises(ValueError, match="disaggregation"):
|
|
asyncio.run(run())
|
|
context.call_tool.assert_not_awaited()
|
|
assert serving.tokenizer_manager.generate_request.call_count == 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main([__file__]))
|