Files
sglang/test/registered/unit/entrypoints/openai/test_serving_chat.py
T

5251 lines
206 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Unit-tests for OpenAIServingChat -- rewritten to use only the std-lib 'unittest'.
Run with either:
python tests/test_serving_chat_unit.py -v
or
python -m unittest discover -s tests -p "test_*unit.py" -v
"""
from sglang.test.test_utils import CustomTestCase, enter_override, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
import asyncio
import gc
import json
import unittest
import uuid
from http import HTTPStatus
from typing import Optional
from unittest.mock import Mock, patch
from fastapi import Request
from transformers.utils.chat_template_utils import _compile_jinja_template
from sglang.srt.entrypoints.openai.chat_encoding import resolve_chat_encoding_spec
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
Function,
MessageProcessingResult,
Tool,
ToolChoice,
ToolChoiceFuncName,
)
from sglang.srt.entrypoints.openai.serving_chat import (
OpenAIServingChat,
normalize_tool_content,
)
from sglang.srt.environ import envs
from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.parser.jinja_template_utils import (
jinja_template_may_reorder_tool_results,
)
from sglang.srt.parser.response_template import (
GRAMMAR_AFTER_REASONING,
GRAMMAR_FROM_START,
ResponseTemplateReasoningDetector,
ResponseTemplateToolDetector,
)
from sglang.srt.parser.template_detection import ReasoningToggleConfig
from sglang.srt.runtime_context import get_context, publish, reset_context
from sglang.srt.sampling.sampling_params import (
REQUEST_REASONING_END_TOKEN_IDS_KEY,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import get_or_create_event_loop
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=13, suite="base-a-test-cpu")
# Every spec resolve_chat_encoding_spec can return; pinned by the guard below.
_ALL_CHAT_ENCODING_SPECS = ("dsv41", "dsv4", "dsv32", "inkling", "kimi_k3")
_RESPONSE_TEMPLATE = {
"start_anchor": "<assistant>",
"fields": {
"content": {"content": "text"},
"tool_calls": {
"open_pattern": r"<call:(?P<name>\w+)>",
"close": "</call>",
"content": "json",
"repeats": True,
"transform": {
"type": "function",
"function": {
"name": "{name}",
"arguments": "{content}",
},
},
},
},
}
# Every field is delimited, so the template gives no field to text outside them.
_DELIMITED_RESPONSE_TEMPLATE = {
"start_anchor": "<assistant>",
"fields": {
"thinking": {"open": "<think>", "close": "</think>"},
"content": {"open": "<answer>", "close": "</answer>"},
"tool_calls": _RESPONSE_TEMPLATE["fields"]["tool_calls"],
},
}
_WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
def _spec_result(index):
return {
"text": f"choice-{index}",
"meta_info": {
"id": "chatcmpl-spec-test",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": 0,
"finish_reason": {"type": "stop"},
"weight_version": "default",
"spec_accept_rate": 0.5,
"spec_accept_length": 2.0,
"spec_cap_length": index + 1.0,
"spec_block_accept_length": index + 0.5,
"spec_num_correct_drafts": 1,
"spec_num_proposed_drafts": 2,
"spec_verify_ct": 1,
"spec_correct_drafts_histogram": [0, 1],
"spec_cap_lens_histogram": [index, 1],
},
"index": index,
}
_DSV4_PREVIEW_ENCODER = 'REASONING_EFFORT_MAX = "preview"\n'
_DSV4_OFFICIAL_ENCODER = (
"REASONING_EFFORT_PROMPTS: Dict[str, str] = "
'{"low": "", "high": "h", "max": "m"}\n'
'DEFAULT_REASONING_EFFORT = "low"\n'
)
_TOOL_RESULT_REORDER_TEMPLATE = """
{%- for assistant in messages
if assistant.role == 'assistant' and assistant.tool_calls -%}
{%- for tool_call in assistant.tool_calls -%}
{%- for result in messages
if result.role == 'tool' and result.tool_call_id == tool_call.id -%}
{%- for part in result.content -%}
{%- if part.type == 'text' -%}
{{- part.text -}}
{%- else -%}
{{- '<' + part.type + '>' -}}
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
{%- endfor -%}
{%- endfor -%}
"""
class _MockTokenizerManager:
"""Minimal mock that satisfies OpenAIServingChat."""
def __init__(self):
self.model_config = Mock(is_multimodal=False)
self.server_args = Mock(
model_path="deepseek-ai/DeepSeek-V4-Flash",
revision=None,
enable_cache_report=False,
tool_call_parser="hermes",
reasoning_parser=None,
stream_response_default_include_usage=False,
default_chat_template_kwargs=None,
return_input_ids=False,
return_output_ids=False,
incremental_streaming_output=False,
)
self.model_path = self.server_args.model_path
# The manager tracks the served name itself; a weight update rewrites it.
self.served_model_name = "test-model"
# Stands in for the context's resolved leaves: an override replaces the
# field's one live value, the seed stays on server_args.
self._config_overrides = {}
# Mock hf_config for _resolve_chat_encoding_spec check
mock_hf_config = Mock()
mock_hf_config.architectures = ["LlamaForCausalLM"]
mock_hf_config.to_dict.return_value = {}
self.model_config.hf_config = mock_hf_config
self.chat_template_name: Optional[str] = "llama-3"
# tokenizer stub
self.tokenizer = Mock()
self.tokenizer.encode.return_value = [1, 2, 3, 4, 5]
self.tokenizer.decode.return_value = "Test response"
self.tokenizer.chat_template = None
self.tokenizer.bos_token_id = 1
# async generator stub for generate_request
async def _mock_generate():
yield {
"text": "Test response",
"meta_info": {
"id": f"chatcmpl-{uuid.uuid4()}",
"prompt_tokens": 10,
"completion_tokens": 5,
"cached_tokens": 0,
"weight_version": "test-version",
"finish_reason": {"type": "stop", "matched": None},
"output_token_logprobs": [(0.1, 1, "Test"), (0.2, 2, "response")],
"output_top_logprobs": None,
},
"index": 0,
}
self.generate_request = Mock(return_value=_mock_generate())
self.create_abort_task = Mock()
self.request_logger = Mock(log_requests=False, log_requests_level=0)
def config_value(self, name: str):
"""The value in effect for one config field."""
if name in self._config_overrides:
return self._config_overrides[name]
return getattr(self.server_args, name)
class _MockTemplateManager:
"""Minimal mock for TemplateManager."""
def __init__(self):
self.chat_template_name: Optional[str] = "llama-3"
self.jinja_template_content_format: Optional[str] = None
self.completion_template_name: Optional[str] = None
self.reasoning_config = None
self.force_reasoning = False
self.jinja_template_may_reorder_tool_results = False
self.python_chat_template = None
class TestChatTemplateCache(CustomTestCase):
def setUp(self):
super().setUp()
reset_context()
self.addCleanup(reset_context)
publish(
ServerArgs(model_path="dummy", default_chat_template_kwargs=None),
role="tokenizer",
)
self.tokenizer_manager = _MockTokenizerManager()
self.chat = OpenAIServingChat(
self.tokenizer_manager,
_MockTemplateManager(),
)
self.tokenizer_manager.tokenizer.apply_chat_template.return_value = "rendered"
self.tokenizer_manager.tokenizer.encode.return_value = [11, 12]
self.tokenizer_manager.tokenizer.decode.return_value = "decoded"
self.tokenizer_manager.tokenizer.reset_mock()
def _render(self, **overrides):
kwargs = {
"messages": [{"role": "user", "content": "same text prefix"}],
"tools": None,
"template_kwargs": {"enable_thinking": False},
"encode_kwargs": {"add_special_tokens": False},
"use_cache": True,
}
kwargs.update(overrides)
return self.chat._render_and_encode_chat_template(**kwargs)
def test_cache_hit_reuses_render_encode_and_returns_an_owned_id_list(self):
first = self._render()
first[0].append(99)
second = self._render()
self.assertEqual(second, ([11, 12], "decoded"))
self.tokenizer_manager.tokenizer.apply_chat_template.assert_called_once()
self.tokenizer_manager.tokenizer.encode.assert_called_once()
self.tokenizer_manager.tokenizer.decode.assert_called_once()
def test_cache_key_includes_template_and_encode_options(self):
self._render()
self._render(template_kwargs={"enable_thinking": True})
self._render(encode_kwargs={"add_special_tokens": True})
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count,
3,
)
self.assertEqual(self.tokenizer_manager.tokenizer.encode.call_count, 3)
def test_cache_key_tracks_tokenizer_chat_template_updates(self):
self.tokenizer_manager.tokenizer.chat_template = "template-v1"
self._render()
self.tokenizer_manager.tokenizer.chat_template = "template-v2"
self._render()
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count,
2,
)
def test_non_serializable_input_bypasses_cache(self):
messages = [{"role": "user", "content": object()}]
self._render(messages=messages)
self._render(messages=messages)
self.assertEqual(
self.tokenizer_manager.tokenizer.apply_chat_template.call_count,
2,
)
self.assertEqual(self.tokenizer_manager.tokenizer.encode.call_count, 2)
self.tokenizer_manager.tokenizer.decode.assert_not_called()
class ServingChatTestCase(CustomTestCase):
# ------------- common fixtures -------------
def setUp(self):
# The serving layer reads its config from the bags, so the fixture has
# to publish one rather than hang the values off a mock manager.
reset_context()
self.addCleanup(reset_context)
# Tests drive coroutines through get_or_create_event_loop(), which
# creates a fresh loop per call and leaves the previous one unclosed.
# Finalize those loops here, between tests: if the cyclic GC collects
# one mid-import, its ResourceWarning imports tracemalloc while the
# outer import still holds the module-lock bookkeeping, which raises
# KeyError from importlib._bootstrap on Python < 3.12.
self.addCleanup(self._close_event_loops)
publish(
ServerArgs(
model_path="dummy",
revision=None,
enable_cache_report=False,
tool_call_parser="hermes",
reasoning_parser=None,
stream_response_default_include_usage=False,
default_chat_template_kwargs=None,
),
role="tokenizer",
)
self.tm = _MockTokenizerManager()
self.template_manager = _MockTemplateManager()
self.chat = OpenAIServingChat(self.tm, self.template_manager)
self.tm.tokenizer.reset_mock()
# frequently reused requests
self.basic_req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
session_id="session-1",
temperature=0.7,
max_tokens=100,
stream=False,
)
self.stream_req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
temperature=0.7,
max_tokens=100,
stream=True,
)
self.fastapi_request = Mock(spec=Request)
self.fastapi_request.headers = {}
@staticmethod
def _close_event_loops():
try:
loop = asyncio.get_event_loop_policy().get_event_loop()
except RuntimeError:
loop = None
if loop is not None and not loop.is_closed():
loop.close()
asyncio.set_event_loop(None)
gc.collect()
@staticmethod
def _render_tool_results_in_call_order(messages, **kwargs):
"""Block-level tool_call_id association, like the GLM chat templates."""
del kwargs
rendered = []
index = 0
while index < len(messages):
message = messages[index]
index += 1
tool_calls = message.get("tool_calls") or []
if message.get("role") != "assistant" or not tool_calls:
continue
run = []
while index < len(messages) and messages[index].get("role") == "tool":
run.append(messages[index])
index += 1
by_id = {result.get("tool_call_id"): result for result in run}
for tool_call in tool_calls:
result = by_id.get(tool_call.get("id"))
if result is None:
continue
for part in result.get("content") or []:
if part.get("type") == "text":
rendered.append(part.get("text", ""))
else:
rendered.append(f"<{part.get('type')}>")
return "".join(rendered)
@staticmethod
def _tool_round(call_ids, result_ids, part_type="image_url"):
assistant = {
"role": "assistant",
"content": "",
"tool_calls": [
{"id": call_id, "function": {"name": call_id, "arguments": {}}}
for call_id in call_ids
],
}
part_key = {"image_url": "image_url", "video_url": "video_url"}[part_type]
results = [
{
"role": "tool",
"tool_call_id": result_id,
"content": [{"type": part_type, part_key: {"url": result_id}}],
}
for result_id in result_ids
]
return [assistant] + results
def test_canonicalize_tool_message_order_sorts_media_runs(self):
messages = self._tool_round(
["call-a", "call-b"], ["call-b", "call-a"]
) + self._tool_round(["call-c", "call-d"], ["call-d", "call-c"], "video_url")
canonical = self.chat._canonicalize_tool_message_order(messages)
self.assertEqual(
[
message["tool_call_id"]
for message in canonical
if message.get("role") == "tool"
],
["call-a", "call-b", "call-c", "call-d"],
)
def test_canonicalize_tool_message_order_keeps_unassociable_runs(self):
cases = {
"unknown_id": (["call-a", "call-b"], ["call-b", "call-z"]),
"duplicate_result_id": (["call-a", "call-b"], ["call-b", "call-b"]),
"missing_call_id": ([None, "call-b"], ["call-b", None]),
}
for name, (call_ids, result_ids) in cases.items():
with self.subTest(name=name):
messages = self._tool_round(call_ids, result_ids)
canonical = self.chat._canonicalize_tool_message_order(messages)
self.assertEqual(canonical, messages)
def test_canonicalize_tool_message_order_keeps_text_only_runs(self):
messages = self._tool_round(["call-a", "call-b"], ["call-b", "call-a"])
for message in messages[1:]:
message["content"] = [{"type": "text", "text": "done"}]
canonical = self.chat._canonicalize_tool_message_order(messages)
self.assertEqual(canonical, messages)
def test_jinja_path_recovers_tool_result_images_only_when_template_needs_it(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "openai"
self.template_manager.jinja_template_may_reorder_tool_results = (
jinja_template_may_reorder_tool_results(_TOOL_RESULT_REORDER_TEMPLATE)
)
self.assertTrue(self.template_manager.jinja_template_may_reorder_tool_results)
self.tm.tokenizer.apply_chat_template.side_effect = (
self._render_tool_results_in_call_order
)
request = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "inspect"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call-a",
"type": "function",
"function": {"name": "a", "arguments": {}},
},
{
"id": "call-b",
"type": "function",
"function": {"name": "b", "arguments": {}},
},
],
},
{
"role": "tool",
"tool_call_id": "call-b",
"content": [{"type": "image_url", "image_url": {"url": "image-b"}}],
},
{
"role": "tool",
"tool_call_id": "call-a",
"content": [{"type": "image_url", "image_url": {"url": "image-a"}}],
},
],
)
result = self.chat._apply_jinja_template(request, None, is_multimodal=True)
self.assertEqual(
[item.url for item in result.image_data], ["image-a", "image-b"]
)
self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1)
rendered_messages = self.tm.tokenizer.apply_chat_template.call_args[0][0]
self.assertEqual(
[m.get("tool_call_id") for m in rendered_messages if "tool_call_id" in m],
["call-a", "call-b"],
)
# An already-ordered request must produce the exact same render input.
ordered_request = ChatCompletionRequest(
model="x",
messages=request.messages[:2] + request.messages[2:][::-1],
)
self.tm.tokenizer.apply_chat_template.reset_mock()
self.chat._apply_jinja_template(ordered_request, None, is_multimodal=True)
self.tm.tokenizer.apply_chat_template.assert_not_called()
self.template_manager.jinja_template_may_reorder_tool_results = False
self.tm.tokenizer.apply_chat_template.reset_mock()
result = self.chat._apply_jinja_template(request, None, is_multimodal=True)
self.assertEqual(
[item.url for item in result.image_data], ["image-b", "image-a"]
)
self.assertEqual(self.tm.tokenizer.apply_chat_template.call_count, 1)
def test_parsers_follow_the_control_plane_overlay(self):
"""Template detection records the parsers through `override`, so they
answer from the bags; `ServerArgs` keeps the launcher's seed."""
self.tm.server_args.tool_call_parser = "auto"
self.tm.server_args.reasoning_parser = "auto"
self.tm._config_overrides.update(
{"tool_call_parser": "qwen25", "reasoning_parser": None}
)
chat = OpenAIServingChat(self.tm, self.template_manager)
self.assertEqual(chat.tool_call_parser, "qwen25")
self.assertIsNone(chat.reasoning_parser)
self.assertEqual(self.tm.server_args.tool_call_parser, "auto")
def test_the_xgrammar_gate_follows_the_overlay(self):
"""A detected `reasoning_parser` must gate xgrammar, not the seed's "auto"."""
self.tm.server_args.reasoning_parser = "auto"
self.tm._config_overrides["reasoning_parser"] = "qwen3"
chat = OpenAIServingChat(self.tm, self.template_manager)
self.assertEqual(chat.reasoning_parser, "qwen3")
# the gate reads the same value the parser was built from
self.assertIsNotNone(chat.reasoning_parser)
def test_text_only_model_rejects_media_before_generation(self):
media_parts = {
"image_url": {
"type": "image_url",
"image_url": {"url": "https://example.com/image.png"},
},
"video_url": {
"type": "video_url",
"video_url": {"url": "https://example.com/video.mp4"},
},
"audio_url": {
"type": "audio_url",
"audio_url": {"url": "https://example.com/audio.wav"},
},
}
for media_type, media_part in media_parts.items():
with self.subTest(media_type=media_type):
request = ChatCompletionRequest(
model="x",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "describe"},
media_part,
],
}
],
)
response = get_or_create_event_loop().run_until_complete(
self.chat.handle_request(request, self.fastapi_request)
)
error = json.loads(response.body)
self.assertEqual(response.status_code, HTTPStatus.BAD_REQUEST)
self.assertEqual(error["type"], "BadRequestError")
self.assertIn(media_type, error["message"])
self.tm.generate_request.assert_not_called()
def test_media_validation_does_not_reject_supported_content(self):
text_request = ChatCompletionRequest(
model="x",
messages=[
{
"role": "user",
"content": [{"type": "tool_reference", "name": "get_weather"}],
}
],
)
self.assertIsNone(self.chat._validate_request(text_request))
self.tm.model_config.is_multimodal = True
multimodal_request = ChatCompletionRequest(
model="x",
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": "https://example.com/image.png"},
}
],
}
],
)
self.assertIsNone(self.chat._validate_request(multimodal_request))
# ------------- conversion tests -------------
def test_convert_to_internal_request_single(self):
with (
patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock,
patch.object(self.chat, "_process_messages") as proc_mock,
):
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_ins.image_data = conv_ins.audio_data = None
conv_ins.modalities = []
conv_ins.stop_str = ["</s>"]
conv_mock.return_value = conv_ins
proc_mock.return_value = MessageProcessingResult(
"Test prompt",
[1, 2, 3],
None,
None,
[],
["</s>"],
None,
)
self.basic_req.return_sampling_mask = True
self.basic_req.sampling_logprobs_mode = "support"
self.basic_req.return_meta_info = True
adapted, processed = self.chat._convert_to_internal_request(self.basic_req)
self.assertIsInstance(adapted, GenerateReqInput)
self.assertFalse(adapted.stream)
self.assertTrue(adapted.return_sampling_mask)
self.assertEqual(adapted.sampling_logprobs_mode, "support")
self.assertEqual(adapted.session_id, "session-1")
self.assertEqual(processed, self.basic_req)
def test_response_template_prefix_uses_prompt_token_ids(self):
class ResponseTemplateAlias(ResponseTemplateToolDetector):
pass
processed_messages = MessageProcessingResult(
"prompt decoded with default spacing",
[1, 2, 3],
None,
None,
[],
[],
None,
uses_response_template=True,
)
self.chat.tool_call_parser = "response-template-alias"
self.basic_req.input_ids = [1, 2, 3]
self.basic_req.tools = [
Tool(function=Function(name="get_weather", parameters={}))
]
self.basic_req.tool_choice = "auto"
self.tm.tokenizer.decode.return_value = "<first><second>"
with (
patch.dict(
FunctionCallParser.ToolCallParserEnum,
{"response-template-alias": ResponseTemplateAlias},
),
patch.object(
self.chat,
"_process_messages",
return_value=processed_messages,
),
):
self.assertTrue(
self.chat._requires_response_template_detokenization(self.basic_req)
)
_, request = self.chat._convert_to_internal_request(self.basic_req)
self.assertEqual(request._response_parser_prefix, "<first><second>")
self.tm.tokenizer.decode.assert_called_once_with(
[1, 2, 3],
skip_special_tokens=False,
spaces_between_special_tokens=False,
)
def test_response_template_prefix_uses_engine_prompt_text(self):
processed_messages = MessageProcessingResult(
"exact engine prompt",
[],
None,
None,
[],
[],
None,
)
self.chat.reasoning_parser = "response_template"
self.chat._reasoning_detector = ResponseTemplateReasoningDetector(
response_template=_RESPONSE_TEMPLATE,
prefix="",
)
self.tm.model_config.is_multimodal = True
def render(request, _):
self.assertIsNone(request.chat_template_kwargs)
return processed_messages
with patch.object(
self.chat,
"_apply_conversation_template",
side_effect=render,
):
adapted, request = self.chat._convert_to_internal_request(self.basic_req)
self.assertEqual(request._response_parser_prefix, "exact engine prompt")
self.assertFalse(request.skip_special_tokens)
self.assertFalse(adapted.sampling_params["spaces_between_special_tokens"])
self.tm.tokenizer.decode.assert_not_called()
def _use_delimited_response_template(
self, reasoning_parser, template=_DELIMITED_RESPONSE_TEMPLATE
):
self.tm.tokenizer.response_template = template
self.tm.tokenizer.encode.side_effect = lambda text, **_: (
[7] if text == "</call>" else [1, 2, 3] if text else []
)
self.tm.model_config.is_multimodal = True
self.chat.tool_call_parser = "response_template"
self.chat.reasoning_parser = reasoning_parser
self.chat._reasoning_detector = (
ResponseTemplateReasoningDetector(response_template=template)
if reasoning_parser
else None
)
# Always-on reasoning: the output grammar starts after the reasoning.
self.template_manager.reasoning_config = (
ReasoningToggleConfig(special_case="always") if reasoning_parser else None
)
@staticmethod
def _stopped_on_token(text, token_id):
return {
"text": text,
"meta_info": {
"id": "chatcmpl-stopped",
"prompt_tokens": 5,
"completion_tokens": 9,
"cached_tokens": 0,
"weight_version": "default",
"finish_reason": {"type": "stop", "matched": token_id},
},
"index": 0,
}
def _convert_rendered_as(self, prompt, request):
rendered = MessageProcessingResult(
prompt=prompt,
prompt_ids=[],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
with patch.object(
self.chat, "_apply_conversation_template", return_value=rendered
):
return self.chat._convert_to_internal_request(request)
def test_response_template_parser_context_records_grammar_start(self):
"""JSON-schema output starts after the reasoning only when the grammar
backend waits for it, which a pattern-only reasoning closer cannot."""
named = {"type": "function", "function": {"name": "get_weather"}}
required = {"tools": [_WEATHER_TOOL], "tool_choice": "required"}
pattern_closed = {
**_DELIMITED_RESPONSE_TEMPLATE,
"fields": {
**_DELIMITED_RESPONSE_TEMPLATE["fields"],
"thinking": {"open": "<think>", "close_pattern": "</think>"},
},
}
cases = {
"auto tool choice": ("response_template", {"tools": [_WEATHER_TOOL]}, None),
"named tool after reasoning": (
"response_template",
{"tools": [_WEATHER_TOOL], "tool_choice": named},
GRAMMAR_AFTER_REASONING,
),
"response_format after reasoning": (
"response_template",
{"response_format": {"type": "json_object"}},
GRAMMAR_AFTER_REASONING,
),
"required tool without reasoning": (None, required, GRAMMAR_FROM_START),
"required tool, reasoning without an end token": (
"response_template",
required,
GRAMMAR_FROM_START,
pattern_closed,
),
}
for name, (reasoning_parser, fields, grammar_start, *template) in cases.items():
with self.subTest(name):
self._use_delimited_response_template(reasoning_parser, *template)
_, request = self._convert_rendered_as(
"<assistant>",
ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather?"}],
**fields,
),
)
self.assertEqual(request._response_parser_grammar_start, grammar_start)
def _chat_outcome(self, adapted, request, output):
"""(calls, reasoning, content, finish reasons) as a client reconstructs them."""
if not request.stream:
choice = self.chat._build_chat_response(request, [output], 0).choices[0]
calls = [
(call.function.name, call.function.arguments)
for call in choice.message.tool_calls or []
]
return (
calls,
choice.message.reasoning_content or "",
choice.message.content or "",
[choice.finish_reason],
)
async def generate():
yield output
self.tm.generate_request = Mock(return_value=generate())
choices = [
choice
for chunk in self._parse_chunks(self._run_chat_stream(adapted, request))
for choice in chunk.get("choices", [])
]
deltas = [choice.get("delta") or {} for choice in choices]
calls: dict[int, list[str]] = {}
for delta in deltas:
for call in delta.get("tool_calls") or []:
name, arguments = calls.setdefault(call["index"], ["", ""])
calls[call["index"]] = [
name + (call["function"].get("name") or ""),
arguments + (call["function"].get("arguments") or ""),
]
return (
[tuple(calls[index]) for index in sorted(calls)],
"".join(delta.get("reasoning_content") or "" for delta in deltas),
"".join(delta.get("content") or "" for delta in deltas),
[
choice["finish_reason"]
for choice in choices
if choice.get("finish_reason")
],
)
def test_required_tool_json_is_parsed_under_delimited_template(self):
"""Required-tool JSON after the reasoning becomes the call, also when it
ends on the tool closer that detokenization keeps."""
payload = '[{"name": "get_weather", "parameters": {"city": "Paris"}}]'
for reasoning_parser in (None, "response_template"):
self._use_delimited_response_template(reasoning_parser)
reasoning = "Check the forecast." if reasoning_parser else ""
text = (f"<think>{reasoning}</think>" if reasoning else "") + payload
text += "</call>"
for stream in (False, True):
with self.subTest(reasoning_parser=reasoning_parser, stream=stream):
adapted, request = self._convert_rendered_as(
"<assistant>",
ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather?"}],
tools=[_WEATHER_TOOL],
tool_choice="required",
stream=stream,
),
)
calls, got_reasoning, content, finish_reasons = self._chat_outcome(
adapted, request, self._stopped_on_token(text, 7)
)
((name, arguments),) = calls
self.assertEqual(name, "get_weather")
self.assertEqual(json.loads(arguments), {"city": "Paris"})
self.assertEqual(got_reasoning, reasoning)
self.assertEqual(content, "")
self.assertEqual(finish_reasons, ["tool_calls"])
def test_json_response_format_with_auto_tools_stays_content(self):
"""A JSON response format with automatic tools must return the JSON
intact; template tool syntax inside its strings is never a call."""
payload = '{"example": "<call:get_weather>{}</call>"}'
for reasoning_parser in (None, "response_template"):
self._use_delimited_response_template(reasoning_parser)
reasoning = "Plan" if reasoning_parser else ""
text = (f"<think>{reasoning}</think>" if reasoning else "") + payload
for stream in (False, True):
with self.subTest(reasoning_parser=reasoning_parser, stream=stream):
adapted, request = self._convert_rendered_as(
"<assistant>",
ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather?"}],
tools=[_WEATHER_TOOL],
tool_choice="auto",
response_format={"type": "json_object"},
stream=stream,
),
)
outcome = self._chat_outcome(
adapted, request, self._stopped_on_token(text, 2)
)
self.assertEqual(outcome, ([], reasoning, payload, ["stop"]))
def test_native_output_under_other_constraints_is_parsed_by_the_template(self):
"""A structural tag, regex or EBNF can spell the template's own framing,
so its content and a call ending on the kept tool closer still parse."""
native = '<answer>Sunny</answer><call:get_weather>{"city": "Paris"}</call>'
structural_tag = {
"type": "structural_tag",
"structures": [
{
"begin": "<call:get_weather>",
"schema": _WEATHER_TOOL["function"]["parameters"],
"end": "</call>",
}
],
"triggers": ["<call:"],
}
regex = "<answer>[^<]+</answer><call:get_weather>[^<]+</call>"
ebnf = 'root ::= "<answer>" [^<]+ "</answer><call:get_weather>" [^<]+ "</call>"'
constraints = {
"structural_tag": {"response_format": structural_tag},
"regex": {"regex": regex},
"ebnf": {"ebnf": ebnf},
}
self._use_delimited_response_template("response_template")
for name, fields in constraints.items():
with self.subTest(name):
_, request = self._convert_rendered_as(
"<assistant>",
ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather?"}],
tools=[_WEATHER_TOOL],
**fields,
),
)
output = self._stopped_on_token("<think>Plan</think>" + native, 7)
choice = self.chat._build_chat_response(request, [output], 0).choices[0]
(call,) = choice.message.tool_calls
self.assertEqual(
(choice.message.reasoning_content, choice.message.content),
("Plan", "Sunny"),
)
self.assertEqual(
(call.function.name, json.loads(call.function.arguments)),
("get_weather", {"city": "Paris"}),
)
self.assertEqual(choice.finish_reason, "tool_calls")
def test_text_only_prompt_reuses_rendered_token_ids(self):
self.tm.model_config.is_multimodal = True
self.chat.chat_encoding_spec = None
self.template_manager.chat_template_name = None
self.chat._tokenizer_auto_adds_specials = False
self.chat._prompt_text_round_trip_is_lossy = False
processed = MessageProcessingResult(
prompt="rendered prompt",
prompt_ids=[11, 22, 33],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
with patch.object(self.chat, "_process_messages", return_value=processed):
adapted, _ = self.chat._convert_to_internal_request(self.basic_req)
self.assertEqual(adapted.input_ids, [11, 22, 33])
self.assertIsNone(adapted.text)
def test_prompt_reuse_keeps_media_path(self):
self.tm.model_config.is_multimodal = True
self.chat.chat_encoding_spec = None
self.template_manager.chat_template_name = None
self.chat._tokenizer_auto_adds_specials = False
self.chat._prompt_text_round_trip_is_lossy = False
processed = MessageProcessingResult(
prompt="rendered prompt",
prompt_ids=[11, 22, 33],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
processed.image_data = ["image"]
with patch.object(self.chat, "_process_messages", return_value=processed):
adapted, _ = self.chat._convert_to_internal_request(self.basic_req)
self.assertEqual(adapted.text, "rendered prompt")
self.assertIsNone(adapted.input_ids)
def test_prompt_reuse_keeps_always_on_processor_path(self):
self.tm.model_config.is_multimodal = True
self.tm.model_config.hf_config.architectures = [
"MossVLForConditionalGeneration"
]
self.chat.chat_encoding_spec = None
self.template_manager.chat_template_name = None
self.chat._tokenizer_auto_adds_specials = False
self.chat._prompt_text_round_trip_is_lossy = False
processed = MessageProcessingResult(
prompt="rendered prompt",
prompt_ids=[11, 22, 33],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
with patch.object(self.chat, "_process_messages", return_value=processed):
adapted, _ = self.chat._convert_to_internal_request(self.basic_req)
self.assertEqual(adapted.text, "rendered prompt")
self.assertIsNone(adapted.input_ids)
def test_chat_applies_pd_header_overrides(self):
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
rid="body-rid",
routed_dp_rank=3,
disagg_prefill_dp_rank=4,
priority=5,
)
self.fastapi_request.headers = {
"x-override-rid": "header-rid",
"x-override-bootstrap-host": "header-host",
"x-override-bootstrap-port": "8998",
"x-override-bootstrap-room": "456",
"x-override-conversation-id": "conversation-1",
"x-override-routed-dp-rank": "6",
"x-override-disagg-prefill-dp-rank": "7",
"x-override-priority": "8",
}
body = request.model_dump()
processed_messages = MessageProcessingResult(
"Test prompt", [1, 2, 3], None, None, [], [], None
)
with (
envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.override(True),
patch.object(
self.chat, "_process_messages", return_value=processed_messages
),
):
response = get_or_create_event_loop().run_until_complete(
self.chat.handle_request(request, self.fastapi_request)
)
self.assertEqual(response.choices[0].message.content, "Test response")
adapted_request = self.tm.generate_request.call_args.args[0]
self.assertEqual(adapted_request.bootstrap_room, 456)
self.assertEqual(adapted_request.bootstrap_host, "header-host")
self.assertEqual(adapted_request.bootstrap_port, 8998)
self.assertEqual(adapted_request.rid, "header-rid")
self.assertEqual(adapted_request.conversation_id, "conversation-1")
self.assertEqual(adapted_request.routed_dp_rank, 6)
self.assertEqual(adapted_request.disagg_prefill_dp_rank, 7)
self.assertEqual(adapted_request.priority, 8)
self.assertEqual(request.model_dump(), body)
self.assertFalse(hasattr(request, "conversation_id"))
def test_convert_to_internal_request_rejects_stream_token_ids(self):
for field in ("return_prompt_token_ids", "return_token_ids"):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
**{field: True},
)
with self.subTest(field=field), self.assertRaisesRegex(ValueError, field):
self.chat._convert_to_internal_request(req, self.fastapi_request)
def test_validate_request_rejects_sampling_mask_without_meta_info(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
return_sampling_mask=True,
)
self.assertEqual(
self.chat._validate_request(req),
"return_sampling_mask requires return_meta_info=true.",
)
def test_validate_request_rejects_flat_output_top_logprobs_without_meta_info(
self,
):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
logprobs=True,
top_logprobs=2,
return_flat_raw_output_top_logprobs=True,
)
self.assertEqual(
self.chat._validate_request(req),
"return_flat_raw_output_top_logprobs requires return_meta_info=true.",
)
def test_convert_to_internal_request_flat_output_top_logprobs_default_to_b64(self):
self.tm.tokenizer = None
# (request flags, expected flat output, expected b64)
cases = (
({"return_flat_raw_output_top_logprobs": True}, True, True),
(
{
"return_flat_raw_output_top_logprobs": True,
"return_flat_raw_top_logprobs_b64": False,
},
True,
False,
),
({}, False, False),
)
for flags, flat, b64 in cases:
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
input_ids=[101, 102, 103],
logprobs=True,
top_logprobs=2,
return_meta_info=True,
**flags,
)
with self.subTest(flags=flags):
adapted, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
# The b64 default must not trip GenerateReqInput validation.
adapted.normalize_batch_and_arguments()
self.assertEqual(adapted.top_logprobs_num, 2)
self.assertEqual(adapted.return_flat_raw_output_top_logprobs, flat)
self.assertEqual(adapted.return_flat_raw_top_logprobs_b64, b64)
def test_convert_to_internal_request_rejects_stream_return_meta_info(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
return_meta_info=True,
)
with self.assertRaisesRegex(
ValueError, "return_meta_info is not supported with streaming"
):
self.chat._convert_to_internal_request(req, self.fastapi_request)
def test_convert_to_internal_request_input_ids_bypasses_template(self):
self.tm.tokenizer = None
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
input_ids=[101, 102, 103],
stop=["STOP"],
return_prompt_token_ids=True,
cache_salt="tenant-a",
extra_key="classification",
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
adapted, processed = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
self.assertEqual(processed, req)
self.assertEqual(adapted.input_ids, [101, 102, 103])
self.assertTrue(adapted.return_prompt_token_ids)
self.assertEqual(adapted.sampling_params["stop"], ["STOP"])
self.assertEqual(adapted.cache_salt, "tenant-a")
self.assertEqual(adapted.extra_key, "classification")
conv_mock.assert_not_called()
def test_kimi_k3_usage_excludes_assistant_generation_stub(self):
self.chat.chat_encoding_spec = "kimi_k3"
ret = [
{
"text": "Answer",
"meta_info": {
"id": "chatcmpl-kimi-k3-usage",
"prompt_tokens": 2075,
"completion_tokens": 1,
"cached_tokens": 0,
"image_tokens": 2035,
"finish_reason": {"type": "stop", "matched": None},
"weight_version": "default",
},
}
]
response = self.chat._build_chat_response(self.basic_req, ret, created=123)
self.assertEqual(response.usage.prompt_tokens, 2072)
self.assertEqual(response.usage.total_tokens, 2073)
self.assertEqual(response.usage.prompt_tokens_details.image_tokens, 2035)
def test_kimi_tool_call_keeps_default_reasoning(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
require_reasoning=True,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertTrue(adapted.require_reasoning)
def test_process_messages_records_template_reasoning_state(self):
self.chat.default_chat_template_kwargs = {"thinking": True}
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=False
)
self.chat.reasoning_parser = "deepseek-v3"
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
rendered = MessageProcessingResult(
prompt="prompt",
prompt_ids=[1, 2, 3],
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=[],
)
with patch.object(
self.chat, "_apply_conversation_template", return_value=rendered
):
processed = self.chat._process_messages(request, is_multimodal=False)
self.assertTrue(processed.require_reasoning)
def test_kimi_tool_call_respects_explicit_reasoning_disable(self):
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": False},
)
with patch.object(self.chat, "_process_messages") as proc_mock:
proc_mock.return_value = MessageProcessingResult(
"",
[1, 2, 3],
None,
None,
[],
[],
None,
)
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertFalse(adapted.require_reasoning)
def test_default_chat_template_kwargs_applied_when_request_unset(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
self.chat.default_chat_template_kwargs = {"enable_thinking": False}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertIs(kwargs["enable_thinking"], False)
def test_default_chat_template_kwargs_overridden_per_request(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
self.chat.default_chat_template_kwargs = {"enable_thinking": False}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
chat_template_kwargs={"enable_thinking": True},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertIs(kwargs["enable_thinking"], True)
def test_default_chat_template_kwargs_mirrors_reasoning_effort(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
self.chat.default_chat_template_kwargs = {"reasoning_effort": "high"}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
self.chat._process_messages(req, is_multimodal=False)
self.assertEqual(req.reasoning_effort, "high")
def test_hunyuan_default_reasoning_effort_is_normalized_for_template(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
special_case="hunyuan_effort"
)
self.chat.reasoning_parser = "hunyuan"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
cases = [
("none", None, "no_think"),
("none", "xhigh", "high"),
]
for default_effort, request_effort, normalized_effort in cases:
with self.subTest(
default_effort=default_effort, request_effort=request_effort
):
self.chat.default_chat_template_kwargs = {
"reasoning_effort": default_effort
}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
reasoning_effort=request_effort,
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertEqual(req.reasoning_effort, normalized_effort)
self.assertEqual(kwargs["reasoning_effort"], normalized_effort)
self.assertNotIn("reasoning_effort", req.chat_template_kwargs)
def test_hunyuan_reasoning_effort_precedence_survives_conversion(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
special_case="hunyuan_effort"
)
self.chat.reasoning_parser = "hunyuan"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
cases = [
("xhigh", "none", "high"),
(None, "no_think", "no_think"),
]
for request_effort, template_effort, normalized_effort in cases:
with self.subTest(
request_effort=request_effort, template_effort=template_effort
):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
reasoning_effort=request_effort,
chat_template_kwargs={"reasoning_effort": template_effort},
)
self.chat._convert_to_internal_request(req)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertEqual(req.reasoning_effort, normalized_effort)
self.assertEqual(kwargs["reasoning_effort"], normalized_effort)
self.assertNotIn("reasoning_effort", req.chat_template_kwargs)
def test_non_hunyuan_default_reasoning_effort_is_unchanged(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
self.chat.default_chat_template_kwargs = {"reasoning_effort": "medium"}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertEqual(req.reasoning_effort, "medium")
self.assertEqual(kwargs["reasoning_effort"], "medium")
self.assertEqual(req.chat_template_kwargs["reasoning_effort"], "medium")
def test_k2_selected_terminator_reaches_sampling_params(self):
self.tm._config_overrides["reasoning_parser"] = "k2_horizon"
self.chat = OpenAIServingChat(self.tm, self.template_manager)
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
self.tm.tokenizer.encode.side_effect = lambda text, **_: (
[8, 9] if text == "</ifm|think_fast>" else [1, 2, 3]
)
req = ChatCompletionRequest(
model="IFM/K2-Horizon-7B",
messages=[{"role": "user", "content": "hi"}],
chat_template_kwargs={"reasoning_effort": "medium"},
)
processed = self.chat._process_messages(req, is_multimodal=False)
self.assertEqual(processed.reasoning_end_token_ids, [8, 9])
with patch.object(self.chat, "_process_messages", return_value=processed):
adapted, _ = self.chat._convert_to_internal_request(req)
self.assertEqual(
adapted.sampling_params["custom_params"][
REQUEST_REASONING_END_TOKEN_IDS_KEY
],
[8, 9],
)
def test_kimi_tool_call_keeps_template_default_thinking(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertNotIn("thinking", kwargs)
def test_kimi_tool_call_keeps_explicit_template_thinking(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": True},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertTrue(kwargs["thinking"])
def test_kimi_tool_call_keeps_explicit_template_thinking_false(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {"a": {"type": "integer"}},
},
},
}
],
tool_choice="required",
chat_template_kwargs={"thinking": False},
)
self.chat._process_messages(req, is_multimodal=False)
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertFalse(kwargs["thinking"])
def test_jinja_uses_openai_tool_schema_first(self):
"""Ensure Jinja chat templates receive OpenAI-shaped tools by default."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"description": "Add two numbers.",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "integer"},
"b": {"type": "integer"},
},
"required": ["a", "b"],
},
},
}
],
)
self.chat._process_messages(req, is_multimodal=False)
expected_tools = [tool.model_dump() for tool in req.tools]
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertEqual(kwargs["tools"], expected_tools)
def test_glm47_without_tools_has_no_tool_call_constraint(self):
"""A plain GLM47 chat must not get the full-assistant EBNF: its
terminal state finishes the request even under ignore_eos."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
self.chat.tool_call_parser = "glm47"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
)
processed = self.chat._process_messages(req, is_multimodal=False)
self.assertIsNone(processed.tool_call_constraint)
def test_jinja_tool_schema_fallback_to_flat_function(self):
"""Fallback to function-only schema when template rejects OpenAI wrapper."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"description": "Add two numbers.",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "integer"},
"b": {"type": "integer"},
},
"required": ["a", "b"],
},
},
}
],
)
self.tm.tokenizer.apply_chat_template.side_effect = [
RuntimeError("template expects flat tools format"),
[1, 2, 3],
]
self.chat._process_messages(req, is_multimodal=False)
first_tools = self.tm.tokenizer.apply_chat_template.call_args_list[0].kwargs[
"tools"
]
second_tools = self.tm.tokenizer.apply_chat_template.call_args_list[1].kwargs[
"tools"
]
self.assertEqual(first_tools, [tool.model_dump() for tool in req.tools])
self.assertEqual(
second_tools, [tool.function.model_dump() for tool in req.tools]
)
def test_glm47_constraint_includes_message_tools(self):
"""Constrain every active message tool, but none when tool use is disabled.
A terminal GLM47 grammar on ``tool_choice="none"`` ends generation early,
while omitting message-scoped tools rejects otherwise valid tool calls.
"""
import xgrammar as xgr
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.tool_call_parser = "glm47"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
compiler = xgr.GrammarCompiler(
xgr.TokenizerInfo(
[bytes([i]) for i in range(256)], vocab_type=xgr.VocabType.RAW
),
max_threads=1,
)
message_tool = {
"type": "function",
"function": {"name": "message_tool", "parameters": {"type": "object"}},
}
top_level_tool = {
"type": "function",
"function": {"name": "top_level_tool", "parameters": {"type": "object"}},
}
named = ToolChoice(function=ToolChoiceFuncName(name="message_tool"))
for role in ("system", "developer"):
for tools in (None, [], [top_level_tool]):
for choice in ("auto", "required", named, "none"):
with self.subTest(role=role, tools=tools, choice=choice):
request = ChatCompletionRequest(
model="x",
messages=[
{"role": role, "content": "", "tools": [message_tool]},
{"role": "user", "content": "Use message_tool."},
],
tools=tools,
tool_choice=choice,
chat_template_kwargs={"enable_thinking": False},
)
result = self.chat._process_messages(
request, is_multimodal=False
)
if choice == "none":
self.assertIsNone(result.tool_call_constraint)
continue
self.assertIsNotNone(result.tool_call_constraint)
kind, ebnf = result.tool_call_constraint
self.assertEqual(kind, "full_assistant_ebnf")
grammar = compiler.compile_grammar(xgr.Grammar.from_ebnf(ebnf))
for name, allowed in (
("message_tool", True),
("top_level_tool", bool(tools) and choice != named),
("unknown_tool", False),
):
matcher = xgr.GrammarMatcher(grammar)
accepted = (
matcher.accept_string(f"<tool_call>{name}</tool_call>")
and matcher.is_completed()
)
self.assertEqual(accepted, allowed, name)
def test_xgrammar_tag_omits_reasoning_when_parser_owns_it(self):
"""ReasonerGrammarBackend owns the thinking prefix when a parser is set."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=True
)
self.tm.server_args.reasoning_parser = "kimi_k2"
self.tm.server_args.tool_call_parser = "kimi_k2"
self.chat.reasoning_parser = "kimi_k2"
self.chat.tool_call_parser = "kimi_k2"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "What is 2+2?"}],
tools=[
{
"type": "function",
"function": {
"name": "add",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "integer"},
"b": {"type": "integer"},
},
"required": ["a", "b"],
},
"strict": True,
},
}
],
tool_choice="required",
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as parser_cls:
parser_cls.ToolCallParserEnum = FunctionCallParser.ToolCallParserEnum
parser = parser_cls.return_value
parser.get_structure_constraint.return_value = ("structural_tag", "tag")
self.chat._process_messages(req, is_multimodal=False)
parser.get_structure_constraint.assert_called_once()
self.assertFalse(
parser.get_structure_constraint.call_args.kwargs["thinking_mode"]
)
def test_kimi_k3_constraint_failure_keeps_native_stop_format(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.chat_encoding_spec = "kimi_k3"
self.chat.tool_call_parser = "kimi_k3"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
tool = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
"required": ["location"],
},
"strict": True,
},
}
cases = (
(None, [TOOLS_CLOSE]),
("USER_STOP", ["USER_STOP", TOOLS_CLOSE]),
(["USER_STOP"], ["USER_STOP", TOOLS_CLOSE]),
([TOOLS_CLOSE], [TOOLS_CLOSE]),
)
for request_stop, expected in cases:
with (
self.subTest(request_stop=request_stop),
patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as parser_cls,
):
parser_cls.ToolCallParserEnum = FunctionCallParser.ToolCallParserEnum
parser = parser_cls.return_value
parser.detector.eot_token = TOOLS_CLOSE
parser.detector.parses_required_natively.return_value = False
parser.get_structure_constraint.return_value = None
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather in Paris?"}],
tools=[tool],
tool_choice="required",
stop=request_stop,
)
original_stop = (
list(request.stop)
if isinstance(request.stop, list)
else request.stop
)
result = self.chat._process_messages(request, is_multimodal=False)
self.assertEqual(result.stop, expected)
self.assertEqual(request.stop, original_stop)
self.assertIsNone(result.tool_call_constraint)
def test_kimi_k3_tool_call_stop_is_scoped_to_active_tools(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.chat_encoding_spec = "kimi_k3"
self.chat.tool_call_parser = "kimi_k3"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Weather in Paris?"}],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object"},
},
}
],
tool_choice="none",
)
result = self.chat._process_messages(request, is_multimodal=False)
self.assertIsNone(result.stop)
def test_dsv_tool_choice_none_omits_tools_from_prompt(self):
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
tool = {
"type": "function",
"function": {
"name": "calculator",
"parameters": {"type": "object"},
},
}
self.chat.chat_encoding_spec = "dsv32"
for tool_choice, expects_tools in (("none", False), ("auto", True)):
with self.subTest(tool_choice=tool_choice):
self.tm.tokenizer.encode.reset_mock()
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Solve 100 / 7."}],
tools=[tool],
tool_choice=tool_choice,
)
self.chat._process_messages(request, is_multimodal=False)
prompt = self.tm.tokenizer.encode.call_args.args[0]
self.assertEqual("calculator" in prompt, expects_tools)
def test_kimi_k3_encoder_receives_wire_request_fields(self):
self.template_manager.chat_template_name = None
self.chat.chat_encoding_spec = "kimi_k3"
self.tm.model_config.is_multimodal = True
self.tm.tokenizer.apply_chat_template.return_value = [7, 8, 9]
tool = {
"type": "function",
"function": {
"name": "weather",
"parameters": {"type": "object"},
},
}
request = ChatCompletionRequest(
model="x",
messages=[
{
"role": "developer",
"content": "<|kimi_image_placeholder|>",
"tools": [tool],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "Explain <|kimi_image_placeholder|>",
},
{"type": "image_url", "image_url": {"url": "image-1"}},
],
},
{
"role": "assistant",
"content": None,
"reasoning_content": "Inspect <|kimi_image_placeholder|>",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {
"name": "inspect",
"arguments": {
"source": "<|kimi_image_placeholder|>",
"nested": ["<|kimi_image_placeholder|>"],
},
},
}
],
},
],
tools=[tool],
tool_choice="required",
response_format={
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {"type": "object"},
"strict": False,
},
},
)
result = self.chat._process_messages(request, is_multimodal=True)
call = self.tm.tokenizer.apply_chat_template.call_args
rendered_messages = call.args[0]
self.assertEqual(rendered_messages[0]["role"], "system")
self.assertEqual(
rendered_messages[0]["content"], "<| kimi_image_placeholder |>"
)
self.assertNotIn("strict", rendered_messages[0]["tools"][0]["function"])
self.assertEqual(
rendered_messages[1]["content"][0]["text"],
"Explain <| kimi_image_placeholder |>",
)
self.assertEqual(
rendered_messages[2]["reasoning_content"],
"Inspect <| kimi_image_placeholder |>",
)
self.assertEqual(
rendered_messages[2]["tool_calls"][0]["function"]["arguments"],
{
"source": "<| kimi_image_placeholder |>",
"nested": ["<| kimi_image_placeholder |>"],
},
)
self.assertEqual(call.kwargs["image_prompts"], ["<|media_pad|>"])
self.assertEqual(call.kwargs["tool_choice"], "required")
self.assertNotIn("strict", call.kwargs["tools"][0]["function"])
self.assertEqual(
call.kwargs["response_format"]["json_schema"]["schema"],
{"type": "object"},
)
self.assertNotIn("schema_", call.kwargs["response_format"]["json_schema"])
self.assertEqual(result.prompt_ids, [7, 8, 9])
self.assertEqual(result.image_data[0].url, "image-1")
def test_kimi_k3_neutralizes_text_only_assistant_history(self):
self.template_manager.chat_template_name = None
self.chat.chat_encoding_spec = "kimi_k3"
self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3]
request = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Run it"},
{
"role": "assistant",
"content": None,
"reasoning_content": "Read <|kimi_image_placeholder|>",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {
"name": "shell",
"arguments": "not-json <|kimi_image_placeholder|>",
},
}
],
},
],
)
self.chat._process_messages(request, is_multimodal=False)
messages = self.tm.tokenizer.apply_chat_template.call_args.args[0]
kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs
self.assertEqual(messages[-1]["role"], "assistant")
self.assertEqual(
messages[-1]["reasoning_content"],
"Read <| kimi_image_placeholder |>",
)
self.assertEqual(
messages[-1]["tool_calls"][0]["function"]["arguments"],
"not-json <| kimi_image_placeholder |>",
)
self.assertNotIn("image_prompts", kwargs)
def test_message_tools_participate_in_validation_across_encodings(self):
tool = {
"type": "function",
"function": {
"name": "weather",
"parameters": {"type": "object"},
},
}
messages = [
{"role": "system", "content": "", "tools": [tool]},
{"role": "user", "content": "Weather?"},
]
for chat_encoding_spec in (None, "dsv4", "dsv32", "kimi_k3"):
with self.subTest(chat_encoding_spec=chat_encoding_spec):
self.chat.chat_encoding_spec = chat_encoding_spec
automatic = ChatCompletionRequest(
model="x", messages=messages, tool_choice=None
)
self.assertEqual(automatic.tool_choice, "auto")
self.assertIsNone(self.chat._validate_request(automatic))
required = ChatCompletionRequest(
model="x", messages=messages, tool_choice="required"
)
self.assertIsNone(self.chat._validate_request(required))
duplicate = ChatCompletionRequest(
model="x",
messages=messages,
tools=[tool],
tool_choice="required",
)
self.assertEqual(
self.chat._validate_request(duplicate),
"Tool names must be unique across request and message tools.",
)
def test_jinja_rejects_non_object_tool_call_arguments(self):
"""History tool call arguments must parse to a JSON object."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
for arguments in ['"Beijing"', '["Beijing"]']:
with self.subTest(arguments=arguments):
self.tm.tokenizer.apply_chat_template.reset_mock()
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": arguments,
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
],
)
with self.assertRaisesRegex(ValueError, "must be a JSON object"):
self.chat._process_messages(req, is_multimodal=False)
self.tm.tokenizer.apply_chat_template.assert_not_called()
def test_jinja_accepts_object_tool_call_arguments_string(self):
"""OpenAI JSON string arguments are converted to dicts for templates."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Beijing"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
],
)
self.chat._process_messages(req, is_multimodal=False)
messages = self.tm.tokenizer.apply_chat_template.call_args.args[0]
self.assertEqual(
messages[1]["tool_calls"][0]["function"]["arguments"],
{"city": "Beijing"},
)
def test_dsv_encoders_reject_non_object_tool_call_arguments(self):
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.chat_encoding_spec = "dsv32"
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '"Beijing"',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
],
)
with self.assertRaisesRegex(ValueError, "must be a JSON object"):
self.chat._process_messages(req, is_multimodal=False)
def test_dsv_encoders_accept_object_tool_call_arguments_string(self):
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
self.template_manager.chat_template_name = None
self.template_manager.jinja_template_content_format = "string"
self.chat.chat_encoding_spec = "dsv32"
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "Where is it raining?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Beijing"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny",
},
],
)
self.chat._process_messages(req, is_multimodal=False)
def test_stop_str_isolation_between_requests(self):
"""Test that stop strings from one request don't affect subsequent requests.
This tests the fix for the bug where conv.stop_str was being mutated globally,
causing stop strings from one request to persist in subsequent requests.
"""
# Mock conversation template with initial stop_str
initial_stop_str = ["\n"]
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
# Create a mock conversation object that will be returned by generate_chat_conv
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_ins.image_data = None
conv_ins.audio_data = None
conv_ins.modalities = []
conv_ins.stop_str = (
initial_stop_str.copy()
) # Template's default stop strings
conv_mock.return_value = conv_ins
# First request with additional stop string
req1 = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "First request"}],
stop=["CUSTOM_STOP"],
)
# Call the actual _apply_conversation_template method (not mocked)
result1 = self.chat._apply_conversation_template(req1, is_multimodal=False)
# Verify first request has both stop strings
expected_stop1 = initial_stop_str + ["CUSTOM_STOP"]
self.assertEqual(result1.stop, expected_stop1)
# Verify the original template's stop_str wasn't mutated after first request
self.assertEqual(conv_ins.stop_str, initial_stop_str)
# Second request without additional stop string
req2 = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Second request"}],
# No custom stop strings
)
result2 = self.chat._apply_conversation_template(req2, is_multimodal=False)
# Verify second request only has original stop strings (no CUSTOM_STOP from req1)
self.assertEqual(result2.stop, initial_stop_str)
self.assertNotIn("CUSTOM_STOP", result2.stop)
self.assertEqual(conv_ins.stop_str, initial_stop_str)
def test_unstreamed_tool_args_completion(self):
"""Test that remaining tool call arguments are sent when generation finishes."""
# Mock FunctionCallParser with detector that has partial tool call data
mock_parser = Mock()
mock_detector = Mock()
# Simulate a tool call that was partially streamed
mock_detector.prev_tool_call_arr = [
{
"name": "get_weather",
"arguments": {"location": "San Francisco", "unit": "celsius"},
}
]
mock_detector.streamed_args_for_tool = [
'{"location": "San Francisco"' # Partial arguments streamed so far
]
mock_parser.detector = mock_detector
content = {
"meta_info": {
"id": "chatcmpl-test123",
}
}
request = ChatCompletionRequest(
model="test",
messages=[{"role": "user", "content": "What's the weather?"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
)
# Test the completion method
result = self.chat._check_for_unstreamed_tool_args(
parser=mock_parser,
content=content,
request=request,
index=0,
created=123,
)
# Should return a chunk with remaining arguments
self.assertIsNotNone(result, "Should return chunk with remaining arguments")
# Parse the result to verify content
self.assertTrue(result.startswith("data: "))
chunk = json.loads(result[6:])
self.assertEqual(chunk["created"], 123)
tool_calls = chunk["choices"][0]["delta"]["tool_calls"]
self.assertEqual(len(tool_calls), 1)
arguments = tool_calls[0]["function"]["arguments"]
self.assertIn(', "unit": "celsius"}', arguments)
self.assertIn(
'"finish_reason":null',
result,
"Should not include finish_reason in completion chunk",
)
def test_unstreamed_tool_args_no_completion_needed(self):
"""Test that no completion chunk is sent when all arguments were already streamed."""
# Mock FunctionCallParser with detector that has complete tool call data
mock_parser = Mock()
mock_detector = Mock()
# Simulate a tool call that was completely streamed
mock_detector.prev_tool_call_arr = [
{"name": "get_weather", "arguments": {"location": "San Francisco"}}
]
mock_detector.streamed_args_for_tool = [
'{"location": "San Francisco"}' # All arguments already streamed
]
mock_parser.detector = mock_detector
content = {
"meta_info": {
"id": "chatcmpl-test123",
}
}
request = ChatCompletionRequest(
model="test",
messages=[{"role": "user", "content": "What's the weather?"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
)
# Test the completion method
result = self.chat._check_for_unstreamed_tool_args(
parser=mock_parser,
content=content,
request=request,
index=0,
created=123,
)
# Should return None since no completion is needed
self.assertIsNone(result, "Should return None when no completion is needed")
def test_unstreamed_tool_args_raw_string_no_completion_needed(self):
"""Test raw JSON string arguments are not JSON-encoded again at finish."""
mock_parser = Mock()
mock_detector = Mock()
mock_detector.prev_tool_call_arr = [
{"name": "report_template_recognition_commit", "arguments": "{}"}
]
mock_detector.streamed_args_for_tool = ["{}"]
mock_parser.detector = mock_detector
content = {
"meta_info": {
"id": "chatcmpl-test123",
}
}
request = ChatCompletionRequest(
model="test",
messages=[{"role": "user", "content": "commit"}],
tools=[
{
"type": "function",
"function": {"name": "report_template_recognition_commit"},
}
],
)
result = self.chat._check_for_unstreamed_tool_args(
parser=mock_parser,
content=content,
request=request,
index=0,
created=123,
)
self.assertIsNone(result, "Should not append encoded quotes")
def test_unstreamed_tool_args_raw_string_completion(self):
"""Test remaining raw JSON string arguments are sent at finish."""
mock_parser = Mock()
mock_detector = Mock()
mock_detector.prev_tool_call_arr = [
{
"name": "get_weather",
"arguments": '{"location": "San Francisco", "unit": "celsius"}',
}
]
mock_detector.streamed_args_for_tool = ['{"location": "San Francisco"']
mock_parser.detector = mock_detector
content = {
"meta_info": {
"id": "chatcmpl-test123",
}
}
request = ChatCompletionRequest(
model="test",
messages=[{"role": "user", "content": "What's the weather?"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
)
result = self.chat._check_for_unstreamed_tool_args(
parser=mock_parser,
content=content,
request=request,
index=0,
created=123,
)
self.assertIsNotNone(result, "Should return chunk with remaining arguments")
chunk = json.loads(result[6:])
tool_calls = chunk["choices"][0]["delta"]["tool_calls"]
self.assertEqual(tool_calls[0]["function"]["arguments"], ', "unit": "celsius"}')
def test_unstreamed_tool_args_no_parser_data(self):
"""Test that no completion chunk is sent when parser has no tool call data."""
# Mock FunctionCallParser with empty detector
mock_parser = Mock()
mock_detector = Mock()
mock_detector.prev_tool_call_arr = []
mock_detector.streamed_args_for_tool = []
mock_parser.detector = mock_detector
content = {
"meta_info": {
"id": "chatcmpl-test123",
}
}
request = ChatCompletionRequest(
model="test",
messages=[{"role": "user", "content": "What's the weather?"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
)
# Test the completion method
result = self.chat._check_for_unstreamed_tool_args(
parser=mock_parser,
content=content,
request=request,
index=0,
created=123,
)
# Should return None since there's no parser data
self.assertIsNone(
result, "Should return None when parser has no tool call data"
)
# ------------- kimi_k2 tool_call_id formatting -------------
def test_kimi_k2_non_streaming_tool_call_id_with_history(self):
"""Ensure non-streaming tool_call.id increase with tool calls history for kimi_k2 parser."""
# Force kimi_k2 parser
self.chat.tool_call_parser = "kimi_k2"
# Prepare request with tool calls history
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "What's the weather today in paris?"},
{
"role": "assistant",
"content": "Let me do some search first.",
"tool_calls": [
{
"id": "functions.get_weather:0",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
}
],
},
{
"role": "tool",
"content": "It's rainy in paris now.",
"tool_call_id": "functions.get_weather:0",
},
{
"role": "assistant",
"content": "It's rainy now.",
},
{
"role": "user",
"content": "What about LA and Tokyo?",
},
],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
stream=False,
)
# Mock FunctionCallParser.parse_non_stream to return one tool call
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as ParserMock:
parser_instance = ParserMock.return_value
# Build a mock ToolCallItem-like object
call_info = Mock()
call_info.name = "get_weather"
call_info.parameters = '{"city":"Loa Angeles"}'
# Kimi-K2 series models might generate fixed number tool_indx,
# ignoring the tool calls history and mess up all the following tool calls
call_info.tool_index = 0
call_info2 = Mock()
call_info2.name = "get_weather"
call_info2.parameters = '{"city":"Tokyo"}'
call_info2.tool_index = 1
parser_instance.has_tool_call.return_value = True
parser_instance.parse_non_stream.return_value = (
"",
[call_info, call_info2],
)
finish_reason = {"type": "stop", "matched": None}
tools = [
{"type": "function", "function": {"name": "get_weather"}},
]
history_tool_calls_cnt = self.chat._get_history_tool_calls_cnt(req)
tool_calls, remaining_text, _ = self.chat._process_tool_calls(
text="<|tool_calls_section_begin|>...",
tools=tools,
finish_reason=finish_reason,
history_tool_calls_cnt=history_tool_calls_cnt,
)
self.assertEqual(history_tool_calls_cnt, 1)
self.assertIsNotNone(tool_calls)
self.assertEqual(len(tool_calls), 2)
self.assertEqual(tool_calls[0].id, "functions.get_weather:1")
self.assertEqual(tool_calls[0].function.name, "get_weather")
self.assertEqual(tool_calls[1].id, "functions.get_weather:2")
self.assertEqual(tool_calls[1].function.name, "get_weather")
def test_non_streaming_tool_call_index_is_the_call_ordinal(self):
"""Two calls to one tool are numbered 0 and 1, as in the streaming deltas,
not by the detector's tool_index (0 for both)."""
self.chat.tool_call_parser = "deepseekv4"
tools = [{"type": "function", "function": {"name": "get_weather"}}]
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as ParserMock:
parser_instance = ParserMock.return_value
calls = []
for city in ("San Francisco", "London"):
call_info = Mock()
call_info.name = "get_weather"
call_info.parameters = json.dumps({"location": city})
call_info.tool_index = 0
calls.append(call_info)
parser_instance.has_tool_call.return_value = True
parser_instance.parse_non_stream.return_value = ("", calls)
tool_calls, _, finish_reason = self.chat._process_tool_calls(
text="<|DSML|tool_calls>...",
tools=tools,
finish_reason={"type": "stop", "matched": None},
history_tool_calls_cnt=0,
)
self.assertEqual([tc.index for tc in tool_calls], [0, 1])
self.assertEqual(
[tc.function.arguments for tc in tool_calls],
[
json.dumps({"location": "San Francisco"}),
json.dumps({"location": "London"}),
],
)
self.assertEqual(finish_reason["type"], "tool_calls")
def test_iquest_release_tool_modes_through_streaming_serving(self):
self.chat.tool_call_parser = "iquest_q1"
self.chat.reasoning_parser = "iquest_q1"
tools = [
{
"type": "function",
"function": {
"name": "run",
"parameters": {
"type": "object",
"properties": {"code": {"type": "string"}},
"required": ["code"],
},
},
}
]
named = ToolChoice(function=ToolChoiceFuncName(name="run"))
payloads = (
(
"auto",
"before<iquest_tool_call>run<arg_key>code</arg_key>"
"<arg_value>你好</arg_value></iquest_tool_call>after",
"beforeafter",
),
("required", '[{"name":"run","parameters":{"code":"你好"}}]', ""),
(named, '{ "code": "你好" }', ""),
)
for choice, payload, expected_content in payloads:
for thinking in (True, False):
wire = ("work</think>" if thinking else "") + payload
for finish in ("stop", "length"):
for size in (1,):
with self.subTest(
choice=choice, thinking=thinking, finish=finish, size=size
):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "run"}],
input_ids=[1],
tools=tools,
tool_choice=choice,
chat_template_kwargs={"thinking": thinking},
stream=True,
)
processed = self.chat._process_messages(req, False)
self.assertFalse(req.skip_special_tokens)
constraint = processed.tool_call_constraint
if choice == "auto":
self.assertIsNone(constraint)
else:
self.assertEqual(constraint[0], "json_schema")
self.assertEqual(
constraint[1]["type"],
"object" if choice == named else "array",
)
async def generate():
for end in range(size, len(wire) + size, size):
yield {
"text": wire[:end],
"meta_info": {
"id": "chatcmpl-iquest-release",
"prompt_tokens": 1,
"completion_tokens": min(end, len(wire)),
"finish_reason": (
{"type": finish, "matched": None}
if end >= len(wire)
else None
),
},
"index": 0,
}
self.tm.generate_request.return_value = generate()
chunks = self._parse_chunks(
self._run_chat_stream(None, req)
)
self.assertFalse(any("error" in chunk for chunk in chunks))
choices = [
c for chunk in chunks for c in chunk.get("choices", [])
]
deltas = [c.get("delta", {}) for c in choices]
self.assertEqual(
"".join(d.get("content") or "" for d in deltas),
expected_content,
)
self.assertEqual(
"".join(
d.get("reasoning_content") or "" for d in deltas
),
"work" if thinking else "",
)
calls = [
c for d in deltas for c in (d.get("tool_calls") or [])
]
self.assertEqual(
[
c["function"]["name"]
for c in calls
if c["function"].get("name")
],
["run"],
)
self.assertTrue(all(c["index"] == 0 for c in calls))
self.assertEqual(
json.loads(
"".join(
c["function"].get("arguments") or ""
for c in calls
)
),
{"code": "你好"},
)
self.assertEqual(
[
c["finish_reason"]
for c in choices
if c.get("finish_reason")
],
["tool_calls" if finish == "stop" else finish],
)
def test_iquest_named_tool_rejects_invalid_arguments(self):
self.chat.tool_call_parser = "iquest_q1"
choice = ToolChoice(function=ToolChoiceFuncName(name="run"))
for payload, finish in (
('{"code":', {"type": "length", "length": 16}),
("42", {"type": "stop", "matched": "stop-marker"}),
):
with self.subTest(payload=payload):
calls, text, finish_reason = self.chat._process_tool_calls(
text=payload,
tools=ChatCompletionRequest(
model="test",
messages=[],
tools=[{"type": "function", "function": {"name": "run"}}],
).tools,
finish_reason=dict(finish),
tool_choice=choice,
)
self.assertIsNone(calls)
self.assertEqual(text, payload)
self.assertEqual(finish_reason, finish)
def test_required_tool_choice_skips_json_fallback_for_native_parser(self):
"""A structural-tag parser owns the output format, so a missing tool
call must not be pushed through the json_schema array fallback."""
self.chat.tool_call_parser = "kimi_k3"
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
texts = {
"prose": "<|open|>response<|sep|>I'll check that.<|close|>response<|sep|>",
"empty": "",
"json_object": '{"name": "get_weather", "parameters": {"city": "Paris"}}',
}
for choice in (
"required",
ToolChoice(function=ToolChoiceFuncName(name="get_weather")),
):
for label, text in texts.items():
with self.subTest(tool_choice=choice, payload=label):
finish_reason = {"type": "stop", "matched": None}
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="WARNING"
) as logs:
tool_calls, remaining, finish_reason = (
self.chat._process_tool_calls(
text=text,
tools=tools,
finish_reason=finish_reason,
tool_choice=choice,
)
)
self.assertIsNone(tool_calls)
self.assertEqual(remaining, text)
self.assertEqual(finish_reason["type"], "stop")
self.assertNotIn("Tool call parsing error", "\n".join(logs.output))
def test_truncated_native_tool_call_logs_and_drops(self):
"""A tools section cut off before its closing tag parses to zero calls
without raising; the sync path used to drop it with no log at all."""
self.chat.tool_call_parser = "kimi_k3"
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
truncated = TOOLS_OPEN + '<|open|>call tool="get_weather" index="1"<|sep|>'
for choice in ("auto", "required"):
with self.subTest(tool_choice=choice):
finish_reason = {"type": "stop", "matched": None}
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="WARNING"
) as logs:
tool_calls, remaining, finish_reason = (
self.chat._process_tool_calls(
text=truncated,
tools=tools,
finish_reason=finish_reason,
tool_choice=choice,
)
)
self.assertIsNone(tool_calls)
self.assertEqual(remaining, "")
self.assertEqual(finish_reason["type"], "stop")
self.assertIn("no complete call", "\n".join(logs.output))
def test_required_tool_choice_json_fallback_tolerates_odd_shapes(self):
"""Parsers without a structural tag keep the JSON array fallback, but a
non-array payload degrades instead of raising an opaque TypeError."""
self.chat.tool_call_parser = "glm45"
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
]
cases = [
(
'[{"name": "get_weather", "parameters": {"city": "Paris"}}]',
'{"city": "Paris"}',
),
(
'{"name": "get_weather", "parameters": {"city": "Paris"}}',
'{"city": "Paris"}',
),
('{"name": "get_weather"}', "{}"),
]
for text, expected_args in cases:
with self.subTest(text=text):
tool_calls, _, finish_reason = self.chat._process_tool_calls(
text=text,
tools=tools,
finish_reason={"type": "stop", "matched": None},
tool_choice="required",
)
self.assertEqual(len(tool_calls), 1)
self.assertEqual(tool_calls[0].function.name, "get_weather")
self.assertEqual(tool_calls[0].function.arguments, expected_args)
self.assertEqual(finish_reason["type"], "tool_calls")
finish_reason = {"type": "stop", "matched": None}
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="ERROR"
):
tool_calls, remaining, finish_reason = self.chat._process_tool_calls(
text='["get_weather"]',
tools=tools,
finish_reason=finish_reason,
tool_choice="required",
)
self.assertIsNone(tool_calls)
self.assertEqual(remaining, '["get_weather"]')
self.assertEqual(finish_reason["type"], "stop")
def test_required_tool_choice_rejects_conflicting_output_constraint(self):
"""response_format and a forced tool call cannot both be honored: the
tool-call constraint was dropped with only a warning, so the model was
constrained to a shape that can never contain a tool call."""
tools = [{"type": "function", "function": {"name": "get_weather"}}]
constraint = ("structural_tag", None)
conflicting = [
{"type": "json_object"},
{
"type": "json_schema",
"json_schema": {"name": "a", "schema": {"type": "object"}},
},
]
for tool_choice in (
"required",
ToolChoice(function=ToolChoiceFuncName(name="get_weather")),
):
for response_format in conflicting:
with self.subTest(tool_choice=tool_choice, rf=response_format["type"]):
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
tools=tools,
tool_choice=tool_choice,
response_format=response_format,
)
with self.assertRaises(ValueError) as ctx:
request.to_sampling_params(
stop=[],
model_generation_config={},
tool_call_constraint=constraint,
)
self.assertIn("cannot be combined", str(ctx.exception))
def test_auto_tool_choice_keeps_response_format_without_raising(self):
""" "auto" means the model need not call a tool, so dropping the
tool-call constraint still leaves a satisfiable request."""
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
tool_choice="auto",
response_format={"type": "json_object"},
)
sampling_params = request.to_sampling_params(
stop=[],
model_generation_config={},
tool_call_constraint=("structural_tag", None),
)
self.assertEqual(sampling_params["json_schema"], '{"type": "object"}')
def test_kimi_k2_streaming_tool_call_id_with_history(self):
"""Ensure streaming first chunk tool_call.id increase with tool calls history for kimi_k2 parser."""
# Force kimi_k2 parser
self.chat.tool_call_parser = "kimi_k2"
# Prepare request with tool calls history
req = ChatCompletionRequest(
model="x",
messages=[
{"role": "user", "content": "What's the weather today in paris?"},
{
"role": "assistant",
"content": "Let me do some search first.",
"tool_calls": [
{
"id": "functions.get_weather:0",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
}
],
},
{
"role": "tool",
"content": "It's rainy in paris now.",
"tool_call_id": "functions.get_weather:0",
},
{
"role": "assistant",
"content": "It's rainy now.",
},
{
"role": "user",
"content": "What about LA?",
},
],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
stream=True,
)
# Patch FunctionCallParser used inside _process_tool_call_stream
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as ParserMock:
parser_instance = ParserMock.return_value
# First call returns one ToolCallItem-like chunk (with name)
first_chunk_call = Mock()
# Kimi-K2 series models might generate fixed number tool_indx,
# ignoring the tool calls history and mess up all the following tool calls
first_chunk_call.tool_index = 0
first_chunk_call.name = "get_weather"
first_chunk_call.parameters = ""
parser_instance.parse_stream_chunk.side_effect = [
("", [first_chunk_call]),
("", []),
]
async def collect_first_tool_chunk():
gen = self.chat._process_tool_call_stream(
index=0,
delta="irrelevant",
parser_dict={},
content={"meta_info": {"id": "chatcmpl-test"}},
request=req,
has_tool_calls={},
created=123,
)
# Get first yielded SSE line
line = None
async for emitted in gen:
line = emitted
break
return line
loop = get_or_create_event_loop()
line = loop.run_until_complete(collect_first_tool_chunk())
self.assertIsNotNone(line)
self.assertTrue(line.startswith("data: "))
payload = json.loads(line[len("data: ") :])
self.assertEqual(payload["created"], 123)
tool_calls = payload["choices"][0]["delta"]["tool_calls"]
self.assertEqual(tool_calls[0]["id"], "functions.get_weather:1")
def test_dpsk_v32_encoding_path(self):
"""Test DeepSeek V3.2 encoding path detection and application."""
from sglang.srt.parser.template_manager import TemplateManager
# Only mock the fields that _use_dpsk_v32_encoding() actually reads:
# tokenizer.chat_template and hf_config.architectures
tm = _MockTokenizerManager()
mock_hf_config = Mock()
mock_hf_config.architectures = ["DeepseekV32ForCausalLM"]
mock_hf_config.to_dict.return_value = {}
tm.model_config.hf_config = mock_hf_config
# Case 1: No chat template + DeepSeek V3.2 arch -> should use dsv32 encoding
tm.tokenizer.chat_template = None
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "dsv32")
# Case 2: Chat template exists -> should NOT use dsv32 encoding
tm.tokenizer.chat_template = "some template"
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertIsNone(serving_chat.chat_encoding_spec)
# Case 3: Not DeepSeek V3.2 architecture -> should NOT use dsv32 encoding
tm.tokenizer.chat_template = None
mock_hf_config.architectures = ["LlamaForCausalLM"]
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertIsNone(serving_chat.chat_encoding_spec)
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
# (release ships a stale V3 jinja we deliberately override).
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
for chat_template in ("stale v3 jinja", None):
tm.tokenizer.chat_template = chat_template
self.assertEqual(
resolve_chat_encoding_spec(
hf_config=mock_hf_config, tokenizer=tm.tokenizer
),
"dsv4",
)
def test_kimi_k3_encoding_detection(self):
from sglang.srt.parser.template_manager import TemplateManager
tm = _MockTokenizerManager()
tm.model_config.hf_config.architectures = ["KimiK3ForConditionalGeneration"]
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "kimi_k3")
tm.model_config.hf_config.architectures = ["LlamaForCausalLM"]
tm.server_args.tool_call_parser = "kimi_k3"
serving_chat = OpenAIServingChat(tm, TemplateManager())
self.assertEqual(serving_chat.chat_encoding_spec, "kimi_k3")
def test_custom_encoders_own_reasoning_history(self):
"""Every custom encoding spec takes reasoning history natively.
The alternative splices a detector's markers into content, which an
encoder that frames its own channels turns into visible raw markers.
A new spec must not silently default to that path.
"""
for spec in _ALL_CHAT_ENCODING_SPECS:
with self.subTest(chat_encoding_spec=spec):
self.chat.chat_encoding_spec = spec
self.assertTrue(self.chat.supports_native_reasoning_history())
# The HF chat-template path keeps the wrap-into-content behaviour.
self.chat.chat_encoding_spec = None
self.assertFalse(self.chat.supports_native_reasoning_history())
# ------------- dsv4 task -------------
def test_dsv4_task_field_schema(self):
"""Top-level `task` accepts the 6 DS task tokens and rejects others."""
for valid in ("action", "query", "authority", "domain", "title", "read_url"):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
task=valid,
)
self.assertEqual(req.task, valid)
# None / unset is fine
self.assertIsNone(self.basic_req.task)
# Bogus value rejected at validation time
from pydantic import ValidationError
with self.assertRaises(ValidationError):
ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
task="bogus",
)
def test_streaming_abort_yields_error(self):
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
err_msg = "Aborted by scheduler"
err_code = HTTPStatus.INTERNAL_SERVER_ERROR
async def _mock_generate_abort():
yield {
"text": "Partial ",
"meta_info": {
"id": "chatcmpl-test",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": 0,
"finish_reason": {
"type": "abort",
"status_code": err_code,
"message": err_msg,
},
"output_token_logprobs": [],
"output_token_logprobs_length": 0,
"output_top_logprobs": [],
},
"index": 0,
}
self.tm.generate_request.return_value = _mock_generate_abort()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
temperature=0.7,
max_tokens=100,
stream=True,
logprobs=True,
top_logprobs=5,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
# Create a mock conversation object
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
async def run_stream():
chunks = []
try:
async for chunk in self.chat._generate_chat_stream(
adapted_request, req, self.fastapi_request
):
chunks.append(chunk)
except Exception as e:
print(f"Error during stream iteration: {e}")
return chunks
loop = get_or_create_event_loop()
chunks = loop.run_until_complete(run_stream())
error_chunk_data = None
for c in chunks:
if "error" in c:
error_chunk_data = json.loads(c[len("data: ") :])
break
self.assertIsNotNone(error_chunk_data, "Error chunk not found in stream")
self.assertEqual(error_chunk_data["error"]["message"], err_msg)
self.assertEqual(error_chunk_data["error"]["code"], err_code.value)
# Ensure the stream stops after the abort error
# The last chunk should be "data: [DONE]\n\n"
self.assertEqual(chunks[-1], "data: [DONE]\n\n")
# Check that there is an error chunk and a DONE chunk
self.assertEqual(len(chunks), 2)
self.assertIn("error", chunks[0])
def test_streaming_abort_with_ids_enabled(self):
"""Test that a terminal abort with input/output ids enabled yields only an error and [DONE]."""
err_msg = "Aborted by scheduler"
err_code = HTTPStatus.INTERNAL_SERVER_ERROR
async def _mock_generate_abort():
yield {
"text": "Partial ",
"prompt_token_ids": [4, 5, 6],
"output_ids": [1, 2, 3],
"meta_info": {
"id": "chatcmpl-test",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": 0,
"finish_reason": {
"type": "abort",
"status_code": err_code,
"message": err_msg,
},
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
self.tm.generate_request.return_value = _mock_generate_abort()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
temperature=0.7,
max_tokens=100,
stream=True,
return_input_ids_in_sglext=True,
return_output_ids_in_sglext=True,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
async def run_stream():
chunks = []
try:
async for chunk in self.chat._generate_chat_stream(
adapted_request, req, self.fastapi_request
):
chunks.append(chunk)
except Exception as e:
print(f"Error during stream iteration: {e}")
return chunks
loop = get_or_create_event_loop()
chunks = loop.run_until_complete(run_stream())
# Exactly one error chunk followed by [DONE]; no sglext ids leak.
self.assertIn("error", chunks[0])
self.assertEqual(chunks[1], "data: [DONE]\n\n")
self.assertFalse(
any("input_ids" in c or "output_ids" in c for c in chunks),
"sglext ids event leaked after abort error",
)
error_chunk_data = json.loads(chunks[0][len("data: ") :])
self.assertEqual(error_chunk_data["error"]["message"], err_msg)
self.assertEqual(error_chunk_data["error"]["code"], err_code.value)
def test_streaming_error_abort_still_finalizes_other_choices(self):
"""An error abort ends generation but still runs finalization, so a
sibling choice's finish chunk and the usage chunk follow the error."""
err_code = HTTPStatus.SERVICE_UNAVAILABLE
async def _mock_generate():
yield {
"text": "hello",
"prompt_token_ids": [4, 5, 6],
"output_ids": [1, 2],
"meta_info": {
"id": "chatcmpl-multi-abort",
"prompt_tokens": 3,
"completion_tokens": 2,
"cached_tokens": 0,
"finish_reason": {"type": "stop"},
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
yield {
"text": "partial",
"prompt_token_ids": [4, 5, 6],
"output_ids": [7],
"meta_info": {
"id": "chatcmpl-multi-abort",
"prompt_tokens": 3,
"completion_tokens": 1,
"cached_tokens": 0,
"finish_reason": {
"type": "abort",
"status_code": err_code,
"message": "Aborted by scheduler",
},
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 1,
}
self.tm.generate_request.return_value = _mock_generate()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
n=2,
stream=True,
stream_options={"include_usage": True},
return_input_ids_in_sglext=True,
return_output_ids_in_sglext=True,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
chunks = self._run_chat_stream(adapted_request, req)
error_idx = next(i for i, c in enumerate(chunks) if "error" in c)
after_error = self._parse_chunks(chunks[error_idx + 1 :])
finish_reasons = [
choice["finish_reason"]
for c in after_error
for choice in c.get("choices", [])
if choice.get("finish_reason") is not None
]
self.assertEqual(finish_reasons, ["stop"])
self.assertTrue(
any(c.get("usage") is not None for c in after_error),
"usage chunk dropped after error abort",
)
def test_truncated_response_template_call_keeps_stop_without_arguments(self):
self.chat.tool_call_parser = "response_template"
self.tm.tokenizer.response_template = _RESPONSE_TEMPLATE
request = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object"},
},
}
],
)
request._response_parser_prefix = "<assistant>"
async def generate():
yield {
"text": '<call:get_weather>{"city":',
"meta_info": {
"id": "chatcmpl-truncated-call",
"prompt_tokens": 5,
"completion_tokens": 3,
"cached_tokens": 0,
"reasoning_tokens": 0,
"finish_reason": {"type": "stop", "matched": "CUSTOM_STOP"},
},
"index": 0,
}
self.tm.generate_request = Mock(return_value=generate())
adapted_request = GenerateReqInput(
text="<assistant>",
sampling_params={},
stream=True,
)
chunks = self._run_chat_stream(adapted_request, request)
parsed = self._parse_chunks(chunks)
arguments = "".join(
tool_call["function"]["arguments"]
for chunk in parsed
for choice in chunk.get("choices", [])
for tool_call in choice.get("delta", {}).get("tool_calls", [])
)
finish = next(
choice
for chunk in parsed
for choice in chunk.get("choices", [])
if choice.get("finish_reason") is not None
)
self.assertEqual(arguments, "")
self.assertEqual(finish["finish_reason"], "stop")
self.assertEqual(finish["matched_stop"], "CUSTOM_STOP")
def _run_chat_stream(self, adapted_request, req):
async def run_stream():
chunks = []
async for chunk in self.chat._generate_chat_stream(
adapted_request, req, self.fastapi_request
):
chunks.append(chunk)
return chunks
return get_or_create_event_loop().run_until_complete(run_stream())
def _parse_chunks(self, chunks):
parsed = []
for c in chunks:
if c.startswith("data: ") and c != "data: [DONE]\n\n":
parsed.append(json.loads(c[len("data: ") :]))
return parsed
def test_streaming_created_is_stable_across_chunks(self):
for spec in (None, "inkling"):
with self.subTest(spec=spec):
if spec == "inkling":
self._stream_inkling_output_in_two_chunks()
self._assert_streaming_created_is_stable()
def _stream_inkling_output_in_two_chunks(self):
from sglang.srt.parser.inkling_renderer import load_tml_renderers
tokenizer = load_tml_renderers().tokenizer
output_ids = [
tokenizer.encode_special("message_model"),
tokenizer.encode_special("content_text"),
*tokenizer.encode_ordinary("Hello there"),
tokenizer.encode_special("end_message"),
]
async def generate():
for count, finish in (
(len(output_ids) - 1, None),
(len(output_ids), {"type": "stop", "matched": None}),
):
yield {
"text": "",
"output_ids": output_ids[:count],
"index": 0,
"meta_info": {
"id": "chatcmpl-inkling",
"prompt_tokens": 2,
"completion_tokens": count,
"cached_tokens": 0,
"finish_reason": finish,
},
}
self.chat.chat_encoding_spec = "inkling"
self.chat.reasoning_parser = "inkling"
self.chat.tool_call_parser = "inkling"
self.tm.tokenizer = None
self.chat._encoding_adapter = self.chat._build_chat_encoding_adapter()
self.tm.generate_request.return_value = generate()
def _assert_streaming_created_is_stable(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
stream_options={"include_usage": True},
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.time.time",
side_effect=[100, 101, 102, 103],
) as current_time:
chunks = self._run_chat_stream(Mock(), req)
parsed = self._parse_chunks(chunks)
self.assertGreater(len(parsed), 1)
identities = {
(chunk["id"], chunk["created"], chunk["model"]) for chunk in parsed
}
self.assertEqual(len(identities), 1)
response_id, created, model = identities.pop()
self.assertTrue(response_id)
self.assertEqual((created, model), (100, "x"))
self.assertTrue(
any(not chunk["choices"] and chunk.get("usage") for chunk in parsed)
)
current_time.assert_called_once_with()
async def _collect_stream_content(self, content, choice_logprobs, req):
chunks = []
async for chunk in self.chat._generate_stream_content(
content=content,
index=0,
request=req,
created=123,
stream_offsets={},
reasoning_parser_dict={},
parser_dict={},
has_tool_calls={},
choice_logprobs=choice_logprobs,
finish_reason_type="stop",
continuous_usage_stats=False,
prompt_tokens={0: 5},
reasoning_tokens={0: 0},
completion_tokens={0: 1},
):
chunks.append(chunk)
return chunks
def test_streaming_top_logprobs_follow_each_token_in_chunk(self):
"""Each token of a multi-token streaming chunk keeps its own alternatives."""
content = {
"meta_info": {
"output_token_logprobs": [
(-0.1, 1, "a"),
(-0.2, 2, "b"),
(-0.3, 3, "c"),
],
"output_top_logprobs": [
[(-0.1, 1, "a"), (-2.0, 9, "x")],
[(-0.2, 2, "b"), (-3.0, 8, "y")],
[(-0.3, 3, "c"), (-4.0, 7, "z")],
],
},
}
choice_logprobs = self.chat._process_streaming_logprobs(content, 1, 3)
tokens = [entry.token for entry in choice_logprobs.content]
alternatives = [
[top.token for top in entry.top_logprobs]
for entry in choice_logprobs.content
]
self.assertEqual(tokens, ["b", "c"])
self.assertEqual(alternatives, [["b", "y"], ["c", "z"]])
def test_streaming_logprobs_attached_with_reasoning_parser(self):
"""Logprobs must ride on the reasoning chunk when a reasoning parser is active."""
self.chat.reasoning_parser = "qwen3"
content = {
"text": "thinking...",
"meta_info": {
"id": "chatcmpl-reasoning",
"prompt_tokens": 5,
"completion_tokens": 2,
"cached_tokens": 0,
"finish_reason": {"type": "stop", "matched": None},
"output_token_logprobs": [(0.1, 1, "think"), (0.2, 2, "ing")],
"output_top_logprobs": [],
"output_token_logprobs_length": 2,
},
"index": 0,
}
choice_logprobs = self.chat._process_streaming_logprobs(
content, 0, 2
).model_dump()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
logprobs=True,
separate_reasoning=True,
)
with patch.object(self.chat, "_process_reasoning_stream") as proc_mock:
proc_mock.return_value = ("Let me think", "")
chunks = get_or_create_event_loop().run_until_complete(
self._collect_stream_content(content, choice_logprobs, req)
)
parsed = self._parse_chunks(chunks)
reasoning_chunks = [
c
for c in parsed
if c["choices"][0]["delta"].get("reasoning_content") is not None
]
self.assertTrue(
reasoning_chunks, "reasoning_parser did not emit a reasoning_content chunk"
)
logprob_chunks = [
c for c in parsed if c["choices"][0].get("logprobs") is not None
]
self.assertTrue(
logprob_chunks,
"logprobs dropped: no chunk carried logprobs with reasoning_parser active",
)
def test_streaming_logprobs_flushed_when_tool_parser_buffers_delta(self):
"""Logprobs must be flushed on a standalone chunk when the tool parser emits no content delta."""
self.chat.tool_call_parser = "hermes"
content = {
"text": "(<",
"meta_info": {
"id": "chatcmpl-tool",
"prompt_tokens": 5,
"completion_tokens": 1,
"cached_tokens": 0,
"finish_reason": {"type": "stop", "matched": None},
"output_token_logprobs": [(0.3, 9, "(<")],
"output_top_logprobs": [],
"output_token_logprobs_length": 1,
},
"index": 0,
}
choice_logprobs = self.chat._process_streaming_logprobs(
content, 0, 1
).model_dump()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
tools=[{"type": "function", "function": {"name": "get_weather"}}],
stream=True,
logprobs=True,
)
async def _empty_tool_stream(*args, **kwargs):
return
yield # make it an async generator
with patch.object(self.chat, "_process_tool_call_stream", _empty_tool_stream):
chunks = get_or_create_event_loop().run_until_complete(
self._collect_stream_content(content, choice_logprobs, req)
)
parsed = self._parse_chunks(chunks)
logprob_chunks = [
c for c in parsed if c["choices"][0].get("logprobs") is not None
]
self.assertTrue(
logprob_chunks,
"logprobs dropped: no flush chunk carried logprobs when tool parser buffered the delta",
)
def test_streaming_logprobs_not_flushed_on_empty_delta_step_without_parser(self):
"""With no parser active, an empty-delta step must not emit a standalone
empty-delta logprobs chunk — clients expect each chunk to carry real
content/reasoning/tool_calls or a finish_reason."""
self.chat.reasoning_parser = None
self.chat.tool_call_parser = None
async def _mock_generate():
yield {
"text": "",
"meta_info": {
"id": "chatcmpl-empty",
"prompt_tokens": 5,
"completion_tokens": 1,
"cached_tokens": 0,
"finish_reason": {"type": "stop", "matched": None},
"output_token_logprobs": [(0.5, 7, "")],
"output_top_logprobs": [],
"output_token_logprobs_length": 1,
},
"index": 0,
}
self.tm.generate_request.return_value = _mock_generate()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
logprobs=True,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
chunks = self._run_chat_stream(adapted_request, req)
parsed = self._parse_chunks(chunks)
empty_logprob_chunks = [
c
for c in parsed
if c["choices"][0].get("logprobs") is not None
and not c["choices"][0]["delta"].get("content")
and not c["choices"][0]["delta"].get("reasoning_content")
and not c["choices"][0]["delta"].get("tool_calls")
and not c["choices"][0].get("finish_reason")
]
self.assertFalse(
empty_logprob_chunks,
"empty-delta logprobs chunk emitted without a parser; would break client chunk-shape assumptions",
)
def test_non_streaming_extension_fields_emit_sglext_without_meta_info(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
return_meta_info=False,
return_routed_experts=True,
return_cached_tokens_details=True,
)
ret = [
{
"text": "Cached response",
"meta_info": {
"id": "chatcmpl-cache-test",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": 6,
"cached_tokens_details": {
"device": 4,
"host": 1,
"storage": 1,
"storage_backend": "file",
},
"routed_experts": "cm91dGUtYQ==",
"finish_reason": {"type": "stop", "matched": None},
"weight_version": "default",
},
}
]
response = self.chat._build_chat_response(req, ret, 1234567890)
self.assertIsNotNone(response.sglext)
self.assertEqual(response.sglext.routed_experts, "cm91dGUtYQ==")
self.assertEqual(
response.sglext.cached_tokens_details.model_dump(exclude_none=True),
{
"device": 4,
"host": 1,
"storage": 1,
"storage_backend": "file",
},
)
self.assertIsNone(response.choices[0].meta_info)
dumped_response = response.model_dump()
self.assertIn("sglext", dumped_response)
self.assertNotIn("meta_info", dumped_response["choices"][0])
def test_non_streaming_meta_info_omits_response_level_routed_experts(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
n=2,
return_meta_info=True,
return_routed_experts=True,
)
ret = [
{
"text": f"Response {index}",
"meta_info": {
"id": "chatcmpl-meta-test",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": index,
"routed_experts": routed_experts,
"finish_reason": {"type": "stop", "matched": None},
"weight_version": "default",
},
}
for index, routed_experts in enumerate(["cm91dGUtYQ==", "cm91dGUtYg=="])
]
response = self.chat._build_chat_response(req, ret, 1234567890)
self.assertIsNone(
response.sglext,
"sglext is absent only when routed_experts is the sole extension",
)
self.assertEqual(
[choice.meta_info for choice in response.choices],
[ret_item["meta_info"] for ret_item in ret],
)
dumped_response = response.model_dump()
self.assertNotIn("sglext", dumped_response)
self.assertEqual(
[choice["meta_info"] for choice in dumped_response["choices"]],
[ret_item["meta_info"] for ret_item in ret],
)
serialized_response = json.dumps(dumped_response)
self.assertEqual(serialized_response.count('"routed_experts"'), 2)
def test_non_streaming_meta_info_preserves_cache_and_spec_in_sglext(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
n=2,
return_meta_info=True,
return_routed_experts=True,
return_cached_tokens_details=True,
return_spec_tokens_details=True,
)
routed_experts = ["cm91dGUtYQ==", "cm91dGUtYg=="]
ret = [_spec_result(index) for index in range(2)]
for index, ret_item in enumerate(ret):
ret_item["meta_info"].update(
{
"cached_tokens_details": {
"device": 4 - index,
"host": index,
},
"routed_experts": routed_experts[index],
}
)
response = self.chat._build_chat_response(req, ret, 1234567890)
self.assertIsNotNone(response.sglext)
self.assertIsNone(response.sglext.routed_experts)
self.assertEqual(
response.sglext.cached_tokens_details.model_dump(exclude_none=True),
{"device": 4, "host": 0},
)
self.assertEqual(
[item.spec_cap_length for item in response.sglext.spec_tokens_details],
[1.0, 2.0],
)
self.assertEqual(
[choice.meta_info for choice in response.choices],
[ret_item["meta_info"] for ret_item in ret],
)
dumped_response = response.model_dump()
self.assertIn("sglext", dumped_response)
self.assertIn("spec_tokens_details", dumped_response["sglext"])
self.assertNotIn("routed_experts", dumped_response["sglext"])
self.assertEqual(
dumped_response["sglext"]["cached_tokens_details"],
{"device": 4, "host": 0},
)
serialized_response = json.dumps(dumped_response)
self.assertEqual(serialized_response.count('"routed_experts"'), 2)
def test_parallel_sampling_returns_spec_details_per_choice(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
n=2,
return_spec_tokens_details=True,
)
ret = [_spec_result(index) for index in range(2)]
response = self.chat._build_chat_response(req, ret, 1234567890)
details = response.sglext.spec_tokens_details
self.assertEqual([item.spec_cap_length for item in details], [1.0, 2.0])
self.assertEqual(
[item.spec_cap_lens_histogram for item in details],
[[0, 1], [1, 1]],
)
single_req = req.model_copy(update={"n": 1})
single_response = self.chat._build_chat_response(
single_req, ret[:1], 1234567890
)
self.assertEqual(
single_response.sglext.spec_tokens_details.spec_cap_length,
1.0,
)
def test_non_streaming_chat_response_returns_requested_token_ids_and_meta_info(
self,
):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
return_prompt_token_ids=True,
return_token_ids=True,
return_meta_info=True,
)
ret = [
{
"text": "Answer",
"output_ids": [21, 22],
"prompt_token_ids": [11, 12, 13],
"meta_info": {
"id": "chatcmpl-token-ids",
"prompt_tokens": 3,
"completion_tokens": 1,
"cached_tokens": 0,
"finish_reason": {"type": "stop", "matched": None},
"weight_version": "default",
},
}
]
response = self.chat._build_chat_response(req, ret, created=123)
choice = response.choices[0]
self.assertEqual(choice.prompt_token_ids, [11, 12, 13])
self.assertEqual(choice.response_token_ids, [21, 22])
self.assertEqual(choice.meta_info, ret[0]["meta_info"])
dumped_choice = response.model_dump()["choices"][0]
self.assertEqual(dumped_choice["prompt_token_ids"], [11, 12, 13])
self.assertEqual(dumped_choice["response_token_ids"], [21, 22])
self.assertEqual(dumped_choice["meta_info"], ret[0]["meta_info"])
def test_streaming_cached_tokens_details_emits_sglext(self):
"""Test that streaming chat responses emit cached token details in sglext."""
async def _mock_generate_with_cached_tokens_details():
yield {
"text": "Cached response",
"meta_info": {
"id": "chatcmpl-cache-test",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": 6,
"cached_tokens_details": {
"device": 4,
"host": 1,
"storage": 1,
"storage_backend": "file",
},
"finish_reason": {"type": "stop", "matched": None},
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
self.tm.generate_request.return_value = (
_mock_generate_with_cached_tokens_details()
)
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
stream=True,
return_cached_tokens_details=True,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
async def run_stream():
chunks = []
async for chunk in self.chat._generate_chat_stream(
adapted_request, req, self.fastapi_request
):
chunks.append(chunk)
return chunks
loop = get_or_create_event_loop()
chunks = loop.run_until_complete(run_stream())
sglext_chunks = []
for chunk in chunks:
if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]":
continue
data = json.loads(chunk[len("data: ") :])
if "sglext" in data:
sglext_chunks.append(data)
self.assertEqual(len(sglext_chunks), 1)
self.assertEqual(sglext_chunks[0]["choices"], [])
self.assertEqual(
sglext_chunks[0]["sglext"]["cached_tokens_details"],
{
"device": 4,
"host": 1,
"storage": 1,
"storage_backend": "file",
},
)
def _output_ids_ret(self, *output_ids_by_choice):
return [
{
"text": "Answer",
"prompt_token_ids": [1, 2, 3],
"output_ids": list(output_ids),
"meta_info": {
"id": "chatcmpl-output-ids",
"prompt_tokens": 3,
"completion_tokens": len(output_ids),
"cached_tokens": 0,
"finish_reason": {"type": "stop", "matched": None},
"weight_version": "default",
},
}
for output_ids in output_ids_by_choice
]
def test_non_streaming_ids_emit_sglext(self):
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
n=2,
return_input_ids_in_sglext=True,
return_output_ids_in_sglext=True,
)
response = self.chat._build_chat_response(
req, self._output_ids_ret([5, 6, 7], [8, 9]), 123
)
self.assertIsNotNone(response.sglext)
self.assertEqual(response.sglext.input_ids, [1, 2, 3])
self.assertEqual(response.sglext.output_ids, [[5, 6, 7], [8, 9]])
dumped = json.loads(response.model_dump_json())
self.assertEqual(dumped["sglext"]["input_ids"], [1, 2, 3])
self.assertEqual(dumped["sglext"]["output_ids"], [[5, 6, 7], [8, 9]])
# None sglext fields stay stripped from the serialized response.
self.assertNotIn("routed_experts", dumped["sglext"])
def test_non_streaming_ids_not_returned_by_default(self):
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
response = self.chat._build_chat_response(
req, self._output_ids_ret([5, 6, 7]), 123
)
self.assertIsNone(response.sglext)
def test_non_streaming_ids_server_default_enables_flag(self):
enter_override(
self,
get_context().override_server_args(
return_input_ids=True, return_output_ids=True
),
)
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
response = self.chat._build_chat_response(
req, self._output_ids_ret([5, 6, 7]), 123
)
self.assertIsNotNone(response.sglext)
self.assertEqual(response.sglext.input_ids, [1, 2, 3])
self.assertEqual(response.sglext.output_ids, [[5, 6, 7]])
def test_ids_headers_enable_flags(self):
self.fastapi_request.headers = {
"x-sglext-return-input-ids": "1",
"x-sglext-return-output-ids": "1",
}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
)
processed_messages = MessageProcessingResult(
"Test prompt",
[1, 2, 3],
None,
None,
[],
["</s>"],
None,
)
with patch.object(
self.chat, "_process_messages", return_value=processed_messages
):
_, processed_request = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
self.assertTrue(processed_request.return_input_ids_in_sglext)
self.assertTrue(processed_request.return_output_ids_in_sglext)
def _run_output_ids_stream(
self,
chunk_output_ids,
incremental,
framed=False,
return_raw=False,
cached_tokens_details=None,
):
"""Stream chunks with incremental or cumulative output_ids;
return parsed sglext chunks, or raw SSE strings when return_raw."""
enter_override(
self,
get_context().override_server_args(
incremental_streaming_output=incremental
),
)
if framed:
self.fastapi_request.headers["x-sglext-ids-framed"] = "1"
async def _mock_generate():
for i, ids in enumerate(chunk_output_ids):
finished = i == len(chunk_output_ids) - 1
yield {
"text": "chunk",
"prompt_token_ids": [1, 2, 3],
"output_ids": list(ids),
"meta_info": {
"id": "chatcmpl-output-ids-stream",
"prompt_tokens": 3,
"completion_tokens": 1 + i,
"cached_tokens": 0,
"cached_tokens_details": cached_tokens_details,
"finish_reason": (
{"type": "stop", "matched": None} if finished else None
),
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
self.tm.generate_request.return_value = _mock_generate()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
stream=True,
return_input_ids_in_sglext=True,
return_output_ids_in_sglext=True,
return_cached_tokens_details=cached_tokens_details is not None,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
chunks = self._run_chat_stream(adapted_request, req)
if return_raw:
return chunks
return [c for c in self._parse_chunks(chunks) if "sglext" in c]
def test_streaming_output_ids_incremental_accumulates_deltas(self):
sglext_chunks = self._run_output_ids_stream([[5, 6], [7]], incremental=True)
self.assertEqual(len(sglext_chunks), 1)
self.assertEqual(sglext_chunks[0]["choices"], [])
# input_ids is captured once as a flat shared prompt, not accumulated.
self.assertEqual(sglext_chunks[0]["sglext"]["input_ids"], [1, 2, 3])
self.assertEqual(sglext_chunks[0]["sglext"]["output_ids"], [[5, 6, 7]])
def test_streaming_output_ids_non_incremental_keeps_latest_full_list(self):
sglext_chunks = self._run_output_ids_stream(
[[5, 6], [5, 6, 7]], incremental=False
)
self.assertEqual(len(sglext_chunks), 1)
self.assertEqual(sglext_chunks[0]["sglext"]["input_ids"], [1, 2, 3])
self.assertEqual(sglext_chunks[0]["sglext"]["output_ids"], [[5, 6, 7]])
_SGLEXT_IDS_EVENT_PREFIX = "event: sglext_ids\ndata: "
def test_streaming_framed_ids_emits_named_event(self):
raw = self._run_output_ids_stream(
[[5, 6], [5, 6, 7]], incremental=False, framed=True, return_raw=True
)
named = [c for c in raw if c.startswith(self._SGLEXT_IDS_EVENT_PREFIX)]
self.assertEqual(len(named), 1)
payload = json.loads(named[0][len(self._SGLEXT_IDS_EVENT_PREFIX) :])
self.assertEqual(payload["choices"], [])
# The named event carries ONLY the id fields.
self.assertEqual(set(payload["sglext"]), {"input_ids", "output_ids"})
self.assertEqual(payload["sglext"]["input_ids"], [1, 2, 3])
self.assertEqual(payload["sglext"]["output_ids"], [[5, 6, 7]])
# All requested sglext fields were ids, so no plain sglext chunk.
self.assertEqual([c for c in self._parse_chunks(raw) if "sglext" in c], [])
def test_streaming_framed_splits_non_id_fields_into_plain_chunk(self):
raw = self._run_output_ids_stream(
[[5, 6], [5, 6, 7]],
incremental=False,
framed=True,
return_raw=True,
cached_tokens_details={"device": 2, "host": 1},
)
named = [c for c in raw if c.startswith(self._SGLEXT_IDS_EVENT_PREFIX)]
self.assertEqual(len(named), 1)
named_payload = json.loads(named[0][len(self._SGLEXT_IDS_EVENT_PREFIX) :])
self.assertEqual(set(named_payload["sglext"]), {"input_ids", "output_ids"})
plain_sglext = [c for c in self._parse_chunks(raw) if "sglext" in c]
self.assertEqual(len(plain_sglext), 1)
self.assertEqual(
plain_sglext[0]["sglext"]["cached_tokens_details"],
{"device": 2, "host": 1},
)
self.assertNotIn("input_ids", plain_sglext[0]["sglext"])
self.assertNotIn("output_ids", plain_sglext[0]["sglext"])
# The plain non-id chunk precedes the named ids event.
plain_raw_idx = next(
i
for i, c in enumerate(raw)
if c.startswith("data: ") and "cached_tokens_details" in c
)
self.assertLess(plain_raw_idx, raw.index(named[0]))
def _run_output_ids_stream_with_graceful_abort(
self, normal_chunks, abort_output_ids, incremental, abort_completion_tokens
):
"""Stream normal chunks then a graceful-abort chunk; return parsed sglext chunks."""
enter_override(
self,
get_context().override_server_args(
incremental_streaming_output=incremental
),
)
async def _mock_generate():
generated = 0
for ids in normal_chunks:
generated = generated + len(ids) if incremental else len(ids)
yield {
"text": "chunk",
"output_ids": list(ids),
"meta_info": {
"id": "chatcmpl-abort-ids-stream",
"prompt_tokens": 3,
"completion_tokens": generated,
"cached_tokens": 0,
"finish_reason": None,
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
# Graceful abort terminal chunk (no status_code): falls through to
# the normal finalization path.
yield {
"text": "chunk",
"output_ids": list(abort_output_ids),
"meta_info": {
"id": "chatcmpl-abort-ids-stream",
"prompt_tokens": 3,
"completion_tokens": abort_completion_tokens,
"cached_tokens": 0,
"finish_reason": {"type": "abort", "message": "Aborted."},
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
self.tm.generate_request.return_value = _mock_generate()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
stream=True,
return_output_ids_in_sglext=True,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
chunks = self._run_chat_stream(adapted_request, req)
return [c for c in self._parse_chunks(chunks) if "sglext" in c]
def test_streaming_output_ids_incremental_ignores_abort_chunk(self):
# Abort re-sends the last token; it must not be appended again.
sglext_chunks = self._run_output_ids_stream_with_graceful_abort(
normal_chunks=[[5, 6], [7]],
abort_output_ids=[7],
incremental=True,
abort_completion_tokens=3,
)
self.assertEqual(len(sglext_chunks), 1)
self.assertEqual(sglext_chunks[0]["sglext"]["output_ids"], [[5, 6, 7]])
def test_streaming_output_ids_incremental_keeps_coalesced_abort_deltas(self):
# Coalesced abort chunk: keep real deltas, drop the repeated last token.
sglext_chunks = self._run_output_ids_stream_with_graceful_abort(
normal_chunks=[[5, 6]],
abort_output_ids=[7, 8, 8],
incremental=True,
abort_completion_tokens=4,
)
self.assertEqual(len(sglext_chunks), 1)
self.assertEqual(sglext_chunks[0]["sglext"]["output_ids"], [[5, 6, 7, 8]])
def test_streaming_output_ids_non_incremental_abort_supersedes_earlier(self):
sglext_chunks = self._run_output_ids_stream_with_graceful_abort(
normal_chunks=[[5, 6]],
abort_output_ids=[5, 6, 7],
incremental=False,
abort_completion_tokens=3,
)
self.assertEqual(len(sglext_chunks), 1)
self.assertEqual(sglext_chunks[0]["sglext"]["output_ids"], [[5, 6, 7]])
def test_streaming_parallel_sampling_orders_spec_details_by_choice(self):
async def mock_generate():
for index in (1, 0):
yield _spec_result(index)
self.tm.generate_request.return_value = mock_generate()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
max_tokens=100,
n=2,
stream=True,
return_spec_tokens_details=True,
)
parsed = self._parse_chunks(self._run_chat_stream(Mock(), req))
details = next(chunk["sglext"] for chunk in parsed if "sglext" in chunk)[
"spec_tokens_details"
]
self.assertEqual([item["spec_cap_length"] for item in details], [1.0, 2.0])
self.assertEqual(
[item["spec_cap_lens_histogram"] for item in details],
[[0, 1], [1, 1]],
)
def _collect_continuous_usage(self, cached_tokens):
content = {
"text": "Hello",
"meta_info": {
"id": "chatcmpl-cont-usage",
"prompt_tokens": 10,
"completion_tokens": 2,
"cached_tokens": cached_tokens,
"finish_reason": {"type": "stop", "matched": None},
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=True,
)
async def _collect():
chunks = []
async for chunk in self.chat._generate_stream_content(
content=content,
index=0,
request=req,
created=123,
stream_offsets={},
reasoning_parser_dict={},
parser_dict={},
has_tool_calls={},
choice_logprobs=None,
finish_reason_type="stop",
continuous_usage_stats=True,
prompt_tokens={0: 10},
reasoning_tokens={0: 0},
completion_tokens={0: 2},
):
chunks.append(chunk)
return chunks
chunks = get_or_create_event_loop().run_until_complete(_collect())
return [c["usage"] for c in self._parse_chunks(chunks) if c.get("usage")]
def test_continuous_usage_reports_cached_tokens(self):
"""continuous_usage_stats chunks include cached tokens when cache reporting is on."""
enter_override(
self, get_context().override_server_args(enable_cache_report=True)
)
for cached_tokens in (0, 6):
with self.subTest(cached_tokens=cached_tokens):
usages = self._collect_continuous_usage(cached_tokens=cached_tokens)
self.assertTrue(usages, "continuous_usage_stats attached no usage")
self.assertEqual(
usages[0]["prompt_tokens_details"]["cached_tokens"],
cached_tokens,
)
def test_continuous_usage_omits_cached_tokens_when_report_disabled(self):
"""With cache reporting off, continuous_usage_stats must not leak cached tokens."""
enter_override(
self, get_context().override_server_args(enable_cache_report=False)
)
usages = self._collect_continuous_usage(cached_tokens=6)
self.assertTrue(usages, "continuous_usage_stats attached no usage")
self.assertIsNone(usages[0].get("prompt_tokens_details"))
# ------------- incremental streaming output tests -------------
def test_incremental_streaming_output_delta(self):
"""Test that streaming with incremental_streaming_output produces correct deltas.
When incremental_streaming_output is enabled, content["text"] is already the
incremental delta (not the full accumulated text). The delta computation must
use content["text"] directly instead of slicing by the accumulated buffer length.
Regression test for https://github.com/sgl-project/sglang/issues/22510.
"""
# Enable incremental_streaming_output on the mock
enter_override(
self, get_context().override_server_args(incremental_streaming_output=True)
)
# Simulate incremental streaming: each yield has ONLY the new text (delta),
# NOT the full accumulated text.
incremental_chunks = [
("I am", None),
(" a large", None),
(" language model", None),
(".", {"type": "stop", "matched": None}),
]
async def _mock_generate_incremental():
for text, finish_reason in incremental_chunks:
yield {
"text": text,
"meta_info": {
"id": "chatcmpl-incr-test",
"prompt_tokens": 10,
"completion_tokens": 5,
"cached_tokens": 0,
"finish_reason": finish_reason,
"output_token_logprobs": None,
"output_top_logprobs": None,
},
"index": 0,
}
self.tm.generate_request.return_value = _mock_generate_incremental()
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
temperature=0.7,
max_tokens=100,
stream=True,
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "Test prompt"
conv_mock.return_value = conv_ins
adapted_request, _ = self.chat._convert_to_internal_request(
req, self.fastapi_request
)
async def run_stream():
chunks = []
async for chunk in self.chat._generate_chat_stream(
adapted_request, req, self.fastapi_request
):
chunks.append(chunk)
return chunks
loop = get_or_create_event_loop()
chunks = loop.run_until_complete(run_stream())
# Extract content deltas from SSE chunks
deltas = []
for c in chunks:
if not c.startswith("data: ") or c.strip() == "data: [DONE]":
continue
data = json.loads(c[len("data: ") :])
if "choices" in data and data["choices"]:
content = data["choices"][0]["delta"].get("content")
if content:
deltas.append(content)
joined = "".join(deltas)
self.assertEqual(
joined,
"I am a large language model.",
f"Streaming deltas produced broken text: {deltas!r}",
)
# ------------- X-Data-Parallel-Rank header tests -------------
def test_extract_routed_dp_rank_from_header_no_header(self):
"""Test that None is returned when no header is present."""
self.fastapi_request.headers = {}
result = self.chat.extract_routed_dp_rank_from_header(
self.fastapi_request, body_routed_dp_rank=None
)
self.assertIsNone(result)
def test_extract_routed_dp_rank_header_overrides_body(self):
"""Test that header value has higher priority than body."""
self.fastapi_request.headers = {"x-data-parallel-rank": "3"}
result = self.chat.extract_routed_dp_rank_from_header(
self.fastapi_request, body_routed_dp_rank=1
)
self.assertEqual(result, 3) # header wins
def test_extract_routed_dp_rank_from_header_invalid(self):
"""Test that invalid header value raises HTTPException."""
from fastapi import HTTPException
self.fastapi_request.headers = {"x-data-parallel-rank": "abc"}
with self.assertRaises(HTTPException) as context:
self.chat.extract_routed_dp_rank_from_header(
self.fastapi_request, body_routed_dp_rank=None
)
self.assertEqual(context.exception.status_code, 400)
self.assertIn("must be an integer", context.exception.detail)
def test_hunyuan_reasoning_effort_dispatch(self):
tm = _MockTokenizerManager()
tm.server_args.reasoning_parser = "hunyuan"
chat = OpenAIServingChat(tm, _MockTemplateManager())
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "hi"}]
)
cases = [
("no_think", False),
("none", False),
(None, False),
("high", True),
("low", True),
]
for effort, expected in cases:
with self.subTest(effort=effort):
req.reasoning_effort = effort
self.assertEqual(chat._get_reasoning_from_request(req), expected)
def _setup_nemotron_super(self):
"""Drive _apply_jinja_template (chat_template_name=None) with a
Nemotron-3 Super reasoning_config carrying effort_kwarg."""
self.tm.server_args.reasoning_parser = "nemotron_3"
self.chat.reasoning_parser = "nemotron_3"
self.tm.server_args.served_model_name = (
"nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
)
self.template_manager.chat_template_name = None
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="enable_thinking",
default_enabled=True,
effort_kwarg="low_effort",
)
self.chat.chat_encoding_spec = None
def _run_jinja_with_effort(self, effort):
req = ChatCompletionRequest(
model="nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16",
messages=[{"role": "user", "content": "hi"}],
reasoning_effort=effort,
)
with (
patch.object(self.chat, "_encode_messages", return_value=None),
patch.object(
self.chat,
"_handle_last_assistant_message",
return_value=([{"role": "user", "content": "hi"}], None),
),
):
self.chat._process_messages(req, False)
_, kwargs = self.tm.tokenizer.apply_chat_template.call_args
return kwargs
def test_nemotron_super_low_effort_mapped_to_kwarg(self):
self._setup_nemotron_super()
kwargs = self._run_jinja_with_effort("low")
self.assertTrue(kwargs.get("low_effort"))
def test_nemotron_super_high_effort_warns_without_kwarg(self):
self._setup_nemotron_super()
with self.assertLogs(
"sglang.srt.entrypoints.openai.serving_chat", level="WARNING"
) as logs:
kwargs = self._run_jinja_with_effort("high")
self.assertNotIn("low_effort", kwargs)
self.assertTrue(any("only 'low' reasoning effort" in m for m in logs.output))
def test_nemotron_nano_no_effort_kwarg(self):
# Nano template has no low_effort, so effort_kwarg stays None and no
# warning is emitted even for non-low effort.
self.tm.server_args.reasoning_parser = "nemotron_3"
self.chat.reasoning_parser = "nemotron_3"
self.template_manager.chat_template_name = None
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
)
self.chat.chat_encoding_spec = None
kwargs = self._run_jinja_with_effort("high")
self.assertNotIn("low_effort", kwargs)
def _use_preserved_reasoning_template(self):
template = _compile_jinja_template(
"{% for m in messages if m.role == 'assistant' %}"
"{% if m.reasoning_content is not string %}"
"{{ raise_exception('Assistant thinking fields must be strings') }}"
"{% endif %}<think>{{ m.reasoning_content }}</think>{% endfor %}"
)
self.tm.tokenizer.apply_chat_template.side_effect = lambda messages, **kwargs: (
template.render(messages=messages)
)
self.template_manager.jinja_template_content_format = "string"
self.template_manager.force_reasoning = True
self.chat.reasoning_parser = self.chat.tool_call_parser = "k2_horizon"
def _assert_empty_reasoning_replays(self, message):
self.assertEqual(message["reasoning_content"], "")
request = ChatCompletionRequest(
model="x",
messages=[*self.basic_req.messages, message, *self.basic_req.messages],
)
self.chat._apply_jinja_template(request, None, False)
self.assertEqual(self.tm.tokenizer.encode.call_args.args[0], "<think></think>")
_K2_TOOL = {
"type": "function",
"function": {"name": "lookup", "parameters": {"type": "object"}},
}
_K2_TOOL_TEXT = '<ifm|tool_call>{"name":"lookup","arguments":{}}</ifm|tool_call>'
def test_empty_reasoning_response_replays(self):
self._use_preserved_reasoning_template()
for tool_only in (False, True):
with self.subTest(tool_only=tool_only):
request = ChatCompletionRequest(
model="x",
messages=self.basic_req.messages,
tools=[self._K2_TOOL] if tool_only else None,
)
ret = _spec_result(0)
ret["text"] = "</ifm|think>" + (
self._K2_TOOL_TEXT if tool_only else "Hello"
)
response = self.chat._build_chat_response(request, [ret], created=123)
message = response.model_dump(mode="json")["choices"][0]["message"]
self.assertEqual(message["content"], "" if tool_only else "Hello")
self.assertEqual(bool(message.get("tool_calls")), tool_only)
self._assert_empty_reasoning_replays(message)
def test_empty_stream_reasoning_each_choice(self):
self.template_manager.force_reasoning = True
self.chat.reasoning_parser = self.chat.tool_call_parser = "k2_horizon"
for tool_only in (False, True):
with self.subTest(tool_only=tool_only):
request = ChatCompletionRequest(
model="x",
messages=self.stream_req.messages,
stream=True,
n=2,
tools=[self._K2_TOOL] if tool_only else None,
)
async def generate():
for text in (
"</ifm|thi",
"</ifm|think>" + (self._K2_TOOL_TEXT if tool_only else "Hello"),
):
for index in range(request.n):
ret = _spec_result(index)
ret["text"] = text
if text == "</ifm|thi":
ret["meta_info"]["finish_reason"] = None
yield ret
self.tm.generate_request.return_value = generate()
chunks = self._parse_chunks(self._run_chat_stream(None, request))
self.assertFalse(any("error" in chunk for chunk in chunks), chunks)
choices = [c for chunk in chunks for c in chunk.get("choices", [])]
for index in range(request.n):
with self.subTest(index=index):
selected = [c for c in choices if c["index"] == index]
self.assertTrue(selected)
self.assertEqual(selected[0]["delta"]["reasoning_content"], "")
for choice in selected:
self.assertIn(
choice["delta"].get("reasoning_content"), (None, "")
)
self.assertEqual(
selected[-1]["finish_reason"],
"tool_calls" if tool_only else "stop",
)
self.assertEqual(
"".join(c["delta"].get("content") or "" for c in selected),
"" if tool_only else "Hello",
)
def test_unparsed_reasoning_remains_null(self):
for parser, separate in ((None, True), ("k2_horizon", False)):
with self.subTest(parser=parser, separate=separate):
self.chat.reasoning_parser = parser
request = self.basic_req.model_copy(
update={"separate_reasoning": separate}
)
response = self.chat._build_chat_response(
request, [_spec_result(0)], created=123
)
self.assertIsNone(response.choices[0].message.reasoning_content)
async def generate():
yield _spec_result(0)
self.tm.generate_request.return_value = generate()
chunks = self._parse_chunks(self._run_chat_stream(None, request))
self.assertFalse(any("error" in chunk for chunk in chunks))
for chunk in chunks:
for choice in chunk.get("choices", []):
self.assertIsNone(choice["delta"].get("reasoning_content"))
def test_empty_reasoning_is_null_for_non_k2_parser(self):
"""A non-K2 parser that finds no reasoning reports reasoning_content as None, not ''."""
self.chat.reasoning_parser = "qwen3"
self.template_manager.force_reasoning = False
request = self.basic_req.model_copy(update={"separate_reasoning": True})
ret = _spec_result(0)
ret["text"] = "Hello"
response = self.chat._build_chat_response(request, [ret], created=123)
message = response.choices[0].message
self.assertEqual(message.content, "Hello")
self.assertIsNone(message.reasoning_content)
def test_missing_reasoning_history_still_rejected(self):
self._use_preserved_reasoning_template()
for reasoning in ({}, {"reasoning_content": None}):
with self.subTest(reasoning=reasoning):
request = ChatCompletionRequest(
model="x",
messages=[
*self.basic_req.messages,
{"role": "assistant", "content": "Hello", **reasoning},
*self.basic_req.messages,
],
)
with self.assertRaisesRegex(
ValueError, "thinking fields must be strings"
):
self.chat._apply_jinja_template(request, None, False)
def test_non_stream_reasoning_response_preserves_payload_whitespace(self):
self.chat.reasoning_parser = "qwen3"
self.template_manager.force_reasoning = False
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
stream=False,
separate_reasoning=True,
)
ret = [
{
"text": "<think>\nLet me think\n</think>\n\nThe answer is 42.\n",
"meta_info": {
"id": "chatcmpl-test",
"prompt_tokens": 5,
"completion_tokens": 8,
"cached_tokens": 0,
"finish_reason": {"type": "stop", "matched": None},
"weight_version": "test",
},
"index": 0,
}
]
response = self.chat._build_chat_response(req, ret, created=123)
message = response.choices[0].message
self.assertEqual(message.reasoning_content, "\nLet me think\n")
self.assertEqual(message.content, "\n\nThe answer is 42.\n")
# ------------- reasoning config tests -------------
def test_get_reasoning_from_request_default_true_toggle(self):
self.tm.server_args.reasoning_parser = "qwen3"
self.chat.reasoning_parser = "qwen3"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
)
enabled_by_default = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
disabled_explicitly = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
chat_template_kwargs={"enable_thinking": False},
)
self.assertTrue(self.chat._get_reasoning_from_request(enabled_by_default))
self.assertFalse(self.chat._get_reasoning_from_request(disabled_explicitly))
def test_get_reasoning_from_request_default_false_toggle(self):
self.tm.server_args.reasoning_parser = "deepseek-v3"
self.chat.reasoning_parser = "deepseek-v3"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="thinking", default_enabled=False
)
disabled_by_default = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
enabled_explicitly = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
chat_template_kwargs={"thinking": True},
)
self.assertFalse(self.chat._get_reasoning_from_request(disabled_by_default))
self.assertTrue(self.chat._get_reasoning_from_request(enabled_explicitly))
def test_get_reasoning_from_request_special_cases(self):
self.tm.server_args.reasoning_parser = "mistral"
self.chat.reasoning_parser = "mistral"
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.template_manager.reasoning_config = ReasoningToggleConfig(
special_case="always"
)
self.assertTrue(self.chat._get_reasoning_from_request(req))
self.template_manager.reasoning_config = ReasoningToggleConfig(
special_case="mistral"
)
self.assertFalse(self.chat._get_reasoning_from_request(req))
req.reasoning_effort = "medium"
self.assertTrue(self.chat._get_reasoning_from_request(req))
# --- fallback path tests (config=None, uses reasoning_default) ---
def _setup_fallback(self, parser_name):
"""Set up reasoning with config=None to exercise the fallback path."""
self.tm.server_args.reasoning_parser = parser_name
self.chat = OpenAIServingChat(self.tm, self.template_manager)
self.chat.reasoning_parser = parser_name
self.template_manager.reasoning_config = None
def test_fallback_always_mode(self):
self._setup_fallback("deepseek-r1")
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.assertTrue(self.chat._get_reasoning_from_request(req))
def test_fallback_mistral_mode(self):
self._setup_fallback("mistral")
req_no_effort = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.assertFalse(self.chat._get_reasoning_from_request(req_no_effort))
req_with_effort = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
reasoning_effort="high",
)
self.assertTrue(self.chat._get_reasoning_from_request(req_with_effort))
def test_fallback_enable_thinking_mode_default_on(self):
self._setup_fallback("qwen3")
req_default = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.assertTrue(self.chat._get_reasoning_from_request(req_default))
req_disabled = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
chat_template_kwargs={"enable_thinking": False},
)
self.assertFalse(self.chat._get_reasoning_from_request(req_disabled))
def test_fallback_explicit_thinking_mode_default_off(self):
self._setup_fallback("deepseek-v3")
req_default = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.assertFalse(self.chat._get_reasoning_from_request(req_default))
req_enabled = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
chat_template_kwargs={"thinking": True},
)
self.assertTrue(self.chat._get_reasoning_from_request(req_enabled))
def test_fallback_explicit_enable_thinking_mode_default_off(self):
self._setup_fallback("mimo")
req_default = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.assertFalse(self.chat._get_reasoning_from_request(req_default))
req_enabled = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "Hi?"}],
chat_template_kwargs={"enable_thinking": True},
)
self.assertTrue(self.chat._get_reasoning_from_request(req_enabled))
def test_fallback_ling3_default_on(self):
"""Ling3 public checkpoints default `thinking_option='on'` in the chat
template when `enable_thinking` is omitted, and the template detector
cannot infer that indirect assignment. The parser fallback must mirror
the template default: omitted kwargs enable reasoning, only an explicit
`enable_thinking=False` disables it. Regression: the detector shipped
with `explicit_enable_thinking`, which left `reasoning_content` null on
default requests while the model was in fact thinking."""
self._setup_fallback("ling3")
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "hi"}]
)
cases = [
(None, True), # no chat_template_kwargs → thinking (template default)
({}, True), # empty kwargs → thinking
({"enable_thinking": True}, True), # explicit on
({"enable_thinking": False}, False), # explicit off
]
for kwargs, expected in cases:
with self.subTest(kwargs=kwargs):
req.chat_template_kwargs = kwargs
self.assertEqual(self.chat._get_reasoning_from_request(req), expected)
def test_fallback_no_detector_returns_false(self):
self.chat.reasoning_parser = "qwen3"
self.chat._reasoning_detector = None
self.template_manager.reasoning_config = None
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "Hi?"}]
)
self.assertFalse(self.chat._get_reasoning_from_request(req))
def test_build_chat_response_qwen3_thinking_forces_reasoning(self):
self.tm.server_args.reasoning_parser = "qwen3-thinking"
self.chat.reasoning_parser = "qwen3-thinking"
self.template_manager.reasoning_config = ReasoningToggleConfig(
toggle_param="enable_thinking", default_enabled=True
)
req = ChatCompletionRequest(
model="Qwen/Qwen3-0.6B",
messages=[{"role": "user", "content": "Hi?"}],
separate_reasoning=True,
chat_template_kwargs={"enable_thinking": False},
)
ret_item = {
"text": "42",
"meta_info": {
"id": f"chatcmpl-{uuid.uuid4()}",
"prompt_tokens": 10,
"completion_tokens": 1,
"weight_version": "default",
"finish_reason": {"type": "stop", "matched": None},
},
"index": 0,
}
response = self.chat._build_chat_response(req, [ret_item], created=0)
msg = response.choices[0].message
self.assertEqual(msg.content, "")
self.assertEqual(msg.reasoning_content, "42")
# --- poolside_v1 (Laguna-XS.2) regression tests ---
def test_poolside_v1_enable_thinking_dispatch(self):
"""Laguna chat template defaults `enable_thinking=false`. Parser must
follow that default — must NOT return True via the generic fallback.
After the reasoning-config refactor, this is driven by
`_PoolsideV1Detector.reasoning_default = "explicit_enable_thinking"`."""
self._setup_fallback("poolside_v1")
req = ChatCompletionRequest(
model="x", messages=[{"role": "user", "content": "hi"}]
)
cases = [
(None, False), # no chat_template_kwargs → non-thinking (default)
({}, False), # empty kwargs → non-thinking
({"enable_thinking": False}, False), # explicit off
({"enable_thinking": True}, True), # explicit on
]
for kwargs, expected in cases:
with self.subTest(kwargs=kwargs):
req.chat_template_kwargs = kwargs
self.assertEqual(self.chat._get_reasoning_from_request(req), expected)
def test_poolside_v1_does_not_double_prepend_think(self):
"""When `enable_thinking=True` for poolside_v1, the HF chat template
already emits `<think>` via add_generation_prompt — server must NOT
append a second `<think>`. After the refactor this is guarded by
`_PoolsideV1Detector.thinks_internally = True` (inherited from Qwen3Detector).
"""
self._setup_fallback("poolside_v1")
req = ChatCompletionRequest(
model="x",
messages=[{"role": "user", "content": "hi"}],
chat_template_kwargs={"enable_thinking": True},
)
with patch(
"sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv"
) as conv_mock:
conv_ins = Mock()
conv_ins.get_prompt.return_value = "BASE_PROMPT"
conv_ins.image_data = conv_ins.audio_data = conv_ins.video_data = None
conv_ins.modalities = []
conv_ins.stop_str = []
conv_mock.return_value = conv_ins
result = self.chat._apply_conversation_template(req, is_multimodal=False)
self.assertEqual(result.prompt, "BASE_PROMPT")
# ------------- hook method tests -------------
def test_encode_messages_returns_none_by_default(self):
"""Default _encode_messages returns None (use standard encoding)."""
result = self.chat._encode_messages([], Mock(), False, image_data=[])
self.assertIsNone(result)
class TestProcessToolCallsWithRequiredToolChoice(unittest.TestCase):
"""Test _process_tool_calls with tool_choice='required' uses model-specific parser."""
def setUp(self):
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
tm = _MockTokenizerManager()
tm.server_args.tool_call_parser = "kimi_k2"
self.chat = OpenAIServingChat(tm, _MockTemplateManager())
def test_required_with_parser_uses_function_call_parser(self):
"""tool_choice='required' should use FunctionCallParser when tool_call_parser is set."""
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as ParserMock:
call_info = Mock()
call_info.name = "get_weather"
call_info.parameters = '{"location":"Tokyo"}'
call_info.tool_index = 0
parser_instance = ParserMock.return_value
parser_instance.has_tool_call.return_value = True
parser_instance.parse_non_stream.return_value = ("", [call_info])
finish_reason = {"type": "stop", "matched": None}
tools = [{"type": "function", "function": {"name": "get_weather"}}]
tool_calls, text, fr = self.chat._process_tool_calls(
text="<|tool_calls_section_begin|>...<|tool_calls_section_end|>",
tools=tools,
finish_reason=finish_reason,
tool_choice="required",
)
self.assertIsNotNone(tool_calls)
self.assertEqual(len(tool_calls), 1)
self.assertEqual(tool_calls[0].function.name, "get_weather")
self.assertEqual(fr["type"], "tool_calls")
def test_empty_parser_result_is_not_reported_as_tool_call(self):
with patch(
"sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser"
) as ParserMock:
parser_instance = ParserMock.return_value
parser_instance.has_tool_call.return_value = True
parser_instance.detector.supports_structural_tag.return_value = True
parser_instance.parse_non_stream.return_value = ("Visible prefix.", [])
finish_reason = {"type": "stop", "matched": None}
tools = [{"type": "function", "function": {"name": "get_weather"}}]
tool_calls, text, fr = self.chat._process_tool_calls(
text="<|malformed_tool_call|>",
tools=tools,
finish_reason=finish_reason,
tool_choice="required",
)
self.assertIsNone(tool_calls)
self.assertEqual(text, "Visible prefix.")
self.assertEqual(fr, {"type": "stop", "matched": None})
def test_required_without_parser_falls_back_to_json(self):
"""tool_choice='required' without parser should parse as JSON array."""
self.chat.tool_call_parser = None
finish_reason = {"type": "stop", "matched": None}
tools = [{"type": "function", "function": {"name": "get_weather"}}]
tool_calls, text, fr = self.chat._process_tool_calls(
text='[{"name":"get_weather","parameters":{"location":"Tokyo"}}]',
tools=tools,
finish_reason=finish_reason,
tool_choice="required",
)
self.assertIsNotNone(tool_calls)
self.assertEqual(len(tool_calls), 1)
self.assertEqual(tool_calls[0].function.name, "get_weather")
def test_required_without_parser_invalid_json_returns_none(self):
"""tool_choice='required' without parser and invalid JSON returns tool_calls=None."""
self.chat.tool_call_parser = None
finish_reason = {"type": "stop", "matched": None}
tools = [{"type": "function", "function": {"name": "get_weather"}}]
tool_calls, text, fr = self.chat._process_tool_calls(
text="<|tool_calls_section_begin|>not json",
tools=tools,
finish_reason=finish_reason,
tool_choice="required",
)
self.assertIsNone(tool_calls)
class TestNormalizeToolContent(unittest.TestCase):
"""Unit tests for normalize_tool_content()."""
def test_multiple_text_parts_joined(self):
result = normalize_tool_content(
"tool",
[{"type": "text", "text": "hello"}, {"type": "text", "text": "world"}],
)
self.assertEqual(result, "hello world")
def test_non_text_part_list_preserved(self):
content = [{"name": "func", "output": "result"}]
result = normalize_tool_content("tool", content)
self.assertIs(result, content)
def test_string_content_unchanged(self):
self.assertEqual(normalize_tool_content("tool", "hello"), "hello")
def test_empty_list_returns_empty_string(self):
self.assertEqual(normalize_tool_content("tool", []), "")
def test_non_tool_role_unchanged(self):
content = [{"type": "text", "text": "hi"}]
result = normalize_tool_content("user", content)
self.assertIs(result, content)
def test_mixed_str_and_dict_parts(self):
result = normalize_tool_content(
"tool", ["plain", {"type": "text", "text": "rich"}]
)
self.assertEqual(result, "plain rich")
class TestRequestChatTemplateTrustGate(CustomTestCase):
def setUp(self):
super().setUp()
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
self.chat = OpenAIServingChat(_MockTokenizerManager(), _MockTemplateManager())
def _request(self, **kwargs):
return ChatCompletionRequest(
model="test-model", messages=[{"role": "user", "content": "hi"}], **kwargs
)
def test_rejected_by_default(self):
for template in ("{{ messages }}", "", None):
with self.subTest(template=template):
error = self.chat._validate_request(
self._request(chat_template_kwargs={"chat_template": template})
)
self.assertIn("--trust-request-chat-template", error)
def test_allowed_when_trusted(self):
with get_context().override_server_args(trust_request_chat_template=True):
self.assertIsNone(
self.chat._validate_request(
self._request(
chat_template_kwargs={"chat_template": "{{ messages }}"}
)
)
)
def test_other_kwargs_unchanged(self):
for kwargs in (None, {}, {"enable_thinking": False}, {"chat_templates": "x"}):
with self.subTest(kwargs=kwargs):
self.assertIsNone(
self.chat._validate_request(
self._request(chat_template_kwargs=kwargs)
)
)
if __name__ == "__main__":
unittest.main(verbosity=2)