Files
sglang/test/registered/unit/entrypoints/anthropic/test_serving.py
T

1656 lines
67 KiB
Python

import asyncio
import json
import unittest
from types import SimpleNamespace
from sglang.test.test_utils import maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede imports that may pull in sgl_kernel
from fastapi.responses import JSONResponse # noqa: E402
from jinja2 import Environment # noqa: E402
from sglang.srt.entrypoints.anthropic.protocol import ( # noqa: E402
AnthropicMessage,
AnthropicMessagesRequest,
)
from sglang.srt.entrypoints.anthropic.serving import AnthropicServing # noqa: E402
from sglang.srt.entrypoints.openai.protocol import ( # noqa: E402
ChatCompletionRequest,
ChatCompletionResponse,
)
from sglang.srt.parser.template_detection import ( # noqa: E402
detect_inline_system_support,
)
from sglang.test.ci.ci_register import register_cpu_ci # noqa: E402
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
class _FakeOpenAIServingChat:
native_reasoning_history = False
def __init__(self, stream_lines=None, chat_template=None):
self.stream_lines = stream_lines or []
self.apply_reasoning_calls: list[bool] = []
self.tokenizer_manager = SimpleNamespace(
tokenizer=SimpleNamespace(chat_template=chat_template)
)
def supports_native_reasoning_history(self):
return self.native_reasoning_history
def _generate_chat_stream(self, adapted_request, processed_request, raw_request):
async def _gen():
for line in self.stream_lines:
yield line
return _gen()
def apply_reasoning_enabled(self, chat_request, enabled):
self.apply_reasoning_calls.append(enabled)
def wrap_reasoning_history(self, text):
return f"<think>\n{text}\n</think>"
class _FakeNonStreamingErrorOpenAI:
"""Returns a configurable error response from the OpenAI handler."""
def __init__(self, status_code=400, body=None, content=None):
self._status_code = status_code
self._body = body
self._content = content
def _validate_request(self, chat_request):
return None
def _convert_to_internal_request(self, chat_request, raw_request):
return SimpleNamespace(), chat_request
async def _handle_non_streaming_request(
self, adapted_request, processed_request, raw_request
):
if self._body is not None:
# Build a response object exposing raw bytes via `.body`.
return SimpleNamespace(status_code=self._status_code, body=self._body)
return JSONResponse(
status_code=self._status_code,
content=self._content
or {
"error": {
"type": "invalid_request_error",
"message": "context length exceeded",
}
},
)
class _FakeNonStreamingOpenAI:
"""Returns a configurable ChatCompletionResponse from the OpenAI handler."""
def __init__(self, response):
self._response = response
def _validate_request(self, chat_request):
return None
def _convert_to_internal_request(self, chat_request, raw_request):
return SimpleNamespace(), chat_request
async def _handle_non_streaming_request(
self, adapted_request, processed_request, raw_request
):
return self._response
def _chunk(choices=None, usage=None):
data = {
"id": "chatcmpl-test",
"object": "chat.completion.chunk",
"created": 0,
"model": "test-model",
"choices": choices or [],
}
if usage is not None:
data["usage"] = usage
return f"data: {json.dumps(data)}\n\n"
def _choice(delta, finish_reason=None):
return {
"index": 0,
"delta": delta,
"finish_reason": finish_reason,
}
async def _collect_anthropic_events(serving, anthropic_request):
events = []
async for sse in serving._generate_anthropic_stream(
adapted_request=object(),
processed_request=object(),
anthropic_request=anthropic_request,
raw_request=object(),
):
for line in sse.splitlines():
if line.startswith("data: "):
events.append(json.loads(line[6:]))
return events
class TestAnthropicServing(unittest.TestCase):
# Renders system at any position (GLM/Kimi/Qwen3) → can pass through.
INLINE_SYSTEM_TEMPLATE = (
"{%- for message in messages %}"
"{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
GLM_TOOL_RESULT_TEMPLATE = """
{%- for message in messages if message.role == "tool" -%}
{%- if loop.first -%}<|observation|>{%- endif -%}
{%- if message.content is string -%}
<tool_response>{{ message.content }}</tool_response>
{%- elif message.content.0.type == "tool_reference" -%}
<tool_response><tools>
{%- for reference in message.content -%}
{%- for tool in tools if tool.function.name == reference.name -%}
{{ tool.function.name }}
{%- endfor -%}
{%- endfor -%}
</tools></tool_response>
{%- endif -%}
{%- endfor -%}
"""
def _serving(self, stream_lines=None, chat_template=None):
return AnthropicServing(_FakeOpenAIServingChat(stream_lines, chat_template))
def _anthropic_request(self, **overrides):
data = {
"model": "test-model",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hello"}],
"stream": True,
}
data.update(overrides)
return AnthropicMessagesRequest.model_validate(data)
def _tool_result_request(self, content, tools=None):
overrides = {
"stream": False,
"messages": [
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_1",
"content": content,
}
],
}
],
}
if tools is not None:
overrides["tools"] = tools
return self._anthropic_request(**overrides)
def test_messages_preserves_pd_rendezvous_through_native_conversion(self):
"""PD routers inject bootstrap fields into /v1/messages bodies; dropping
them in the Chat Completions conversion strands the decode request."""
request = self._anthropic_request(
bootstrap_host="prefill.internal",
bootstrap_port=8998,
bootstrap_room=2**62 + 17,
routed_dp_rank=3,
disagg_prefill_dp_rank=2,
)
converted = self._serving()._convert_to_chat_completion_request(request)
self.assertEqual(converted.bootstrap_host, "prefill.internal")
self.assertEqual(converted.bootstrap_port, 8998)
self.assertEqual(converted.bootstrap_room, 2**62 + 17)
self.assertEqual(converted.routed_dp_rank, 3)
self.assertEqual(converted.disagg_prefill_dp_rank, 2)
def test_stream_closes_tool_block_before_text_delta(self):
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": ""})]),
_chunk(
[
_choice(
{
"tool_calls": [
{
"index": 0,
"id": "call_1",
"type": "function",
"function": {
"name": "lookup",
"arguments": '{"query"',
},
}
]
}
)
]
),
_chunk(
[
_choice(
{
"tool_calls": [
{
"index": 0,
"type": "function",
"function": {"arguments": ': "sglang"}'},
}
]
}
)
]
),
_chunk([_choice({"content": "done"})]),
_chunk([_choice({}, finish_reason="stop")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
block_events = [
(event["type"], event.get("content_block", {}).get("type"))
for event in events
if event["type"].startswith("content_block")
]
self.assertEqual(
block_events,
[
("content_block_start", "tool_use"),
("content_block_delta", None),
("content_block_delta", None),
("content_block_stop", None),
("content_block_start", "text"),
("content_block_delta", None),
("content_block_stop", None),
],
)
text_delta = [
event
for event in events
if event["type"] == "content_block_delta"
and event["delta"].get("type") == "text_delta"
][0]
self.assertEqual(text_delta["index"], 1)
def test_stream_reasoning_content_uses_thinking_block(self):
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": ""})]),
_chunk([_choice({"reasoning_content": "think first"})]),
_chunk([_choice({"content": "answer"})]),
_chunk([_choice({}, finish_reason="stop")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
content_events = [
event for event in events if event["type"].startswith("content_block")
]
self.assertEqual(content_events[0]["content_block"]["type"], "thinking")
# Signature is absent (None and excluded) — never emit empty
# string, which would fail downstream Anthropic signature verifiers.
self.assertNotIn("signature", content_events[0]["content_block"])
self.assertEqual(content_events[1]["delta"]["type"], "thinking_delta")
self.assertEqual(content_events[1]["delta"]["thinking"], "think first")
# No empty signature_delta event between thinking_delta and content_block_stop.
self.assertEqual(content_events[2]["type"], "content_block_stop")
self.assertEqual(content_events[3]["content_block"]["type"], "text")
# Confirm no signature_delta event was emitted in the entire stream.
sig_deltas = [
event
for event in events
if event["type"] == "content_block_delta"
and event.get("delta", {}).get("type") == "signature_delta"
]
self.assertEqual(sig_deltas, [])
def test_stream_usage_subtracts_cache_read_and_omits_final_input_tokens(self):
usage = {
"prompt_tokens": 10,
"completion_tokens": 0,
"total_tokens": 10,
"prompt_tokens_details": {"cached_tokens": 4},
}
final_usage = {
"prompt_tokens": 10,
"completion_tokens": 2,
"total_tokens": 12,
"prompt_tokens_details": {"cached_tokens": 4},
}
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": ""})]),
_chunk([_choice({"content": "hi"})], usage=usage),
_chunk([], usage=final_usage),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
message_start = [event for event in events if event["type"] == "message_start"][
0
]
message_delta = [event for event in events if event["type"] == "message_delta"][
0
]
self.assertEqual(message_start["message"]["usage"]["input_tokens"], 6)
self.assertEqual(
message_start["message"]["usage"]["cache_read_input_tokens"], 4
)
self.assertNotIn("input_tokens", message_delta["usage"])
self.assertEqual(message_delta["usage"]["output_tokens"], 2)
def test_non_streaming_usage_subtracts_cache_read_tokens(self):
response = ChatCompletionResponse.model_validate(
{
"id": "chatcmpl-test",
"model": "test-model",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 2,
"total_tokens": 12,
"prompt_tokens_details": {"cached_tokens": 4},
},
}
)
anthropic_response = self._serving()._convert_response(response)
self.assertEqual(anthropic_response.usage.input_tokens, 6)
self.assertEqual(anthropic_response.usage.output_tokens, 2)
self.assertEqual(anthropic_response.usage.cache_read_input_tokens, 4)
def test_tool_result_search_result_content_is_flattened(self):
request = AnthropicMessagesRequest.model_validate(
{
"model": "test-model",
"max_tokens": 16,
"messages": [
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_1",
"content": [
{
"type": "search_result",
"title": "SGLang docs",
"source": "https://docs.sglang.ai",
"content": [
{
"type": "text",
"text": "Anthropic API notes",
}
],
}
],
}
],
}
],
}
)
chat_request = self._serving()._convert_to_chat_completion_request(request)
tool_message = [
msg
for msg in chat_request.model_dump()["messages"]
if msg["role"] == "tool"
][0]
self.assertIn("SGLang docs", tool_message["content"])
self.assertIn("https://docs.sglang.ai", tool_message["content"])
self.assertIn("Anthropic API notes", tool_message["content"])
def test_mixed_tool_reference_content_preserves_part_order(self):
request = self._tool_result_request(
[
{"type": "text", "text": "Tool loaded: Bash"},
{"type": "tool_reference", "tool_name": "Bash"},
{"type": "text", "text": "Ready"},
]
)
chat_request = self._serving()._convert_to_chat_completion_request(request)
messages = chat_request.model_dump(exclude_none=True)["messages"]
self.assertEqual([message["role"] for message in messages], ["tool"] * 3)
self.assertEqual(
[message["tool_call_id"] for message in messages], ["call_1"] * 3
)
self.assertEqual(messages[0]["content"], "Tool loaded: Bash")
self.assertEqual(
messages[1]["content"],
[{"type": "tool_reference", "name": "Bash"}],
)
self.assertEqual(messages[2]["content"], "Ready")
def test_mixed_tool_reference_content_renders_text_and_schema(self):
template = Environment().from_string(self.GLM_TOOL_RESULT_TEMPLATE)
request = self._tool_result_request(
[
{"type": "text", "text": "Tool loaded: Bash"},
{"type": "tool_reference", "tool_name": "Bash"},
],
tools=[
{
"name": "Bash",
"description": "Run a shell command",
"input_schema": {"type": "object", "properties": {}},
"defer_loading": True,
}
],
)
chat_request = self._serving()._convert_to_chat_completion_request(request)
payload = chat_request.model_dump(exclude_none=True)
prompt = template.render(messages=payload["messages"], tools=payload["tools"])
self.assertIn("<tool_response>Tool loaded: Bash</tool_response>", prompt)
self.assertIn("<tools>Bash</tools>", prompt)
self.assertEqual(prompt.count("<|observation|>"), 1)
def test_reference_only_tool_result_remains_one_message(self):
request = self._tool_result_request(
[
{"type": "tool_reference", "tool_name": "Bash"},
{"type": "tool_reference", "tool_name": "Read"},
]
)
chat_request = self._serving()._convert_to_chat_completion_request(request)
messages = chat_request.model_dump(exclude_none=True)["messages"]
self.assertEqual(len(messages), 1)
self.assertEqual(
messages[0]["content"],
[
{"type": "tool_reference", "name": "Bash"},
{"type": "tool_reference", "name": "Read"},
],
)
def test_builtin_web_search_tool_without_schema_is_skipped(self):
request = AnthropicMessagesRequest.model_validate(
{
"model": "test-model",
"max_tokens": 16,
"messages": [{"role": "user", "content": "search sglang"}],
"tools": [{"name": "web_search", "type": "web_search_20250305"}],
"tool_choice": {"type": "auto"},
}
)
chat_request = self._serving()._convert_to_chat_completion_request(request)
self.assertIsNone(chat_request.tools)
self.assertEqual(chat_request.tool_choice, "none")
def test_custom_tool_without_schema_is_rejected(self):
# With the discriminated union, an AnthropicCustomTool variant must
# carry an input_schema. The check fires at request-parse time
# (Pydantic raises ValidationError, a subclass of ValueError).
with self.assertRaisesRegex(ValueError, "input_schema"):
AnthropicMessagesRequest.model_validate(
{
"model": "test-model",
"max_tokens": 16,
"messages": [{"role": "user", "content": "call a tool"}],
"tools": [{"name": "custom_without_schema"}],
}
)
def test_non_streaming_openai_error_response_is_forwarded(self):
serving = AnthropicServing(_FakeNonStreamingErrorOpenAI())
chat_request = ChatCompletionRequest(
model="test-model",
max_tokens=16,
messages=[{"role": "user", "content": "hello"}],
)
anthropic_request = self._anthropic_request(stream=False)
response = asyncio.run(
serving._handle_non_streaming(chat_request, anthropic_request, object())
)
payload = json.loads(response.body)
self.assertEqual(response.status_code, 400)
self.assertEqual(payload["error"]["type"], "invalid_request_error")
self.assertEqual(payload["error"]["message"], "context length exceeded")
# ------------------------------------------------------------------
# Edge-case coverage added in the review-fix pass
# ------------------------------------------------------------------
def test_stream_text_then_tool_use_closes_text_block(self):
"""Text deltas followed by tool_use must close the text block before opening tool_use index."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": "Hello"})]),
_chunk([_choice({"content": " world"})]),
_chunk(
[
_choice(
{
"tool_calls": [
{
"index": 0,
"id": "call_1",
"type": "function",
"function": {
"name": "search",
"arguments": '{"q":"hello"}',
},
}
]
}
)
]
),
_chunk([_choice({}, finish_reason="tool_calls")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
block_events = [
(event["type"], event.get("content_block", {}).get("type"))
for event in events
if event["type"].startswith("content_block")
]
# text block (start, delta, delta, stop) then tool_use (start, delta, stop)
self.assertEqual(block_events[0], ("content_block_start", "text"))
text_stop_idx = next(
i for i, ev in enumerate(block_events) if ev == ("content_block_stop", None)
)
tool_start_idx = next(
i
for i, ev in enumerate(block_events)
if ev == ("content_block_start", "tool_use")
)
self.assertLess(text_stop_idx, tool_start_idx)
def test_stream_tool_use_without_arguments_is_not_empty_completion(self):
"""A zero-argument tool call is valid content even without input_json_delta."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": ""})]),
_chunk(
[
_choice(
{
"tool_calls": [
{
"index": 0,
"id": "call_1",
"type": "function",
"function": {"name": "ping", "arguments": ""},
}
]
}
)
]
),
_chunk([_choice({}, finish_reason="tool_calls")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
self.assertFalse(any(event["type"] == "error" for event in events))
tool_start = next(
event
for event in events
if event["type"] == "content_block_start"
and event["content_block"]["type"] == "tool_use"
)
self.assertEqual(tool_start["content_block"]["name"], "ping")
message_delta = next(
event for event in events if event["type"] == "message_delta"
)
self.assertEqual(message_delta["delta"]["stop_reason"], "tool_use")
def test_stream_no_usage_chunk_emits_error_event(self):
"""Stream that yields only [DONE] (no content delta) must surface as an error event."""
serving = self._serving(["data: [DONE]\n\n"])
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
types = [event["type"] for event in events]
# Sequence: message_start, error, message_stop
self.assertEqual(types, ["message_start", "error", "message_stop"])
error_event = events[1]
self.assertEqual(error_event["error"]["type"], "api_error")
self.assertIn("no content", error_event["error"]["message"].lower())
def test_cache_read_exceeds_prompt_tokens_clamps_to_zero(self):
"""When cached_tokens > prompt_tokens, input_tokens clamps to 0 instead of going negative."""
usage = {
"prompt_tokens": 4,
"completion_tokens": 0,
"total_tokens": 4,
"prompt_tokens_details": {"cached_tokens": 10},
}
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": ""})]),
_chunk([_choice({"content": "ok"})], usage=usage),
_chunk([_choice({}, finish_reason="stop")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
message_start = next(e for e in events if e["type"] == "message_start")
usage_out = message_start["message"]["usage"]
self.assertEqual(usage_out["input_tokens"], 0)
self.assertEqual(usage_out["cache_read_input_tokens"], 10)
def test_usage_without_prompt_tokens_details(self):
"""Usage object without prompt_tokens_details must omit cache_read_input_tokens cleanly."""
usage = {"prompt_tokens": 5, "completion_tokens": 0, "total_tokens": 5}
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": ""})]),
_chunk([_choice({"content": "ok"})], usage=usage),
_chunk([_choice({}, finish_reason="stop")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
message_start = next(e for e in events if e["type"] == "message_start")
usage_out = message_start["message"]["usage"]
self.assertEqual(usage_out["input_tokens"], 5)
self.assertNotIn("cache_read_input_tokens", usage_out)
def test_non_streaming_error_with_non_json_body(self):
"""Non-JSON upstream error body falls back to body[:500] as the message (for 4xx)."""
serving = AnthropicServing(
_FakeNonStreamingErrorOpenAI(
status_code=400,
body=b"<html>upstream gateway rejected: bad payload</html>",
)
)
chat_request = ChatCompletionRequest(
model="test-model",
max_tokens=16,
messages=[{"role": "user", "content": "hello"}],
)
anthropic_request = self._anthropic_request(stream=False)
response = asyncio.run(
serving._handle_non_streaming(chat_request, anthropic_request, object())
)
payload = json.loads(response.body)
self.assertEqual(response.status_code, 400)
self.assertEqual(payload["error"]["type"], "invalid_request_error")
self.assertIn("upstream gateway rejected", payload["error"]["message"])
def test_non_streaming_error_5xx_scrubs_message(self):
"""5xx errors always return a generic message regardless of upstream payload."""
for status_code, expected_type in [
(500, "api_error"),
(502, "api_error"),
(503, "overloaded_error"),
(504, "api_error"),
]:
serving = AnthropicServing(
_FakeNonStreamingErrorOpenAI(
status_code=status_code,
body=b'{"error":{"message":"sensitive internals: /opt/secret","type":"internal"}}',
)
)
chat_request = ChatCompletionRequest(
model="test-model",
max_tokens=16,
messages=[{"role": "user", "content": "hello"}],
)
anthropic_request = self._anthropic_request(stream=False)
response = asyncio.run(
serving._handle_non_streaming(chat_request, anthropic_request, object())
)
payload = json.loads(response.body)
self.assertEqual(
response.status_code,
status_code,
f"status {status_code} should be preserved",
)
self.assertEqual(
payload["error"]["type"],
expected_type,
f"status {status_code} should map to {expected_type}",
)
self.assertEqual(
payload["error"]["message"],
"Internal server error",
f"status {status_code} must scrub the message; got {payload['error']['message']!r}",
)
def test_non_streaming_response_includes_thinking_block(self):
"""When the OpenAI response carries reasoning_content, the Anthropic response has a thinking block first."""
response = ChatCompletionResponse.model_validate(
{
"id": "chatcmpl-test",
"model": "test-model",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "the answer is 4",
"reasoning_content": "2 + 2 = 4",
},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 3,
"completion_tokens": 5,
"total_tokens": 8,
},
}
)
anthropic_response = self._serving()._convert_response(response)
# thinking block first, then text block
self.assertEqual(anthropic_response.content[0].type, "thinking")
self.assertEqual(anthropic_response.content[0].thinking, "2 + 2 = 4")
self.assertEqual(anthropic_response.content[1].type, "text")
self.assertEqual(anthropic_response.content[1].text, "the answer is 4")
def test_request_thinking_disabled_invokes_apply_reasoning_enabled(self):
"""``thinking={"type": "disabled"}`` must flip the reasoning toggle off."""
serving = self._serving()
request = self._anthropic_request(thinking={"type": "disabled"}, stream=False)
serving._convert_to_chat_completion_request(request)
self.assertEqual(serving.openai_serving_chat.apply_reasoning_calls, [False])
def test_request_thinking_enabled_with_budget_tokens_logs_warning(self):
"""SDK shape: ``enabled`` requires ``budget_tokens``. We accept it
(the SDK would), but log a WARNING because the local backend has
no equivalent hard-cap knob — the budget is not enforced."""
import logging
serving = self._serving()
request = self._anthropic_request(
thinking={"type": "enabled", "budget_tokens": 2048}, stream=False
)
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level=logging.WARNING
) as log:
serving._convert_to_chat_completion_request(request)
self.assertEqual(serving.openai_serving_chat.apply_reasoning_calls, [True])
self.assertTrue(
any("budget_tokens=2048" in r and "not enforced" in r for r in log.output),
f"expected unenforced-budget warning: {log.output}",
)
def test_request_thinking_enabled_requires_budget_tokens(self):
"""SDK requires ``budget_tokens`` for ``type=enabled`` — Pydantic 400."""
from pydantic import ValidationError
with self.assertRaises(ValidationError) as ctx:
self._anthropic_request(thinking={"type": "enabled"}, stream=False)
self.assertIn("budget_tokens", str(ctx.exception))
def test_request_thinking_enabled_budget_below_min_is_rejected(self):
"""SDK doc: ``budget_tokens`` must be >= 1024."""
from pydantic import ValidationError
with self.assertRaises(ValidationError) as ctx:
self._anthropic_request(
thinking={"type": "enabled", "budget_tokens": 512}, stream=False
)
self.assertIn("1024", str(ctx.exception))
def test_request_thinking_disabled_with_display_is_rejected(self):
"""SDK ``ThinkingConfigDisabledParam`` has no ``display`` field."""
from pydantic import ValidationError
with self.assertRaises(ValidationError) as ctx:
self._anthropic_request(
thinking={"type": "disabled", "display": "omitted"}, stream=False
)
self.assertIn("display", str(ctx.exception))
def test_request_thinking_disabled_with_budget_is_rejected(self):
"""SDK ``ThinkingConfigDisabledParam`` has no ``budget_tokens`` field."""
from pydantic import ValidationError
with self.assertRaises(ValidationError) as ctx:
self._anthropic_request(
thinking={"type": "disabled", "budget_tokens": 2048}, stream=False
)
self.assertIn("budget_tokens", str(ctx.exception))
def test_request_thinking_adaptive_with_budget_is_rejected(self):
"""SDK ``ThinkingConfigAdaptiveParam`` has no ``budget_tokens`` field."""
from pydantic import ValidationError
with self.assertRaises(ValidationError) as ctx:
self._anthropic_request(
thinking={"type": "adaptive", "budget_tokens": 2048}, stream=False
)
self.assertIn("budget_tokens", str(ctx.exception))
def test_request_thinking_adaptive_is_treated_as_enabled(self):
"""Claude 4.7 ``thinking.type='adaptive'`` (the SDK default for
unknown models) must be accepted and routed to ``apply_reasoning_enabled(True)``.
"""
serving = self._serving()
request = self._anthropic_request(thinking={"type": "adaptive"}, stream=False)
serving._convert_to_chat_completion_request(request)
self.assertEqual(serving.openai_serving_chat.apply_reasoning_calls, [True])
def test_request_thinking_display_omitted_logs_warning_but_still_enables(self):
"""``thinking.display='omitted'`` is accepted; reasoning stays on
because we cannot suppress reasoning text from the OpenAI stream.
``enabled`` requires ``budget_tokens`` per SDK shape."""
import logging
serving = self._serving()
request = self._anthropic_request(
thinking={
"type": "enabled",
"budget_tokens": 1024,
"display": "omitted",
},
stream=False,
)
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level=logging.WARNING
) as log:
serving._convert_to_chat_completion_request(request)
self.assertEqual(serving.openai_serving_chat.apply_reasoning_calls, [True])
self.assertTrue(any("omitted" in r for r in log.output))
def test_request_output_config_effort_maps_to_reasoning_effort(self):
"""``output_config.effort`` rows map onto ``reasoning_effort``."""
for anthropic_effort, openai_effort in [
("low", "low"),
("medium", "medium"),
("high", "high"),
("xhigh", "max"), # OpenAI Literal has no xhigh
("max", "max"),
]:
with self.subTest(anthropic_effort=anthropic_effort):
serving = self._serving()
request = self._anthropic_request(
output_config={"effort": anthropic_effort}, stream=False
)
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(chat_request.reasoning_effort, openai_effort)
def test_request_output_config_task_budget_is_logged_not_enforced(self):
"""``task_budget`` is a soft hint; ``max_tokens`` is the hard cap."""
import logging
serving = self._serving()
request = self._anthropic_request(
output_config={"task_budget": {"type": "tokens", "total": 32768}},
stream=False,
)
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level=logging.INFO
) as log:
chat_request = serving._convert_to_chat_completion_request(request)
# max_tokens is untouched
self.assertEqual(chat_request.max_tokens, 16)
self.assertTrue(any("task_budget" in r and "32768" in r for r in log.output))
def test_request_betas_is_accepted_and_logged(self):
"""``betas`` is accepted and logged; the local backend has no beta system."""
import logging
serving = self._serving()
request = self._anthropic_request(betas=["thinking-2025-08-04"], stream=False)
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level=logging.INFO
) as log:
serving._convert_to_chat_completion_request(request)
self.assertTrue(any("thinking-2025-08-04" in r for r in log.output))
def test_assistant_thinking_history_is_rewrapped_for_chat_template(self):
"""Past-turn thinking blocks get re-emitted via wrap_reasoning_history."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "ponder"},
{"type": "text", "text": "hello"},
],
},
{"role": "user", "content": "again"},
],
)
chat_request = serving._convert_to_chat_completion_request(request)
# ``ChatCompletionRequest.messages`` is a list of Pydantic
# ChatCompletionMessage*Param instances; access via attributes.
assistant_msg = next(m for m in chat_request.messages if m.role == "assistant")
content = assistant_msg.content
# Reasoning history sits in front; the thinking block itself is dropped
# from the prompt so its text is not duplicated.
if isinstance(content, list):
texts = []
for part in content:
if isinstance(part, dict):
texts.append(part.get("text", ""))
else:
texts.append(getattr(part, "text", "") or "")
else:
texts = [content]
joined = "\n".join(texts)
self.assertIn("<think>", joined)
self.assertIn("ponder", joined)
self.assertNotIn("<think>\nponder\n</think>\nponder", joined)
def test_assistant_thinking_history_uses_native_reasoning_content(self):
"""Channel-framing encoders take thinking history as ``reasoning_content``.
Splicing the detector's markers into content would nest a reasoning block
inside the content channel and leave the real one empty, training the
model to emit raw markers as visible text.
"""
class _NativeOpenAI(_FakeOpenAIServingChat):
native_reasoning_history = True
def wrap_reasoning_history(self, text):
raise AssertionError("must not rewrap for a native-history encoder")
serving = AnthropicServing(_NativeOpenAI())
request = self._anthropic_request(
stream=False,
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "ponder"},
{"type": "text", "text": "hello"},
],
},
{"role": "user", "content": "again"},
],
)
chat_request = serving._convert_to_chat_completion_request(request)
assistant_msg = next(m for m in chat_request.messages if m.role == "assistant")
self.assertEqual(assistant_msg.reasoning_content, "ponder")
self.assertEqual(assistant_msg.content, "hello")
def test_thinking_only_turn_keeps_native_reasoning_content(self):
"""An assistant turn that is only thinking still carries its reasoning."""
class _NativeOpenAI(_FakeOpenAIServingChat):
native_reasoning_history = True
serving = AnthropicServing(_NativeOpenAI())
request = self._anthropic_request(
stream=False,
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"content": [{"type": "thinking", "thinking": "ponder"}],
},
{"role": "user", "content": "again"},
],
)
chat_request = serving._convert_to_chat_completion_request(request)
assistant_msg = next(m for m in chat_request.messages if m.role == "assistant")
self.assertEqual(assistant_msg.reasoning_content, "ponder")
self.assertEqual(assistant_msg.content, "")
def test_redacted_thinking_history_is_rejected(self):
"""``redacted_thinking`` cannot be rendered by local parsers."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
messages=[
{
"role": "assistant",
"content": [
{"type": "redacted_thinking", "data": "opaque"},
],
},
],
)
with self.assertRaises(ValueError):
serving._convert_to_chat_completion_request(request)
def test_stream_text_then_thinking_closes_text_block(self):
"""Text deltas followed by reasoning_content must close the text block before opening thinking."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": "Direct"})]),
_chunk([_choice({"reasoning_content": "but wait, let me think"})]),
_chunk([_choice({}, finish_reason="stop")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
block_events = [
(event["type"], event.get("content_block", {}).get("type"))
for event in events
if event["type"].startswith("content_block")
]
# text (start, delta, stop) then thinking (start, delta, stop)
self.assertEqual(block_events[0], ("content_block_start", "text"))
text_stop_idx = next(
i for i, ev in enumerate(block_events) if ev == ("content_block_stop", None)
)
thinking_start_idx = next(
i
for i, ev in enumerate(block_events)
if ev == ("content_block_start", "thinking")
)
self.assertLess(text_stop_idx, thinking_start_idx)
def test_stream_consecutive_tool_calls_get_separate_blocks(self):
"""Two tool_use calls in sequence must occupy distinct content_block indices."""
serving = self._serving(
[
_chunk(
[
_choice(
{
"tool_calls": [
{
"index": 0,
"id": "call_a",
"function": {
"name": "alpha",
"arguments": '{"x":1}',
},
}
]
}
)
]
),
_chunk(
[
_choice(
{
"tool_calls": [
{
"index": 1,
"id": "call_b",
"function": {
"name": "beta",
"arguments": '{"y":2}',
},
}
]
}
)
]
),
_chunk([_choice({}, finish_reason="tool_calls")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
starts = [
(e["index"], e["content_block"]["name"])
for e in events
if e["type"] == "content_block_start"
]
stops = [e["index"] for e in events if e["type"] == "content_block_stop"]
deltas = [
(e["index"], e["delta"].get("partial_json"))
for e in events
if e["type"] == "content_block_delta"
and e["delta"].get("type") == "input_json_delta"
]
# Each tool gets its own start, its own stop, and its own
# argument delta — without the fix, beta's args were appended
# to alpha's index 0 block.
self.assertEqual(starts, [(0, "alpha"), (1, "beta")])
self.assertEqual(stops, [0, 1])
self.assertEqual(deltas, [(0, '{"x":1}'), (1, '{"y":2}')])
def test_stream_finish_chunk_with_payload_emits_delta(self):
"""A chunk carrying both finish_reason and content must not drop the content."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant"})]),
_chunk([_choice({"content": "last token"}, finish_reason="stop")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
text_deltas = [
e["delta"]["text"]
for e in events
if e["type"] == "content_block_delta"
and e["delta"].get("type") == "text_delta"
]
self.assertEqual(text_deltas, ["last token"])
# And stop_reason still travels via message_delta
message_delta = next(e for e in events if e["type"] == "message_delta")
self.assertEqual(message_delta["delta"]["stop_reason"], "end_turn")
def test_stream_empty_completion_with_finish_reason_emits_message_delta(self):
"""An empty stream with a finish_reason is a legitimate stop, not api_error."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant"})]),
_chunk([_choice({}, finish_reason="length")]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
types = [e["type"] for e in events]
self.assertIn("message_start", types)
self.assertIn("message_delta", types)
self.assertIn("message_stop", types)
self.assertNotIn("error", types)
message_delta = next(e for e in events if e["type"] == "message_delta")
self.assertEqual(message_delta["delta"]["stop_reason"], "max_tokens")
def test_stream_no_finish_no_content_still_emits_api_error(self):
"""Backend that drops both content and finish_reason is genuinely broken."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant"})]),
"data: [DONE]\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
types = [e["type"] for e in events]
self.assertIn("error", types)
err = next(e for e in events if e["type"] == "error")
self.assertEqual(err["error"]["type"], "api_error")
def test_stream_upstream_error_envelope_is_forwarded(self):
"""OpenAI handler streaming-error JSON must surface real type/message."""
upstream_error = {
"error": {
"object": "error",
"message": "context length exceeded",
"type": "BadRequestError",
"code": 400,
}
}
serving = self._serving(
[
_chunk([_choice({"role": "assistant"})]),
f"data: {json.dumps(upstream_error)}\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
err = next(e for e in events if e["type"] == "error")
self.assertEqual(err["error"]["type"], "invalid_request_error")
self.assertEqual(err["error"]["message"], "context length exceeded")
# message_stop must still close the stream
self.assertEqual(events[-1]["type"], "message_stop")
def test_stream_parse_failure_closes_open_content_block(self):
"""Unparsable mid-stream chunk must still close any open content_block."""
serving = self._serving(
[
_chunk([_choice({"role": "assistant", "content": "first"})]),
"data: {not-json\n\n",
]
)
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
types = [e["type"] for e in events]
# Sequence: message_start, content_block_start, content_block_delta,
# content_block_stop, error, message_stop
self.assertIn("content_block_start", types)
self.assertEqual(
types.count("content_block_stop"),
types.count("content_block_start"),
f"unbalanced block events: {types}",
)
self.assertIn("error", types)
self.assertEqual(types[-1], "message_stop")
def test_stream_pre_first_chunk_value_error_emits_envelope(self):
"""ValueError before any chunk must yield a clean Anthropic error sequence."""
class _RaisingOpenAI(_FakeOpenAIServingChat):
def _generate_chat_stream(
self, adapted_request, processed_request, raw_request
):
async def _gen():
raise ValueError("tokenization failed")
yield # pragma: no cover
return _gen()
serving = AnthropicServing(_RaisingOpenAI())
events = asyncio.run(
_collect_anthropic_events(serving, self._anthropic_request())
)
types = [e["type"] for e in events]
self.assertEqual(types[0], "message_start")
self.assertIn("error", types)
err = next(e for e in events if e["type"] == "error")
self.assertEqual(err["error"]["type"], "invalid_request_error")
self.assertIn("tokenization failed", err["error"]["message"])
self.assertEqual(types[-1], "message_stop")
def test_server_tool_only_with_tool_choice_any_raises_400(self):
"""A request with only server-side tools cannot honor tool_choice=any."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
tools=[{"type": "web_search_20250305", "name": "web_search"}],
tool_choice={"type": "any"},
)
with self.assertRaises(ValueError) as ctx:
serving._convert_to_chat_completion_request(request)
self.assertIn("tool_choice", str(ctx.exception))
def test_tool_choice_named_custom_tool_is_resolved(self):
"""tool_choice={type:'tool', name:'X'} where X is a custom tool wires through."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
tools=[
{
"type": "custom",
"name": "lookup",
"input_schema": {
"type": "object",
"properties": {"q": {"type": "string"}},
},
}
],
tool_choice={"type": "tool", "name": "lookup"},
)
# Must not AttributeError: Tool.function is a Pydantic model, not a
# dict — access must be via .name, never .get("name").
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(chat_request.tool_choice.type, "function")
self.assertEqual(chat_request.tool_choice.function.name, "lookup")
def test_tool_choice_named_unknown_tool_raises_400(self):
"""tool_choice={type:'tool', name:'X'} where X is missing must raise."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
tools=[
{
"type": "custom",
"name": "lookup",
"input_schema": {"type": "object", "properties": {}},
}
],
tool_choice={"type": "tool", "name": "nonexistent"},
)
with self.assertRaises(ValueError) as ctx:
serving._convert_to_chat_completion_request(request)
self.assertIn("nonexistent", str(ctx.exception))
def test_convert_response_non_streaming_empty_content_keeps_block(self):
"""Empty-string completion must still produce a content list of len 1."""
response = ChatCompletionResponse.model_validate(
{
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 0,
"model": "test-model",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": ""},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 5,
"completion_tokens": 0,
"total_tokens": 5,
},
}
)
serving = self._serving()
anthropic_response = serving._convert_response(response)
self.assertEqual(len(anthropic_response.content), 1)
self.assertEqual(anthropic_response.content[0].type, "text")
self.assertEqual(anthropic_response.content[0].text, "")
def test_error_response_does_not_leak_exception_name(self):
"""``error.type`` must stay in Anthropic's documented literal set."""
serving = self._serving()
response = serving._error_response(
status_code=500,
error_type="api_error",
message="Internal server error",
exception_name="KeyError",
)
body = json.loads(bytes(response.body).decode())
self.assertEqual(body["error"]["type"], "api_error")
def test_user_message_text_tool_text_preserves_order(self):
"""User message [text, tool_result, text] must stay user→tool→user on the wire."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "first"},
{
"type": "tool_result",
"tool_use_id": "call_x",
"content": "ok",
},
{"type": "text", "text": "second"},
],
}
],
)
chat_request = serving._convert_to_chat_completion_request(request)
# chat_request.messages items are Pydantic ChatCompletionMessage*Param
# variants — use attribute access, not subscripts.
roles = [m.role for m in chat_request.messages]
self.assertEqual(roles, ["user", "tool", "user"])
self.assertEqual(chat_request.messages[0].content, "first")
self.assertEqual(chat_request.messages[2].content, "second")
def test_empty_text_assistant_turn_preserves_role_alternation(self):
"""Assistant turn with only empty text must NOT vanish from the wire."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
messages=[
{"role": "user", "content": "u1"},
{"role": "assistant", "content": [{"type": "text", "text": ""}]},
{"role": "user", "content": "u2"},
],
)
chat_request = serving._convert_to_chat_completion_request(request)
roles = [m.role for m in chat_request.messages]
# Without the fix this collapses to ['user', 'user'] and breaks
# strict role-alternation chat templates (qwen, llama, mistral).
self.assertEqual(roles, ["user", "assistant", "user"])
def test_in_messages_system_merged_when_template_requires_first(self):
"""When the chat template rejects mid-conversation ``role: "system"``
(e.g. Qwen's system-first guard), the converter folds the inline
system turn into the leading system block so the template doesn't
400. The request object itself is no longer mutated — detection runs
in the serving layer on conversion."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
messages=[
{"role": "user", "content": "hi"},
{"role": "system", "content": "Reply with exactly: OK"},
{"role": "user", "content": "go"},
],
)
self.assertIsNone(request.system)
self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages], ["system", "user", "user"]
)
self.assertEqual(chat_request.messages[0].content, "Reply with exactly: OK")
def test_in_messages_system_merged_with_top_level_when_merge(self):
"""On the merge path, a top-level ``system`` field and a mid-conversation
system turn are joined into the leading system block; top-level text
comes first."""
serving = self._serving()
request = self._anthropic_request(
stream=False,
system="You are terse.",
messages=[
{"role": "user", "content": "hi"},
{"role": "system", "content": "One word only."},
{"role": "user", "content": "go"},
],
)
self.assertEqual(request.system, "You are terse.")
self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages], ["system", "user", "user"]
)
self.assertEqual(
chat_request.messages[0].content, "You are terse.\nOne word only."
)
def test_in_messages_system_passed_through_when_template_allows_inline(self):
"""When the chat template renders ``role: "system"`` at any position
(GLM / Kimi / Qwen3), the inline system turn stays at its original
position — preserving the prefix cache and the request's structure."""
serving = self._serving(chat_template=self.INLINE_SYSTEM_TEMPLATE)
self.assertFalse(serving._merge_inline_system)
request = self._anthropic_request(
stream=False,
system="You are terse.",
messages=[
{"role": "user", "content": "hi"},
{"role": "system", "content": "Reply with exactly: OK"},
{"role": "user", "content": "go"},
],
)
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages],
["system", "user", "system", "user"],
)
self.assertEqual(chat_request.messages[0].content, "You are terse.")
self.assertEqual(chat_request.messages[2].content, "Reply with exactly: OK")
def test_top_level_system_only_is_unchanged(self):
"""A request with only the top-level ``system`` field (no in-messages
system turn) is unaffected on both detection paths: the system field is
preserved verbatim and the dialogue order is untouched. Guards the
common multi-turn path against regressions."""
for template in (None, self.INLINE_SYSTEM_TEMPLATE):
serving = self._serving(chat_template=template)
request = self._anthropic_request(
stream=False,
system="You are a helpful assistant.",
messages=[
{"role": "user", "content": "My name is Alice."},
{"role": "assistant", "content": "Hello Alice!"},
{"role": "user", "content": "What is my name?"},
],
)
self.assertEqual(request.system, "You are a helpful assistant.")
self.assertEqual(
[m.role for m in request.messages], ["user", "assistant", "user"]
)
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages],
["system", "user", "assistant", "user"],
)
self.assertEqual(
chat_request.messages[0].content, "You are a helpful assistant."
)
def test_constructed_message_objects_merged_on_merge_path(self):
"""Requests built programmatically with ``AnthropicMessage`` objects
(e.g. ``handle_count_tokens``) also get inline system folded into the
leading block on the merge path."""
serving = self._serving()
request = AnthropicMessagesRequest(
model="m",
max_tokens=8,
messages=[
AnthropicMessage(role="user", content="hi"),
AnthropicMessage(role="system", content="be terse"),
AnthropicMessage(role="user", content="go"),
],
)
self.assertEqual([m.role for m in request.messages], ["user", "system", "user"])
chat_request = serving._convert_to_chat_completion_request(request)
self.assertEqual(
[m.role for m in chat_request.messages], ["system", "user", "user"]
)
self.assertEqual(chat_request.messages[0].content, "be terse")
def test_thinking_history_drop_on_missing_detector(self):
"""Replaying a thinking block on a non-reasoning model should not 400."""
class _NoDetectorOpenAI(_FakeOpenAIServingChat):
def wrap_reasoning_history(self, text):
raise ValueError("no reasoning detector is configured")
serving = AnthropicServing(_NoDetectorOpenAI())
request = self._anthropic_request(
stream=False,
messages=[
{
"role": "assistant",
"content": [{"type": "thinking", "thinking": "I think..."}],
},
{"role": "user", "content": "follow-up"},
],
)
# Must convert successfully; the thinking block is silently dropped.
chat_request = serving._convert_to_chat_completion_request(request)
roles = [m.role for m in chat_request.messages]
self.assertIn("user", roles)
# The assistant turn was rendered (as empty placeholder) so
# alternation is preserved.
self.assertIn("assistant", roles)
def test_stop_reason_content_filter_falls_back_with_warning(self):
"""Unmapped OpenAI finish_reasons default to 'end_turn' + log a warning.
``content_filter`` and ``abort`` have no entry in STOP_REASON_MAP
because Anthropic's ``stop_reason`` Literal (end_turn/max_tokens/
stop_sequence/tool_use) has no perfect target. The fallback must
produce a spec-valid stop_reason and a WARNING so operators don't
silently lose the safety/abort signal.
"""
import logging
response = ChatCompletionResponse.model_validate(
{
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 0,
"model": "test-model",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "hi"},
"finish_reason": "content_filter",
}
],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
}
)
serving = self._serving()
with self.assertLogs(
"sglang.srt.entrypoints.anthropic.serving", level=logging.WARNING
) as log:
anthropic_response = serving._convert_response(response)
self.assertEqual(anthropic_response.stop_reason, "end_turn")
self.assertTrue(
any("content_filter" in rec for rec in log.output),
f"expected a warning mentioning the unmapped finish_reason: {log.output}",
)
class TestDetectInlineSystemSupport(unittest.TestCase):
"""Chat-template detection for mid-conversation system messages (#28883)."""
def test_guarded_template_not_supported(self):
guarded = (
"{%- for message in messages %}"
"{%- if message.role == 'system' and not loop.first %}"
"{{- raise_exception('system must be first') }}"
"{%- endif %}"
"{%- endfor %}"
)
self.assertFalse(detect_inline_system_support(guarded))
def test_inline_template_supported(self):
inline = (
"{%- for message in messages %}"
"{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
self.assertTrue(detect_inline_system_support(inline))
def test_silent_drop_template_not_supported(self):
# Renders only the leading system; silently ignores later system turns.
silent_drop = (
"{%- if messages[0].role == 'system' %}"
"{{ messages[0].content }}\n"
"{%- endif %}"
"{%- for message in messages %}"
"{%- if message.role in ('user', 'assistant') %}"
"{{ message.role }}: {{ message.content }}\n"
"{%- endif %}"
"{%- endfor %}"
)
self.assertFalse(detect_inline_system_support(silent_drop))
def test_no_template_not_supported(self):
self.assertFalse(detect_inline_system_support(None))
self.assertFalse(detect_inline_system_support(""))
if __name__ == "__main__":
unittest.main()