[CI] Fold duplicate-server suites and prune the retract matrix on 1-gpu-5090 (#33745)
This commit is contained in:
Liangsheng Yin
@@ -1,20 +1,24 @@
|
||||
import unittest
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
|
||||
from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
|
||||
from sglang.test.kits.json_mode_kit import JSONModeMixin
|
||||
from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=120, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=179, suite="stage-b-test-1-gpu-small-amd")
|
||||
register_cuda_ci(est_time=135, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=220, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class ServerWithGrammar(CustomTestCase):
|
||||
@@ -35,12 +39,16 @@ class ServerWithGrammar(CustomTestCase):
|
||||
if cls.disable_overlap:
|
||||
launch_args += ["--disable-overlap-schedule"]
|
||||
|
||||
if is_in_amd_ci():
|
||||
launch_args.append("--constrained-json-disable-any-whitespace")
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=launch_args,
|
||||
)
|
||||
cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
@@ -50,19 +58,21 @@ class ServerWithGrammar(CustomTestCase):
|
||||
class TestXGrammarBackend(
|
||||
ServerWithGrammar,
|
||||
JSONConstrainedMixin,
|
||||
JSONModeMixin,
|
||||
EBNFConstrainedMixin,
|
||||
RegexConstrainedMixin,
|
||||
):
|
||||
backend = "xgrammar"
|
||||
|
||||
|
||||
class TestOutlinesBackend(ServerWithGrammar, JSONConstrainedMixin):
|
||||
class TestOutlinesBackend(ServerWithGrammar, JSONConstrainedMixin, JSONModeMixin):
|
||||
backend = "outlines"
|
||||
|
||||
|
||||
class TestLLGuidanceBackend(
|
||||
ServerWithGrammar,
|
||||
JSONConstrainedMixin,
|
||||
JSONModeMixin,
|
||||
EBNFConstrainedMixin,
|
||||
RegexConstrainedMixin,
|
||||
):
|
||||
|
||||
@@ -27,7 +27,7 @@ DEFAULT_MODEL = "Qwen/Qwen3-0.6B"
|
||||
# file per suite, TestWeightCacheDaemonTP2 self-skips when fewer than 2 GPUs are
|
||||
# visible (i.e. on the 1-gpu runner).
|
||||
register_cuda_ci(est_time=100, stage="extra-a", runner_config="2-gpu-large")
|
||||
register_cuda_ci(est_time=100, stage="base-b", runner_config="1-gpu-small")
|
||||
register_cuda_ci(est_time=45, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
# Capture the client server's logs so test_loaded_via_ipc can assert the IPC
|
||||
# load path actually ran (and did not silently fall back to disk).
|
||||
|
||||
@@ -1,582 +0,0 @@
|
||||
"""
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_simple_messages
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_simple_messages_stream
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_multi_turn_messages
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_system_message_string
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_system_message_blocks
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_max_tokens
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_temperature
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_stop_sequences
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_invalid_max_tokens
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_empty_messages
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_non_streaming
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_streaming
|
||||
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_tool_result_image_content_conversion
|
||||
"""
|
||||
|
||||
import json
|
||||
import unittest
|
||||
|
||||
import anthropic
|
||||
import requests
|
||||
|
||||
from sglang.srt.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from sglang.srt.entrypoints.anthropic.serving import AnthropicServing
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=40, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=140, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestAnthropicServer(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.api_key = "sk-123456"
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
api_key=cls.api_key,
|
||||
)
|
||||
cls.messages_url = cls.base_url + "/v1/messages"
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def _make_request(self, payload, stream=False):
|
||||
"""Send a request to the /v1/messages endpoint."""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
return requests.post(
|
||||
self.messages_url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
def _default_payload(self, **overrides):
|
||||
"""Build a default Anthropic Messages request payload."""
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the capital of France? Answer in a few words.",
|
||||
}
|
||||
],
|
||||
}
|
||||
payload.update(overrides)
|
||||
return payload
|
||||
|
||||
# ---- Non-streaming tests ----
|
||||
|
||||
def test_tool_result_image_content_conversion(self):
|
||||
"""Tool-result image blocks should be preserved as OpenAI image_url content."""
|
||||
anthropic_request = AnthropicMessagesRequest(
|
||||
model=self.model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I have called read_file to get an image. What color is it?",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "call_123",
|
||||
"name": "read_file",
|
||||
"input": {"file_path": "/test.png"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "call_123",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": "abcd",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
serving = AnthropicServing(openai_serving_chat=object())
|
||||
chat_request = serving._convert_to_chat_completion_request(anthropic_request)
|
||||
converted = chat_request.model_dump()
|
||||
|
||||
tool_messages = [m for m in converted["messages"] if m.get("role") == "tool"]
|
||||
self.assertEqual(
|
||||
len(tool_messages),
|
||||
1,
|
||||
f"Expected one tool message, got: {converted['messages']}",
|
||||
)
|
||||
|
||||
tool_message = tool_messages[0]
|
||||
self.assertEqual(tool_message["tool_call_id"], "call_123")
|
||||
self.assertIsInstance(tool_message["content"], list)
|
||||
self.assertEqual(len(tool_message["content"]), 1)
|
||||
self.assertEqual(tool_message["content"][0]["type"], "image_url")
|
||||
self.assertEqual(
|
||||
tool_message["content"][0]["image_url"]["url"],
|
||||
"data:image/png;base64,abcd",
|
||||
)
|
||||
|
||||
def test_simple_messages(self):
|
||||
"""Test basic non-streaming message request."""
|
||||
payload = self._default_payload()
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertEqual(body["role"], "assistant")
|
||||
self.assertIn("content", body)
|
||||
self.assertIsInstance(body["content"], list)
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
self.assertEqual(body["content"][0]["type"], "text")
|
||||
self.assertIsInstance(body["content"][0]["text"], str)
|
||||
self.assertTrue(len(body["content"][0]["text"]) > 0)
|
||||
|
||||
# Verify stop reason
|
||||
self.assertIn(body["stop_reason"], ["end_turn", "max_tokens", "stop_sequence"])
|
||||
|
||||
# Verify usage
|
||||
self.assertIn("usage", body)
|
||||
self.assertIsInstance(body["usage"]["input_tokens"], int)
|
||||
self.assertIsInstance(body["usage"]["output_tokens"], int)
|
||||
self.assertGreater(body["usage"]["input_tokens"], 0)
|
||||
self.assertGreater(body["usage"]["output_tokens"], 0)
|
||||
|
||||
# Verify id format (must be msg_*) and model
|
||||
self.assertIn("id", body)
|
||||
self.assertIsInstance(body["id"], str)
|
||||
self.assertTrue(
|
||||
body["id"].startswith("msg_"),
|
||||
f"ID should start with 'msg_', got: {body['id']}",
|
||||
)
|
||||
self.assertIn("model", body)
|
||||
|
||||
def test_multi_turn_messages(self):
|
||||
"""Test multi-turn conversation."""
|
||||
payload = self._default_payload(
|
||||
messages=[
|
||||
{"role": "user", "content": "My name is Alice."},
|
||||
{"role": "assistant", "content": "Hello Alice! Nice to meet you."},
|
||||
{"role": "user", "content": "What is my name?"},
|
||||
]
|
||||
)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
self.assertEqual(body["content"][0]["type"], "text")
|
||||
self.assertIsInstance(body["content"][0]["text"], str)
|
||||
|
||||
def test_system_message_string(self):
|
||||
"""Test system message as a string."""
|
||||
payload = self._default_payload(
|
||||
system="You are a helpful assistant. Always respond in French.",
|
||||
)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
|
||||
def test_system_message_blocks(self):
|
||||
"""Test system message as content blocks."""
|
||||
payload = self._default_payload(
|
||||
system=[
|
||||
{"type": "text", "text": "You are a helpful assistant."},
|
||||
{"type": "text", "text": "Always be concise."},
|
||||
],
|
||||
)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
|
||||
def test_in_messages_system_role(self):
|
||||
"""A ``role: "system"`` turn inside ``messages`` (emitted by some
|
||||
clients, e.g. Claude Code) must be accepted — not rejected with 400.
|
||||
Uses the Anthropic SDK the way a real client would."""
|
||||
client = anthropic.Anthropic(
|
||||
base_url=self.base_url,
|
||||
auth_token=self.api_key, # Bearer header — SGLang's --api-key checks Authorization
|
||||
)
|
||||
message = client.messages.create(
|
||||
model=self.model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
{"role": "system", "content": "Always respond in French."},
|
||||
{"role": "user", "content": "Answer in a few words."},
|
||||
],
|
||||
)
|
||||
self.assertEqual(message.role, "assistant")
|
||||
self.assertTrue(len(message.content) > 0)
|
||||
self.assertEqual(message.content[0].type, "text")
|
||||
|
||||
def test_max_tokens(self):
|
||||
"""Test max_tokens limits output length."""
|
||||
payload = self._default_payload(
|
||||
max_tokens=5,
|
||||
messages=[
|
||||
{"role": "user", "content": "Tell me a long story about a dragon."}
|
||||
],
|
||||
)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
# With very small max_tokens the model should hit the limit
|
||||
self.assertIn(body["stop_reason"], ["max_tokens", "end_turn"])
|
||||
self.assertGreater(body["usage"]["output_tokens"], 0)
|
||||
|
||||
def test_temperature(self):
|
||||
"""Test temperature parameter is accepted."""
|
||||
payload = self._default_payload(temperature=0.0)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
|
||||
def test_stop_sequences(self):
|
||||
"""Test stop_sequences parameter is accepted."""
|
||||
payload = self._default_payload(
|
||||
stop_sequences=["\n"],
|
||||
max_tokens=128,
|
||||
)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
|
||||
def test_top_p_and_top_k(self):
|
||||
"""Test top_p and top_k parameters."""
|
||||
payload = self._default_payload(top_p=0.9, top_k=40)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
|
||||
# ---- Streaming tests ----
|
||||
|
||||
def test_simple_messages_stream(self):
|
||||
"""Test basic streaming message request."""
|
||||
payload = self._default_payload(stream=True)
|
||||
resp = self._make_request(payload, stream=True)
|
||||
self.assertEqual(resp.status_code, 200, f"Status: {resp.status_code}")
|
||||
|
||||
events = self._parse_sse_events(resp)
|
||||
|
||||
# Verify event sequence
|
||||
event_types = [e["type"] for e in events]
|
||||
self.assertIn("message_start", event_types)
|
||||
self.assertIn("message_stop", event_types)
|
||||
|
||||
# Verify message_start
|
||||
message_start = next(e for e in events if e["type"] == "message_start")
|
||||
self.assertIn("message", message_start)
|
||||
self.assertEqual(message_start["message"]["type"], "message")
|
||||
self.assertEqual(message_start["message"]["role"], "assistant")
|
||||
self.assertIn("usage", message_start["message"])
|
||||
|
||||
# Verify we got content deltas
|
||||
content_deltas = [e for e in events if e["type"] == "content_block_delta"]
|
||||
self.assertTrue(
|
||||
len(content_deltas) > 0, "Expected at least one content_block_delta event"
|
||||
)
|
||||
|
||||
# Verify all text deltas have correct structure
|
||||
for delta_event in content_deltas:
|
||||
self.assertIn("delta", delta_event)
|
||||
self.assertEqual(delta_event["delta"]["type"], "text_delta")
|
||||
self.assertIn("text", delta_event["delta"])
|
||||
|
||||
# Reconstruct the full text
|
||||
full_text = "".join(
|
||||
e["delta"]["text"]
|
||||
for e in content_deltas
|
||||
if e["delta"].get("type") == "text_delta"
|
||||
)
|
||||
self.assertTrue(len(full_text) > 0, "Reconstructed text should not be empty")
|
||||
|
||||
# Verify content_block_start/stop
|
||||
block_starts = [e for e in events if e["type"] == "content_block_start"]
|
||||
block_stops = [e for e in events if e["type"] == "content_block_stop"]
|
||||
self.assertTrue(len(block_starts) > 0, "Expected content_block_start")
|
||||
self.assertTrue(len(block_stops) > 0, "Expected content_block_stop")
|
||||
self.assertEqual(block_starts[0]["content_block"]["type"], "text")
|
||||
|
||||
# Verify message_delta with stop_reason
|
||||
message_deltas = [e for e in events if e["type"] == "message_delta"]
|
||||
self.assertTrue(len(message_deltas) > 0, "Expected message_delta event")
|
||||
last_delta = message_deltas[-1]
|
||||
self.assertIn("delta", last_delta)
|
||||
self.assertIn("stop_reason", last_delta["delta"])
|
||||
self.assertIn(
|
||||
last_delta["delta"]["stop_reason"],
|
||||
["end_turn", "max_tokens", "stop_sequence", "tool_use"],
|
||||
)
|
||||
|
||||
# Verify usage in message_delta
|
||||
self.assertIn("usage", last_delta)
|
||||
self.assertIsInstance(last_delta["usage"]["output_tokens"], int)
|
||||
|
||||
def test_stream_multi_turn(self):
|
||||
"""Test streaming with multi-turn conversation."""
|
||||
payload = self._default_payload(
|
||||
stream=True,
|
||||
messages=[
|
||||
{"role": "user", "content": "Say hello."},
|
||||
{"role": "assistant", "content": "Hello!"},
|
||||
{"role": "user", "content": "Say goodbye."},
|
||||
],
|
||||
)
|
||||
resp = self._make_request(payload, stream=True)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
events = self._parse_sse_events(resp)
|
||||
event_types = [e["type"] for e in events]
|
||||
self.assertIn("message_start", event_types)
|
||||
self.assertIn("message_stop", event_types)
|
||||
|
||||
def test_stream_with_system(self):
|
||||
"""Test streaming with system message."""
|
||||
payload = self._default_payload(
|
||||
stream=True,
|
||||
system="You are a pirate. Respond in pirate speak.",
|
||||
)
|
||||
resp = self._make_request(payload, stream=True)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
events = self._parse_sse_events(resp)
|
||||
event_types = [e["type"] for e in events]
|
||||
self.assertIn("message_start", event_types)
|
||||
self.assertIn("message_stop", event_types)
|
||||
|
||||
# ---- Error handling tests ----
|
||||
|
||||
def test_error_invalid_max_tokens(self):
|
||||
"""Test error response for invalid max_tokens."""
|
||||
payload = self._default_payload(max_tokens=-1)
|
||||
resp = self._make_request(payload)
|
||||
self.assertIn(resp.status_code, [400, 422])
|
||||
|
||||
def test_error_empty_messages(self):
|
||||
"""Test error response for empty messages list."""
|
||||
payload = self._default_payload(messages=[])
|
||||
resp = self._make_request(payload)
|
||||
self.assertIn(resp.status_code, [400, 422])
|
||||
|
||||
def test_error_missing_content_type(self):
|
||||
"""Test error when Content-Type is not application/json."""
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
resp = requests.post(
|
||||
self.messages_url,
|
||||
headers=headers,
|
||||
data="not json",
|
||||
)
|
||||
self.assertIn(resp.status_code, [400, 415, 422])
|
||||
|
||||
# ---- Raw HTTP tests ----
|
||||
|
||||
def test_raw_http_non_streaming(self):
|
||||
"""Test raw HTTP request/response format for non-streaming."""
|
||||
payload = self._default_payload(temperature=0)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
# Verify response content type
|
||||
self.assertIn("application/json", resp.headers.get("content-type", ""))
|
||||
|
||||
body = resp.json()
|
||||
# Verify all required fields per Anthropic spec
|
||||
required_fields = ["id", "type", "role", "content", "model", "usage"]
|
||||
for field in required_fields:
|
||||
self.assertIn(field, body, f"Missing required field: {field}")
|
||||
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertEqual(body["role"], "assistant")
|
||||
|
||||
def test_raw_http_streaming(self):
|
||||
"""Test raw HTTP request/response format for streaming."""
|
||||
payload = self._default_payload(stream=True, temperature=0)
|
||||
resp = self._make_request(payload, stream=True)
|
||||
self.assertEqual(resp.status_code, 200)
|
||||
|
||||
# Verify streaming content type
|
||||
self.assertIn("text/event-stream", resp.headers.get("content-type", ""))
|
||||
|
||||
# Verify we get proper SSE events
|
||||
events = self._parse_sse_events(resp)
|
||||
self.assertTrue(len(events) > 0, "Expected at least some SSE events")
|
||||
|
||||
# Verify event ordering: message_start should be first
|
||||
self.assertEqual(
|
||||
events[0]["type"], "message_start", "First event should be message_start"
|
||||
)
|
||||
|
||||
# Verify message_stop is last data event
|
||||
data_events = [e for e in events if e["type"] != "ping"]
|
||||
self.assertEqual(
|
||||
data_events[-1]["type"],
|
||||
"message_stop",
|
||||
"Last data event should be message_stop",
|
||||
)
|
||||
|
||||
# ---- Content block tests ----
|
||||
|
||||
def test_content_blocks_message(self):
|
||||
"""Test sending messages with explicit content blocks."""
|
||||
payload = self._default_payload(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is 2+2?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
resp = self._make_request(payload)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertEqual(body["type"], "message")
|
||||
self.assertTrue(len(body["content"]) > 0)
|
||||
self.assertEqual(body["content"][0]["type"], "text")
|
||||
|
||||
# ---- Count tokens tests ----
|
||||
|
||||
def test_count_tokens(self):
|
||||
"""Test /v1/messages/count_tokens endpoint."""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, how are you?"},
|
||||
],
|
||||
}
|
||||
resp = requests.post(
|
||||
self.base_url + "/v1/messages/count_tokens",
|
||||
headers=headers,
|
||||
json=payload,
|
||||
)
|
||||
self.assertEqual(resp.status_code, 200, f"Response: {resp.text}")
|
||||
|
||||
body = resp.json()
|
||||
self.assertIn("input_tokens", body)
|
||||
self.assertIsInstance(body["input_tokens"], int)
|
||||
self.assertGreater(body["input_tokens"], 0)
|
||||
|
||||
def test_count_tokens_with_system(self):
|
||||
"""Test count_tokens with system message."""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
}
|
||||
payload_no_system = {
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
}
|
||||
payload_with_system = {
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
"system": "You are a helpful assistant with a very long system prompt that adds tokens.",
|
||||
}
|
||||
resp1 = requests.post(
|
||||
self.base_url + "/v1/messages/count_tokens",
|
||||
headers=headers,
|
||||
json=payload_no_system,
|
||||
)
|
||||
resp2 = requests.post(
|
||||
self.base_url + "/v1/messages/count_tokens",
|
||||
headers=headers,
|
||||
json=payload_with_system,
|
||||
)
|
||||
self.assertEqual(resp1.status_code, 200)
|
||||
self.assertEqual(resp2.status_code, 200)
|
||||
|
||||
# System message should increase the token count
|
||||
tokens_no_system = resp1.json()["input_tokens"]
|
||||
tokens_with_system = resp2.json()["input_tokens"]
|
||||
self.assertGreater(
|
||||
tokens_with_system,
|
||||
tokens_no_system,
|
||||
"Adding system message should increase token count",
|
||||
)
|
||||
|
||||
# ---- Helpers ----
|
||||
|
||||
def _parse_sse_events(self, response):
|
||||
"""Parse SSE events from a streaming response."""
|
||||
events = []
|
||||
|
||||
for line in response.iter_lines(decode_unicode=True):
|
||||
if not line:
|
||||
continue
|
||||
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:].strip()
|
||||
if data_str == "[DONE]":
|
||||
continue
|
||||
try:
|
||||
data = json.loads(data_str)
|
||||
events.append(data)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return events
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -7,6 +7,7 @@ python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test
|
||||
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Optional
|
||||
@@ -18,6 +19,7 @@ from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.anthropic_messages_kit import AnthropicMessagesMixin
|
||||
from sglang.test.runners import TEST_RERANK_QUERY_DOCS
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST,
|
||||
@@ -29,11 +31,11 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=240, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=200, suite="stage-b-test-1-gpu-small-amd")
|
||||
register_cuda_ci(est_time=300, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=280, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestOpenAIServer(CustomTestCase):
|
||||
class TestOpenAIServer(CustomTestCase, AnthropicMessagesMixin):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
@@ -52,6 +54,107 @@ class TestOpenAIServer(CustomTestCase):
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_ignore_eos(self):
|
||||
"""ignore_eos=True must keep generating past EOS up to max_tokens."""
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
max_tokens = 200
|
||||
|
||||
response_default = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Count from 1 to 20."},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=max_tokens,
|
||||
extra_body={"ignore_eos": False},
|
||||
)
|
||||
|
||||
response_ignore_eos = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Count from 1 to 20."},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=max_tokens,
|
||||
extra_body={"ignore_eos": True},
|
||||
)
|
||||
|
||||
default_tokens = len(
|
||||
self.tokenizer.encode(response_default.choices[0].message.content)
|
||||
)
|
||||
ignore_eos_tokens = len(
|
||||
self.tokenizer.encode(response_ignore_eos.choices[0].message.content)
|
||||
)
|
||||
|
||||
# Check if ignore_eos resulted in more tokens or exactly max_tokens
|
||||
# The ignore_eos response should either:
|
||||
# 1. Have more tokens than the default response (if default stopped at EOS before max_tokens)
|
||||
# 2. Have exactly max_tokens (if it reached the max_tokens limit)
|
||||
self.assertTrue(
|
||||
ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens,
|
||||
f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
response_ignore_eos.choices[0].finish_reason,
|
||||
"length",
|
||||
f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}",
|
||||
)
|
||||
|
||||
def test_ebnf(self):
|
||||
"""`ebnf` in extra_body must be enforced by the grammar backend."""
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
ebnf_grammar = r"""
|
||||
root ::= "Hello" | "Hi" | "Hey"
|
||||
"""
|
||||
pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful EBNF test bot."},
|
||||
{"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=32,
|
||||
extra_body={"ebnf": ebnf_grammar},
|
||||
)
|
||||
text = response.choices[0].message.content.strip()
|
||||
self.assertTrue(len(text) > 0, "Got empty text from EBNF generation")
|
||||
self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices")
|
||||
|
||||
def test_ebnf_strict_json(self):
|
||||
"""Stricter EBNF: exact {"name":"Alice"} shape, no extra fields."""
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
ebnf_grammar = r"""
|
||||
root ::= "{" pair "}"
|
||||
pair ::= "\"name\"" ":" string
|
||||
string ::= "\"" [A-Za-z]+ "\""
|
||||
"""
|
||||
pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$')
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "EBNF mini-JSON generator."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Generate single key JSON with only letters.",
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=64,
|
||||
extra_body={"ebnf": ebnf_grammar},
|
||||
)
|
||||
text = response.choices[0].message.content.strip()
|
||||
self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test")
|
||||
self.assertRegex(
|
||||
text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape"
|
||||
)
|
||||
|
||||
def run_completion(
|
||||
self, echo, logprobs, use_list_input, parallel_sample_num, token_input
|
||||
):
|
||||
@@ -149,7 +252,6 @@ class TestOpenAIServer(CustomTestCase):
|
||||
|
||||
is_firsts = {}
|
||||
for response in generator:
|
||||
print(f"{response=}")
|
||||
usage = response.usage
|
||||
if usage is not None:
|
||||
assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero"
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
import json
|
||||
import unittest
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=118, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=180, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class JSONModeMixin:
|
||||
"""Mixin class containing JSON mode test methods"""
|
||||
|
||||
def test_json_mode_response(self):
|
||||
"""Test that response_format json_object (also known as "json mode") produces valid JSON, even without a system prompt that mentions JSON."""
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
# We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful AI assistant that gives a short answer.",
|
||||
},
|
||||
{"role": "user", "content": "What is the capital of Bulgaria?"},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=128,
|
||||
response_format={"type": "json_object"},
|
||||
)
|
||||
text = response.choices[0].message.content
|
||||
|
||||
print(f"Response ({len(text)} characters): {text}")
|
||||
|
||||
# Verify the response is valid JSON
|
||||
try:
|
||||
js_obj = json.loads(text)
|
||||
except json.JSONDecodeError as e:
|
||||
self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}")
|
||||
|
||||
# Verify it's actually an object (dict)
|
||||
self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}")
|
||||
|
||||
def test_json_mode_with_streaming(self):
|
||||
"""Test that streaming with json_object response (also known as "json mode") format works correctly, even without a system prompt that mentions JSON."""
|
||||
stream = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
# We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful AI assistant that gives a short answer.",
|
||||
},
|
||||
{"role": "user", "content": "What is the capital of Bulgaria?"},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=128,
|
||||
response_format={"type": "json_object"},
|
||||
stream=True,
|
||||
)
|
||||
|
||||
# Collect all chunks
|
||||
chunks = []
|
||||
for chunk in stream:
|
||||
if chunk.choices[0].delta.content is not None:
|
||||
chunks.append(chunk.choices[0].delta.content)
|
||||
full_response = "".join(chunks)
|
||||
|
||||
print(
|
||||
f"Concatenated Response ({len(full_response)} characters): {full_response}"
|
||||
)
|
||||
|
||||
# Verify the combined response is valid JSON
|
||||
try:
|
||||
js_obj = json.loads(full_response)
|
||||
except json.JSONDecodeError as e:
|
||||
self.fail(
|
||||
f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}"
|
||||
)
|
||||
|
||||
self.assertIsInstance(js_obj, dict)
|
||||
|
||||
|
||||
class ServerWithGrammarBackend(CustomTestCase):
|
||||
"""Base class for tests requiring a grammar backend server"""
|
||||
|
||||
backend = "xgrammar"
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
other_args = [
|
||||
"--max-running-requests",
|
||||
"10",
|
||||
"--grammar-backend",
|
||||
cls.backend,
|
||||
]
|
||||
|
||||
if is_in_amd_ci():
|
||||
other_args.append("--constrained-json-disable-any-whitespace")
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
)
|
||||
cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
|
||||
class TestJSONModeXGrammar(ServerWithGrammarBackend, JSONModeMixin):
|
||||
backend = "xgrammar"
|
||||
|
||||
|
||||
class TestJSONModeOutlines(ServerWithGrammarBackend, JSONModeMixin):
|
||||
backend = "outlines"
|
||||
|
||||
|
||||
class TestJSONModeLLGuidance(ServerWithGrammarBackend, JSONModeMixin):
|
||||
backend = "llguidance"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,108 +0,0 @@
|
||||
import re
|
||||
|
||||
import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=44, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=20, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# EBNF Test Class: TestOpenAIServerEBNF
|
||||
# Launches the server with xgrammar, has only EBNF tests
|
||||
# -------------------------------------------------------------------------
|
||||
class TestOpenAIServerEBNF(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.api_key = "sk-123456"
|
||||
|
||||
# passing xgrammar specifically
|
||||
other_args = ["--grammar-backend", "xgrammar"]
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
api_key=cls.api_key,
|
||||
other_args=other_args,
|
||||
)
|
||||
cls.base_url += "/v1"
|
||||
cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_ebnf(self):
|
||||
"""
|
||||
Ensure we can pass `ebnf` to the local openai server
|
||||
and that it enforces the grammar.
|
||||
"""
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
ebnf_grammar = r"""
|
||||
root ::= "Hello" | "Hi" | "Hey"
|
||||
"""
|
||||
pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful EBNF test bot."},
|
||||
{"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=32,
|
||||
extra_body={"ebnf": ebnf_grammar},
|
||||
)
|
||||
text = response.choices[0].message.content.strip()
|
||||
self.assertTrue(len(text) > 0, "Got empty text from EBNF generation")
|
||||
self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices")
|
||||
|
||||
def test_ebnf_strict_json(self):
|
||||
"""
|
||||
A stricter EBNF that produces exactly {"name":"Alice"} format
|
||||
with no trailing punctuation or extra fields.
|
||||
"""
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
ebnf_grammar = r"""
|
||||
root ::= "{" pair "}"
|
||||
pair ::= "\"name\"" ":" string
|
||||
string ::= "\"" [A-Za-z]+ "\""
|
||||
"""
|
||||
pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$')
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "EBNF mini-JSON generator."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Generate single key JSON with only letters.",
|
||||
},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=64,
|
||||
extra_body={"ebnf": ebnf_grammar},
|
||||
)
|
||||
text = response.choices[0].message.content.strip()
|
||||
self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test")
|
||||
self.assertRegex(
|
||||
text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import unittest
|
||||
|
||||
unittest.main()
|
||||
@@ -1,94 +0,0 @@
|
||||
import openai
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=44, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=47, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestOpenAIServerIgnoreEOS(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.api_key = "sk-123456"
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
api_key=cls.api_key,
|
||||
)
|
||||
cls.base_url += "/v1"
|
||||
cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def test_ignore_eos(self):
|
||||
"""
|
||||
Test that ignore_eos=True allows generation to continue beyond EOS token
|
||||
and reach the max_tokens limit.
|
||||
"""
|
||||
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
|
||||
|
||||
max_tokens = 200
|
||||
|
||||
response_default = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Count from 1 to 20."},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=max_tokens,
|
||||
extra_body={"ignore_eos": False},
|
||||
)
|
||||
|
||||
response_ignore_eos = client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Count from 1 to 20."},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=max_tokens,
|
||||
extra_body={"ignore_eos": True},
|
||||
)
|
||||
|
||||
default_tokens = len(
|
||||
self.tokenizer.encode(response_default.choices[0].message.content)
|
||||
)
|
||||
ignore_eos_tokens = len(
|
||||
self.tokenizer.encode(response_ignore_eos.choices[0].message.content)
|
||||
)
|
||||
|
||||
# Check if ignore_eos resulted in more tokens or exactly max_tokens
|
||||
# The ignore_eos response should either:
|
||||
# 1. Have more tokens than the default response (if default stopped at EOS before max_tokens)
|
||||
# 2. Have exactly max_tokens (if it reached the max_tokens limit)
|
||||
self.assertTrue(
|
||||
ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens,
|
||||
f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
response_ignore_eos.choices[0].finish_reason,
|
||||
"length",
|
||||
f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import unittest
|
||||
|
||||
unittest.main()
|
||||
@@ -17,8 +17,8 @@ from sglang.test.test_utils import (
|
||||
)
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=353, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=600, suite="stage-b-test-1-gpu-small-amd")
|
||||
register_cuda_ci(est_time=215, stage="base-b", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestRetractDecode(CustomTestCase):
|
||||
@@ -62,18 +62,6 @@ class TestRetractDecode(CustomTestCase):
|
||||
assert self.process.poll() is None, "Server crashed during test"
|
||||
|
||||
|
||||
class TestRetractDecodePaged(TestRetractDecode):
|
||||
"""python -m unittest test_retract_decode.TestRetractDecodePaged"""
|
||||
|
||||
other_args = ["--page-size", "16"]
|
||||
|
||||
|
||||
class TestRetractDecodeChunkCache(TestRetractDecode):
|
||||
"""python -m unittest test_retract_decode.TestRetractDecodeChunkCache"""
|
||||
|
||||
other_args = ["--disable-radix-cache"]
|
||||
|
||||
|
||||
class TestRetractDecodeChunkCachePaged(TestRetractDecode):
|
||||
"""python -m unittest test_retract_decode.TestRetractDecodeChunkCachePaged"""
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.test.kits.spec_server_kits import (
|
||||
)
|
||||
from sglang.test.server_fixtures.spec_eagle_fixture import Eagle3Base, EagleLlama2Base
|
||||
|
||||
register_cuda_ci(est_time=480, stage="base-b", runner_config="1-gpu-small")
|
||||
register_cuda_ci(est_time=350, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestEagle3Triton(
|
||||
@@ -33,7 +33,7 @@ class TestEagle3Triton(
|
||||
attention_backend = "triton"
|
||||
max_running_requests = 64
|
||||
cuda_graph_max_bs_decode = 64
|
||||
gsm8k_num_examples = 1000
|
||||
gsm8k_num_examples = 200
|
||||
gsm8k_check_accept_len = False
|
||||
env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),)
|
||||
|
||||
|
||||
@@ -154,7 +154,6 @@ class TestSkipTokenizerInit(CustomTestCase):
|
||||
|
||||
response_stream_json = []
|
||||
for line in response_stream.iter_lines():
|
||||
print(line)
|
||||
if line.startswith(b"data: ") and line[6:] != b"[DONE]":
|
||||
response_stream_json.append(json.loads(line[6:]))
|
||||
out_stream_ids = []
|
||||
|
||||
Reference in New Issue
Block a user