minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Fold duplicate-server suites and prune the retract matrix on 1-gpu-5090 (#33745)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-05 12:41:51 -07:00
committed by GitHub
parent 36853b8ffc
commit de34dd11e9
11 changed files with 215 additions and 418 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/openai_server/basic/test_anthropic_server.py → python/sglang/test/kits/anthropic_messages_kit.py
+16 -52
View File
@@ -1,61 +1,29 @@
"""
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_simple_messages
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_simple_messages_stream
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_multi_turn_messages
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_system_message_string
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_system_message_blocks
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_max_tokens
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_temperature
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_stop_sequences
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_invalid_max_tokens
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_empty_messages
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_non_streaming
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_streaming
python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_tool_result_image_content_conversion
"""Anthropic /v1/messages API test mixin.
Host class must provide ``self.base_url`` (with or without a trailing /v1),
``self.api_key``, and ``self.model``.
"""
import json
import unittest
import anthropic
import requests
from sglang.srt.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from sglang.srt.entrypoints.anthropic.serving import AnthropicServing
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=40, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=140, suite="stage-b-test-1-gpu-small-amd")
class TestAnthropicServer(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
)
cls.messages_url = cls.base_url + "/v1/messages"
class AnthropicMessagesMixin:
@property
def anthropic_base_url(self):
base = self.base_url
return base[: -len("/v1")] if base.endswith("/v1") else base
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
@property
def messages_url(self):
return self.anthropic_base_url + "/v1/messages"
def _make_request(self, payload, stream=False):
"""Send a request to the /v1/messages endpoint."""
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}",
@@ -233,7 +201,7 @@ class TestAnthropicServer(CustomTestCase):
clients, e.g. Claude Code) must be accepted — not rejected with 400.
Uses the Anthropic SDK the way a real client would."""
client = anthropic.Anthropic(
base_url=self.base_url,
base_url=self.anthropic_base_url,
auth_token=self.api_key, # Bearer header — SGLang's --api-key checks Authorization
)
message = client.messages.create(
@@ -503,7 +471,7 @@ class TestAnthropicServer(CustomTestCase):
],
}
resp = requests.post(
self.base_url + "/v1/messages/count_tokens",
self.anthropic_base_url + "/v1/messages/count_tokens",
headers=headers,
json=payload,
)
@@ -534,12 +502,12 @@ class TestAnthropicServer(CustomTestCase):
"system": "You are a helpful assistant with a very long system prompt that adds tokens.",
}
resp1 = requests.post(
self.base_url + "/v1/messages/count_tokens",
self.anthropic_base_url + "/v1/messages/count_tokens",
headers=headers,
json=payload_no_system,
)
resp2 = requests.post(
self.base_url + "/v1/messages/count_tokens",
self.anthropic_base_url + "/v1/messages/count_tokens",
headers=headers,
json=payload_with_system,
)
@@ -576,7 +544,3 @@ class TestAnthropicServer(CustomTestCase):
pass
return events
if __name__ == "__main__":
unittest.main()
python/sglang/test/kits/json_mode_kit.py
+75
View File
@@ -0,0 +1,75 @@
"""JSON mode (response_format json_object) test mixin.
Host class must provide ``self.client`` (openai.Client) and ``self.model``.
"""
import json
class JSONModeMixin:
def test_json_mode_response(self):
"""json_object without a JSON-mentioning system prompt must still
produce valid JSON."""
response = self.client.chat.completions.create(
model=self.model,
messages=[
# No JSON hint in the prompt on purpose -- the format must be
# enforced by response_format, not by the instruction.
{
"role": "system",
"content": "You are a helpful AI assistant that gives a short answer.",
},
{"role": "user", "content": "What is the capital of Bulgaria?"},
],
temperature=0,
max_tokens=128,
response_format={"type": "json_object"},
)
text = response.choices[0].message.content
print(f"Response ({len(text)} characters): {text}")
try:
js_obj = json.loads(text)
except json.JSONDecodeError as e:
self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}")
self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}")
def test_json_mode_with_streaming(self):
"""Same contract over a stream: the concatenated chunks must parse."""
stream = self.client.chat.completions.create(
model=self.model,
messages=[
# No JSON hint in the prompt on purpose -- the format must be
# enforced by response_format, not by the instruction.
{
"role": "system",
"content": "You are a helpful AI assistant that gives a short answer.",
},
{"role": "user", "content": "What is the capital of Bulgaria?"},
],
temperature=0,
max_tokens=128,
response_format={"type": "json_object"},
stream=True,
)
chunks = []
for chunk in stream:
if chunk.choices[0].delta.content is not None:
chunks.append(chunk.choices[0].delta.content)
full_response = "".join(chunks)
print(
f"Concatenated Response ({len(full_response)} characters): {full_response}"
)
try:
js_obj = json.loads(full_response)
except json.JSONDecodeError as e:
self.fail(
f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}"
)
self.assertIsInstance(js_obj, dict)
test/registered/constrained_decoding/test_constrained_decoding.py
+13 -3
View File
@@ -1,20 +1,24 @@
import unittest
import openai
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
from sglang.test.kits.json_mode_kit import JSONModeMixin
from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
popen_launch_server,
)
register_cuda_ci(est_time=120, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=179, suite="stage-b-test-1-gpu-small-amd")
register_cuda_ci(est_time=135, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=220, suite="stage-b-test-1-gpu-small-amd")
class ServerWithGrammar(CustomTestCase):
@@ -35,12 +39,16 @@ class ServerWithGrammar(CustomTestCase):
if cls.disable_overlap:
launch_args += ["--disable-overlap-schedule"]
if is_in_amd_ci():
launch_args.append("--constrained-json-disable-any-whitespace")
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=launch_args,
)
cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1")
@classmethod
def tearDownClass(cls):
@@ -50,19 +58,21 @@ class ServerWithGrammar(CustomTestCase):
class TestXGrammarBackend(
ServerWithGrammar,
JSONConstrainedMixin,
JSONModeMixin,
EBNFConstrainedMixin,
RegexConstrainedMixin,
):
backend = "xgrammar"
class TestOutlinesBackend(ServerWithGrammar, JSONConstrainedMixin):
class TestOutlinesBackend(ServerWithGrammar, JSONConstrainedMixin, JSONModeMixin):
backend = "outlines"
class TestLLGuidanceBackend(
ServerWithGrammar,
JSONConstrainedMixin,
JSONModeMixin,
EBNFConstrainedMixin,
RegexConstrainedMixin,
):
test/registered/model_loading/test_weight_cache_daemon.py
+1 -1
View File
@@ -27,7 +27,7 @@ DEFAULT_MODEL = "Qwen/Qwen3-0.6B"
# file per suite, TestWeightCacheDaemonTP2 self-skips when fewer than 2 GPUs are
# visible (i.e. on the 1-gpu runner).
register_cuda_ci(est_time=100, stage="extra-a", runner_config="2-gpu-large")
register_cuda_ci(est_time=100, stage="base-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=45, stage="base-b", runner_config="1-gpu-small")
# Capture the client server's logs so test_loaded_via_ipc can assert the IPC
# load path actually ran (and did not silently fall back to disk).
test/registered/openai_server/basic/test_openai_server.py
+106 -4
View File
@@ -7,6 +7,7 @@ python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test
import json
import random
import re
import unittest
from concurrent.futures import ThreadPoolExecutor
from typing import Optional
@@ -18,6 +19,7 @@ from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kits.anthropic_messages_kit import AnthropicMessagesMixin
from sglang.test.runners import TEST_RERANK_QUERY_DOCS
from sglang.test.test_utils import (
DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST,
@@ -29,11 +31,11 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=240, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=200, suite="stage-b-test-1-gpu-small-amd")
register_cuda_ci(est_time=300, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=280, suite="stage-b-test-1-gpu-small-amd")
class TestOpenAIServer(CustomTestCase):
class TestOpenAIServer(CustomTestCase, AnthropicMessagesMixin):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
@@ -52,6 +54,107 @@ class TestOpenAIServer(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_ignore_eos(self):
"""ignore_eos=True must keep generating past EOS up to max_tokens."""
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
max_tokens = 200
response_default = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Count from 1 to 20."},
],
temperature=0,
max_tokens=max_tokens,
extra_body={"ignore_eos": False},
)
response_ignore_eos = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Count from 1 to 20."},
],
temperature=0,
max_tokens=max_tokens,
extra_body={"ignore_eos": True},
)
default_tokens = len(
self.tokenizer.encode(response_default.choices[0].message.content)
)
ignore_eos_tokens = len(
self.tokenizer.encode(response_ignore_eos.choices[0].message.content)
)
# Check if ignore_eos resulted in more tokens or exactly max_tokens
# The ignore_eos response should either:
# 1. Have more tokens than the default response (if default stopped at EOS before max_tokens)
# 2. Have exactly max_tokens (if it reached the max_tokens limit)
self.assertTrue(
ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens,
f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}",
)
self.assertEqual(
response_ignore_eos.choices[0].finish_reason,
"length",
f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}",
)
def test_ebnf(self):
"""`ebnf` in extra_body must be enforced by the grammar backend."""
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
ebnf_grammar = r"""
root ::= "Hello" | "Hi" | "Hey"
"""
pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$")
response = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful EBNF test bot."},
{"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."},
],
temperature=0,
max_tokens=32,
extra_body={"ebnf": ebnf_grammar},
)
text = response.choices[0].message.content.strip()
self.assertTrue(len(text) > 0, "Got empty text from EBNF generation")
self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices")
def test_ebnf_strict_json(self):
"""Stricter EBNF: exact {"name":"Alice"} shape, no extra fields."""
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
ebnf_grammar = r"""
root ::= "{" pair "}"
pair ::= "\"name\"" ":" string
string ::= "\"" [A-Za-z]+ "\""
"""
pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$')
response = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "EBNF mini-JSON generator."},
{
"role": "user",
"content": "Generate single key JSON with only letters.",
},
],
temperature=0,
max_tokens=64,
extra_body={"ebnf": ebnf_grammar},
)
text = response.choices[0].message.content.strip()
self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test")
self.assertRegex(
text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape"
)
def run_completion(
self, echo, logprobs, use_list_input, parallel_sample_num, token_input
):
@@ -149,7 +252,6 @@ class TestOpenAIServer(CustomTestCase):
is_firsts = {}
for response in generator:
print(f"{response=}")
usage = response.usage
if usage is not None:
assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero"
test/registered/openai_server/features/test_json_mode.py
-139
View File
@@ -1,139 +0,0 @@
import json
import unittest
import openai
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
popen_launch_server,
)
register_cuda_ci(est_time=118, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=180, suite="stage-b-test-1-gpu-small-amd")
class JSONModeMixin:
"""Mixin class containing JSON mode test methods"""
def test_json_mode_response(self):
"""Test that response_format json_object (also known as "json mode") produces valid JSON, even without a system prompt that mentions JSON."""
response = self.client.chat.completions.create(
model=self.model,
messages=[
# We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results
{
"role": "system",
"content": "You are a helpful AI assistant that gives a short answer.",
},
{"role": "user", "content": "What is the capital of Bulgaria?"},
],
temperature=0,
max_tokens=128,
response_format={"type": "json_object"},
)
text = response.choices[0].message.content
print(f"Response ({len(text)} characters): {text}")
# Verify the response is valid JSON
try:
js_obj = json.loads(text)
except json.JSONDecodeError as e:
self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}")
# Verify it's actually an object (dict)
self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}")
def test_json_mode_with_streaming(self):
"""Test that streaming with json_object response (also known as "json mode") format works correctly, even without a system prompt that mentions JSON."""
stream = self.client.chat.completions.create(
model=self.model,
messages=[
# We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results
{
"role": "system",
"content": "You are a helpful AI assistant that gives a short answer.",
},
{"role": "user", "content": "What is the capital of Bulgaria?"},
],
temperature=0,
max_tokens=128,
response_format={"type": "json_object"},
stream=True,
)
# Collect all chunks
chunks = []
for chunk in stream:
if chunk.choices[0].delta.content is not None:
chunks.append(chunk.choices[0].delta.content)
full_response = "".join(chunks)
print(
f"Concatenated Response ({len(full_response)} characters): {full_response}"
)
# Verify the combined response is valid JSON
try:
js_obj = json.loads(full_response)
except json.JSONDecodeError as e:
self.fail(
f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}"
)
self.assertIsInstance(js_obj, dict)
class ServerWithGrammarBackend(CustomTestCase):
"""Base class for tests requiring a grammar backend server"""
backend = "xgrammar"
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--max-running-requests",
"10",
"--grammar-backend",
cls.backend,
]
if is_in_amd_ci():
other_args.append("--constrained-json-disable-any-whitespace")
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1")
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestJSONModeXGrammar(ServerWithGrammarBackend, JSONModeMixin):
backend = "xgrammar"
class TestJSONModeOutlines(ServerWithGrammarBackend, JSONModeMixin):
backend = "outlines"
class TestJSONModeLLGuidance(ServerWithGrammarBackend, JSONModeMixin):
backend = "llguidance"
if __name__ == "__main__":
unittest.main()
test/registered/openai_server/features/test_openai_server_ebnf.py
-108
View File
@@ -1,108 +0,0 @@
import re
import openai
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=44, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=20, suite="stage-b-test-1-gpu-small-amd")
# -------------------------------------------------------------------------
# EBNF Test Class: TestOpenAIServerEBNF
# Launches the server with xgrammar, has only EBNF tests
# -------------------------------------------------------------------------
class TestOpenAIServerEBNF(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
# passing xgrammar specifically
other_args = ["--grammar-backend", "xgrammar"]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
other_args=other_args,
)
cls.base_url += "/v1"
cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_ebnf(self):
"""
Ensure we can pass `ebnf` to the local openai server
and that it enforces the grammar.
"""
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
ebnf_grammar = r"""
root ::= "Hello" | "Hi" | "Hey"
"""
pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$")
response = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful EBNF test bot."},
{"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."},
],
temperature=0,
max_tokens=32,
extra_body={"ebnf": ebnf_grammar},
)
text = response.choices[0].message.content.strip()
self.assertTrue(len(text) > 0, "Got empty text from EBNF generation")
self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices")
def test_ebnf_strict_json(self):
"""
A stricter EBNF that produces exactly {"name":"Alice"} format
with no trailing punctuation or extra fields.
"""
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
ebnf_grammar = r"""
root ::= "{" pair "}"
pair ::= "\"name\"" ":" string
string ::= "\"" [A-Za-z]+ "\""
"""
pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$')
response = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "EBNF mini-JSON generator."},
{
"role": "user",
"content": "Generate single key JSON with only letters.",
},
],
temperature=0,
max_tokens=64,
extra_body={"ebnf": ebnf_grammar},
)
text = response.choices[0].message.content.strip()
self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test")
self.assertRegex(
text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape"
)
if __name__ == "__main__":
import unittest
unittest.main()
test/registered/openai_server/validation/test_openai_server_ignore_eos.py
-94
View File
@@ -1,94 +0,0 @@
import openai
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=44, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=47, suite="stage-b-test-1-gpu-small-amd")
class TestOpenAIServerIgnoreEOS(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.api_key = "sk-123456"
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
api_key=cls.api_key,
)
cls.base_url += "/v1"
cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_ignore_eos(self):
"""
Test that ignore_eos=True allows generation to continue beyond EOS token
and reach the max_tokens limit.
"""
client = openai.Client(api_key=self.api_key, base_url=self.base_url)
max_tokens = 200
response_default = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Count from 1 to 20."},
],
temperature=0,
max_tokens=max_tokens,
extra_body={"ignore_eos": False},
)
response_ignore_eos = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Count from 1 to 20."},
],
temperature=0,
max_tokens=max_tokens,
extra_body={"ignore_eos": True},
)
default_tokens = len(
self.tokenizer.encode(response_default.choices[0].message.content)
)
ignore_eos_tokens = len(
self.tokenizer.encode(response_ignore_eos.choices[0].message.content)
)
# Check if ignore_eos resulted in more tokens or exactly max_tokens
# The ignore_eos response should either:
# 1. Have more tokens than the default response (if default stopped at EOS before max_tokens)
# 2. Have exactly max_tokens (if it reached the max_tokens limit)
self.assertTrue(
ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens,
f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}",
)
self.assertEqual(
response_ignore_eos.choices[0].finish_reason,
"length",
f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}",
)
if __name__ == "__main__":
import unittest
unittest.main()
test/registered/scheduler/test_retract_decode.py
+2 -14
View File
@@ -17,8 +17,8 @@ from sglang.test.test_utils import (
)
from sglang.utils import is_in_ci
register_cuda_ci(est_time=353, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=600, suite="stage-b-test-1-gpu-small-amd")
register_cuda_ci(est_time=215, stage="base-b", runner_config="1-gpu-small")
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
class TestRetractDecode(CustomTestCase):
@@ -62,18 +62,6 @@ class TestRetractDecode(CustomTestCase):
assert self.process.poll() is None, "Server crashed during test"
class TestRetractDecodePaged(TestRetractDecode):
"""python -m unittest test_retract_decode.TestRetractDecodePaged"""
other_args = ["--page-size", "16"]
class TestRetractDecodeChunkCache(TestRetractDecode):
"""python -m unittest test_retract_decode.TestRetractDecodeChunkCache"""
other_args = ["--disable-radix-cache"]
class TestRetractDecodeChunkCachePaged(TestRetractDecode):
"""python -m unittest test_retract_decode.TestRetractDecodeChunkCachePaged"""
test/registered/spec/eagle/test_spec_eagle_triton.py
+2 -2
View File
@@ -17,7 +17,7 @@ from sglang.test.kits.spec_server_kits import (
)
from sglang.test.server_fixtures.spec_eagle_fixture import Eagle3Base, EagleLlama2Base
register_cuda_ci(est_time=480, stage="base-b", runner_config="1-gpu-small")
register_cuda_ci(est_time=350, stage="base-b", runner_config="1-gpu-small")
class TestEagle3Triton(
@@ -33,7 +33,7 @@ class TestEagle3Triton(
attention_backend = "triton"
max_running_requests = 64
cuda_graph_max_bs_decode = 64
gsm8k_num_examples = 1000
gsm8k_num_examples = 200
gsm8k_check_accept_len = False
env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),)
test/registered/tokenizer/test_skip_tokenizer_init.py
-1
View File
@@ -154,7 +154,6 @@ class TestSkipTokenizerInit(CustomTestCase):
response_stream_json = []
for line in response_stream.iter_lines():
print(line)
if line.startswith(b"data: ") and line[6:] != b"[DONE]":
response_stream_json.append(json.loads(line[6:]))
out_stream_ids = []