From de34dd11e9793379d1229813595be77b8d488608 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 5 Aug 2026 12:41:51 -0700 Subject: [PATCH] [CI] Fold duplicate-server suites and prune the retract matrix on 1-gpu-5090 (#33745) --- .../test/kits/anthropic_messages_kit.py | 68 ++------- python/sglang/test/kits/json_mode_kit.py | 75 ++++++++++ .../test_constrained_decoding.py | 16 +- .../model_loading/test_weight_cache_daemon.py | 2 +- .../openai_server/basic/test_openai_server.py | 110 +++++++++++++- .../openai_server/features/test_json_mode.py | 139 ------------------ .../features/test_openai_server_ebnf.py | 108 -------------- .../test_openai_server_ignore_eos.py | 94 ------------ .../scheduler/test_retract_decode.py | 16 +- .../spec/eagle/test_spec_eagle_triton.py | 4 +- .../tokenizer/test_skip_tokenizer_init.py | 1 - 11 files changed, 215 insertions(+), 418 deletions(-) rename test/registered/openai_server/basic/test_anthropic_server.py => python/sglang/test/kits/anthropic_messages_kit.py (87%) create mode 100644 python/sglang/test/kits/json_mode_kit.py delete mode 100644 test/registered/openai_server/features/test_json_mode.py delete mode 100644 test/registered/openai_server/features/test_openai_server_ebnf.py delete mode 100644 test/registered/openai_server/validation/test_openai_server_ignore_eos.py diff --git a/test/registered/openai_server/basic/test_anthropic_server.py b/python/sglang/test/kits/anthropic_messages_kit.py similarity index 87% rename from test/registered/openai_server/basic/test_anthropic_server.py rename to python/sglang/test/kits/anthropic_messages_kit.py index 2aad292f2..51feebdd7 100644 --- a/test/registered/openai_server/basic/test_anthropic_server.py +++ b/python/sglang/test/kits/anthropic_messages_kit.py @@ -1,61 +1,29 @@ -""" -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_simple_messages -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_simple_messages_stream -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_multi_turn_messages -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_system_message_string -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_system_message_blocks -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_max_tokens -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_temperature -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_stop_sequences -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_invalid_max_tokens -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_error_empty_messages -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_non_streaming -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_raw_http_streaming -python3 -m unittest openai_server.basic.test_anthropic_server.TestAnthropicServer.test_tool_result_image_content_conversion +"""Anthropic /v1/messages API test mixin. + +Host class must provide ``self.base_url`` (with or without a trailing /v1), +``self.api_key``, and ``self.model``. """ import json -import unittest import anthropic import requests from sglang.srt.entrypoints.anthropic.protocol import AnthropicMessagesRequest from sglang.srt.entrypoints.anthropic.serving import AnthropicServing -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_cuda_ci(est_time=40, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=140, suite="stage-b-test-1-gpu-small-amd") -class TestAnthropicServer(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.api_key = "sk-123456" - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - api_key=cls.api_key, - ) - cls.messages_url = cls.base_url + "/v1/messages" +class AnthropicMessagesMixin: + @property + def anthropic_base_url(self): + base = self.base_url + return base[: -len("/v1")] if base.endswith("/v1") else base - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + @property + def messages_url(self): + return self.anthropic_base_url + "/v1/messages" def _make_request(self, payload, stream=False): - """Send a request to the /v1/messages endpoint.""" headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}", @@ -233,7 +201,7 @@ class TestAnthropicServer(CustomTestCase): clients, e.g. Claude Code) must be accepted — not rejected with 400. Uses the Anthropic SDK the way a real client would.""" client = anthropic.Anthropic( - base_url=self.base_url, + base_url=self.anthropic_base_url, auth_token=self.api_key, # Bearer header — SGLang's --api-key checks Authorization ) message = client.messages.create( @@ -503,7 +471,7 @@ class TestAnthropicServer(CustomTestCase): ], } resp = requests.post( - self.base_url + "/v1/messages/count_tokens", + self.anthropic_base_url + "/v1/messages/count_tokens", headers=headers, json=payload, ) @@ -534,12 +502,12 @@ class TestAnthropicServer(CustomTestCase): "system": "You are a helpful assistant with a very long system prompt that adds tokens.", } resp1 = requests.post( - self.base_url + "/v1/messages/count_tokens", + self.anthropic_base_url + "/v1/messages/count_tokens", headers=headers, json=payload_no_system, ) resp2 = requests.post( - self.base_url + "/v1/messages/count_tokens", + self.anthropic_base_url + "/v1/messages/count_tokens", headers=headers, json=payload_with_system, ) @@ -576,7 +544,3 @@ class TestAnthropicServer(CustomTestCase): pass return events - - -if __name__ == "__main__": - unittest.main() diff --git a/python/sglang/test/kits/json_mode_kit.py b/python/sglang/test/kits/json_mode_kit.py new file mode 100644 index 000000000..351029127 --- /dev/null +++ b/python/sglang/test/kits/json_mode_kit.py @@ -0,0 +1,75 @@ +"""JSON mode (response_format json_object) test mixin. + +Host class must provide ``self.client`` (openai.Client) and ``self.model``. +""" + +import json + + +class JSONModeMixin: + def test_json_mode_response(self): + """json_object without a JSON-mentioning system prompt must still + produce valid JSON.""" + response = self.client.chat.completions.create( + model=self.model, + messages=[ + # No JSON hint in the prompt on purpose -- the format must be + # enforced by response_format, not by the instruction. + { + "role": "system", + "content": "You are a helpful AI assistant that gives a short answer.", + }, + {"role": "user", "content": "What is the capital of Bulgaria?"}, + ], + temperature=0, + max_tokens=128, + response_format={"type": "json_object"}, + ) + text = response.choices[0].message.content + + print(f"Response ({len(text)} characters): {text}") + + try: + js_obj = json.loads(text) + except json.JSONDecodeError as e: + self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}") + + self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}") + + def test_json_mode_with_streaming(self): + """Same contract over a stream: the concatenated chunks must parse.""" + stream = self.client.chat.completions.create( + model=self.model, + messages=[ + # No JSON hint in the prompt on purpose -- the format must be + # enforced by response_format, not by the instruction. + { + "role": "system", + "content": "You are a helpful AI assistant that gives a short answer.", + }, + {"role": "user", "content": "What is the capital of Bulgaria?"}, + ], + temperature=0, + max_tokens=128, + response_format={"type": "json_object"}, + stream=True, + ) + + chunks = [] + for chunk in stream: + if chunk.choices[0].delta.content is not None: + chunks.append(chunk.choices[0].delta.content) + full_response = "".join(chunks) + + print( + f"Concatenated Response ({len(full_response)} characters): {full_response}" + ) + + try: + js_obj = json.loads(full_response) + except json.JSONDecodeError as e: + self.fail( + f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}" + ) + + self.assertIsInstance(js_obj, dict) diff --git a/test/registered/constrained_decoding/test_constrained_decoding.py b/test/registered/constrained_decoding/test_constrained_decoding.py index 976445b10..8d967fb14 100644 --- a/test/registered/constrained_decoding/test_constrained_decoding.py +++ b/test/registered/constrained_decoding/test_constrained_decoding.py @@ -1,20 +1,24 @@ import unittest +import openai + from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin +from sglang.test.kits.json_mode_kit import JSONModeMixin from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin from sglang.test.test_utils import ( DEFAULT_SMALL_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, + is_in_amd_ci, popen_launch_server, ) -register_cuda_ci(est_time=120, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=179, suite="stage-b-test-1-gpu-small-amd") +register_cuda_ci(est_time=135, stage="base-b", runner_config="1-gpu-small") +register_amd_ci(est_time=220, suite="stage-b-test-1-gpu-small-amd") class ServerWithGrammar(CustomTestCase): @@ -35,12 +39,16 @@ class ServerWithGrammar(CustomTestCase): if cls.disable_overlap: launch_args += ["--disable-overlap-schedule"] + if is_in_amd_ci(): + launch_args.append("--constrained-json-disable-any-whitespace") + cls.process = popen_launch_server( cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=launch_args, ) + cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") @classmethod def tearDownClass(cls): @@ -50,19 +58,21 @@ class ServerWithGrammar(CustomTestCase): class TestXGrammarBackend( ServerWithGrammar, JSONConstrainedMixin, + JSONModeMixin, EBNFConstrainedMixin, RegexConstrainedMixin, ): backend = "xgrammar" -class TestOutlinesBackend(ServerWithGrammar, JSONConstrainedMixin): +class TestOutlinesBackend(ServerWithGrammar, JSONConstrainedMixin, JSONModeMixin): backend = "outlines" class TestLLGuidanceBackend( ServerWithGrammar, JSONConstrainedMixin, + JSONModeMixin, EBNFConstrainedMixin, RegexConstrainedMixin, ): diff --git a/test/registered/model_loading/test_weight_cache_daemon.py b/test/registered/model_loading/test_weight_cache_daemon.py index a49dfc446..00669caab 100644 --- a/test/registered/model_loading/test_weight_cache_daemon.py +++ b/test/registered/model_loading/test_weight_cache_daemon.py @@ -27,7 +27,7 @@ DEFAULT_MODEL = "Qwen/Qwen3-0.6B" # file per suite, TestWeightCacheDaemonTP2 self-skips when fewer than 2 GPUs are # visible (i.e. on the 1-gpu runner). register_cuda_ci(est_time=100, stage="extra-a", runner_config="2-gpu-large") -register_cuda_ci(est_time=100, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=45, stage="base-b", runner_config="1-gpu-small") # Capture the client server's logs so test_loaded_via_ipc can assert the IPC # load path actually ran (and did not silently fall back to disk). diff --git a/test/registered/openai_server/basic/test_openai_server.py b/test/registered/openai_server/basic/test_openai_server.py index d84bd1a0f..d4042ce28 100644 --- a/test/registered/openai_server/basic/test_openai_server.py +++ b/test/registered/openai_server/basic/test_openai_server.py @@ -7,6 +7,7 @@ python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test import json import random +import re import unittest from concurrent.futures import ThreadPoolExecutor from typing import Optional @@ -18,6 +19,7 @@ from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor from sglang.srt.utils import kill_process_tree from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.anthropic_messages_kit import AnthropicMessagesMixin from sglang.test.runners import TEST_RERANK_QUERY_DOCS from sglang.test.test_utils import ( DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST, @@ -29,11 +31,11 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=240, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=200, suite="stage-b-test-1-gpu-small-amd") +register_cuda_ci(est_time=300, stage="base-b", runner_config="1-gpu-small") +register_amd_ci(est_time=280, suite="stage-b-test-1-gpu-small-amd") -class TestOpenAIServer(CustomTestCase): +class TestOpenAIServer(CustomTestCase, AnthropicMessagesMixin): @classmethod def setUpClass(cls): cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST @@ -52,6 +54,107 @@ class TestOpenAIServer(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) + def test_ignore_eos(self): + """ignore_eos=True must keep generating past EOS up to max_tokens.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + max_tokens = 200 + + response_default = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": False}, + ) + + response_ignore_eos = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": True}, + ) + + default_tokens = len( + self.tokenizer.encode(response_default.choices[0].message.content) + ) + ignore_eos_tokens = len( + self.tokenizer.encode(response_ignore_eos.choices[0].message.content) + ) + + # Check if ignore_eos resulted in more tokens or exactly max_tokens + # The ignore_eos response should either: + # 1. Have more tokens than the default response (if default stopped at EOS before max_tokens) + # 2. Have exactly max_tokens (if it reached the max_tokens limit) + self.assertTrue( + ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens, + f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}", + ) + + self.assertEqual( + response_ignore_eos.choices[0].finish_reason, + "length", + f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}", + ) + + def test_ebnf(self): + """`ebnf` in extra_body must be enforced by the grammar backend.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "Hello" | "Hi" | "Hey" + """ + pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful EBNF test bot."}, + {"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."}, + ], + temperature=0, + max_tokens=32, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + self.assertTrue(len(text) > 0, "Got empty text from EBNF generation") + self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices") + + def test_ebnf_strict_json(self): + """Stricter EBNF: exact {"name":"Alice"} shape, no extra fields.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "{" pair "}" + pair ::= "\"name\"" ":" string + string ::= "\"" [A-Za-z]+ "\"" + """ + pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$') + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "EBNF mini-JSON generator."}, + { + "role": "user", + "content": "Generate single key JSON with only letters.", + }, + ], + temperature=0, + max_tokens=64, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test") + self.assertRegex( + text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape" + ) + def run_completion( self, echo, logprobs, use_list_input, parallel_sample_num, token_input ): @@ -149,7 +252,6 @@ class TestOpenAIServer(CustomTestCase): is_firsts = {} for response in generator: - print(f"{response=}") usage = response.usage if usage is not None: assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero" diff --git a/test/registered/openai_server/features/test_json_mode.py b/test/registered/openai_server/features/test_json_mode.py deleted file mode 100644 index 5cbb68a3d..000000000 --- a/test/registered/openai_server/features/test_json_mode.py +++ /dev/null @@ -1,139 +0,0 @@ -import json -import unittest - -import openai - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_amd_ci, - popen_launch_server, -) - -register_cuda_ci(est_time=118, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=180, suite="stage-b-test-1-gpu-small-amd") - - -class JSONModeMixin: - """Mixin class containing JSON mode test methods""" - - def test_json_mode_response(self): - """Test that response_format json_object (also known as "json mode") produces valid JSON, even without a system prompt that mentions JSON.""" - response = self.client.chat.completions.create( - model=self.model, - messages=[ - # We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results - { - "role": "system", - "content": "You are a helpful AI assistant that gives a short answer.", - }, - {"role": "user", "content": "What is the capital of Bulgaria?"}, - ], - temperature=0, - max_tokens=128, - response_format={"type": "json_object"}, - ) - text = response.choices[0].message.content - - print(f"Response ({len(text)} characters): {text}") - - # Verify the response is valid JSON - try: - js_obj = json.loads(text) - except json.JSONDecodeError as e: - self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}") - - # Verify it's actually an object (dict) - self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}") - - def test_json_mode_with_streaming(self): - """Test that streaming with json_object response (also known as "json mode") format works correctly, even without a system prompt that mentions JSON.""" - stream = self.client.chat.completions.create( - model=self.model, - messages=[ - # We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results - { - "role": "system", - "content": "You are a helpful AI assistant that gives a short answer.", - }, - {"role": "user", "content": "What is the capital of Bulgaria?"}, - ], - temperature=0, - max_tokens=128, - response_format={"type": "json_object"}, - stream=True, - ) - - # Collect all chunks - chunks = [] - for chunk in stream: - if chunk.choices[0].delta.content is not None: - chunks.append(chunk.choices[0].delta.content) - full_response = "".join(chunks) - - print( - f"Concatenated Response ({len(full_response)} characters): {full_response}" - ) - - # Verify the combined response is valid JSON - try: - js_obj = json.loads(full_response) - except json.JSONDecodeError as e: - self.fail( - f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}" - ) - - self.assertIsInstance(js_obj, dict) - - -class ServerWithGrammarBackend(CustomTestCase): - """Base class for tests requiring a grammar backend server""" - - backend = "xgrammar" - - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - - other_args = [ - "--max-running-requests", - "10", - "--grammar-backend", - cls.backend, - ] - - if is_in_amd_ci(): - other_args.append("--constrained-json-disable-any-whitespace") - - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestJSONModeXGrammar(ServerWithGrammarBackend, JSONModeMixin): - backend = "xgrammar" - - -class TestJSONModeOutlines(ServerWithGrammarBackend, JSONModeMixin): - backend = "outlines" - - -class TestJSONModeLLGuidance(ServerWithGrammarBackend, JSONModeMixin): - backend = "llguidance" - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/openai_server/features/test_openai_server_ebnf.py b/test/registered/openai_server/features/test_openai_server_ebnf.py deleted file mode 100644 index c81a1492a..000000000 --- a/test/registered/openai_server/features/test_openai_server_ebnf.py +++ /dev/null @@ -1,108 +0,0 @@ -import re - -import openai - -from sglang.srt.utils import kill_process_tree -from sglang.srt.utils.hf_transformers_utils import get_tokenizer -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_cuda_ci(est_time=44, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=20, suite="stage-b-test-1-gpu-small-amd") - - -# ------------------------------------------------------------------------- -# EBNF Test Class: TestOpenAIServerEBNF -# Launches the server with xgrammar, has only EBNF tests -# ------------------------------------------------------------------------- -class TestOpenAIServerEBNF(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.api_key = "sk-123456" - - # passing xgrammar specifically - other_args = ["--grammar-backend", "xgrammar"] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - api_key=cls.api_key, - other_args=other_args, - ) - cls.base_url += "/v1" - cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_ebnf(self): - """ - Ensure we can pass `ebnf` to the local openai server - and that it enforces the grammar. - """ - client = openai.Client(api_key=self.api_key, base_url=self.base_url) - ebnf_grammar = r""" - root ::= "Hello" | "Hi" | "Hey" - """ - pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$") - - response = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": "You are a helpful EBNF test bot."}, - {"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."}, - ], - temperature=0, - max_tokens=32, - extra_body={"ebnf": ebnf_grammar}, - ) - text = response.choices[0].message.content.strip() - self.assertTrue(len(text) > 0, "Got empty text from EBNF generation") - self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices") - - def test_ebnf_strict_json(self): - """ - A stricter EBNF that produces exactly {"name":"Alice"} format - with no trailing punctuation or extra fields. - """ - client = openai.Client(api_key=self.api_key, base_url=self.base_url) - ebnf_grammar = r""" - root ::= "{" pair "}" - pair ::= "\"name\"" ":" string - string ::= "\"" [A-Za-z]+ "\"" - """ - pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$') - - response = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": "EBNF mini-JSON generator."}, - { - "role": "user", - "content": "Generate single key JSON with only letters.", - }, - ], - temperature=0, - max_tokens=64, - extra_body={"ebnf": ebnf_grammar}, - ) - text = response.choices[0].message.content.strip() - self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test") - self.assertRegex( - text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape" - ) - - -if __name__ == "__main__": - import unittest - - unittest.main() diff --git a/test/registered/openai_server/validation/test_openai_server_ignore_eos.py b/test/registered/openai_server/validation/test_openai_server_ignore_eos.py deleted file mode 100644 index ddd8ffcec..000000000 --- a/test/registered/openai_server/validation/test_openai_server_ignore_eos.py +++ /dev/null @@ -1,94 +0,0 @@ -import openai - -from sglang.srt.utils import kill_process_tree -from sglang.srt.utils.hf_transformers_utils import get_tokenizer -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_cuda_ci(est_time=44, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=47, suite="stage-b-test-1-gpu-small-amd") - - -class TestOpenAIServerIgnoreEOS(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST - cls.base_url = DEFAULT_URL_FOR_TEST - cls.api_key = "sk-123456" - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - api_key=cls.api_key, - ) - cls.base_url += "/v1" - cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_ignore_eos(self): - """ - Test that ignore_eos=True allows generation to continue beyond EOS token - and reach the max_tokens limit. - """ - client = openai.Client(api_key=self.api_key, base_url=self.base_url) - - max_tokens = 200 - - response_default = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Count from 1 to 20."}, - ], - temperature=0, - max_tokens=max_tokens, - extra_body={"ignore_eos": False}, - ) - - response_ignore_eos = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Count from 1 to 20."}, - ], - temperature=0, - max_tokens=max_tokens, - extra_body={"ignore_eos": True}, - ) - - default_tokens = len( - self.tokenizer.encode(response_default.choices[0].message.content) - ) - ignore_eos_tokens = len( - self.tokenizer.encode(response_ignore_eos.choices[0].message.content) - ) - - # Check if ignore_eos resulted in more tokens or exactly max_tokens - # The ignore_eos response should either: - # 1. Have more tokens than the default response (if default stopped at EOS before max_tokens) - # 2. Have exactly max_tokens (if it reached the max_tokens limit) - self.assertTrue( - ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens, - f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}", - ) - - self.assertEqual( - response_ignore_eos.choices[0].finish_reason, - "length", - f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}", - ) - - -if __name__ == "__main__": - import unittest - - unittest.main() diff --git a/test/registered/scheduler/test_retract_decode.py b/test/registered/scheduler/test_retract_decode.py index 31b9f6715..8eafe6afd 100644 --- a/test/registered/scheduler/test_retract_decode.py +++ b/test/registered/scheduler/test_retract_decode.py @@ -17,8 +17,8 @@ from sglang.test.test_utils import ( ) from sglang.utils import is_in_ci -register_cuda_ci(est_time=353, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=600, suite="stage-b-test-1-gpu-small-amd") +register_cuda_ci(est_time=215, stage="base-b", runner_config="1-gpu-small") +register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd") class TestRetractDecode(CustomTestCase): @@ -62,18 +62,6 @@ class TestRetractDecode(CustomTestCase): assert self.process.poll() is None, "Server crashed during test" -class TestRetractDecodePaged(TestRetractDecode): - """python -m unittest test_retract_decode.TestRetractDecodePaged""" - - other_args = ["--page-size", "16"] - - -class TestRetractDecodeChunkCache(TestRetractDecode): - """python -m unittest test_retract_decode.TestRetractDecodeChunkCache""" - - other_args = ["--disable-radix-cache"] - - class TestRetractDecodeChunkCachePaged(TestRetractDecode): """python -m unittest test_retract_decode.TestRetractDecodeChunkCachePaged""" diff --git a/test/registered/spec/eagle/test_spec_eagle_triton.py b/test/registered/spec/eagle/test_spec_eagle_triton.py index d8f611f73..a36f9b571 100644 --- a/test/registered/spec/eagle/test_spec_eagle_triton.py +++ b/test/registered/spec/eagle/test_spec_eagle_triton.py @@ -17,7 +17,7 @@ from sglang.test.kits.spec_server_kits import ( ) from sglang.test.server_fixtures.spec_eagle_fixture import Eagle3Base, EagleLlama2Base -register_cuda_ci(est_time=480, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=350, stage="base-b", runner_config="1-gpu-small") class TestEagle3Triton( @@ -33,7 +33,7 @@ class TestEagle3Triton( attention_backend = "triton" max_running_requests = 64 cuda_graph_max_bs_decode = 64 - gsm8k_num_examples = 1000 + gsm8k_num_examples = 200 gsm8k_check_accept_len = False env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/tokenizer/test_skip_tokenizer_init.py b/test/registered/tokenizer/test_skip_tokenizer_init.py index 2b93f380a..4bc255bab 100644 --- a/test/registered/tokenizer/test_skip_tokenizer_init.py +++ b/test/registered/tokenizer/test_skip_tokenizer_init.py @@ -154,7 +154,6 @@ class TestSkipTokenizerInit(CustomTestCase): response_stream_json = [] for line in response_stream.iter_lines(): - print(line) if line.startswith(b"data: ") and line[6:] != b"[DONE]": response_stream_json.append(json.loads(line[6:])) out_stream_ids = []