minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

ci: migrate remaining spec/eagle tests to test/registered/spec/ (#15800)

Browse Source
This commit is contained in:
Alison Shao
2025-12-29 14:00:00 -08:00
committed by GitHub
parent 9c4eb46099
commit f4ec6f8e17
9 changed files with 59 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/srt/run_suite.py
-5
View File
@@ -86,11 +86,9 @@ suites = {
TestFile("test_retract_decode.py", 259),
TestFile("test_score_api.py", 260),
TestFile("test_server_args.py", 9),
TestFile("test_speculative_registry.py", 8),
TestFile("test_skip_tokenizer_init.py", 77),
TestFile("test_srt_endpoint.py", 127),
TestFile("test_srt_engine.py", 252),
TestFile("test_standalone_speculative_decoding.py", 150),
TestFile("test_start_profile.py", 41),
TestFile("test_profile_merger.py", 8),
TestFile("test_profile_merger_http_api.py", 9),
@@ -116,13 +114,11 @@ suites = {
TestFile("models/test_glm4_moe_models.py", 100),
TestFile("models/test_kimi_linear_models.py", 90),
TestFile("rl/test_update_weights_from_distributed.py", 103),
TestFile("test_constrained_decoding_spec_reasoning.py", 60),
TestFile("test_data_parallelism.py", 73),
TestFile("test_disaggregation_basic.py", 400),
TestFile("test_dp_attention.py", 350),
TestFile("test_load_weights_from_remote_instance.py", 72),
TestFile("test_patch_torch.py", 19),
TestFile("test_eagle_dp_attention.py", 200),
],
"per-commit-4-gpu": [
TestFile("models/test_qwen3_next_models.py", 650),
@@ -152,7 +148,6 @@ suites = {
TestFile("test_fp8_blockwise_gemm.py", 280),
TestFile("test_gpt_oss_4gpu.py", 700),
TestFile("test_llama31_fp4.py", 90),
TestFile("test_eagle_infer_beta_dp_attention.py", 300),
],
# "per-commit-8-gpu-b200": [
# TestFile("test_mistral_large3_basic.py", 275), # Moved to nightly - large model
test/srt/test_constrained_decoding_spec_reasoning.py
-100
View File
@@ -1,100 +0,0 @@
import json
import unittest
import openai
from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
class ServerWithGrammar(CustomTestCase):
json_schema = json.dumps(
{
"type": "object",
"properties": {
"name": {"type": "string", "pattern": "^[\\w]+$"},
"population": {"type": "integer"},
"languages": {
"type": "array",
"items": {"type": "string"},
"minItems": 1,
},
"has_held_olympics": {"type": "boolean"},
},
"required": ["name", "population", "languages", "has_held_olympics"],
"additionalProperties": False,
}
)
@classmethod
def setUpClass(cls):
cls.model = "openai/gpt-oss-120b"
cls.base_url = DEFAULT_URL_FOR_TEST
launch_args = [
"--trust-remote-code",
"--tp=2",
"--reasoning-parser=gpt-oss",
"--speculative-algorithm=EAGLE3",
"--speculative-draft-model-path=lmsys/EAGLE3-gpt-oss-120b-bf16",
"--speculative-num-steps=5",
"--speculative-eagle-topk=4",
"--speculative-num-draft-tokens=8",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=launch_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_json_openai(self):
client = openai.Client(api_key="EMPTY", base_url=f"{self.base_url}/v1")
response = client.chat.completions.create(
model=self.model,
messages=[
{"role": "system", "content": "You are a helpful AI assistant"},
{
"role": "user",
"content": "Introduce the capital of France. Return in a JSON format. "
"The JSON Schema is: " + json.dumps(self.json_schema),
},
],
temperature=0,
max_tokens=1024,
response_format={
"type": "json_schema",
"json_schema": {"name": "foo", "schema": json.loads(self.json_schema)},
},
)
text = response.choices[0].message.content
print("\n=== Reasoning Content ===")
reasoning_content = response.choices[0].message.reasoning_content
assert reasoning_content is not None and len(reasoning_content) > 0
print(reasoning_content)
try:
js_obj = json.loads(text)
print("\n=== Parsed JSON Content ===")
print(json.dumps(js_obj))
except (TypeError, json.decoder.JSONDecodeError):
print("JSONDecodeError", text)
raise
self.assertIsInstance(js_obj["name"], str)
self.assertIsInstance(js_obj["population"], int)
if __name__ == "__main__":
unittest.main()
test/srt/test_eagle_dp_attention.py
-129
View File
@@ -1,129 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN,
DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
kill_process_tree,
popen_launch_server,
write_github_step_summary,
)
class TestEAGLE3EngineDPAttention(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--trust-remote-code",
"--speculative-algorithm",
"EAGLE3",
"--speculative-num-steps",
"6",
"--speculative-eagle-topk",
"10",
"--speculative-num-draft-tokens",
"32",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN,
"--tp-size",
"2",
"--dp-size",
"2",
"--enable-dp-attention",
"--enable-dp-lm-head",
"--moe-dense-tp-size",
"1",
"--attention-backend",
"fa3",
"--mem-fraction-static",
"0.75",
"--cuda-graph-max-bs",
"64",
]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
"""Test GSM8K evaluation - append 'a' to run first alphabetically"""
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
server_data = server_info.json()
# Try to get avg_spec_accept_length
avg_spec_accept_length = None
if "internal_states" in server_data and len(server_data["internal_states"]) > 0:
internal_state = server_data["internal_states"][0]
if "avg_spec_accept_length" in internal_state:
avg_spec_accept_length = internal_state["avg_spec_accept_length"]
elif "spec_accept_length" in internal_state:
avg_spec_accept_length = internal_state["spec_accept_length"]
print(f"{avg_spec_accept_length=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (EAGLE3 DP Attention)\n"
f'{metrics["accuracy"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], 0.91)
if avg_spec_accept_length is not None:
self.assertGreater(avg_spec_accept_length, 2.5)
def test_bs_1_speed(self):
"""Test batch size 1 speed with EAGLE3 DP Attention"""
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
acc_length, speed = send_one_prompt(args)
print(f"{acc_length=:.2f} {speed=:.2f}")
if is_in_ci():
write_github_step_summary(
f"### test_bs_1_speed (EAGLE3 DP Attention)\n"
f"{acc_length=:.2f}\n"
f"{speed=:.2f} token/s\n"
)
if is_in_amd_ci():
self.assertGreater(acc_length, 2.0)
else:
self.assertGreater(acc_length, 2.3)
if is_in_amd_ci():
self.assertGreater(speed, 10)
else:
self.assertGreater(speed, 40)
if __name__ == "__main__":
unittest.main()
test/srt/test_eagle_infer_beta_dp_attention.py
-83
View File
@@ -1,83 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST_MLA,
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
def test_gsm8k(base_url: str):
requests.get(base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
server_info = requests.get(base_url + "/get_server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{metrics=}")
print(f"{avg_spec_accept_length=}")
return metrics, avg_spec_accept_length
class TestEagleDPAttnServerSmall(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--tp-size",
"2",
"--dp-size",
"2",
"--enable-dp-attention",
"--speculative-draft-model-path",
DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
]
with envs.SGLANG_ENABLE_SPEC_V2.override(True):
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_a_gsm8k(self):
metrics, avg_spec_accept_length = test_gsm8k(self.base_url)
self.assertGreater(metrics["accuracy"], 0.64)
self.assertGreater(avg_spec_accept_length, 1.4)
if __name__ == "__main__":
unittest.main()
test/srt/test_speculative_registry.py
-149
View File
@@ -1,149 +0,0 @@
import unittest
from sglang.srt.speculative import spec_info as spec_info_module
from sglang.srt.speculative.spec_info import (
SpeculativeAlgorithm,
register_speculative_algorithm,
)
class DummyWorker:
def __init__(self, **kwargs):
self.kwargs = kwargs
class SpeculativeRegistryTests(unittest.TestCase):
def test_nextn_alias_maps_to_eagle(self):
eagle = SpeculativeAlgorithm.from_string("EAGLE")
alias = SpeculativeAlgorithm.from_string("NEXTN")
self.assertIs(alias, eagle)
def test_register_speculative_algorithm_registers_worker_and_flags(self):
original_next_value = SpeculativeAlgorithm._next_value
algo = register_speculative_algorithm(
"TEST_SPEC_ALGO",
DummyWorker,
aliases=("TEST_SPEC_ALIAS",),
flags=("EAGLE",),
override_worker=True,
)
self.addCleanup(self._cleanup_registered_algorithm, algo, ("TEST_SPEC_ALIAS",))
self.addCleanup(
setattr, SpeculativeAlgorithm, "_next_value", original_next_value
)
self.assertIs(SpeculativeAlgorithm.from_string("TEST_SPEC_ALGO"), algo)
self.assertIs(SpeculativeAlgorithm.from_string("TEST_SPEC_ALIAS"), algo)
self.assertTrue(algo.is_eagle())
self.assertIs(SpeculativeAlgorithm.from_value(int(algo)), algo)
self.assertIn(algo, list(spec_info_module._REGISTERED_WORKERS))
worker = algo.create_draft_worker(example_arg=42)
self.assertIsInstance(worker, DummyWorker)
self.assertEqual(worker.kwargs["example_arg"], 42)
def test_builtin_algorithms_flags_and_factories(self):
cases = {
"NONE": {
"is_none": True,
"is_eagle": False,
"is_eagle3": False,
"is_standalone": False,
"is_ngram": False,
"has_factory": False,
},
"EAGLE": {
"is_none": False,
"is_eagle": True,
"is_eagle3": False,
"is_standalone": False,
"is_ngram": False,
"has_factory": True,
},
"EAGLE3": {
"is_none": False,
"is_eagle": True,
"is_eagle3": True,
"is_standalone": False,
"is_ngram": False,
"has_factory": True,
},
"STANDALONE": {
"is_none": False,
"is_eagle": False,
"is_eagle3": False,
"is_standalone": True,
"is_ngram": False,
"has_factory": True,
},
"NGRAM": {
"is_none": False,
"is_eagle": False,
"is_eagle3": False,
"is_standalone": False,
"is_ngram": True,
"has_factory": True,
},
}
for name, expectations in cases.items():
with self.subTest(name=name):
algo = SpeculativeAlgorithm.from_string(name)
self.assertEqual(algo.name, name)
self.assertEqual(algo.is_none(), expectations["is_none"])
self.assertEqual(algo.is_eagle(), expectations["is_eagle"])
self.assertEqual(algo.is_eagle3(), expectations["is_eagle3"])
self.assertEqual(algo.is_standalone(), expectations["is_standalone"])
self.assertEqual(algo.is_ngram(), expectations["is_ngram"])
has_factory = algo._draft_worker_factory is not None
self.assertEqual(has_factory, expectations["has_factory"])
self.assertIs(SpeculativeAlgorithm.from_value(int(algo)), algo)
self.assertIs(SpeculativeAlgorithm.from_string(None), SpeculativeAlgorithm.NONE)
def test_iteration_returns_registration_order(self):
names = [algo.name for algo in SpeculativeAlgorithm._registration_order]
for required in ["NONE", "EAGLE", "EAGLE3", "STANDALONE", "NGRAM"]:
self.assertIn(required, names)
def test_create_draft_worker_returns_none_for_none_algorithm(self):
self.assertIsNone(SpeculativeAlgorithm.NONE.create_draft_worker())
def test_register_draft_worker_override(self):
algo = SpeculativeAlgorithm.from_string("EAGLE")
original_factory = algo._draft_worker_factory
def dummy_factory(_: SpeculativeAlgorithm, **kwargs):
return "dummy"
SpeculativeAlgorithm.register_draft_worker(algo, dummy_factory)
self.addCleanup(
SpeculativeAlgorithm.register_draft_worker, algo, original_factory
)
self.assertEqual(algo.create_draft_worker(), "dummy")
def _cleanup_registered_algorithm(self, algorithm: SpeculativeAlgorithm, aliases):
name = algorithm.name
SpeculativeAlgorithm._registry_by_value.pop(algorithm.value, None)
SpeculativeAlgorithm._registry_by_name.pop(name, None)
if hasattr(SpeculativeAlgorithm, name):
delattr(SpeculativeAlgorithm, name)
for alias in aliases:
SpeculativeAlgorithm._registry_by_name.pop(alias, None)
try:
SpeculativeAlgorithm._registration_order.remove(algorithm)
except ValueError:
pass
for flag_values in SpeculativeAlgorithm._flags.values():
flag_values.discard(algorithm.value)
spec_info_module._REGISTERED_WORKERS.pop(algorithm, None)
if __name__ == "__main__":
unittest.main()
test/srt/test_standalone_speculative_decoding.py
-115
View File
@@ -1,115 +0,0 @@
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_STANDALONE,
DEFAULT_TARGET_MODEL_STANDALONE,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
GSM_DATASET_PATH = None
# Default server arguments shared across all tests
DEFAULT_SERVER_ARGS = [
"--trust-remote-code",
"--cuda-graph-max-bs",
"8",
"--speculative-algorithm",
"STANDALONE",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_STANDALONE,
"--speculative-num-steps",
"4",
"--speculative-eagle-topk",
"2",
"--speculative-num-draft-tokens",
"7",
"--mem-fraction-static",
0.7,
]
class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
model = DEFAULT_TARGET_MODEL_STANDALONE
draft_model = DEFAULT_DRAFT_MODEL_STANDALONE
base_url = DEFAULT_URL_FOR_TEST
accuracy_threshold = 0.7 # derived tests need to override this
spec_decode_threshold = 3.6 # derived spec decoding tests need to override this
@classmethod
def get_server_args(cls):
"""Return the arguments for the server launch. Override in subclasses."""
return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"]
@classmethod
def setUpClass(cls):
# disable deep gemm precompile to make launch server faster
# please don't do this if you want to make your inference workload faster
envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False)
envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False)
model = cls.model
cls.process = popen_launch_server(
model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=cls.get_server_args(),
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=4,
num_questions=100,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
data_path=GSM_DATASET_PATH,
)
metrics = run_eval_few_shot_gsm8k(args)
print(f"{metrics=}")
# Use the appropriate metric key based on the test class
metric_key = "accuracy"
self.assertGreater(metrics[metric_key], self.accuracy_threshold)
server_info = requests.get(self.base_url + "/get_server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold)
class TestStandaloneSpeculativeDecodingTriton(TestStandaloneSpeculativeDecodingBase):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"]
class TestStandaloneSpeculativeDecodingFlashinfer(
TestStandaloneSpeculativeDecodingBase
):
@classmethod
def get_server_args(cls):
return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"]
if __name__ == "__main__":
unittest.main()