minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Test] Consolidate eval accuracy test mixins into eval_accuracy_kit (#21047)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-03-26 14:26:46 -07:00
committed by GitHub
parent e5dd411f64
commit fb90c9d298
25 changed files with 276 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/skills/write-sglang-test/SKILL.md
+16
View File
@@ -379,6 +379,22 @@ python/sglang/jit_kernel/
---
## Eval Accuracy Mixins
**Design philosophy**: Most test files don't care about eval logic — they only need a "does this feature break model output quality?" sanity check. The mixin pattern separates **what to test** (threshold) from **how to test** (run_eval, assertions, CI summary). Test classes declare thresholds as class attributes; the mixin provides the `test_*` method. Override when you need extra assertions (e.g. EAGLE accept length).
Available mixins in `python/sglang/test/kits/eval_accuracy_kit.py`: `MMLUMixin`, `HumanEvalMixin`, `MGSMEnMixin`, `GSM8KMixin`. Can be combined freely. Read the source for attrs and defaults.
```python
class TestMyFeature(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
# test_mmlu is inherited — no code needed
```
---
## Key Utilities
```python
python/sglang/test/kits/eval_accuracy_kit.py
+165
View File
@@ -0,0 +1,165 @@
from types import SimpleNamespace
from typing import Optional
import requests
from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
_THRESHOLD_NOT_SET = float("nan")
def _check_accept_length(test_case, base_url, threshold):
"""Check speculative decoding accept length from server info."""
server_info = requests.get(base_url + "/get_server_info").json()
avg_spec_accept_length = server_info["internal_states"][0]["avg_spec_accept_length"]
print(f"{avg_spec_accept_length=}")
test_case.assertGreater(avg_spec_accept_length, threshold)
class GSM8KMixin:
"""Mixin for few-shot GSM8K evaluation.
Required attributes on the test class:
base_url: str
gsm8k_accuracy_thres: float
"""
gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET
gsm8k_accept_length_thres: Optional[float] = None
gsm8k_num_questions: int = 200
gsm8k_parallel: int = 128
def test_gsm8k(self):
assert (
self.gsm8k_accuracy_thres == self.gsm8k_accuracy_thres
), f"{type(self).__name__} must set gsm8k_accuracy_thres"
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=self.gsm8k_num_questions,
max_new_tokens=512,
parallel=self.gsm8k_parallel,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_gsm8k(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres)
if self.gsm8k_accept_length_thres is not None:
_check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres)
class MMLUMixin:
"""Mixin for MMLU evaluation.
Required attributes on the test class:
base_url: str
model: str
mmlu_score_threshold: float
"""
mmlu_score_threshold: float = _THRESHOLD_NOT_SET
mmlu_accept_length_thres: Optional[float] = None
mmlu_num_examples: int = 5000
mmlu_num_threads: int = 1024
def test_mmlu(self):
assert (
self.mmlu_score_threshold == self.mmlu_score_threshold
), f"{type(self).__name__} must set mmlu_score_threshold"
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_mmlu\n{metrics['score']=:.4f}\n")
self.assertGreaterEqual(metrics["score"], self.mmlu_score_threshold)
if self.mmlu_accept_length_thres is not None:
_check_accept_length(self, self.base_url, self.mmlu_accept_length_thres)
class HumanEvalMixin:
"""Mixin for HumanEval evaluation.
Required attributes on the test class:
base_url: str
model: str
humaneval_score_threshold: float
"""
humaneval_score_threshold: float = _THRESHOLD_NOT_SET
humaneval_score_threshold_amd: Optional[float] = None
humaneval_num_threads: int = 1024
def test_human_eval(self):
assert (
self.humaneval_score_threshold == self.humaneval_score_threshold
), f"{type(self).__name__} must set humaneval_score_threshold"
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="humaneval",
num_examples=None,
num_threads=self.humaneval_num_threads,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_human_eval\n{metrics['score']=:.4f}\n")
threshold = self.humaneval_score_threshold
if is_in_amd_ci() and self.humaneval_score_threshold_amd is not None:
threshold = self.humaneval_score_threshold_amd
self.assertGreaterEqual(metrics["score"], threshold)
class MGSMEnMixin:
"""Mixin for MGSM English evaluation.
Required attributes on the test class:
base_url: str
model: str
mgsm_en_score_threshold: float
"""
mgsm_en_score_threshold: float = _THRESHOLD_NOT_SET
mgsm_en_num_examples: Optional[int] = None
mgsm_en_num_threads: int = 1024
def test_mgsm_en(self):
assert (
self.mgsm_en_score_threshold == self.mgsm_en_score_threshold
), f"{type(self).__name__} must set mgsm_en_score_threshold"
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=self.mgsm_en_num_examples,
num_threads=self.mgsm_en_num_threads,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_mgsm_en\n{metrics['score']=:.4f}\n")
self.assertGreaterEqual(metrics["score"], self.mgsm_en_score_threshold)
python/sglang/test/kits/gsm8k_accuracy_kit.py
-37
View File
@@ -1,37 +0,0 @@
from types import SimpleNamespace
from typing import Optional
import requests
from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k
class GSM8KMixin:
gsm8k_accuracy_thres: float
gsm8k_accept_length_thres: Optional[float] = None
gsm8k_num_questions: int = 200
gsm8k_parallel: int = 128
def test_gsm8k(self):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=self.gsm8k_num_questions,
max_new_tokens=512,
parallel=self.gsm8k_parallel,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_gsm8k(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres)
if self.gsm8k_accept_length_thres is not None:
server_info = requests.get(self.base_url + "/server_info")
avg_spec_accept_length = server_info.json()["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, self.gsm8k_accept_length_thres)
test/registered/4-gpu-models/test_qwen3_next_models.py
+1 -1
View File
@@ -1,7 +1,7 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
test/registered/4-gpu-models/test_qwen3_next_models_mtp.py
+1 -1
View File
@@ -2,7 +2,7 @@ import unittest
from sglang.srt.environ import envs
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin
from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
test/registered/8-gpu-models/test_mimo_models.py
+1 -1
View File
@@ -1,7 +1,7 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
test/registered/backends/test_torch_compile.py
+6 -15
View File
@@ -1,12 +1,11 @@
import time
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -20,7 +19,11 @@ register_cuda_ci(est_time=144, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
class TestTorchCompile(CustomTestCase):
class TestTorchCompile(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
@@ -36,18 +39,6 @@ class TestTorchCompile(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
def run_decode(self, max_new_tokens):
response = requests.post(
self.base_url + "/generate",
test/registered/core/test_cpp_radix_cache.py
+6 -16
View File
@@ -1,10 +1,9 @@
import unittest
from types import SimpleNamespace
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -17,7 +16,11 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=60, suite="nightly-1-gpu", nightly=True)
class TestCppRadixCache(CustomTestCase):
class TestCppRadixCache(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
envs.SGLANG_EXPERIMENTAL_CPP_RADIX_TREE.set(True)
@@ -33,19 +36,6 @@ class TestCppRadixCache(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["score"], 0.65)
if __name__ == "__main__":
unittest.main()
test/registered/core/test_page_size.py
+6 -15
View File
@@ -1,10 +1,9 @@
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -17,7 +16,11 @@ register_cuda_ci(est_time=60, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd")
class TestPageSize(CustomTestCase):
class TestPageSize(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
os.environ["SGLANG_DEBUG_MEMORY_POOL"] = "1"
@@ -34,18 +37,6 @@ class TestPageSize(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
if __name__ == "__main__":
unittest.main()
test/registered/distributed/test_data_parallelism.py
+6 -15
View File
@@ -1,12 +1,11 @@
import time
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -19,7 +18,11 @@ register_cuda_ci(est_time=73, suite="stage-b-test-2-gpu-large")
register_amd_ci(est_time=73, suite="stage-b-test-2-gpu-large-amd")
class TestDataParallelism(CustomTestCase):
class TestDataParallelism(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
@@ -35,18 +38,6 @@ class TestDataParallelism(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
def test_update_weight(self):
response = requests.post(
self.base_url + "/update_weights_from_disk",
test/registered/distributed/test_dp_attention.py
+4 -14
View File
@@ -9,10 +9,10 @@ from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin
from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_IMAGE_URL,
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -30,10 +30,13 @@ register_cuda_ci(est_time=350, suite="stage-b-test-2-gpu-large")
class TestDPAttentionDP2TP2(
CustomTestCase,
MGSMEnMixin,
JSONConstrainedMixin,
EBNFConstrainedMixin,
RegexConstrainedMixin,
):
mgsm_en_score_threshold = 0.8
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST
@@ -64,19 +67,6 @@ class TestDPAttentionDP2TP2(
kill_process_tree(cls.process.pid)
cls._env_override.__exit__(None, None, None)
def test_mgsm_en(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.8)
class TestDPRetract(
CustomTestCase,
test/registered/eval/test_eval_accuracy_large.py
+7 -61
View File
@@ -4,27 +4,28 @@ python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu
"""
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import HumanEvalMixin, MGSMEnMixin, MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=300, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-small-amd")
class TestEvalAccuracyLarge(CustomTestCase):
class TestEvalAccuracyLarge(CustomTestCase, MMLUMixin, HumanEvalMixin, MGSMEnMixin):
mmlu_score_threshold = 0.70
humaneval_score_threshold = 0.64
humaneval_score_threshold_amd = 0.60
mgsm_en_score_threshold = 0.835
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
@@ -40,61 +41,6 @@ class TestEvalAccuracyLarge(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=5000,
num_threads=1024,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n')
self.assertGreater(metrics["score"], 0.70)
def test_human_eval(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="humaneval",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(
f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n'
)
if is_in_amd_ci():
self.assertGreater(metrics["score"], 0.60)
else:
self.assertGreater(metrics["score"], 0.64)
def test_mgsm_en(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(
f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n'
)
self.assertGreater(metrics["score"], 0.835)
if __name__ == "__main__":
unittest.main()
test/registered/eval/test_moe_eval_accuracy_large.py
+6 -54
View File
@@ -5,27 +5,28 @@ python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mm
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import HumanEvalMixin, MGSMEnMixin, MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
register_cuda_ci(est_time=500, suite="stage-b-test-2-gpu-large")
register_amd_ci(est_time=500, suite="stage-b-test-2-gpu-large-amd")
class TestMoEEvalAccuracyLarge(CustomTestCase):
class TestMoEEvalAccuracyLarge(CustomTestCase, MMLUMixin, HumanEvalMixin, MGSMEnMixin):
mmlu_score_threshold = 0.62
humaneval_score_threshold = 0.40
mgsm_en_score_threshold = 0.61
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST
@@ -56,55 +57,6 @@ class TestMoEEvalAccuracyLarge(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=5000,
num_threads=1024,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], 0.62)
if is_in_ci():
write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n')
def test_human_eval(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="humaneval",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], 0.40)
if is_in_ci():
write_github_step_summary(
f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n'
)
def test_mgsm_en(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], 0.61)
if is_in_ci():
write_github_step_summary(
f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n'
)
if __name__ == "__main__":
unittest.main()
test/registered/hicache/test_hicache_storage.py
+6 -15
View File
@@ -5,10 +5,9 @@ register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
import time
import unittest
from types import SimpleNamespace
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -20,7 +19,11 @@ from sglang.test.test_utils import (
_is_hip = is_hip()
class TestHiCache(CustomTestCase):
class TestHiCache(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
@@ -47,18 +50,6 @@ class TestHiCache(CustomTestCase):
kill_process_tree(cls.process.pid)
time.sleep(5)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
if __name__ == "__main__":
unittest.main()
test/registered/hicache/test_hicache_variants.py
+19 -66
View File
@@ -8,13 +8,10 @@ Tests HiCache with different configurations: standard, MLA, EAGLE, and page size
"""
import unittest
from types import SimpleNamespace
import requests
from sglang.benchmark.utils import get_tokenizer
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin, MMLUMixin
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE3,
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -29,44 +26,11 @@ from sglang.test.test_utils import (
_is_hip = is_hip()
class HiCacheEvalMixin:
"""Mixin class containing common HiCache evaluation test methods"""
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score)
class HiCacheMGSMEvalMixin:
"""Mixin for tests that also run MGSM evaluation"""
def test_mgsm_en(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], 0.8)
class HiCacheBaseServer(CustomTestCase):
"""Base class for HiCache tests with configurable server setup"""
model_name = DEFAULT_MODEL_NAME_FOR_TEST
hicache_args = []
expected_mmlu_score = 0.65
@classmethod
def setUpClass(cls):
@@ -89,7 +53,7 @@ class HiCacheBaseServer(CustomTestCase):
kill_process_tree(cls.process.pid)
class TestHiCacheStandard(HiCacheBaseServer, HiCacheEvalMixin):
class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):
"""Standard HiCache configuration tests"""
model_name = DEFAULT_MODEL_NAME_FOR_TEST
@@ -100,10 +64,12 @@ class TestHiCacheStandard(HiCacheBaseServer, HiCacheEvalMixin):
"--hicache-size",
100 if not _is_hip else 200,
]
expected_mmlu_score = 0.65
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin):
class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin):
"""HiCache with MLA model tests"""
model_name = DEFAULT_MLA_MODEL_NAME_FOR_TEST
@@ -111,11 +77,14 @@ class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin):
"--trust-remote-code",
"--enable-hierarchical-cache",
] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2])
expected_mmlu_score = 0.5
mmlu_score_threshold = 0.5
mmlu_num_examples = 64
mmlu_num_threads = 32
mgsm_en_score_threshold = 0.8
@unittest.skipIf(is_hip(), "Disabled for AMD-aiter")
class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin):
"""HiCache with EAGLE speculative decoding tests"""
model_name = DEFAULT_TARGET_MODEL_EAGLE3
@@ -141,31 +110,13 @@ class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin):
"--chunked-prefill-size",
1024,
]
expected_mmlu_score = 0.72
def test_mmlu(self):
"""Override to add EAGLE-specific assertions"""
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score)
# EAGLE-specific check
server_info = requests.get(self.base_url + "/get_server_info").json()
avg_spec_accept_length = server_info["internal_states"][0][
"avg_spec_accept_length"
]
print(f"{avg_spec_accept_length=}")
self.assertGreater(avg_spec_accept_length, 2.26)
mmlu_score_threshold = 0.72
mmlu_num_examples = 64
mmlu_num_threads = 32
mmlu_accept_length_thres = 2.26
class TestHiCachePage(HiCacheBaseServer, HiCacheEvalMixin):
class TestHiCachePage(HiCacheBaseServer, MMLUMixin):
"""HiCache with custom page size tests"""
model_name = DEFAULT_MODEL_NAME_FOR_TEST
@@ -176,7 +127,9 @@ class TestHiCachePage(HiCacheBaseServer, HiCacheEvalMixin):
"--hicache-write-policy",
"write_back",
]
expected_mmlu_score = 0.65
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
if __name__ == "__main__":
test/registered/mla/test_mla.py
+4 -15
View File
@@ -1,9 +1,8 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -17,7 +16,9 @@ register_cuda_ci(est_time=194, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
class TestMLA(CustomTestCase):
class TestMLA(CustomTestCase, MGSMEnMixin):
mgsm_en_score_threshold = 0.8
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST
@@ -40,18 +41,6 @@ class TestMLA(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_en(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
self.assertGreater(metrics["score"], 0.8)
if __name__ == "__main__":
unittest.main()
test/registered/mla/test_mla_fp8.py
+4 -15
View File
@@ -1,9 +1,8 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin
from sglang.test.test_utils import (
DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -17,7 +16,9 @@ register_cuda_ci(est_time=77, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=800, suite="stage-b-test-1-gpu-small-amd")
class TestMLA(CustomTestCase):
class TestMLA(CustomTestCase, MGSMEnMixin):
mgsm_en_score_threshold = 0.8
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST
@@ -37,18 +38,6 @@ class TestMLA(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_en(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
assert metrics["score"] >= 0.8
if __name__ == "__main__":
unittest.main()
test/registered/models/test_ministral3_models.py
+1 -1
View File
@@ -1,7 +1,7 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
test/registered/models/test_nvidia_nemotron_nano_v2.py
+1 -1
View File
@@ -2,7 +2,7 @@ import unittest
from sglang.srt.utils import is_blackwell
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_cuda_ci(est_time=132, suite="stage-b-test-2-gpu-large")
test/registered/models/test_nvidia_nemotron_nano_v2_vl.py
+1 -1
View File
@@ -1,7 +1,7 @@
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.mmmu_vlm_kit import MMMUMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase
test/registered/models/test_qwen3_next_models_fp4.py
+1 -1
View File
@@ -2,7 +2,7 @@ import unittest
from sglang.srt.utils import get_device_sm
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_cuda_ci(est_time=500, suite="nightly-4-gpu-b200", nightly=True)
test/registered/models/test_qwen3_next_models_pcg.py
+1 -1
View File
@@ -5,7 +5,7 @@ Qwen3 Next piecewise CUDA graph tests.
import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
register_cuda_ci(
test/registered/quant/test_torchao.py
+6 -15
View File
@@ -1,5 +1,4 @@
import unittest
from types import SimpleNamespace
import requests
@@ -10,7 +9,7 @@ register_cuda_ci(est_time=103, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=230, suite="stage-b-test-1-gpu-small-amd")
from sglang.lang.chat_template import get_chat_template_by_model_path
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_IMAGE_URL,
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -23,7 +22,11 @@ from sglang.test.test_utils import (
)
class TestTorchAO(CustomTestCase):
class TestTorchAO(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.60
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
@@ -39,18 +42,6 @@ class TestTorchAO(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
assert metrics["score"] >= 0.60
def run_decode(self, max_new_tokens):
response = requests.post(
self.base_url + "/generate",
test/registered/spec/test_ngram_speculative_decoding.py
+1 -1
View File
@@ -3,7 +3,7 @@ import unittest
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.test_utils import (
DEFAULT_TARGET_MODEL_NGRAM,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
test/registered/tokenizer/test_multi_tokenizer.py
+6 -15
View File
@@ -1,9 +1,8 @@
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -21,8 +20,11 @@ register_cuda_ci(est_time=230, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=345, suite="stage-b-test-1-gpu-small-amd")
class TestMultiTokenizer(CustomTestCase):
# from test_hicache.py
class TestMultiTokenizer(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
@@ -43,17 +45,6 @@ class TestMultiTokenizer(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmlu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
def test_multi_tokenizer_ttft(self):
# from test_bench_serving.py run_bench_serving
args = get_benchmark_args(