From fb90c9d298eeeb15fde5de1c7676cfc3d9a4b6ee Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 26 Mar 2026 14:26:46 -0700 Subject: [PATCH] [Test] Consolidate eval accuracy test mixins into eval_accuracy_kit (#21047) --- .claude/skills/write-sglang-test/SKILL.md | 16 ++ python/sglang/test/kits/eval_accuracy_kit.py | 165 ++++++++++++++++++ python/sglang/test/kits/gsm8k_accuracy_kit.py | 37 ---- .../4-gpu-models/test_qwen3_next_models.py | 2 +- .../test_qwen3_next_models_mtp.py | 2 +- .../8-gpu-models/test_mimo_models.py | 2 +- .../registered/backends/test_torch_compile.py | 21 +-- test/registered/core/test_cpp_radix_cache.py | 22 +-- test/registered/core/test_page_size.py | 21 +-- .../distributed/test_data_parallelism.py | 21 +-- .../distributed/test_dp_attention.py | 18 +- .../eval/test_eval_accuracy_large.py | 68 +------- .../eval/test_moe_eval_accuracy_large.py | 60 +------ .../hicache/test_hicache_storage.py | 21 +-- .../hicache/test_hicache_variants.py | 85 ++------- test/registered/mla/test_mla.py | 19 +- test/registered/mla/test_mla_fp8.py | 19 +- .../models/test_ministral3_models.py | 2 +- .../models/test_nvidia_nemotron_nano_v2.py | 2 +- .../models/test_nvidia_nemotron_nano_v2_vl.py | 2 +- .../models/test_qwen3_next_models_fp4.py | 2 +- .../models/test_qwen3_next_models_pcg.py | 2 +- test/registered/quant/test_torchao.py | 21 +-- .../spec/test_ngram_speculative_decoding.py | 2 +- .../tokenizer/test_multi_tokenizer.py | 21 +-- 25 files changed, 276 insertions(+), 377 deletions(-) create mode 100644 python/sglang/test/kits/eval_accuracy_kit.py delete mode 100644 python/sglang/test/kits/gsm8k_accuracy_kit.py diff --git a/.claude/skills/write-sglang-test/SKILL.md b/.claude/skills/write-sglang-test/SKILL.md index d70c31a7a..d24524468 100644 --- a/.claude/skills/write-sglang-test/SKILL.md +++ b/.claude/skills/write-sglang-test/SKILL.md @@ -379,6 +379,22 @@ python/sglang/jit_kernel/ --- +## Eval Accuracy Mixins + +**Design philosophy**: Most test files don't care about eval logic — they only need a "does this feature break model output quality?" sanity check. The mixin pattern separates **what to test** (threshold) from **how to test** (run_eval, assertions, CI summary). Test classes declare thresholds as class attributes; the mixin provides the `test_*` method. Override when you need extra assertions (e.g. EAGLE accept length). + +Available mixins in `python/sglang/test/kits/eval_accuracy_kit.py`: `MMLUMixin`, `HumanEvalMixin`, `MGSMEnMixin`, `GSM8KMixin`. Can be combined freely. Read the source for attrs and defaults. + +```python +class TestMyFeature(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + # test_mmlu is inherited — no code needed +``` + +--- + ## Key Utilities ```python diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py new file mode 100644 index 000000000..cab5d4860 --- /dev/null +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -0,0 +1,165 @@ +from types import SimpleNamespace +from typing import Optional + +import requests + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary + +_THRESHOLD_NOT_SET = float("nan") + + +def _check_accept_length(test_case, base_url, threshold): + """Check speculative decoding accept length from server info.""" + server_info = requests.get(base_url + "/get_server_info").json() + avg_spec_accept_length = server_info["internal_states"][0]["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + test_case.assertGreater(avg_spec_accept_length, threshold) + + +class GSM8KMixin: + """Mixin for few-shot GSM8K evaluation. + + Required attributes on the test class: + base_url: str + gsm8k_accuracy_thres: float + """ + + gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET + gsm8k_accept_length_thres: Optional[float] = None + gsm8k_num_questions: int = 200 + gsm8k_parallel: int = 128 + + def test_gsm8k(self): + assert ( + self.gsm8k_accuracy_thres == self.gsm8k_accuracy_thres + ), f"{type(self).__name__} must set gsm8k_accuracy_thres" + + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=self.gsm8k_num_questions, + max_new_tokens=512, + parallel=self.gsm8k_parallel, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_gsm8k(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres) + + if self.gsm8k_accept_length_thres is not None: + _check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres) + + +class MMLUMixin: + """Mixin for MMLU evaluation. + + Required attributes on the test class: + base_url: str + model: str + mmlu_score_threshold: float + """ + + mmlu_score_threshold: float = _THRESHOLD_NOT_SET + mmlu_accept_length_thres: Optional[float] = None + mmlu_num_examples: int = 5000 + mmlu_num_threads: int = 1024 + + def test_mmlu(self): + assert ( + self.mmlu_score_threshold == self.mmlu_score_threshold + ), f"{type(self).__name__} must set mmlu_score_threshold" + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=self.mmlu_num_examples, + num_threads=self.mmlu_num_threads, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n{metrics['score']=:.4f}\n") + + self.assertGreaterEqual(metrics["score"], self.mmlu_score_threshold) + + if self.mmlu_accept_length_thres is not None: + _check_accept_length(self, self.base_url, self.mmlu_accept_length_thres) + + +class HumanEvalMixin: + """Mixin for HumanEval evaluation. + + Required attributes on the test class: + base_url: str + model: str + humaneval_score_threshold: float + """ + + humaneval_score_threshold: float = _THRESHOLD_NOT_SET + humaneval_score_threshold_amd: Optional[float] = None + humaneval_num_threads: int = 1024 + + def test_human_eval(self): + assert ( + self.humaneval_score_threshold == self.humaneval_score_threshold + ), f"{type(self).__name__} must set humaneval_score_threshold" + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=self.humaneval_num_threads, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary(f"### test_human_eval\n{metrics['score']=:.4f}\n") + + threshold = self.humaneval_score_threshold + if is_in_amd_ci() and self.humaneval_score_threshold_amd is not None: + threshold = self.humaneval_score_threshold_amd + + self.assertGreaterEqual(metrics["score"], threshold) + + +class MGSMEnMixin: + """Mixin for MGSM English evaluation. + + Required attributes on the test class: + base_url: str + model: str + mgsm_en_score_threshold: float + """ + + mgsm_en_score_threshold: float = _THRESHOLD_NOT_SET + mgsm_en_num_examples: Optional[int] = None + mgsm_en_num_threads: int = 1024 + + def test_mgsm_en(self): + assert ( + self.mgsm_en_score_threshold == self.mgsm_en_score_threshold + ), f"{type(self).__name__} must set mgsm_en_score_threshold" + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=self.mgsm_en_num_examples, + num_threads=self.mgsm_en_num_threads, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary(f"### test_mgsm_en\n{metrics['score']=:.4f}\n") + + self.assertGreaterEqual(metrics["score"], self.mgsm_en_score_threshold) diff --git a/python/sglang/test/kits/gsm8k_accuracy_kit.py b/python/sglang/test/kits/gsm8k_accuracy_kit.py deleted file mode 100644 index 5503d0004..000000000 --- a/python/sglang/test/kits/gsm8k_accuracy_kit.py +++ /dev/null @@ -1,37 +0,0 @@ -from types import SimpleNamespace -from typing import Optional - -import requests - -from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k - - -class GSM8KMixin: - gsm8k_accuracy_thres: float - gsm8k_accept_length_thres: Optional[float] = None - gsm8k_num_questions: int = 200 - gsm8k_parallel: int = 128 - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=self.gsm8k_num_questions, - max_new_tokens=512, - parallel=self.gsm8k_parallel, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval_gsm8k(args) - print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres) - - if self.gsm8k_accept_length_thres is not None: - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, self.gsm8k_accept_length_thres) diff --git a/test/registered/4-gpu-models/test_qwen3_next_models.py b/test/registered/4-gpu-models/test_qwen3_next_models.py index db3140cf5..b18c463b3 100644 --- a/test/registered/4-gpu-models/test_qwen3_next_models.py +++ b/test/registered/4-gpu-models/test_qwen3_next_models.py @@ -1,7 +1,7 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase diff --git a/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py b/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py index 215e96125..0ac3c5f62 100644 --- a/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py +++ b/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py @@ -2,7 +2,7 @@ import unittest from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase diff --git a/test/registered/8-gpu-models/test_mimo_models.py b/test/registered/8-gpu-models/test_mimo_models.py index dd111133b..f9dc8165d 100644 --- a/test/registered/8-gpu-models/test_mimo_models.py +++ b/test/registered/8-gpu-models/test_mimo_models.py @@ -1,7 +1,7 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase diff --git a/test/registered/backends/test_torch_compile.py b/test/registered/backends/test_torch_compile.py index a426b79ce..6afce40e8 100644 --- a/test/registered/backends/test_torch_compile.py +++ b/test/registered/backends/test_torch_compile.py @@ -1,12 +1,11 @@ import time import unittest -from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -20,7 +19,11 @@ register_cuda_ci(est_time=144, suite="stage-b-test-1-gpu-large") register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd") -class TestTorchCompile(CustomTestCase): +class TestTorchCompile(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST @@ -36,18 +39,6 @@ class TestTorchCompile(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - def run_decode(self, max_new_tokens): response = requests.post( self.base_url + "/generate", diff --git a/test/registered/core/test_cpp_radix_cache.py b/test/registered/core/test_cpp_radix_cache.py index faf7d2d45..c8fad0637 100644 --- a/test/registered/core/test_cpp_radix_cache.py +++ b/test/registered/core/test_cpp_radix_cache.py @@ -1,10 +1,9 @@ import unittest -from types import SimpleNamespace from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -17,7 +16,11 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=60, suite="nightly-1-gpu", nightly=True) -class TestCppRadixCache(CustomTestCase): +class TestCppRadixCache(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): envs.SGLANG_EXPERIMENTAL_CPP_RADIX_TREE.set(True) @@ -33,19 +36,6 @@ class TestCppRadixCache(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - print(metrics) - self.assertGreaterEqual(metrics["score"], 0.65) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/core/test_page_size.py b/test/registered/core/test_page_size.py index b3e1b333e..1cf8cbcb6 100644 --- a/test/registered/core/test_page_size.py +++ b/test/registered/core/test_page_size.py @@ -1,10 +1,9 @@ import os import unittest -from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -17,7 +16,11 @@ register_cuda_ci(est_time=60, suite="stage-b-test-1-gpu-small") register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd") -class TestPageSize(CustomTestCase): +class TestPageSize(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): os.environ["SGLANG_DEBUG_MEMORY_POOL"] = "1" @@ -34,18 +37,6 @@ class TestPageSize(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/distributed/test_data_parallelism.py b/test/registered/distributed/test_data_parallelism.py index b93af4584..70323f563 100644 --- a/test/registered/distributed/test_data_parallelism.py +++ b/test/registered/distributed/test_data_parallelism.py @@ -1,12 +1,11 @@ import time import unittest -from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -19,7 +18,11 @@ register_cuda_ci(est_time=73, suite="stage-b-test-2-gpu-large") register_amd_ci(est_time=73, suite="stage-b-test-2-gpu-large-amd") -class TestDataParallelism(CustomTestCase): +class TestDataParallelism(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST @@ -35,18 +38,6 @@ class TestDataParallelism(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - def test_update_weight(self): response = requests.post( self.base_url + "/update_weights_from_disk", diff --git a/test/registered/distributed/test_dp_attention.py b/test/registered/distributed/test_dp_attention.py index 0d7f1838c..a2a883227 100644 --- a/test/registered/distributed/test_dp_attention.py +++ b/test/registered/distributed/test_dp_attention.py @@ -9,10 +9,10 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin +from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin -from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_IMAGE_URL, DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -30,10 +30,13 @@ register_cuda_ci(est_time=350, suite="stage-b-test-2-gpu-large") class TestDPAttentionDP2TP2( CustomTestCase, + MGSMEnMixin, JSONConstrainedMixin, EBNFConstrainedMixin, RegexConstrainedMixin, ): + mgsm_en_score_threshold = 0.8 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST @@ -64,19 +67,6 @@ class TestDPAttentionDP2TP2( kill_process_tree(cls.process.pid) cls._env_override.__exit__(None, None, None) - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreater(metrics["score"], 0.8) - class TestDPRetract( CustomTestCase, diff --git a/test/registered/eval/test_eval_accuracy_large.py b/test/registered/eval/test_eval_accuracy_large.py index fa6b40bd0..dab7bb6be 100644 --- a/test/registered/eval/test_eval_accuracy_large.py +++ b/test/registered/eval/test_eval_accuracy_large.py @@ -4,27 +4,28 @@ python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu """ import unittest -from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import HumanEvalMixin, MGSMEnMixin, MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_amd_ci, - is_in_ci, popen_launch_server, - write_github_step_summary, ) register_cuda_ci(est_time=300, suite="stage-b-test-1-gpu-small") register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-small-amd") -class TestEvalAccuracyLarge(CustomTestCase): +class TestEvalAccuracyLarge(CustomTestCase, MMLUMixin, HumanEvalMixin, MGSMEnMixin): + mmlu_score_threshold = 0.70 + humaneval_score_threshold = 0.64 + humaneval_score_threshold_amd = 0.60 + mgsm_en_score_threshold = 0.835 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST @@ -40,61 +41,6 @@ class TestEvalAccuracyLarge(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=5000, - num_threads=1024, - ) - - metrics = run_eval(args) - - if is_in_ci(): - write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') - - self.assertGreater(metrics["score"], 0.70) - - def test_human_eval(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="humaneval", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - - if is_in_ci(): - write_github_step_summary( - f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' - ) - - if is_in_amd_ci(): - self.assertGreater(metrics["score"], 0.60) - else: - self.assertGreater(metrics["score"], 0.64) - - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - - if is_in_ci(): - write_github_step_summary( - f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' - ) - - self.assertGreater(metrics["score"], 0.835) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/eval/test_moe_eval_accuracy_large.py b/test/registered/eval/test_moe_eval_accuracy_large.py index 759b66ad1..edfb48fd2 100644 --- a/test/registered/eval/test_moe_eval_accuracy_large.py +++ b/test/registered/eval/test_moe_eval_accuracy_large.py @@ -5,27 +5,28 @@ python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mm import os import unittest -from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import HumanEvalMixin, MGSMEnMixin, MMLUMixin from sglang.test.test_utils import ( DEFAULT_MOE_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, is_in_amd_ci, - is_in_ci, popen_launch_server, - write_github_step_summary, ) register_cuda_ci(est_time=500, suite="stage-b-test-2-gpu-large") register_amd_ci(est_time=500, suite="stage-b-test-2-gpu-large-amd") -class TestMoEEvalAccuracyLarge(CustomTestCase): +class TestMoEEvalAccuracyLarge(CustomTestCase, MMLUMixin, HumanEvalMixin, MGSMEnMixin): + mmlu_score_threshold = 0.62 + humaneval_score_threshold = 0.40 + mgsm_en_score_threshold = 0.61 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST @@ -56,55 +57,6 @@ class TestMoEEvalAccuracyLarge(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=5000, - num_threads=1024, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.62) - - if is_in_ci(): - write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') - - def test_human_eval(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="humaneval", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.40) - - if is_in_ci(): - write_github_step_summary( - f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' - ) - - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.61) - - if is_in_ci(): - write_github_step_summary( - f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' - ) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/hicache/test_hicache_storage.py b/test/registered/hicache/test_hicache_storage.py index 00640de91..d6470b198 100644 --- a/test/registered/hicache/test_hicache_storage.py +++ b/test/registered/hicache/test_hicache_storage.py @@ -5,10 +5,9 @@ register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd") import time import unittest -from types import SimpleNamespace from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -20,7 +19,11 @@ from sglang.test.test_utils import ( _is_hip = is_hip() -class TestHiCache(CustomTestCase): +class TestHiCache(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST @@ -47,18 +50,6 @@ class TestHiCache(CustomTestCase): kill_process_tree(cls.process.pid) time.sleep(5) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/hicache/test_hicache_variants.py b/test/registered/hicache/test_hicache_variants.py index 6934a32fc..89c9006a1 100644 --- a/test/registered/hicache/test_hicache_variants.py +++ b/test/registered/hicache/test_hicache_variants.py @@ -8,13 +8,10 @@ Tests HiCache with different configurations: standard, MLA, EAGLE, and page size """ import unittest -from types import SimpleNamespace - -import requests from sglang.benchmark.utils import get_tokenizer from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin, MMLUMixin from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -29,44 +26,11 @@ from sglang.test.test_utils import ( _is_hip = is_hip() -class HiCacheEvalMixin: - """Mixin class containing common HiCache evaluation test methods""" - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) - - -class HiCacheMGSMEvalMixin: - """Mixin for tests that also run MGSM evaluation""" - - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.8) - - class HiCacheBaseServer(CustomTestCase): """Base class for HiCache tests with configurable server setup""" model_name = DEFAULT_MODEL_NAME_FOR_TEST hicache_args = [] - expected_mmlu_score = 0.65 @classmethod def setUpClass(cls): @@ -89,7 +53,7 @@ class HiCacheBaseServer(CustomTestCase): kill_process_tree(cls.process.pid) -class TestHiCacheStandard(HiCacheBaseServer, HiCacheEvalMixin): +class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin): """Standard HiCache configuration tests""" model_name = DEFAULT_MODEL_NAME_FOR_TEST @@ -100,10 +64,12 @@ class TestHiCacheStandard(HiCacheBaseServer, HiCacheEvalMixin): "--hicache-size", 100 if not _is_hip else 200, ] - expected_mmlu_score = 0.65 + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 -class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin): +class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin): """HiCache with MLA model tests""" model_name = DEFAULT_MLA_MODEL_NAME_FOR_TEST @@ -111,11 +77,14 @@ class TestHiCacheMLA(HiCacheBaseServer, HiCacheEvalMixin, HiCacheMGSMEvalMixin): "--trust-remote-code", "--enable-hierarchical-cache", ] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2]) - expected_mmlu_score = 0.5 + mmlu_score_threshold = 0.5 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + mgsm_en_score_threshold = 0.8 @unittest.skipIf(is_hip(), "Disabled for AMD-aiter") -class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin): +class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin): """HiCache with EAGLE speculative decoding tests""" model_name = DEFAULT_TARGET_MODEL_EAGLE3 @@ -141,31 +110,13 @@ class TestHiCacheEagle(HiCacheBaseServer, HiCacheEvalMixin): "--chunked-prefill-size", 1024, ] - expected_mmlu_score = 0.72 - - def test_mmlu(self): - """Override to add EAGLE-specific assertions""" - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], self.expected_mmlu_score) - - # EAGLE-specific check - server_info = requests.get(self.base_url + "/get_server_info").json() - avg_spec_accept_length = server_info["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 2.26) + mmlu_score_threshold = 0.72 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + mmlu_accept_length_thres = 2.26 -class TestHiCachePage(HiCacheBaseServer, HiCacheEvalMixin): +class TestHiCachePage(HiCacheBaseServer, MMLUMixin): """HiCache with custom page size tests""" model_name = DEFAULT_MODEL_NAME_FOR_TEST @@ -176,7 +127,9 @@ class TestHiCachePage(HiCacheBaseServer, HiCacheEvalMixin): "--hicache-write-policy", "write_back", ] - expected_mmlu_score = 0.65 + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 if __name__ == "__main__": diff --git a/test/registered/mla/test_mla.py b/test/registered/mla/test_mla.py index 55140c2ae..03fce55f4 100644 --- a/test/registered/mla/test_mla.py +++ b/test/registered/mla/test_mla.py @@ -1,9 +1,8 @@ import unittest -from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -17,7 +16,9 @@ register_cuda_ci(est_time=194, suite="stage-b-test-1-gpu-large") register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd") -class TestMLA(CustomTestCase): +class TestMLA(CustomTestCase, MGSMEnMixin): + mgsm_en_score_threshold = 0.8 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST @@ -40,18 +41,6 @@ class TestMLA(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.8) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/mla/test_mla_fp8.py b/test/registered/mla/test_mla_fp8.py index 80528a545..a256aad9d 100644 --- a/test/registered/mla/test_mla_fp8.py +++ b/test/registered/mla/test_mla_fp8.py @@ -1,9 +1,8 @@ import unittest -from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin from sglang.test.test_utils import ( DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -17,7 +16,9 @@ register_cuda_ci(est_time=77, suite="stage-b-test-1-gpu-large") register_amd_ci(est_time=800, suite="stage-b-test-1-gpu-small-amd") -class TestMLA(CustomTestCase): +class TestMLA(CustomTestCase, MGSMEnMixin): + mgsm_en_score_threshold = 0.8 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST @@ -37,18 +38,6 @@ class TestMLA(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mgsm_en(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mgsm_en", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - assert metrics["score"] >= 0.8 - if __name__ == "__main__": unittest.main() diff --git a/test/registered/models/test_ministral3_models.py b/test/registered/models/test_ministral3_models.py index 49184a05d..327b49f14 100644 --- a/test/registered/models/test_ministral3_models.py +++ b/test/registered/models/test_ministral3_models.py @@ -1,7 +1,7 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.mmmu_vlm_kit import MMMUMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase diff --git a/test/registered/models/test_nvidia_nemotron_nano_v2.py b/test/registered/models/test_nvidia_nemotron_nano_v2.py index 7e510bcd9..e5b0150ac 100644 --- a/test/registered/models/test_nvidia_nemotron_nano_v2.py +++ b/test/registered/models/test_nvidia_nemotron_nano_v2.py @@ -2,7 +2,7 @@ import unittest from sglang.srt.utils import is_blackwell from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase register_cuda_ci(est_time=132, suite="stage-b-test-2-gpu-large") diff --git a/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py b/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py index abf6d91d0..77fc1dab5 100644 --- a/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py +++ b/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py @@ -1,7 +1,7 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.mmmu_vlm_kit import MMMUMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase diff --git a/test/registered/models/test_qwen3_next_models_fp4.py b/test/registered/models/test_qwen3_next_models_fp4.py index f08d99be1..2c5d6bdf2 100644 --- a/test/registered/models/test_qwen3_next_models_fp4.py +++ b/test/registered/models/test_qwen3_next_models_fp4.py @@ -2,7 +2,7 @@ import unittest from sglang.srt.utils import get_device_sm from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase register_cuda_ci(est_time=500, suite="nightly-4-gpu-b200", nightly=True) diff --git a/test/registered/models/test_qwen3_next_models_pcg.py b/test/registered/models/test_qwen3_next_models_pcg.py index 1bb5f003b..593b6d700 100644 --- a/test/registered/models/test_qwen3_next_models_pcg.py +++ b/test/registered/models/test_qwen3_next_models_pcg.py @@ -5,7 +5,7 @@ Qwen3 Next piecewise CUDA graph tests. import unittest from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase register_cuda_ci( diff --git a/test/registered/quant/test_torchao.py b/test/registered/quant/test_torchao.py index 21b9d30a3..c5f6ad599 100644 --- a/test/registered/quant/test_torchao.py +++ b/test/registered/quant/test_torchao.py @@ -1,5 +1,4 @@ import unittest -from types import SimpleNamespace import requests @@ -10,7 +9,7 @@ register_cuda_ci(est_time=103, suite="stage-b-test-1-gpu-small") register_amd_ci(est_time=230, suite="stage-b-test-1-gpu-small-amd") from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_IMAGE_URL, DEFAULT_MODEL_NAME_FOR_TEST, @@ -23,7 +22,11 @@ from sglang.test.test_utils import ( ) -class TestTorchAO(CustomTestCase): +class TestTorchAO(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.60 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST @@ -39,18 +42,6 @@ class TestTorchAO(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - assert metrics["score"] >= 0.60 - def run_decode(self, max_new_tokens): response = requests.post( self.base_url + "/generate", diff --git a/test/registered/spec/test_ngram_speculative_decoding.py b/test/registered/spec/test_ngram_speculative_decoding.py index 2b16c1ba2..63c75b116 100644 --- a/test/registered/spec/test_ngram_speculative_decoding.py +++ b/test/registered/spec/test_ngram_speculative_decoding.py @@ -3,7 +3,7 @@ import unittest from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.gsm8k_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.test_utils import ( DEFAULT_TARGET_MODEL_NGRAM, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, diff --git a/test/registered/tokenizer/test_multi_tokenizer.py b/test/registered/tokenizer/test_multi_tokenizer.py index 2e07a2f73..b367cd4f8 100644 --- a/test/registered/tokenizer/test_multi_tokenizer.py +++ b/test/registered/tokenizer/test_multi_tokenizer.py @@ -1,9 +1,8 @@ import unittest -from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.kits.eval_accuracy_kit import MMLUMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -21,8 +20,11 @@ register_cuda_ci(est_time=230, suite="stage-b-test-1-gpu-large") register_amd_ci(est_time=345, suite="stage-b-test-1-gpu-small-amd") -class TestMultiTokenizer(CustomTestCase): - # from test_hicache.py +class TestMultiTokenizer(CustomTestCase, MMLUMixin): + mmlu_score_threshold = 0.65 + mmlu_num_examples = 64 + mmlu_num_threads = 32 + @classmethod def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST @@ -43,17 +45,6 @@ class TestMultiTokenizer(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) - def test_multi_tokenizer_ttft(self): # from test_bench_serving.py run_bench_serving args = get_benchmark_args(