From 1c4b130bf7fb2ec6f080f8b190755f233161d7f9 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 16 Sep 2026 22:19:05 -0700 Subject: [PATCH] [CI] Unify basic and speculative sanity accuracy checks with MMLU (#39906) --- python/sglang/test/kits/eval_accuracy_kit.py | 28 ++++++++++++++++++- test/registered/core/test_basic_sanity.py | 15 ++-------- .../core/test_basic_sanity_dflash.py | 8 ++---- .../core/test_basic_sanity_dspark.py | 9 +++--- .../core/test_basic_sanity_eagle3.py | 7 ++--- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index 034927578..81eedf8b7 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -97,6 +97,7 @@ def _run_sgl_eval( num_examples: Optional[int] = None, num_threads: int = 512, thinking: bool = True, + chat_template_kwargs: Optional[dict] = None, reasoning_effort: Optional[str] = None, max_tokens: Optional[int] = None, temperature: Optional[float] = None, @@ -134,7 +135,11 @@ def _run_sgl_eval( gen_kwargs = dict( max_tokens=max_tokens, reasoning_effort=reasoning_effort, - chat_template_kwargs={"thinking": True} if thinking else None, + chat_template_kwargs=( + chat_template_kwargs + if chat_template_kwargs is not None + else ({"thinking": True} if thinking else None) + ), ) if temperature is not None: gen_kwargs["temperature"] = temperature @@ -163,6 +168,27 @@ def _run_sgl_eval( return result +class MMLUSanityMixin: + """Short MMLU accuracy gate shared by ordinary and speculative sanity tests.""" + + mmlu_score_threshold: float = 0.60 + mmlu_accept_length_thres: Optional[float] = None + + def test_accuracy_floor(self): + _run_sgl_eval( + self, + eval_name="mmlu", + score_threshold=self.mmlu_score_threshold, + num_examples=200, + num_threads=64, + thinking=False, + chat_template_kwargs={"enable_thinking": False}, + max_tokens=1024, + temperature=0, + accept_length_thres=self.mmlu_accept_length_thres, + ) + + class GSM8KMixin: """Mixin for GSM8K evaluation. diff --git a/test/registered/core/test_basic_sanity.py b/test/registered/core/test_basic_sanity.py index 9e11feeae..4dca71625 100644 --- a/test/registered/core/test_basic_sanity.py +++ b/test/registered/core/test_basic_sanity.py @@ -10,7 +10,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin -from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval +from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, @@ -29,6 +29,7 @@ class TestBasicSanity( BasicDecodeCorrectnessMixin, BasicSchedulerStressMixin, FwdOccupancyMixin, + MMLUSanityMixin, CustomTestCase, ): served_model_name = DEFAULT_MODEL_NAME_FOR_TEST @@ -56,18 +57,6 @@ class TestBasicSanity( env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"}, ) - def test_accuracy_floor(self): - _run_sgl_eval( - self, - eval_name="mmlu", - score_threshold=0.60, - num_examples=200, - num_threads=64, - thinking=False, - max_tokens=256, - temperature=0, - ) - @classmethod def tearDownClass(cls): kill_process_tree(cls.process.pid) diff --git a/test/registered/core/test_basic_sanity_dflash.py b/test/registered/core/test_basic_sanity_dflash.py index 40048d8c2..d37085677 100644 --- a/test/registered/core/test_basic_sanity_dflash.py +++ b/test/registered/core/test_basic_sanity_dflash.py @@ -9,7 +9,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_DFLASH, @@ -28,7 +28,7 @@ class TestBasicSanityDFlash( BasicDecodeCorrectnessMixin, BasicSchedulerStressMixin, FwdOccupancyMixin, - GSM8KMixin, + MMLUSanityMixin, CustomTestCase, ): served_model_name = DEFAULT_TARGET_MODEL_DFLASH @@ -39,9 +39,7 @@ class TestBasicSanityDFlash( fwd_occupancy_acc_length_threshold: float = 2.0 model = DEFAULT_TARGET_MODEL_DFLASH - gsm8k_num_questions = 1400 - gsm8k_accuracy_thres = 0.74 - gsm8k_accept_length_thres = 2.8 + mmlu_accept_length_thres = 2.8 attention_backend = "triton" draft_attention_backend = "triton" diff --git a/test/registered/core/test_basic_sanity_dspark.py b/test/registered/core/test_basic_sanity_dspark.py index 9e5fad7ab..bc0d55a27 100644 --- a/test/registered/core/test_basic_sanity_dspark.py +++ b/test/registered/core/test_basic_sanity_dspark.py @@ -5,7 +5,7 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.spec_server_kits import SpecGrammarKit, SpecLogprobKit @@ -35,7 +35,7 @@ class TestBasicSanityDSpark( BasicDecodeCorrectnessMixin, BasicSchedulerStressMixin, FwdOccupancyMixin, - GSM8KMixin, + MMLUSanityMixin, JSONConstrainedMixin, SpecGrammarKit, SpecLogprobKit, @@ -48,9 +48,8 @@ class TestBasicSanityDSpark( fwd_occupancy_max_new_tokens = 4096 fwd_occupancy_acc_length_threshold: float = 2.0 - gsm8k_num_questions = 200 - gsm8k_accuracy_thres = 0.80 - gsm8k_accept_length_thres = 2.0 + mmlu_score_threshold = 0.70 + mmlu_accept_length_thres = 3.0 attention_backend = ATTENTION_BACKEND draft_attention_backend = DRAFT_ATTENTION_BACKEND diff --git a/test/registered/core/test_basic_sanity_eagle3.py b/test/registered/core/test_basic_sanity_eagle3.py index a45430c43..1ad5185a1 100644 --- a/test/registered/core/test_basic_sanity_eagle3.py +++ b/test/registered/core/test_basic_sanity_eagle3.py @@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, @@ -29,7 +29,7 @@ class TestBasicSanityEagle3( BasicDecodeCorrectnessMixin, BasicSchedulerStressMixin, FwdOccupancyMixin, - GSM8KMixin, + MMLUSanityMixin, CustomTestCase, ): served_model_name = DEFAULT_TARGET_MODEL_EAGLE3 @@ -42,8 +42,7 @@ class TestBasicSanityEagle3( fwd_occupancy_acc_length_threshold: float = 1.6 model = DEFAULT_TARGET_MODEL_EAGLE3 - gsm8k_num_questions = 1400 - gsm8k_accuracy_thres = 0.74 + mmlu_accept_length_thres = 1.5 @classmethod def setUpClass(cls):