[CI] Unify basic and speculative sanity accuracy checks with MMLU (#39906)

python/sglang/test/kits/eval_accuracy_kit.py

@@ -97,6 +97,7 @@ def _run_sgl_eval(
     num_examples: Optional[int] = None,
     num_threads: int = 512,
     thinking: bool = True,
+    chat_template_kwargs: Optional[dict] = None,
     reasoning_effort: Optional[str] = None,
     max_tokens: Optional[int] = None,
     temperature: Optional[float] = None,
@@ -134,7 +135,11 @@ def _run_sgl_eval(
     gen_kwargs = dict(
         max_tokens=max_tokens,
         reasoning_effort=reasoning_effort,
-        chat_template_kwargs={"thinking": True} if thinking else None,
+        chat_template_kwargs=(
+            chat_template_kwargs
+            if chat_template_kwargs is not None
+            else ({"thinking": True} if thinking else None)
+        ),
     )
     if temperature is not None:
         gen_kwargs["temperature"] = temperature
@@ -163,6 +168,27 @@ def _run_sgl_eval(
     return result
 
 
+class MMLUSanityMixin:
+    """Short MMLU accuracy gate shared by ordinary and speculative sanity tests."""
+
+    mmlu_score_threshold: float = 0.60
+    mmlu_accept_length_thres: Optional[float] = None
+
+    def test_accuracy_floor(self):
+        _run_sgl_eval(
+            self,
+            eval_name="mmlu",
+            score_threshold=self.mmlu_score_threshold,
+            num_examples=200,
+            num_threads=64,
+            thinking=False,
+            chat_template_kwargs={"enable_thinking": False},
+            max_tokens=1024,
+            temperature=0,
+            accept_length_thres=self.mmlu_accept_length_thres,
+        )
+
+
 class GSM8KMixin:
     """Mixin for GSM8K evaluation.
 

test/registered/core/test_basic_sanity.py

@@ -10,7 +10,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin
 from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin
 from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin
-from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval
+from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin
 from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -29,6 +29,7 @@ class TestBasicSanity(
     BasicDecodeCorrectnessMixin,
     BasicSchedulerStressMixin,
     FwdOccupancyMixin,
+    MMLUSanityMixin,
     CustomTestCase,
 ):
     served_model_name = DEFAULT_MODEL_NAME_FOR_TEST
@@ -56,18 +57,6 @@ class TestBasicSanity(
             env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"},
         )
 
-    def test_accuracy_floor(self):
-        _run_sgl_eval(
-            self,
-            eval_name="mmlu",
-            score_threshold=0.60,
-            num_examples=200,
-            num_threads=64,
-            thinking=False,
-            max_tokens=256,
-            temperature=0,
-        )
-
     @classmethod
     def tearDownClass(cls):
         kill_process_tree(cls.process.pid)

test/registered/core/test_basic_sanity_dflash.py

@@ -9,7 +9,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin
 from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin
 from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin
-from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
+from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin
 from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_DFLASH,
@@ -28,7 +28,7 @@ class TestBasicSanityDFlash(
     BasicDecodeCorrectnessMixin,
     BasicSchedulerStressMixin,
     FwdOccupancyMixin,
-    GSM8KMixin,
+    MMLUSanityMixin,
     CustomTestCase,
 ):
     served_model_name = DEFAULT_TARGET_MODEL_DFLASH
@@ -39,9 +39,7 @@ class TestBasicSanityDFlash(
     fwd_occupancy_acc_length_threshold: float = 2.0
 
     model = DEFAULT_TARGET_MODEL_DFLASH
-    gsm8k_num_questions = 1400
+    mmlu_accept_length_thres = 2.8
-    gsm8k_accuracy_thres = 0.74
-    gsm8k_accept_length_thres = 2.8
 
     attention_backend = "triton"
     draft_attention_backend = "triton"

test/registered/core/test_basic_sanity_dspark.py

@@ -5,7 +5,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin
 from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin
 from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin
-from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
+from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin
 from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin
 from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
 from sglang.test.kits.spec_server_kits import SpecGrammarKit, SpecLogprobKit
@@ -35,7 +35,7 @@ class TestBasicSanityDSpark(
     BasicDecodeCorrectnessMixin,
     BasicSchedulerStressMixin,
     FwdOccupancyMixin,
-    GSM8KMixin,
+    MMLUSanityMixin,
     JSONConstrainedMixin,
     SpecGrammarKit,
     SpecLogprobKit,
@@ -48,9 +48,8 @@ class TestBasicSanityDSpark(
     fwd_occupancy_max_new_tokens = 4096
     fwd_occupancy_acc_length_threshold: float = 2.0
 
-    gsm8k_num_questions = 200
+    mmlu_score_threshold = 0.70
-    gsm8k_accuracy_thres = 0.80
+    mmlu_accept_length_thres = 3.0
-    gsm8k_accept_length_thres = 2.0
 
     attention_backend = ATTENTION_BACKEND
     draft_attention_backend = DRAFT_ATTENTION_BACKEND

test/registered/core/test_basic_sanity_eagle3.py

@@ -8,7 +8,7 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 from sglang.test.kits.basic_api_contract_kit import BasicAPIContractMixin
 from sglang.test.kits.basic_decode_correctness_kit import BasicDecodeCorrectnessMixin
 from sglang.test.kits.basic_scheduler_stress_kit import BasicSchedulerStressMixin
-from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
+from sglang.test.kits.eval_accuracy_kit import MMLUSanityMixin
 from sglang.test.kits.fwd_occupancy_kit import FwdOccupancyMixin
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE3,
@@ -29,7 +29,7 @@ class TestBasicSanityEagle3(
     BasicDecodeCorrectnessMixin,
     BasicSchedulerStressMixin,
     FwdOccupancyMixin,
-    GSM8KMixin,
+    MMLUSanityMixin,
     CustomTestCase,
 ):
     served_model_name = DEFAULT_TARGET_MODEL_EAGLE3
@@ -42,8 +42,7 @@ class TestBasicSanityEagle3(
     fwd_occupancy_acc_length_threshold: float = 1.6
 
     model = DEFAULT_TARGET_MODEL_EAGLE3
-    gsm8k_num_questions = 1400
+    mmlu_accept_length_thres = 1.5
-    gsm8k_accuracy_thres = 0.74
 
     @classmethod
     def setUpClass(cls):