[CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477)

test/registered/attention/test_torch_native_attention_backend.py

@@ -38,12 +38,12 @@ class TestTorchNativeAttnBackend(CustomTestCase):
                 base_url=base_url,
                 model=model,
                 eval_name="mmlu",
-                num_examples=64,
+                num_examples=256,
                 num_threads=32,
             )
 
             metrics = run_eval(args)
-            self.assertGreaterEqual(metrics["score"], 0.65)
+            self.assertGreaterEqual(metrics["score"], 0.64)
         finally:
             kill_process_tree(process.pid)
 

test/registered/attention/test_triton_sliding_window.py

@@ -10,7 +10,6 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
-    is_in_amd_ci,
     is_in_ci,
     popen_launch_server,
 )
@@ -53,17 +52,16 @@ class TestSlidingWindowAttentionTriton(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=200,
+            num_examples=256,
             num_threads=32,
         )
 
         metrics = run_eval(args)
         print(f"MMLU metrics with sliding window: {metrics}")
 
-        if is_in_amd_ci():
-            self.assertGreaterEqual(metrics["score"], 0.55)
-        else:
-            self.assertGreaterEqual(metrics["score"], 0.60)
+        # gemma-3-4b-it scores 0.59 over 256 questions under sgl-eval's grader,
+        # minus the 0.05 margin the other eval thresholds use.
+        self.assertGreaterEqual(metrics["score"], 0.54)
 
     def _test_short_context_generation(self):
         response = requests.post(

test/registered/backends/test_torch_compile.py

@@ -20,8 +20,8 @@ register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
 
 
 class TestTorchCompile(CustomTestCase, MMLUMixin):
-    mmlu_score_threshold = 0.65
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
     @classmethod

test/registered/disaggregation/test_disaggregation_decode_offload.py

@@ -139,7 +139,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
             base_url=f"http://{self.base_host}:{self.lb_port}",
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
@@ -166,8 +166,8 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
         metrics2 = run_eval(args)
 
         # Assert score is above a minimum threshold for both rounds
-        self.assertGreater(metrics1["score"], 0.65)
-        self.assertGreater(metrics2["score"], 0.65)
+        self.assertGreater(metrics1["score"], 0.64)
+        self.assertGreater(metrics2["score"], 0.64)
 
         # Score should be consistent: round 2 should be >= round 1, or at least within a 0.05 margin if slightly lower
         self.assertGreaterEqual(metrics2["score"], metrics1["score"] - 0.05)

test/registered/gb300/test_kimi_k25_nvfp4.py

@@ -63,8 +63,17 @@ class TestKimiK25Nvfp4(unittest.TestCase):
         run_combined_tests(
             models=variants,
             test_name="Kimi-K2.5-NVFP4",
+            # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
+            # its `:1` suffix means temperature 0.7, not greedy -- so the baseline
+            # carries over unchanged. Do not "simplify" these away.
             accuracy_params=AccuracyTestParams(
-                dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
+                dataset="mmmu_pro_vision",
+                baseline_accuracy=0.69,
+                repeat=1,
+                max_tokens=32768,
+                temperature=0.7,
+                seed=0,
+                sgl_eval_thinking=False,
             ),
             performance_params=PerformanceTestParams(
                 result_dir="performance_results_gb300",

test/registered/gb300/test_qwen35_fp8.py

@@ -61,8 +61,17 @@ class TestQwen35Fp8(unittest.TestCase):
         run_combined_tests(
             models=variants,
             test_name="Qwen3.5-397B-FP8",
+            # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
+            # its `:1` suffix means temperature 0.7, not greedy -- so the baseline
+            # carries over unchanged. Do not "simplify" these away.
             accuracy_params=AccuracyTestParams(
-                dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768
+                dataset="mmmu_pro_vision",
+                baseline_accuracy=0.76,
+                repeat=1,
+                max_tokens=32768,
+                temperature=0.7,
+                seed=0,
+                sgl_eval_thinking=False,
             ),
             performance_params=PerformanceTestParams(
                 result_dir="performance_results_gb300",

test/registered/hicache/test_hicache_storage.py

@@ -21,8 +21,8 @@ _is_hip = is_hip()
 
 
 class TestHiCache(CustomTestCase, MMLUMixin):
-    mmlu_score_threshold = 0.65
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
     @classmethod

test/registered/hicache/test_hicache_variants.py

@@ -64,8 +64,8 @@ class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):
         "--hicache-size",
         100 if not _is_hip else 200,
     ]
-    mmlu_score_threshold = 0.65
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
 
@@ -77,8 +77,8 @@ class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin):
         "--trust-remote-code",
         "--enable-hierarchical-cache",
     ] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2])
-    mmlu_score_threshold = 0.5
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.54
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
     mgsm_en_score_threshold = 0.8
 
@@ -110,8 +110,8 @@ class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin):
         "--chunked-prefill-size",
         1024,
     ]
-    mmlu_score_threshold = 0.72
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
     mmlu_accept_length_thres = 2.26
 
@@ -127,8 +127,8 @@ class TestHiCachePage(HiCacheBaseServer, MMLUMixin):
         "--hicache-write-policy",
         "write_back",
     ]
-    mmlu_score_threshold = 0.65
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
 

test/registered/models_e2e/test_transformers_models.py

@@ -36,7 +36,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
             timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
             other_args=["--model-impl", "transformers"],
         )
-        cls.mmlu_lower_bound = 0.63
+        cls.mmlu_lower_bound = 0.64
         cls.gsm8k_lower_bound = 0.65
 
     @classmethod
@@ -48,7 +48,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
         metrics = run_eval(args)

test/registered/models_e2e/test_zaya.py

@@ -1,127 +0,0 @@
-"""End-to-end server test for Zyphra ZAYA1 (hybrid CCA attention + MoE).
-
-This test boots a real ``Zyphra/ZAYA1-base`` SGLang server via
-``popen_launch_server``, sends a handful of completions through the HTTP API,
-and finishes with a small MMLU sanity slice.
-
-The test is gated behind ``RUN_ZAYA_E2E=1`` so the registered suite does not
-have to download the full ZAYA1-base checkpoint (≈17 GB) on every run; the CI
-job that owns this test sets the variable explicitly.
-"""
-
-import os
-import unittest
-from types import SimpleNamespace
-
-from sglang.srt.utils import is_hip, kill_process_tree
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-# ZAYA1-base is a heavyweight launch (≈120 transformer layers with MoE), so
-# the estimated time is set generously to keep the CI scheduler from preempting
-# the job before the server finishes warming up.
-register_cuda_ci(est_time=420, stage="extra-a", runner_config="1-gpu-large")
-register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-large-amd")
-
-
-_MODEL_PATH = os.environ.get("ZAYA_MODEL_PATH", "Zyphra/ZAYA1-base")
-
-
-def _zaya_enabled() -> bool:
-    return os.environ.get("RUN_ZAYA_E2E", "0") == "1"
-
-
-@unittest.skipUnless(
-    _zaya_enabled(),
-    "Set RUN_ZAYA_E2E=1 to enable the ZAYA1 end-to-end server test "
-    "(requires downloading the model weights).",
-)
-class TestZayaServer(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = _MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-
-        other_args = [
-            "--mem-fraction-static",
-            "0.5",
-            "--max-running-requests",
-            "8",
-        ]
-        if is_hip():
-            other_args += ["--attention-backend", "triton"]
-
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        if getattr(cls, "process", None) is not None:
-            kill_process_tree(cls.process.pid)
-
-    def test_generation_basic(self):
-        """Send three prompts through the ``/generate`` endpoint and require
-        non-empty completions for each."""
-        import requests
-
-        prompts = [
-            "The capital of France is",
-            "1 + 2 + 3 + 4 + 5 =",
-            "Write a haiku about silicon:",
-        ]
-        for prompt in prompts:
-            resp = requests.post(
-                f"{self.base_url}/generate",
-                json={
-                    "text": prompt,
-                    "sampling_params": {
-                        "temperature": 0.0,
-                        "max_new_tokens": 16,
-                    },
-                },
-                timeout=60,
-            )
-            self.assertEqual(resp.status_code, 200, resp.text)
-            data = resp.json()
-            self.assertIn("text", data, data)
-            self.assertGreater(len(data["text"].strip()), 0, data)
-
-    def test_mmlu_sanity(self):
-        """32-example MMLU sanity slice.
-
-        ZAYA1-base is a pretrained (non instruction-tuned) checkpoint that
-        emits long ``<think>…</think>`` reasoning blocks before settling on a
-        final letter, so ``max_tokens`` must be large enough for the evaluator
-        to see the chosen answer. The threshold sits just above chance: it is
-        a regression sanity check rather than a production-quality gate. An
-        instruction-tuned ZAYA1 checkpoint scores meaningfully higher and
-        should raise this bound when wired in.
-        """
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            model=self.model,
-            eval_name="mmlu",
-            num_examples=32,
-            num_threads=8,
-            max_tokens=1024,
-        )
-        metrics = run_eval(args)
-        self.assertGreaterEqual(
-            metrics["score"],
-            0.30,
-            f"MMLU sanity below threshold: {metrics}",
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/moe/test_torch_compile_moe.py

@@ -41,12 +41,13 @@ class TestTorchCompileMoe(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
         metrics = run_eval(args)
-        self.assertGreaterEqual(metrics["score"], 0.50)
+        # 0.48 measured, minus the 0.05 margin the other eval thresholds use.
+        self.assertGreaterEqual(metrics["score"], 0.43)
 
     def run_decode(self, max_new_tokens):
         response = requests.post(

test/registered/quant/test_autoround_quantization.py

@@ -26,9 +26,11 @@ from sglang.test.test_utils import (
 
 register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large")
 
-MMLU_NUM_EXAMPLES = 32
+MMLU_NUM_EXAMPLES = 256
 MMLU_NUM_THREADS = 32
-MMLU_SCORE_THRESHOLD = 22 / MMLU_NUM_EXAMPLES
+# The unquantized model scores 0.68-0.70 over 256 questions; int8 gives up a
+# point or two, then the 0.05 margin the other eval thresholds use.
+MMLU_SCORE_THRESHOLD = 0.63
 
 
 class TestAutoRoundQuantization(CustomTestCase):

test/registered/quant/test_awq.py

@@ -38,7 +38,7 @@ class TestAWQ(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
@@ -72,7 +72,7 @@ class TestAWQMarlinBfloat16(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 

test/registered/sampling/test_pytorch_sampling_backend.py

@@ -40,13 +40,13 @@ class TestPyTorchSamplingBackend(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
             temperature=0.1,
         )
 
         metrics = run_eval(args)
-        self.assertGreaterEqual(metrics["score"], 0.65)
+        self.assertGreaterEqual(metrics["score"], 0.64)
 
     @unittest.skipIf(
         is_in_amd_ci(),

test/registered/scheduler/test_retract_decode.py

@@ -51,12 +51,12 @@ class TestRetractDecode(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
         metrics = run_eval(args)
-        self.assertGreaterEqual(metrics["score"], 0.65)
+        self.assertGreaterEqual(metrics["score"], 0.64)
         time.sleep(1)  # wait for mem check
 
         assert self.process.poll() is None, "Server crashed during test"

test/registered/tokenizer/test_multi_tokenizer.py

@@ -27,8 +27,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
     """One server covering both worker pools: multi-tokenizer and
     multi-detokenizer (the flags are orthogonal)."""
 
-    mmlu_score_threshold = 0.65
-    mmlu_num_examples = 64
+    mmlu_score_threshold = 0.64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
     @classmethod

test/registered/unit/bench/test_simple_eval_gsm8k.py

@@ -129,6 +129,56 @@ class TestRunSglEval(CustomTestCase):
 
         self.assertNotIn("--num-examples", captured["cmd"])
 
+    def _capture_cmd(self, eval_name="gsm8k", **overrides):
+        captured = {}
+
+        def fake_run(cmd, **kwargs):
+            captured["cmd"] = cmd
+            out_dir = Path(cmd[cmd.index("--out-dir") + 1])
+            _write_fake_metrics(
+                out_dir,
+                eval_name,
+                {
+                    "model": "test-model",
+                    "latency_seconds": 1.0,
+                    "output_throughput_tps": 1.0,
+                    "aggregate": {"score": 0.5},
+                },
+            )
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+        with tempfile.TemporaryDirectory() as td:
+            args = self._args(td, **overrides)
+            with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
+                _run_sgl_eval(eval_name, args)
+        return captured["cmd"]
+
+    def test_omits_sampling_flags_when_unset(self):
+        """Unset top_p / seed / repeat must not reach the CLI -- sgl-eval's own
+        defaults differ from a forced value (seed unset != seed 0)."""
+        cmd = self._capture_cmd()
+        for flag in ("--top-p", "--seed", "--n-repeats"):
+            self.assertNotIn(flag, cmd)
+
+    def test_forwards_sampling_flags_when_set(self):
+        cmd = self._capture_cmd(top_p=0.95, seed=0, repeat=1)
+        for flag, value in (("--top-p", "0.95"), ("--seed", "0"), ("--n-repeats", "1")):
+            self.assertIn(flag, cmd)
+            self.assertEqual(cmd[cmd.index(flag) + 1], value)
+
+    def test_thinking_auto_detected_from_model_name(self):
+        self.assertIn(
+            "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8")
+        )
+
+    def test_explicit_thinking_false_suppresses_auto_detect(self):
+        """A caller matching a harness that sent no chat_template_kwargs has to be
+        able to turn the model-name heuristic off."""
+        cmd = self._capture_cmd(
+            model="Qwen/Qwen3.5-397B-A17B-FP8", sgl_eval_thinking=False
+        )
+        self.assertNotIn("--thinking", cmd)
+
     def test_raises_on_nonzero_exit(self):
         def fake_run(cmd, **kwargs):
             return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")