[CI] Add Kimi-K3 MMMU-Pro accuracy coverage (#36284)

python/sglang/test/kits/eval_accuracy_kit.py

@@ -44,7 +44,7 @@ def _run_accuracy_eval(
     eval_name: str,
     score_threshold: float,
     num_examples: Optional[int],
-    num_threads: int,
+    num_threads: Optional[int],
     accept_length_thres: Optional[float] = None,
     summary_label: Optional[str] = None,
     **eval_overrides,
@@ -64,9 +64,10 @@ def _run_accuracy_eval(
         score_threshold == score_threshold
     ), f"{type(test_case).__name__} must set the {eval_name} score threshold"
 
+    model = eval_overrides.pop("model", getattr(test_case, "model", None))
     kwargs = dict(
         base_url=test_case.base_url,
-        model=getattr(test_case, "model", None),
+        model=model,
         eval_name=eval_name,
         num_examples=num_examples,
         num_threads=num_threads,
@@ -272,6 +273,41 @@ class MMLUMixin:
             )
 
 
+class MMMUProMixin:
+    """Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval.
+
+    The model preset supplies the endpoint model and all generation settings.
+    Leaving those values to sgl-eval is important for reasoning models whose
+    recommended token budget and sampling settings differ from run_eval defaults.
+
+    Required attributes on the test class:
+        base_url: str
+        mmmu_pro_score_threshold: float
+        mmmu_pro_load_preset_from_model_id: str
+    """
+
+    mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET
+    mmmu_pro_accept_length_thres: Optional[float] = None
+    mmmu_pro_num_examples: Optional[int] = 300
+    mmmu_pro_num_threads: Optional[int] = None
+    mmmu_pro_load_preset_from_model_id: Optional[str] = None
+
+    def test_mmmu_pro(self):
+        assert self.mmmu_pro_load_preset_from_model_id, (
+            f"{type(self).__name__} must set " "mmmu_pro_load_preset_from_model_id"
+        )
+        _run_accuracy_eval(
+            self,
+            eval_name="mmmu_pro",
+            score_threshold=self.mmmu_pro_score_threshold,
+            num_examples=self.mmmu_pro_num_examples,
+            num_threads=self.mmmu_pro_num_threads,
+            accept_length_thres=self.mmmu_pro_accept_length_thres,
+            model=None,
+            load_preset_from_model_id=self.mmmu_pro_load_preset_from_model_id,
+        )
+
+
 class GPQAMixin:
     """Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
 

python/sglang/test/run_eval.py

@@ -66,12 +66,15 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
         if value is not None:
             extra_body[param_name] = value
 
+    max_tokens = getattr(args, "max_tokens", None)
+    top_p = getattr(args, "top_p", None)
+    temperature = getattr(args, "temperature", None)
     common_kwargs = dict(
         model=getattr(args, "model", None),
-        max_tokens=getattr(args, "max_tokens", 2048),
+        max_tokens=2048 if max_tokens is None else max_tokens,
-        top_p=getattr(args, "top_p", 1.0),
+        top_p=1.0 if top_p is None else top_p,
         base_url=base_url,
-        temperature=getattr(args, "temperature", 0.0),
+        temperature=0.0 if temperature is None else temperature,
     )
 
     api_mode = getattr(args, "api", "chat")
@@ -119,25 +122,32 @@ def _run_sgl_eval(eval_name, args) -> dict:
     ).expanduser()
     out_parent.mkdir(parents=True, exist_ok=True)
 
+    model_preset_id = getattr(args, "load_preset_from_model_id", None)
     cmd = [
         "sgl-eval",
         "run",
         eval_name,
         "--base-url",
         base_url,
-        "--num-threads",
-        str(getattr(args, "num_threads", 64)),
-        "--temperature",
-        str(getattr(args, "temperature", 0.0)),
         "--out-dir",
         str(out_parent),
     ]
+    if model_preset_id:
+        cmd += ["--load-preset-from-model-id", model_preset_id]
     if getattr(args, "model", None):
         cmd += ["--model", args.model]
     if getattr(args, "num_examples", None) is not None:
         cmd += ["--num-examples", str(args.num_examples)]
+    if getattr(args, "num_threads", None) is not None:
+        cmd += ["--num-threads", str(args.num_threads)]
+    if getattr(args, "temperature", None) is not None:
+        cmd += ["--temperature", str(args.temperature)]
+    elif not model_preset_id:
+        cmd += ["--temperature", "0.0"]
     if getattr(args, "top_p", None) is not None:
         cmd += ["--top-p", str(args.top_p)]
+    elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False):
+        cmd += ["--top-p", "1.0"]
     # Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
     if getattr(args, "seed", None) is not None:
         cmd += ["--seed", str(args.seed)]
@@ -146,15 +156,17 @@ def _run_sgl_eval(eval_name, args) -> dict:
     # Bound generation length so long-reasoning models don't stall the eval.
     if getattr(args, "max_tokens", None) is not None:
         cmd += ["--max-tokens", str(args.max_tokens)]
-    else:
+    elif not model_preset_id:
         cmd += ["--max-tokens", "2048"]
     # Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
     # without --thinking their message.content is empty and sgl-eval scores 0.
-    if getattr(args, "sgl_eval_thinking", None) is None:
+    sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None)
+    if sgl_eval_thinking is None:
+        if not model_preset_id:
             model_l = (getattr(args, "model", None) or "").lower()
             if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
                 cmd += ["--thinking"]
-    elif args.sgl_eval_thinking:
+    elif sgl_eval_thinking:
         cmd += ["--thinking"]
 
     try:
@@ -308,6 +320,9 @@ def run_eval(args):
             args.num_threads,
             response_answer_regex=getattr(args, "response_answer_regex", None),
         )
+    elif args.eval_name in ("mmmu_pro", "mmmu-pro"):
+        # Canonical sgl-eval name for MMMU-Pro's standard 10-option split.
+        return _run_sgl_eval("mmmu_pro", args)
     elif args.eval_name == "mmmu_pro_vision":
         # sgl-eval owns this benchmark's dataset, prompt and grader; there is no
         # simple_eval implementation to fall back to.
@@ -465,6 +480,12 @@ if __name__ == "__main__":
         type=str,
         help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
     )
+    parser.add_argument(
+        "--load-preset-from-model-id",
+        type=str,
+        default=None,
+        help="Load repository-maintained sgl-eval generation defaults for this model ID.",
+    )
     parser.add_argument(
         "--repeat", type=int, default=1, help="repeat the evaluation n times"
     )
@@ -478,9 +499,9 @@ if __name__ == "__main__":
     )
     parser.add_argument("--num-examples", type=int)
     parser.add_argument("--num-threads", type=int, default=512)
-    parser.add_argument("--max-tokens", type=int, default=2048)
+    parser.add_argument("--max-tokens", type=int, default=None)
-    parser.add_argument("--temperature", type=float, default=0.0)
+    parser.add_argument("--temperature", type=float, default=None)
-    parser.add_argument("--top-p", type=float, default=1.0)
+    parser.add_argument("--top-p", type=float, default=None)
     parser.add_argument(
         "--top-k", type=int, default=None, help="Top-k sampling parameter"
     )
@@ -551,5 +572,6 @@ if __name__ == "__main__":
     )
 
     args = parser.parse_args()
+    args._sgl_eval_from_cli = True
 
     run_eval(args)

scripts/ci/utils/sgl_eval_ref.sh

@@ -9,5 +9,5 @@
 # MODEL_SCORE_THRESHOLDS in
 # test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds
 # of run_eval's other callers, before changing this.
-SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d"
+SGL_EVAL_REF="a231b7a439b235090ff7baa30778fa2b514309ae"
 SGL_EVAL_SPEC="sgl-eval@git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"

test/registered/models_e2e/test_kimi_k3_b300.py

@@ -1,8 +1,7 @@
 """B300 per-commit CI coverage for Kimi-K3 serving recipes.
 
-Runs the Low Latency DSPARK, Balanced DCP/HiCache, and MegaMoE recipes on
+Runs the Balanced DCP/HiCache and MegaMoE recipes on eight B300 GPUs, retaining
-eight B300 GPUs. Each server must preserve basic model quality on GSM8K, and
+their GSM8K accuracy gates.
-the Low Latency recipe must also preserve single-request decode performance.
 """
 
 import unittest
@@ -10,7 +9,6 @@ import unittest
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
-from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
@@ -37,59 +35,6 @@ def _stop_server(process):
         _wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT)
 
 
-class TestKimiK3B300LowLatency(GSM8KMixin, SpecDecodingMixin, CustomTestCase):
-    """TP8 Low Latency recipe with DSPARK linear ReplaySSM speculation."""
-
-    gsm8k_score_threshold = 0.95
-    gsm8k_num_examples = 200
-    gsm8k_num_threads = 37
-    # Gated on GSM8K rather than on test_bs_1_speed below: a 200-question
-    # average holds steady when a numerics change moves where the single
-    # greedy prompt hits EOS.
-    gsm8k_accept_length_thres = 4.5
-    # Both scale with how far that one greedy prompt runs, and speed is
-    # end-to-end, so launch and TTFT are amortized over the output -- it sits
-    # well below the steady decode rate the server logs. Coarse guards only.
-    accept_length_thres = 4.0
-    bs_1_speed_thres = 300
-
-    @classmethod
-    def setUpClass(cls):
-        cls.model = MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=SERVER_LAUNCH_TIMEOUT,
-            other_args=[
-                "--trust-remote-code",
-                "--tp-size",
-                "8",
-                "--mem-fraction-static",
-                "0.85",
-                "--model-loader-extra-config",
-                MODEL_LOADER_EXTRA_CONFIG,
-                "--reasoning-parser",
-                "kimi_k3",
-                "--tool-call-parser",
-                "kimi_k3",
-                "--mamba-full-memory-ratio",
-                "0.86",
-                "--speculative-algorithm",
-                "DSPARK",
-                "--speculative-draft-model-path",
-                DSPARK_DRAFT_MODEL,
-                "--speculative-dspark-block-size",
-                "7",
-                "--enable-linear-replayssm-spec",
-            ],
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        _stop_server(getattr(cls, "process", None))
-
-
 class TestKimiK3B300Balanced(GSM8KMixin, CustomTestCase):
     """TP8/DCP8 Balanced recipe with hierarchical cache."""
 

test/registered/models_e2e/test_kimi_k3_b300_low_latency.py

@@ -0,0 +1,89 @@
+"""B300 per-commit CI coverage for the Kimi-K3 Low Latency recipe.
+
+Runs the TP8 DSPARK recipe on eight B300 GPUs and checks MMMU-Pro quality,
+speculative acceptance, and single-request decode performance.
+"""
+
+import unittest
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.kits.eval_accuracy_kit import MMMUProMixin
+from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
+from sglang.test.test_utils import (
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    _wait_for_gpu_idle_in_ci,
+    popen_launch_server,
+)
+
+register_cuda_ci(est_time=1800, stage="base-c", runner_config="8-gpu-b300")
+
+MODEL_PATH = "moonshotai/Kimi-K3"
+DSPARK_DRAFT_MODEL = "RadixArk/Kimi-K3-DSpark"
+MODEL_LOADER_EXTRA_CONFIG = '{"enable_multithread_load": true, "num_threads": 12}'
+SERVER_LAUNCH_TIMEOUT = 3600
+GPU_IDLE_TIMEOUT = 120
+
+
+def _stop_server(process):
+    if process:
+        kill_process_tree(process.pid)
+        _wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT)
+
+
+class TestKimiK3B300LowLatency(MMMUProMixin, SpecDecodingMixin, CustomTestCase):
+    """TP8 Low Latency recipe with DSPARK linear ReplaySSM speculation."""
+
+    mmmu_pro_score_threshold = 0.75
+    mmmu_pro_num_examples = 200
+    mmmu_pro_load_preset_from_model_id = MODEL_PATH
+    # MMMU-Pro's long multimodal reasoning has a lower speculative average than
+    # GSM8K (2.62 in the first B300 run). Keep a workload-specific regression
+    # gate here; test_bs_1_speed below retains the stricter single-prompt gate.
+    mmmu_pro_accept_length_thres = 2.4
+    # Both scale with how far that one greedy prompt runs, and speed is
+    # end-to-end, so launch and TTFT are amortized over the output -- it sits
+    # well below the steady decode rate the server logs. Coarse guards only.
+    accept_length_thres = 4.0
+    bs_1_speed_thres = 300
+
+    @classmethod
+    def setUpClass(cls):
+        cls.model = MODEL_PATH
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.process = popen_launch_server(
+            cls.model,
+            cls.base_url,
+            timeout=SERVER_LAUNCH_TIMEOUT,
+            other_args=[
+                "--trust-remote-code",
+                "--tp-size",
+                "8",
+                "--mem-fraction-static",
+                "0.85",
+                "--model-loader-extra-config",
+                MODEL_LOADER_EXTRA_CONFIG,
+                "--reasoning-parser",
+                "kimi_k3",
+                "--tool-call-parser",
+                "kimi_k3",
+                "--mamba-full-memory-ratio",
+                "0.86",
+                "--speculative-algorithm",
+                "DSPARK",
+                "--speculative-draft-model-path",
+                DSPARK_DRAFT_MODEL,
+                "--speculative-dspark-block-size",
+                "7",
+                "--enable-linear-replayssm-spec",
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        _stop_server(getattr(cls, "process", None))
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/unit/bench/test_simple_eval_gsm8k.py

@@ -7,7 +7,7 @@ from types import SimpleNamespace
 from unittest.mock import patch
 
 from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.run_eval import _run_sgl_eval
+from sglang.test.run_eval import _run_sgl_eval, run_eval
 from sglang.test.test_utils import CustomTestCase
 
 register_cpu_ci(est_time=6, suite="base-b-test-cpu")
@@ -166,6 +166,53 @@ class TestRunSglEval(CustomTestCase):
             self.assertIn(flag, cmd)
             self.assertEqual(cmd[cmd.index(flag) + 1], value)
 
+    def test_model_preset_owns_model_and_sampling_defaults(self):
+        cmd = self._capture_cmd(
+            eval_name="mmmu_pro",
+            model=None,
+            num_examples=300,
+            num_threads=None,
+            temperature=None,
+            load_preset_from_model_id="moonshotai/Kimi-K3",
+        )
+
+        self.assertEqual(cmd[:3], ["sgl-eval", "run", "mmmu_pro"])
+        self.assertIn("--load-preset-from-model-id", cmd)
+        self.assertEqual(
+            cmd[cmd.index("--load-preset-from-model-id") + 1],
+            "moonshotai/Kimi-K3",
+        )
+        self.assertEqual(cmd[cmd.index("--num-examples") + 1], "300")
+        for flag in (
+            "--model",
+            "--num-threads",
+            "--temperature",
+            "--top-p",
+            "--max-tokens",
+            "--thinking",
+        ):
+            self.assertNotIn(flag, cmd)
+
+    def test_non_preset_cli_keeps_legacy_top_p_default(self):
+        cmd = self._capture_cmd(top_p=None, _sgl_eval_from_cli=True)
+
+        self.assertIn("--top-p", cmd)
+        self.assertEqual(cmd[cmd.index("--top-p") + 1], "1.0")
+
+    @patch("sglang.test.run_eval._run_sgl_eval", return_value={"score": 0.8})
+    def test_run_eval_dispatches_hyphenated_mmmu_pro_name(self, mock_sgl_eval):
+        args = SimpleNamespace(
+            base_url="http://127.0.0.1:30000",
+            eval_name="mmmu-pro",
+        )
+
+        try:
+            result = run_eval(args)
+        except ValueError as exc:
+            self.fail(f"mmmu-pro must dispatch to sgl-eval: {exc}")
+        self.assertEqual(result, {"score": 0.8})
+        mock_sgl_eval.assert_called_once_with("mmmu_pro", args)
+
     def test_thinking_auto_detected_from_model_name(self):
         self.assertIn(
             "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8")

test/registered/unit/test_eval_accuracy_kit_sgl_eval.py

@@ -1,4 +1,4 @@
-"""Unit tests for the GSM8K backend dispatch + sgl-eval skip in eval_accuracy_kit.
+"""Unit tests for sgl-eval-backed accuracy mixin dispatch.
 
 Hermetic (no server, no real sgl-eval install). These guard the behavior that
 existing consumers rely on -- not the sgl-eval happy path, which the live
@@ -8,7 +8,8 @@ accuracy runs already cover:
      the ~47 existing GSM8K consumers must never be silently rerouted.
   2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail
      gate when the canonical ``gsm8k_score_threshold`` is unset.
-  3. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent,
+  3. MMMU-Pro delegates model and sampling selection to a built-in model preset.
+  4. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent,
      so CI without the optional dependency stays green.
 """
 
@@ -20,7 +21,7 @@ import requests
 
 from sglang.test.ci.ci_register import register_cpu_ci
 from sglang.test.kits import eval_accuracy_kit as kit
-from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin
+from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin, MMMUProMixin
 from sglang.test.test_utils import CustomTestCase
 
 register_cpu_ci(est_time=5, suite="base-a-test-cpu")
@@ -95,6 +96,39 @@ class TestEvalKitBackendDispatch(CustomTestCase):
             with self.assertRaises(unittest.SkipTest):
                 host.test_gpqa()
 
+    def _run_mmmu_pro(self, score):
+        captured = {}
+
+        def fake_run_eval(args):
+            captured["args"] = args
+            return {"score": score}
+
+        host = _make_host(MMMUProMixin, "test_mmmu_pro")
+        host.base_url = "http://127.0.0.1:0"
+        host.model = "deployment-model"
+        host.mmmu_pro_score_threshold = 0.75
+        host.mmmu_pro_load_preset_from_model_id = "moonshotai/Kimi-K3"
+        with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object(
+            kit.requests, "get", side_effect=_fake_get
+        ):
+            host.test_mmmu_pro()
+        return captured["args"]
+
+    def test_mmmu_pro_uses_kimi_preset_and_300_examples(self):
+        args = self._run_mmmu_pro(0.80)
+
+        self.assertEqual(args.eval_name, "mmmu_pro")
+        self.assertEqual(args.load_preset_from_model_id, "moonshotai/Kimi-K3")
+        self.assertEqual(args.num_examples, 300)
+        self.assertIsNone(args.num_threads)
+        self.assertIsNone(args.model)
+        for attr in ("temperature", "top_p", "max_tokens", "reasoning_effort"):
+            self.assertFalse(hasattr(args, attr))
+
+    def test_mmmu_pro_score_threshold_gates_result(self):
+        with self.assertRaises(AssertionError):
+            self._run_mmmu_pro(0.74)
+
 
 if __name__ == "__main__":
     unittest.main()