diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index b865dba9d..ae9509c2a 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -44,7 +44,7 @@ def _run_accuracy_eval( eval_name: str, score_threshold: float, num_examples: Optional[int], - num_threads: int, + num_threads: Optional[int], accept_length_thres: Optional[float] = None, summary_label: Optional[str] = None, **eval_overrides, @@ -64,9 +64,10 @@ def _run_accuracy_eval( score_threshold == score_threshold ), f"{type(test_case).__name__} must set the {eval_name} score threshold" + model = eval_overrides.pop("model", getattr(test_case, "model", None)) kwargs = dict( base_url=test_case.base_url, - model=getattr(test_case, "model", None), + model=model, eval_name=eval_name, num_examples=num_examples, num_threads=num_threads, @@ -272,6 +273,41 @@ class MMLUMixin: ) +class MMMUProMixin: + """Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval. + + The model preset supplies the endpoint model and all generation settings. + Leaving those values to sgl-eval is important for reasoning models whose + recommended token budget and sampling settings differ from run_eval defaults. + + Required attributes on the test class: + base_url: str + mmmu_pro_score_threshold: float + mmmu_pro_load_preset_from_model_id: str + """ + + mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET + mmmu_pro_accept_length_thres: Optional[float] = None + mmmu_pro_num_examples: Optional[int] = 300 + mmmu_pro_num_threads: Optional[int] = None + mmmu_pro_load_preset_from_model_id: Optional[str] = None + + def test_mmmu_pro(self): + assert self.mmmu_pro_load_preset_from_model_id, ( + f"{type(self).__name__} must set " "mmmu_pro_load_preset_from_model_id" + ) + _run_accuracy_eval( + self, + eval_name="mmmu_pro", + score_threshold=self.mmmu_pro_score_threshold, + num_examples=self.mmmu_pro_num_examples, + num_threads=self.mmmu_pro_num_threads, + accept_length_thres=self.mmmu_pro_accept_length_thres, + model=None, + load_preset_from_model_id=self.mmmu_pro_load_preset_from_model_id, + ) + + class GPQAMixin: """Mixin for GPQA-Diamond evaluation (graduate-level multiple choice). diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index f3b0ceadc..6a759ebb0 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -66,12 +66,15 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict: if value is not None: extra_body[param_name] = value + max_tokens = getattr(args, "max_tokens", None) + top_p = getattr(args, "top_p", None) + temperature = getattr(args, "temperature", None) common_kwargs = dict( model=getattr(args, "model", None), - max_tokens=getattr(args, "max_tokens", 2048), - top_p=getattr(args, "top_p", 1.0), + max_tokens=2048 if max_tokens is None else max_tokens, + top_p=1.0 if top_p is None else top_p, base_url=base_url, - temperature=getattr(args, "temperature", 0.0), + temperature=0.0 if temperature is None else temperature, ) api_mode = getattr(args, "api", "chat") @@ -119,25 +122,32 @@ def _run_sgl_eval(eval_name, args) -> dict: ).expanduser() out_parent.mkdir(parents=True, exist_ok=True) + model_preset_id = getattr(args, "load_preset_from_model_id", None) cmd = [ "sgl-eval", "run", eval_name, "--base-url", base_url, - "--num-threads", - str(getattr(args, "num_threads", 64)), - "--temperature", - str(getattr(args, "temperature", 0.0)), "--out-dir", str(out_parent), ] + if model_preset_id: + cmd += ["--load-preset-from-model-id", model_preset_id] if getattr(args, "model", None): cmd += ["--model", args.model] if getattr(args, "num_examples", None) is not None: cmd += ["--num-examples", str(args.num_examples)] + if getattr(args, "num_threads", None) is not None: + cmd += ["--num-threads", str(args.num_threads)] + if getattr(args, "temperature", None) is not None: + cmd += ["--temperature", str(args.temperature)] + elif not model_preset_id: + cmd += ["--temperature", "0.0"] if getattr(args, "top_p", None) is not None: cmd += ["--top-p", str(args.top_p)] + elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False): + cmd += ["--top-p", "1.0"] # Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it. if getattr(args, "seed", None) is not None: cmd += ["--seed", str(args.seed)] @@ -146,15 +156,17 @@ def _run_sgl_eval(eval_name, args) -> dict: # Bound generation length so long-reasoning models don't stall the eval. if getattr(args, "max_tokens", None) is not None: cmd += ["--max-tokens", str(args.max_tokens)] - else: + elif not model_preset_id: cmd += ["--max-tokens", "2048"] # Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel; # without --thinking their message.content is empty and sgl-eval scores 0. - if getattr(args, "sgl_eval_thinking", None) is None: - model_l = (getattr(args, "model", None) or "").lower() - if "qwen3.5" in model_l or "qwen3-thinking" in model_l: - cmd += ["--thinking"] - elif args.sgl_eval_thinking: + sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None) + if sgl_eval_thinking is None: + if not model_preset_id: + model_l = (getattr(args, "model", None) or "").lower() + if "qwen3.5" in model_l or "qwen3-thinking" in model_l: + cmd += ["--thinking"] + elif sgl_eval_thinking: cmd += ["--thinking"] try: @@ -308,6 +320,9 @@ def run_eval(args): args.num_threads, response_answer_regex=getattr(args, "response_answer_regex", None), ) + elif args.eval_name in ("mmmu_pro", "mmmu-pro"): + # Canonical sgl-eval name for MMMU-Pro's standard 10-option split. + return _run_sgl_eval("mmmu_pro", args) elif args.eval_name == "mmmu_pro_vision": # sgl-eval owns this benchmark's dataset, prompt and grader; there is no # simple_eval implementation to fall back to. @@ -465,6 +480,12 @@ if __name__ == "__main__": type=str, help="Name or path of the model. If not set, the default model will request /v1/models for conf.", ) + parser.add_argument( + "--load-preset-from-model-id", + type=str, + default=None, + help="Load repository-maintained sgl-eval generation defaults for this model ID.", + ) parser.add_argument( "--repeat", type=int, default=1, help="repeat the evaluation n times" ) @@ -478,9 +499,9 @@ if __name__ == "__main__": ) parser.add_argument("--num-examples", type=int) parser.add_argument("--num-threads", type=int, default=512) - parser.add_argument("--max-tokens", type=int, default=2048) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--top-p", type=float, default=1.0) + parser.add_argument("--max-tokens", type=int, default=None) + parser.add_argument("--temperature", type=float, default=None) + parser.add_argument("--top-p", type=float, default=None) parser.add_argument( "--top-k", type=int, default=None, help="Top-k sampling parameter" ) @@ -551,5 +572,6 @@ if __name__ == "__main__": ) args = parser.parse_args() + args._sgl_eval_from_cli = True run_eval(args) diff --git a/scripts/ci/utils/sgl_eval_ref.sh b/scripts/ci/utils/sgl_eval_ref.sh index a83d3a8a3..1e4044516 100644 --- a/scripts/ci/utils/sgl_eval_ref.sh +++ b/scripts/ci/utils/sgl_eval_ref.sh @@ -9,5 +9,5 @@ # MODEL_SCORE_THRESHOLDS in # test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds # of run_eval's other callers, before changing this. -SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d" +SGL_EVAL_REF="a231b7a439b235090ff7baa30778fa2b514309ae" SGL_EVAL_SPEC="sgl-eval@git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" diff --git a/test/registered/models_e2e/test_kimi_k3_b300.py b/test/registered/models_e2e/test_kimi_k3_b300.py index 41b2ca4e3..7fab627e2 100644 --- a/test/registered/models_e2e/test_kimi_k3_b300.py +++ b/test/registered/models_e2e/test_kimi_k3_b300.py @@ -1,8 +1,7 @@ """B300 per-commit CI coverage for Kimi-K3 serving recipes. -Runs the Low Latency DSPARK, Balanced DCP/HiCache, and MegaMoE recipes on -eight B300 GPUs. Each server must preserve basic model quality on GSM8K, and -the Low Latency recipe must also preserve single-request decode performance. +Runs the Balanced DCP/HiCache and MegaMoE recipes on eight B300 GPUs, retaining +their GSM8K accuracy gates. """ import unittest @@ -10,7 +9,6 @@ import unittest from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin -from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -37,59 +35,6 @@ def _stop_server(process): _wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT) -class TestKimiK3B300LowLatency(GSM8KMixin, SpecDecodingMixin, CustomTestCase): - """TP8 Low Latency recipe with DSPARK linear ReplaySSM speculation.""" - - gsm8k_score_threshold = 0.95 - gsm8k_num_examples = 200 - gsm8k_num_threads = 37 - # Gated on GSM8K rather than on test_bs_1_speed below: a 200-question - # average holds steady when a numerics change moves where the single - # greedy prompt hits EOS. - gsm8k_accept_length_thres = 4.5 - # Both scale with how far that one greedy prompt runs, and speed is - # end-to-end, so launch and TTFT are amortized over the output -- it sits - # well below the steady decode rate the server logs. Coarse guards only. - accept_length_thres = 4.0 - bs_1_speed_thres = 300 - - @classmethod - def setUpClass(cls): - cls.model = MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=[ - "--trust-remote-code", - "--tp-size", - "8", - "--mem-fraction-static", - "0.85", - "--model-loader-extra-config", - MODEL_LOADER_EXTRA_CONFIG, - "--reasoning-parser", - "kimi_k3", - "--tool-call-parser", - "kimi_k3", - "--mamba-full-memory-ratio", - "0.86", - "--speculative-algorithm", - "DSPARK", - "--speculative-draft-model-path", - DSPARK_DRAFT_MODEL, - "--speculative-dspark-block-size", - "7", - "--enable-linear-replayssm-spec", - ], - ) - - @classmethod - def tearDownClass(cls): - _stop_server(getattr(cls, "process", None)) - - class TestKimiK3B300Balanced(GSM8KMixin, CustomTestCase): """TP8/DCP8 Balanced recipe with hierarchical cache.""" diff --git a/test/registered/models_e2e/test_kimi_k3_b300_low_latency.py b/test/registered/models_e2e/test_kimi_k3_b300_low_latency.py new file mode 100644 index 000000000..5a5baa94a --- /dev/null +++ b/test/registered/models_e2e/test_kimi_k3_b300_low_latency.py @@ -0,0 +1,89 @@ +"""B300 per-commit CI coverage for the Kimi-K3 Low Latency recipe. + +Runs the TP8 DSPARK recipe on eight B300 GPUs and checks MMMU-Pro quality, +speculative acceptance, and single-request decode performance. +""" + +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import MMMUProMixin +from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + _wait_for_gpu_idle_in_ci, + popen_launch_server, +) + +register_cuda_ci(est_time=1800, stage="base-c", runner_config="8-gpu-b300") + +MODEL_PATH = "moonshotai/Kimi-K3" +DSPARK_DRAFT_MODEL = "RadixArk/Kimi-K3-DSpark" +MODEL_LOADER_EXTRA_CONFIG = '{"enable_multithread_load": true, "num_threads": 12}' +SERVER_LAUNCH_TIMEOUT = 3600 +GPU_IDLE_TIMEOUT = 120 + + +def _stop_server(process): + if process: + kill_process_tree(process.pid) + _wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT) + + +class TestKimiK3B300LowLatency(MMMUProMixin, SpecDecodingMixin, CustomTestCase): + """TP8 Low Latency recipe with DSPARK linear ReplaySSM speculation.""" + + mmmu_pro_score_threshold = 0.75 + mmmu_pro_num_examples = 200 + mmmu_pro_load_preset_from_model_id = MODEL_PATH + # MMMU-Pro's long multimodal reasoning has a lower speculative average than + # GSM8K (2.62 in the first B300 run). Keep a workload-specific regression + # gate here; test_bs_1_speed below retains the stricter single-prompt gate. + mmmu_pro_accept_length_thres = 2.4 + # Both scale with how far that one greedy prompt runs, and speed is + # end-to-end, so launch and TTFT are amortized over the output -- it sits + # well below the steady decode rate the server logs. Coarse guards only. + accept_length_thres = 4.0 + bs_1_speed_thres = 300 + + @classmethod + def setUpClass(cls): + cls.model = MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=[ + "--trust-remote-code", + "--tp-size", + "8", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + MODEL_LOADER_EXTRA_CONFIG, + "--reasoning-parser", + "kimi_k3", + "--tool-call-parser", + "kimi_k3", + "--mamba-full-memory-ratio", + "0.86", + "--speculative-algorithm", + "DSPARK", + "--speculative-draft-model-path", + DSPARK_DRAFT_MODEL, + "--speculative-dspark-block-size", + "7", + "--enable-linear-replayssm-spec", + ], + ) + + @classmethod + def tearDownClass(cls): + _stop_server(getattr(cls, "process", None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/bench/test_simple_eval_gsm8k.py b/test/registered/unit/bench/test_simple_eval_gsm8k.py index f5bf0fb43..4c58c5333 100644 --- a/test/registered/unit/bench/test_simple_eval_gsm8k.py +++ b/test/registered/unit/bench/test_simple_eval_gsm8k.py @@ -7,7 +7,7 @@ from types import SimpleNamespace from unittest.mock import patch from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.run_eval import _run_sgl_eval +from sglang.test.run_eval import _run_sgl_eval, run_eval from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=6, suite="base-b-test-cpu") @@ -166,6 +166,53 @@ class TestRunSglEval(CustomTestCase): self.assertIn(flag, cmd) self.assertEqual(cmd[cmd.index(flag) + 1], value) + def test_model_preset_owns_model_and_sampling_defaults(self): + cmd = self._capture_cmd( + eval_name="mmmu_pro", + model=None, + num_examples=300, + num_threads=None, + temperature=None, + load_preset_from_model_id="moonshotai/Kimi-K3", + ) + + self.assertEqual(cmd[:3], ["sgl-eval", "run", "mmmu_pro"]) + self.assertIn("--load-preset-from-model-id", cmd) + self.assertEqual( + cmd[cmd.index("--load-preset-from-model-id") + 1], + "moonshotai/Kimi-K3", + ) + self.assertEqual(cmd[cmd.index("--num-examples") + 1], "300") + for flag in ( + "--model", + "--num-threads", + "--temperature", + "--top-p", + "--max-tokens", + "--thinking", + ): + self.assertNotIn(flag, cmd) + + def test_non_preset_cli_keeps_legacy_top_p_default(self): + cmd = self._capture_cmd(top_p=None, _sgl_eval_from_cli=True) + + self.assertIn("--top-p", cmd) + self.assertEqual(cmd[cmd.index("--top-p") + 1], "1.0") + + @patch("sglang.test.run_eval._run_sgl_eval", return_value={"score": 0.8}) + def test_run_eval_dispatches_hyphenated_mmmu_pro_name(self, mock_sgl_eval): + args = SimpleNamespace( + base_url="http://127.0.0.1:30000", + eval_name="mmmu-pro", + ) + + try: + result = run_eval(args) + except ValueError as exc: + self.fail(f"mmmu-pro must dispatch to sgl-eval: {exc}") + self.assertEqual(result, {"score": 0.8}) + mock_sgl_eval.assert_called_once_with("mmmu_pro", args) + def test_thinking_auto_detected_from_model_name(self): self.assertIn( "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8") diff --git a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py b/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py index 70fc3a20d..48d075073 100644 --- a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py +++ b/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py @@ -1,4 +1,4 @@ -"""Unit tests for the GSM8K backend dispatch + sgl-eval skip in eval_accuracy_kit. +"""Unit tests for sgl-eval-backed accuracy mixin dispatch. Hermetic (no server, no real sgl-eval install). These guard the behavior that existing consumers rely on -- not the sgl-eval happy path, which the live @@ -8,7 +8,8 @@ accuracy runs already cover: the ~47 existing GSM8K consumers must never be silently rerouted. 2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail gate when the canonical ``gsm8k_score_threshold`` is unset. - 3. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent, + 3. MMMU-Pro delegates model and sampling selection to a built-in model preset. + 4. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent, so CI without the optional dependency stays green. """ @@ -20,7 +21,7 @@ import requests from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.kits import eval_accuracy_kit as kit -from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin +from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin, MMMUProMixin from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -95,6 +96,39 @@ class TestEvalKitBackendDispatch(CustomTestCase): with self.assertRaises(unittest.SkipTest): host.test_gpqa() + def _run_mmmu_pro(self, score): + captured = {} + + def fake_run_eval(args): + captured["args"] = args + return {"score": score} + + host = _make_host(MMMUProMixin, "test_mmmu_pro") + host.base_url = "http://127.0.0.1:0" + host.model = "deployment-model" + host.mmmu_pro_score_threshold = 0.75 + host.mmmu_pro_load_preset_from_model_id = "moonshotai/Kimi-K3" + with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object( + kit.requests, "get", side_effect=_fake_get + ): + host.test_mmmu_pro() + return captured["args"] + + def test_mmmu_pro_uses_kimi_preset_and_300_examples(self): + args = self._run_mmmu_pro(0.80) + + self.assertEqual(args.eval_name, "mmmu_pro") + self.assertEqual(args.load_preset_from_model_id, "moonshotai/Kimi-K3") + self.assertEqual(args.num_examples, 300) + self.assertIsNone(args.num_threads) + self.assertIsNone(args.model) + for attr in ("temperature", "top_p", "max_tokens", "reasoning_effort"): + self.assertFalse(hasattr(args, attr)) + + def test_mmmu_pro_score_threshold_gates_result(self): + with self.assertRaises(AssertionError): + self._run_mmmu_pro(0.74) + if __name__ == "__main__": unittest.main()