minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Add Kimi-K3 MMMU-Pro accuracy coverage (#36284)

Browse Source
This commit is contained in:
Baizhou Zhang
2026-08-25 16:33:46 -07:00
committed by GitHub
parent aa718f7343
commit 2d88c79b3e
7 changed files with 253 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/kits/eval_accuracy_kit.py
+38 -2
View File
@@ -44,7 +44,7 @@ def _run_accuracy_eval(
eval_name: str,
score_threshold: float,
num_examples: Optional[int],
num_threads: int,
num_threads: Optional[int],
accept_length_thres: Optional[float] = None,
summary_label: Optional[str] = None,
**eval_overrides,
@@ -64,9 +64,10 @@ def _run_accuracy_eval(
score_threshold == score_threshold
), f"{type(test_case).__name__} must set the {eval_name} score threshold"
model = eval_overrides.pop("model", getattr(test_case, "model", None))
kwargs = dict(
base_url=test_case.base_url,
model=getattr(test_case, "model", None),
model=model,
eval_name=eval_name,
num_examples=num_examples,
num_threads=num_threads,
@@ -272,6 +273,41 @@ class MMLUMixin:
)
class MMMUProMixin:
"""Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval.
The model preset supplies the endpoint model and all generation settings.
Leaving those values to sgl-eval is important for reasoning models whose
recommended token budget and sampling settings differ from run_eval defaults.
Required attributes on the test class:
base_url: str
mmmu_pro_score_threshold: float
mmmu_pro_load_preset_from_model_id: str
"""
mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET
mmmu_pro_accept_length_thres: Optional[float] = None
mmmu_pro_num_examples: Optional[int] = 300
mmmu_pro_num_threads: Optional[int] = None
mmmu_pro_load_preset_from_model_id: Optional[str] = None
def test_mmmu_pro(self):
assert self.mmmu_pro_load_preset_from_model_id, (
f"{type(self).__name__} must set " "mmmu_pro_load_preset_from_model_id"
)
_run_accuracy_eval(
self,
eval_name="mmmu_pro",
score_threshold=self.mmmu_pro_score_threshold,
num_examples=self.mmmu_pro_num_examples,
num_threads=self.mmmu_pro_num_threads,
accept_length_thres=self.mmmu_pro_accept_length_thres,
model=None,
load_preset_from_model_id=self.mmmu_pro_load_preset_from_model_id,
)
class GPQAMixin:
"""Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
python/sglang/test/run_eval.py
+38 -16
View File
@@ -66,12 +66,15 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
if value is not None:
extra_body[param_name] = value
max_tokens = getattr(args, "max_tokens", None)
top_p = getattr(args, "top_p", None)
temperature = getattr(args, "temperature", None)
common_kwargs = dict(
model=getattr(args, "model", None),
max_tokens=getattr(args, "max_tokens", 2048),
top_p=getattr(args, "top_p", 1.0),
max_tokens=2048 if max_tokens is None else max_tokens,
top_p=1.0 if top_p is None else top_p,
base_url=base_url,
temperature=getattr(args, "temperature", 0.0),
temperature=0.0 if temperature is None else temperature,
)
api_mode = getattr(args, "api", "chat")
@@ -119,25 +122,32 @@ def _run_sgl_eval(eval_name, args) -> dict:
).expanduser()
out_parent.mkdir(parents=True, exist_ok=True)
model_preset_id = getattr(args, "load_preset_from_model_id", None)
cmd = [
"sgl-eval",
"run",
eval_name,
"--base-url",
base_url,
"--num-threads",
str(getattr(args, "num_threads", 64)),
"--temperature",
str(getattr(args, "temperature", 0.0)),
"--out-dir",
str(out_parent),
]
if model_preset_id:
cmd += ["--load-preset-from-model-id", model_preset_id]
if getattr(args, "model", None):
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
if getattr(args, "num_threads", None) is not None:
cmd += ["--num-threads", str(args.num_threads)]
if getattr(args, "temperature", None) is not None:
cmd += ["--temperature", str(args.temperature)]
elif not model_preset_id:
cmd += ["--temperature", "0.0"]
if getattr(args, "top_p", None) is not None:
cmd += ["--top-p", str(args.top_p)]
elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False):
cmd += ["--top-p", "1.0"]
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
if getattr(args, "seed", None) is not None:
cmd += ["--seed", str(args.seed)]
@@ -146,15 +156,17 @@ def _run_sgl_eval(eval_name, args) -> dict:
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
else:
elif not model_preset_id:
cmd += ["--max-tokens", "2048"]
# Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
# without --thinking their message.content is empty and sgl-eval scores 0.
if getattr(args, "sgl_eval_thinking", None) is None:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif args.sgl_eval_thinking:
sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None)
if sgl_eval_thinking is None:
if not model_preset_id:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif sgl_eval_thinking:
cmd += ["--thinking"]
try:
@@ -308,6 +320,9 @@ def run_eval(args):
args.num_threads,
response_answer_regex=getattr(args, "response_answer_regex", None),
)
elif args.eval_name in ("mmmu_pro", "mmmu-pro"):
# Canonical sgl-eval name for MMMU-Pro's standard 10-option split.
return _run_sgl_eval("mmmu_pro", args)
elif args.eval_name == "mmmu_pro_vision":
# sgl-eval owns this benchmark's dataset, prompt and grader; there is no
# simple_eval implementation to fall back to.
@@ -465,6 +480,12 @@ if __name__ == "__main__":
type=str,
help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
)
parser.add_argument(
"--load-preset-from-model-id",
type=str,
default=None,
help="Load repository-maintained sgl-eval generation defaults for this model ID.",
)
parser.add_argument(
"--repeat", type=int, default=1, help="repeat the evaluation n times"
)
@@ -478,9 +499,9 @@ if __name__ == "__main__":
)
parser.add_argument("--num-examples", type=int)
parser.add_argument("--num-threads", type=int, default=512)
parser.add_argument("--max-tokens", type=int, default=2048)
parser.add_argument("--temperature", type=float, default=0.0)
parser.add_argument("--top-p", type=float, default=1.0)
parser.add_argument("--max-tokens", type=int, default=None)
parser.add_argument("--temperature", type=float, default=None)
parser.add_argument("--top-p", type=float, default=None)
parser.add_argument(
"--top-k", type=int, default=None, help="Top-k sampling parameter"
)
@@ -551,5 +572,6 @@ if __name__ == "__main__":
)
args = parser.parse_args()
args._sgl_eval_from_cli = True
run_eval(args)
scripts/ci/utils/sgl_eval_ref.sh
+1 -1
View File
@@ -9,5 +9,5 @@
# MODEL_SCORE_THRESHOLDS in
# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds
# of run_eval's other callers, before changing this.
SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d"
SGL_EVAL_REF="a231b7a439b235090ff7baa30778fa2b514309ae"
SGL_EVAL_SPEC="sgl-eval@git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"
test/registered/models_e2e/test_kimi_k3_b300.py
+2 -57
View File
@@ -1,8 +1,7 @@
"""B300 per-commit CI coverage for Kimi-K3 serving recipes.
Runs the Low Latency DSPARK, Balanced DCP/HiCache, and MegaMoE recipes on
eight B300 GPUs. Each server must preserve basic model quality on GSM8K, and
the Low Latency recipe must also preserve single-request decode performance.
Runs the Balanced DCP/HiCache and MegaMoE recipes on eight B300 GPUs, retaining
their GSM8K accuracy gates.
"""
import unittest
@@ -10,7 +9,6 @@ import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
@@ -37,59 +35,6 @@ def _stop_server(process):
_wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT)
class TestKimiK3B300LowLatency(GSM8KMixin, SpecDecodingMixin, CustomTestCase):
"""TP8 Low Latency recipe with DSPARK linear ReplaySSM speculation."""
gsm8k_score_threshold = 0.95
gsm8k_num_examples = 200
gsm8k_num_threads = 37
# Gated on GSM8K rather than on test_bs_1_speed below: a 200-question
# average holds steady when a numerics change moves where the single
# greedy prompt hits EOS.
gsm8k_accept_length_thres = 4.5
# Both scale with how far that one greedy prompt runs, and speed is
# end-to-end, so launch and TTFT are amortized over the output -- it sits
# well below the steady decode rate the server logs. Coarse guards only.
accept_length_thres = 4.0
bs_1_speed_thres = 300
@classmethod
def setUpClass(cls):
cls.model = MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=[
"--trust-remote-code",
"--tp-size",
"8",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
MODEL_LOADER_EXTRA_CONFIG,
"--reasoning-parser",
"kimi_k3",
"--tool-call-parser",
"kimi_k3",
"--mamba-full-memory-ratio",
"0.86",
"--speculative-algorithm",
"DSPARK",
"--speculative-draft-model-path",
DSPARK_DRAFT_MODEL,
"--speculative-dspark-block-size",
"7",
"--enable-linear-replayssm-spec",
],
)
@classmethod
def tearDownClass(cls):
_stop_server(getattr(cls, "process", None))
class TestKimiK3B300Balanced(GSM8KMixin, CustomTestCase):
"""TP8/DCP8 Balanced recipe with hierarchical cache."""
test/registered/models_e2e/test_kimi_k3_b300_low_latency.py
+89
View File
@@ -0,0 +1,89 @@
"""B300 per-commit CI coverage for the Kimi-K3 Low Latency recipe.
Runs the TP8 DSPARK recipe on eight B300 GPUs and checks MMMU-Pro quality,
speculative acceptance, and single-request decode performance.
"""
import unittest
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import MMMUProMixin
from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
_wait_for_gpu_idle_in_ci,
popen_launch_server,
)
register_cuda_ci(est_time=1800, stage="base-c", runner_config="8-gpu-b300")
MODEL_PATH = "moonshotai/Kimi-K3"
DSPARK_DRAFT_MODEL = "RadixArk/Kimi-K3-DSpark"
MODEL_LOADER_EXTRA_CONFIG = '{"enable_multithread_load": true, "num_threads": 12}'
SERVER_LAUNCH_TIMEOUT = 3600
GPU_IDLE_TIMEOUT = 120
def _stop_server(process):
if process:
kill_process_tree(process.pid)
_wait_for_gpu_idle_in_ci(timeout=GPU_IDLE_TIMEOUT)
class TestKimiK3B300LowLatency(MMMUProMixin, SpecDecodingMixin, CustomTestCase):
"""TP8 Low Latency recipe with DSPARK linear ReplaySSM speculation."""
mmmu_pro_score_threshold = 0.75
mmmu_pro_num_examples = 200
mmmu_pro_load_preset_from_model_id = MODEL_PATH
# MMMU-Pro's long multimodal reasoning has a lower speculative average than
# GSM8K (2.62 in the first B300 run). Keep a workload-specific regression
# gate here; test_bs_1_speed below retains the stricter single-prompt gate.
mmmu_pro_accept_length_thres = 2.4
# Both scale with how far that one greedy prompt runs, and speed is
# end-to-end, so launch and TTFT are amortized over the output -- it sits
# well below the steady decode rate the server logs. Coarse guards only.
accept_length_thres = 4.0
bs_1_speed_thres = 300
@classmethod
def setUpClass(cls):
cls.model = MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=[
"--trust-remote-code",
"--tp-size",
"8",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
MODEL_LOADER_EXTRA_CONFIG,
"--reasoning-parser",
"kimi_k3",
"--tool-call-parser",
"kimi_k3",
"--mamba-full-memory-ratio",
"0.86",
"--speculative-algorithm",
"DSPARK",
"--speculative-draft-model-path",
DSPARK_DRAFT_MODEL,
"--speculative-dspark-block-size",
"7",
"--enable-linear-replayssm-spec",
],
)
@classmethod
def tearDownClass(cls):
_stop_server(getattr(cls, "process", None))
if __name__ == "__main__":
unittest.main()
test/registered/unit/bench/test_simple_eval_gsm8k.py
+48 -1
View File
@@ -7,7 +7,7 @@ from types import SimpleNamespace
from unittest.mock import patch
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.run_eval import _run_sgl_eval
from sglang.test.run_eval import _run_sgl_eval, run_eval
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=6, suite="base-b-test-cpu")
@@ -166,6 +166,53 @@ class TestRunSglEval(CustomTestCase):
self.assertIn(flag, cmd)
self.assertEqual(cmd[cmd.index(flag) + 1], value)
def test_model_preset_owns_model_and_sampling_defaults(self):
cmd = self._capture_cmd(
eval_name="mmmu_pro",
model=None,
num_examples=300,
num_threads=None,
temperature=None,
load_preset_from_model_id="moonshotai/Kimi-K3",
)
self.assertEqual(cmd[:3], ["sgl-eval", "run", "mmmu_pro"])
self.assertIn("--load-preset-from-model-id", cmd)
self.assertEqual(
cmd[cmd.index("--load-preset-from-model-id") + 1],
"moonshotai/Kimi-K3",
)
self.assertEqual(cmd[cmd.index("--num-examples") + 1], "300")
for flag in (
"--model",
"--num-threads",
"--temperature",
"--top-p",
"--max-tokens",
"--thinking",
):
self.assertNotIn(flag, cmd)
def test_non_preset_cli_keeps_legacy_top_p_default(self):
cmd = self._capture_cmd(top_p=None, _sgl_eval_from_cli=True)
self.assertIn("--top-p", cmd)
self.assertEqual(cmd[cmd.index("--top-p") + 1], "1.0")
@patch("sglang.test.run_eval._run_sgl_eval", return_value={"score": 0.8})
def test_run_eval_dispatches_hyphenated_mmmu_pro_name(self, mock_sgl_eval):
args = SimpleNamespace(
base_url="http://127.0.0.1:30000",
eval_name="mmmu-pro",
)
try:
result = run_eval(args)
except ValueError as exc:
self.fail(f"mmmu-pro must dispatch to sgl-eval: {exc}")
self.assertEqual(result, {"score": 0.8})
mock_sgl_eval.assert_called_once_with("mmmu_pro", args)
def test_thinking_auto_detected_from_model_name(self):
self.assertIn(
"--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8")
test/registered/unit/test_eval_accuracy_kit_sgl_eval.py
+37 -3
View File
@@ -1,4 +1,4 @@
"""Unit tests for the GSM8K backend dispatch + sgl-eval skip in eval_accuracy_kit.
"""Unit tests for sgl-eval-backed accuracy mixin dispatch.
Hermetic (no server, no real sgl-eval install). These guard the behavior that
existing consumers rely on -- not the sgl-eval happy path, which the live
@@ -8,7 +8,8 @@ accuracy runs already cover:
the ~47 existing GSM8K consumers must never be silently rerouted.
2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail
gate when the canonical ``gsm8k_score_threshold`` is unset.
3. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent,
3. MMMU-Pro delegates model and sampling selection to a built-in model preset.
4. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent,
so CI without the optional dependency stays green.
"""
@@ -20,7 +21,7 @@ import requests
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.kits import eval_accuracy_kit as kit
from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin
from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin, MMMUProMixin
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
@@ -95,6 +96,39 @@ class TestEvalKitBackendDispatch(CustomTestCase):
with self.assertRaises(unittest.SkipTest):
host.test_gpqa()
def _run_mmmu_pro(self, score):
captured = {}
def fake_run_eval(args):
captured["args"] = args
return {"score": score}
host = _make_host(MMMUProMixin, "test_mmmu_pro")
host.base_url = "http://127.0.0.1:0"
host.model = "deployment-model"
host.mmmu_pro_score_threshold = 0.75
host.mmmu_pro_load_preset_from_model_id = "moonshotai/Kimi-K3"
with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object(
kit.requests, "get", side_effect=_fake_get
):
host.test_mmmu_pro()
return captured["args"]
def test_mmmu_pro_uses_kimi_preset_and_300_examples(self):
args = self._run_mmmu_pro(0.80)
self.assertEqual(args.eval_name, "mmmu_pro")
self.assertEqual(args.load_preset_from_model_id, "moonshotai/Kimi-K3")
self.assertEqual(args.num_examples, 300)
self.assertIsNone(args.num_threads)
self.assertIsNone(args.model)
for attr in ("temperature", "top_p", "max_tokens", "reasoning_effort"):
self.assertFalse(hasattr(args, attr))
def test_mmmu_pro_score_threshold_gates_result(self):
with self.assertRaises(AssertionError):
self._run_mmmu_pro(0.74)
if __name__ == "__main__":
unittest.main()