minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Add Kimi-K3 MMMU-Pro accuracy coverage (#36284)

Browse Source
This commit is contained in:
Baizhou Zhang
2026-08-25 16:33:46 -07:00
committed by GitHub
parent aa718f7343
commit 2d88c79b3e
7 changed files with 253 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/kits/eval_accuracy_kit.py
+38 -2
View File
@@ -44,7 +44,7 @@ def _run_accuracy_eval(
eval_name: str,
score_threshold: float,
num_examples: Optional[int],
num_threads: int,
num_threads: Optional[int],
accept_length_thres: Optional[float] = None,
summary_label: Optional[str] = None,
**eval_overrides,
@@ -64,9 +64,10 @@ def _run_accuracy_eval(
score_threshold == score_threshold
), f"{type(test_case).__name__} must set the {eval_name} score threshold"
model = eval_overrides.pop("model", getattr(test_case, "model", None))
kwargs = dict(
base_url=test_case.base_url,
model=getattr(test_case, "model", None),
model=model,
eval_name=eval_name,
num_examples=num_examples,
num_threads=num_threads,
@@ -272,6 +273,41 @@ class MMLUMixin:
)
class MMMUProMixin:
"""Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval.
The model preset supplies the endpoint model and all generation settings.
Leaving those values to sgl-eval is important for reasoning models whose
recommended token budget and sampling settings differ from run_eval defaults.
Required attributes on the test class:
base_url: str
mmmu_pro_score_threshold: float
mmmu_pro_load_preset_from_model_id: str
"""
mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET
mmmu_pro_accept_length_thres: Optional[float] = None
mmmu_pro_num_examples: Optional[int] = 300
mmmu_pro_num_threads: Optional[int] = None
mmmu_pro_load_preset_from_model_id: Optional[str] = None
def test_mmmu_pro(self):
assert self.mmmu_pro_load_preset_from_model_id, (
f"{type(self).__name__} must set " "mmmu_pro_load_preset_from_model_id"
)
_run_accuracy_eval(
self,
eval_name="mmmu_pro",
score_threshold=self.mmmu_pro_score_threshold,
num_examples=self.mmmu_pro_num_examples,
num_threads=self.mmmu_pro_num_threads,
accept_length_thres=self.mmmu_pro_accept_length_thres,
model=None,
load_preset_from_model_id=self.mmmu_pro_load_preset_from_model_id,
)
class GPQAMixin:
"""Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
python/sglang/test/run_eval.py
+38 -16
View File
@@ -66,12 +66,15 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
if value is not None:
extra_body[param_name] = value
max_tokens = getattr(args, "max_tokens", None)
top_p = getattr(args, "top_p", None)
temperature = getattr(args, "temperature", None)
common_kwargs = dict(
model=getattr(args, "model", None),
max_tokens=getattr(args, "max_tokens", 2048),
top_p=getattr(args, "top_p", 1.0),
max_tokens=2048 if max_tokens is None else max_tokens,
top_p=1.0 if top_p is None else top_p,
base_url=base_url,
temperature=getattr(args, "temperature", 0.0),
temperature=0.0 if temperature is None else temperature,
)
api_mode = getattr(args, "api", "chat")
@@ -119,25 +122,32 @@ def _run_sgl_eval(eval_name, args) -> dict:
).expanduser()
out_parent.mkdir(parents=True, exist_ok=True)
model_preset_id = getattr(args, "load_preset_from_model_id", None)
cmd = [
"sgl-eval",
"run",
eval_name,
"--base-url",
base_url,
"--num-threads",
str(getattr(args, "num_threads", 64)),
"--temperature",
str(getattr(args, "temperature", 0.0)),
"--out-dir",
str(out_parent),
]
if model_preset_id:
cmd += ["--load-preset-from-model-id", model_preset_id]
if getattr(args, "model", None):
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
if getattr(args, "num_threads", None) is not None:
cmd += ["--num-threads", str(args.num_threads)]
if getattr(args, "temperature", None) is not None:
cmd += ["--temperature", str(args.temperature)]
elif not model_preset_id:
cmd += ["--temperature", "0.0"]
if getattr(args, "top_p", None) is not None:
cmd += ["--top-p", str(args.top_p)]
elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False):
cmd += ["--top-p", "1.0"]
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
if getattr(args, "seed", None) is not None:
cmd += ["--seed", str(args.seed)]
@@ -146,15 +156,17 @@ def _run_sgl_eval(eval_name, args) -> dict:
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
else:
elif not model_preset_id:
cmd += ["--max-tokens", "2048"]
# Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
# without --thinking their message.content is empty and sgl-eval scores 0.
if getattr(args, "sgl_eval_thinking", None) is None:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif args.sgl_eval_thinking:
sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None)
if sgl_eval_thinking is None:
if not model_preset_id:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif sgl_eval_thinking:
cmd += ["--thinking"]
try:
@@ -308,6 +320,9 @@ def run_eval(args):
args.num_threads,
response_answer_regex=getattr(args, "response_answer_regex", None),
)
elif args.eval_name in ("mmmu_pro", "mmmu-pro"):
# Canonical sgl-eval name for MMMU-Pro's standard 10-option split.
return _run_sgl_eval("mmmu_pro", args)
elif args.eval_name == "mmmu_pro_vision":
# sgl-eval owns this benchmark's dataset, prompt and grader; there is no
# simple_eval implementation to fall back to.
@@ -465,6 +480,12 @@ if __name__ == "__main__":
type=str,
help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
)
parser.add_argument(
"--load-preset-from-model-id",
type=str,
default=None,
help="Load repository-maintained sgl-eval generation defaults for this model ID.",
)
parser.add_argument(
"--repeat", type=int, default=1, help="repeat the evaluation n times"
)
@@ -478,9 +499,9 @@ if __name__ == "__main__":
)
parser.add_argument("--num-examples", type=int)
parser.add_argument("--num-threads", type=int, default=512)
parser.add_argument("--max-tokens", type=int, default=2048)
parser.add_argument("--temperature", type=float, default=0.0)
parser.add_argument("--top-p", type=float, default=1.0)
parser.add_argument("--max-tokens", type=int, default=None)
parser.add_argument("--temperature", type=float, default=None)
parser.add_argument("--top-p", type=float, default=None)
parser.add_argument(
"--top-k", type=int, default=None, help="Top-k sampling parameter"
)
@@ -551,5 +572,6 @@ if __name__ == "__main__":
)
args = parser.parse_args()
args._sgl_eval_from_cli = True
run_eval(args)