minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-12 19:33:21 -07:00
committed by GitHub
parent cfdfd31826
commit 50cc1aa241
25 changed files with 182 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/accuracy_test_runner.py
+29 -323
View File
@@ -8,7 +8,6 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
dump_metric,
popen_launch_server,
write_github_step_summary,
)
@@ -31,6 +30,10 @@ class AccuracyTestParams:
top_k: Optional[int] = None
repeat: Optional[int] = None
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
seed: Optional[int] = None # pin for reproducibility when temperature > 0
# sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead
# of letting _run_sgl_eval infer it from the model name.
sgl_eval_thinking: Optional[bool] = None
@dataclass
@@ -89,6 +92,8 @@ def _run_simple_eval(
top_k: Optional[int] = None,
repeat: Optional[int] = None,
api: Optional[str] = None,
seed: Optional[int] = None,
sgl_eval_thinking: Optional[bool] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using simple_eval backend (run_eval.py).
@@ -137,6 +142,12 @@ def _run_simple_eval(
if repeat is not None:
args.repeat = repeat
if seed is not None:
args.seed = seed
if sgl_eval_thinking is not None:
args.sgl_eval_thinking = sgl_eval_thinking
result = run_eval(args)
# Handle result format (run_eval can return metrics or (metrics, latency))
@@ -156,299 +167,6 @@ def _run_simple_eval(
kill_process_tree(process.pid)
# Cached uv venv for NeMo Skills (persists across variants within a process).
_nemo_venv_dir: Optional[str] = None
_nemo_data_prepared: set = set()
def _get_nemo_venv() -> Tuple[str, dict]:
"""Get or create a uv venv with nemo_skills installed.
Returns (venv_python_path, env_dict) reusable across calls.
"""
import os
import subprocess
import tempfile
global _nemo_venv_dir
if _nemo_venv_dir is not None:
venv_python = f"{_nemo_venv_dir}/venv/bin/python"
env = {
**dict(os.environ),
"NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1",
"OPENAI_API_KEY": "dummy",
"VIRTUAL_ENV": f"{_nemo_venv_dir}/venv",
"PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""),
}
return venv_python, env
_nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_")
print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...")
# Create venv
result = subprocess.run(
["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"],
capture_output=True,
text=True,
)
if result.returncode != 0:
subprocess.run(
["uv", "venv", f"{_nemo_venv_dir}/venv"],
capture_output=True,
text=True,
)
# Install nemo_skills.
# Pinned: NeMo-Skills main after PR #1433 pins litellm==1.83.14 (httpx==0.28.1),
# which is unsatisfiable against nemo-run's transitive leptonai dep.
nemo_skills_ref = "589294c"
print(f"Installing nemo_skills (pinned to {nemo_skills_ref})...")
pip_result = subprocess.run(
[
"uv",
"pip",
"install",
"--python",
f"{_nemo_venv_dir}/venv/bin/python",
f"git+https://github.com/NVIDIA/NeMo-Skills.git@{nemo_skills_ref}",
"mcp<2",
"typer<0.27",
],
capture_output=True,
text=True,
timeout=300,
)
if pip_result.returncode != 0:
raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}")
print("NeMo Skills installed successfully")
return _get_nemo_venv()
def _ensure_nemo_data_prepared(
venv_python: str, env: dict, dataset: str
) -> Tuple[bool, Optional[str]]:
"""Prepare NeMo Skills dataset data if not already done.
Uses the venv python so data lands inside the venv's nemo_skills package.
"""
import subprocess
if dataset in _nemo_data_prepared:
return True, None
print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...")
result = subprocess.run(
[venv_python, "-m", "nemo_skills.dataset.prepare", dataset],
text=True,
timeout=600,
env=env,
)
if result.returncode != 0:
return False, f"Failed to prepare {dataset} data (exit {result.returncode})"
_nemo_data_prepared.add(dataset)
return True, None
def _run_nemo_skills_eval(
model: ModelLaunchSettings,
base_url: str,
dataset: str,
max_tokens: Optional[int] = None,
repeat: Optional[int] = None,
temperature: Optional[float] = None,
top_p: Optional[float] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro.
Uses an isolated uv venv (shared across variants) so nemo_skills dependencies
don't interfere with the system python / sglang server.
Returns:
Tuple of (success, error_message, metrics_dict)
"""
import subprocess
import tempfile
process = None
try:
# Get or create the shared venv (once per process)
venv_python, env = _get_nemo_venv()
# Prepare dataset (once per process, cached)
ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset)
if not ok:
return False, err, None
process = popen_launch_server(
model.model_path,
base_url,
other_args=model.extra_args,
timeout=model.launch_timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
env=model.env,
)
port = int(base_url.split(":")[-1])
server_address = f"http://127.0.0.1:{port}/v1"
repeat_val = repeat or 1
max_tokens_val = max_tokens or 32768
benchmark_spec = f"{dataset}:{repeat_val}"
# Build ns eval command using venv python
# Note: nemo_skills.pipeline.eval requires the "eval" subcommand
output_dir = tempfile.mkdtemp(prefix="ns_eval_output_")
cmd = [
venv_python,
"-m",
"nemo_skills.pipeline.eval",
"eval",
f"--benchmarks={benchmark_spec}",
"--server_type=sglang",
f"--model={model.model_path}",
f"--server_address={server_address}",
f"--output_dir={output_dir}",
f"++inference.tokens_to_generate={max_tokens_val}",
]
if temperature is not None:
cmd.append(f"++inference.temperature={temperature}")
if top_p is not None:
cmd.append(f"++inference.top_p={top_p}")
# Add VLM-specific config
if dataset in ("mmmu-pro", "mmmu_pro"):
cmd.append("++prompt_config=vlm/mmmu-pro")
cmd.append("++max_concurrent_requests=512")
cmd.append("++max_samples=500")
print(f"Running: {' '.join(cmd)}")
eval_result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=7200,
env=env,
)
print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)")
if eval_result.stderr:
print(eval_result.stderr[-1000:])
if eval_result.returncode != 0:
return (
False,
f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}",
None,
)
# Parse results
summarize_result = subprocess.run(
[
venv_python,
"-m",
"nemo_skills.pipeline.summarize_results",
f"{output_dir}/eval-results",
],
capture_output=True,
text=True,
timeout=60,
env=env,
)
output = summarize_result.stdout + "\n" + eval_result.stdout
print(f"Summary: {summarize_result.stdout[:1000]}")
# Parse accuracy from output (format varies, look for common patterns)
import re
score = None
for line in output.split("\n"):
match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE)
if match:
score = float(match.group(1))
if score is None:
# Try to find it in eval-results directory
import glob
import json
for result_file in glob.glob(
f"{output_dir}/eval-results/**/*.json", recursive=True
):
try:
with open(result_file) as f:
data = json.load(f)
if isinstance(data, dict):
score = (
data.get("accuracy")
or data.get("score")
or data.get("mean_score")
)
if score is not None:
break
except (json.JSONDecodeError, KeyError):
continue
if score is None:
# Last resort: compute accuracy directly from JSONL output
import glob
import json
for jsonl_file in sorted(
glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True)
):
correct = 0
total = 0
try:
with open(jsonl_file) as f:
for line in f:
line = line.strip()
if not line:
continue
entry = json.loads(line)
expected = entry.get("expected_answer", "")
generation = entry.get("generation", "")
# Extract "Answer: X" from the end of generation
answer_match = re.search(
r"Answer:\s*([A-J])", generation, re.IGNORECASE
)
if answer_match:
predicted = answer_match.group(1).upper()
if predicted == expected.upper():
correct += 1
total += 1
except (json.JSONDecodeError, KeyError, OSError):
continue
if total > 0:
score = correct / total
print(
f"Computed accuracy from {jsonl_file}: "
f"{correct}/{total} = {score:.4f}"
)
break
if score is None:
return False, "Could not parse accuracy from ns eval output", None
dump_metric(
f"{dataset}_score",
score,
labels={"model": model.model_path, "eval": dataset, "api": "nemo-skills"},
)
return True, None, {"score": score}
except subprocess.TimeoutExpired:
return False, "NeMo Skills eval timed out", None
except Exception as e:
return False, f"NeMo Skills eval exception: {str(e)}", None
finally:
if process:
kill_process_tree(process.pid)
def run_accuracy_test(
model: ModelLaunchSettings,
params: AccuracyTestParams,
@@ -472,35 +190,23 @@ def run_accuracy_test(
print(f" Baseline: {params.baseline_accuracy}")
print(f"{'='*60}\n")
# Run evaluation based on dataset type
# - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
# - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.)
if params.dataset in ("mmmu-pro", "mmmu_pro"):
success, error, metrics = _run_nemo_skills_eval(
model=model,
base_url=base_url,
dataset="mmmu-pro",
max_tokens=params.max_tokens,
repeat=params.repeat or 1,
temperature=params.temperature,
top_p=params.top_p,
)
else:
success, error, metrics = _run_simple_eval(
model=model,
base_url=base_url,
dataset=params.dataset,
num_examples=params.num_examples,
num_threads=params.num_threads,
max_tokens=params.max_tokens,
return_latency=params.return_latency,
thinking_mode=params.thinking_mode,
temperature=params.temperature,
top_p=params.top_p,
top_k=params.top_k,
repeat=params.repeat,
api=params.api,
)
success, error, metrics = _run_simple_eval(
model=model,
base_url=base_url,
dataset=params.dataset,
num_examples=params.num_examples,
num_threads=params.num_threads,
max_tokens=params.max_tokens,
return_latency=params.return_latency,
thinking_mode=params.thinking_mode,
temperature=params.temperature,
top_p=params.top_p,
top_k=params.top_k,
repeat=params.repeat,
api=params.api,
seed=params.seed,
sgl_eval_thinking=params.sgl_eval_thinking,
)
if not success:
print(f"✗ Accuracy test failed for {model.model_path}: {error}")
python/sglang/test/kits/eval_accuracy_kit.py
+3 -3
View File
@@ -231,9 +231,9 @@ class GSM8KMixin:
class MMLUMixin:
"""Mixin for MMLU evaluation.
Backend is selectable via ``mmlu_backend`` (default ``"run_eval"``; or
``"sgl_eval"``: sgl-eval multichoice grader, skipped if sgl-eval is not
installed).
Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it
in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch
picks the call mechanism, not the grader.
Required attributes on the test class:
base_url: str
python/sglang/test/run_eval.py
+15 -4
View File
@@ -136,6 +136,13 @@ def _run_sgl_eval(eval_name, args) -> dict:
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
if getattr(args, "top_p", None) is not None:
cmd += ["--top-p", str(args.top_p)]
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
if getattr(args, "seed", None) is not None:
cmd += ["--seed", str(args.seed)]
if getattr(args, "repeat", None) is not None:
cmd += ["--n-repeats", str(args.repeat)]
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
@@ -242,10 +249,10 @@ def run_eval(args):
)
if args.eval_name == "mmlu":
from sglang.test.simple_eval_mmlu import MMLUEval
filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
eval_obj = MMLUEval(filename, args.num_examples, args.num_threads)
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
return _run_sgl_eval("mmlu", args)
elif args.eval_name == "math":
from sglang.test.simple_eval_math import MathEval
@@ -301,6 +308,10 @@ def run_eval(args):
args.num_threads,
response_answer_regex=getattr(args, "response_answer_regex", None),
)
elif args.eval_name == "mmmu_pro_vision":
# sgl-eval owns this benchmark's dataset, prompt and grader; there is no
# simple_eval implementation to fall back to.
return _run_sgl_eval("mmmu_pro_vision", args)
elif args.eval_name == "aime25":
from sglang.test.simple_eval_aime25 import AIME25Eval