minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-12 19:33:21 -07:00
committed by GitHub
parent cfdfd31826
commit 50cc1aa241
25 changed files with 182 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_pr-test-stage-cpu.yml
+2
View File
@@ -119,6 +119,8 @@ jobs:
UV_SYSTEM_PYTHON: "1"
run: |
uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow
source scripts/ci/utils/sgl_eval_ref.sh
uv pip install "$SGL_EVAL_SPEC" --index-strategy unsafe-best-match
# Hosted runners are ephemeral, so models are re-fetched every run and the
# Hub occasionally returns 429s. Persist the HF cache in GitHub's cache
python/sglang/test/accuracy_test_runner.py
+29 -323
View File
@@ -8,7 +8,6 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
dump_metric,
popen_launch_server,
write_github_step_summary,
)
@@ -31,6 +30,10 @@ class AccuracyTestParams:
top_k: Optional[int] = None
repeat: Optional[int] = None
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
seed: Optional[int] = None # pin for reproducibility when temperature > 0
# sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead
# of letting _run_sgl_eval infer it from the model name.
sgl_eval_thinking: Optional[bool] = None
@dataclass
@@ -89,6 +92,8 @@ def _run_simple_eval(
top_k: Optional[int] = None,
repeat: Optional[int] = None,
api: Optional[str] = None,
seed: Optional[int] = None,
sgl_eval_thinking: Optional[bool] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using simple_eval backend (run_eval.py).
@@ -137,6 +142,12 @@ def _run_simple_eval(
if repeat is not None:
args.repeat = repeat
if seed is not None:
args.seed = seed
if sgl_eval_thinking is not None:
args.sgl_eval_thinking = sgl_eval_thinking
result = run_eval(args)
# Handle result format (run_eval can return metrics or (metrics, latency))
@@ -156,299 +167,6 @@ def _run_simple_eval(
kill_process_tree(process.pid)
# Cached uv venv for NeMo Skills (persists across variants within a process).
_nemo_venv_dir: Optional[str] = None
_nemo_data_prepared: set = set()
def _get_nemo_venv() -> Tuple[str, dict]:
"""Get or create a uv venv with nemo_skills installed.
Returns (venv_python_path, env_dict) reusable across calls.
"""
import os
import subprocess
import tempfile
global _nemo_venv_dir
if _nemo_venv_dir is not None:
venv_python = f"{_nemo_venv_dir}/venv/bin/python"
env = {
**dict(os.environ),
"NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1",
"OPENAI_API_KEY": "dummy",
"VIRTUAL_ENV": f"{_nemo_venv_dir}/venv",
"PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""),
}
return venv_python, env
_nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_")
print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...")
# Create venv
result = subprocess.run(
["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"],
capture_output=True,
text=True,
)
if result.returncode != 0:
subprocess.run(
["uv", "venv", f"{_nemo_venv_dir}/venv"],
capture_output=True,
text=True,
)
# Install nemo_skills.
# Pinned: NeMo-Skills main after PR #1433 pins litellm==1.83.14 (httpx==0.28.1),
# which is unsatisfiable against nemo-run's transitive leptonai dep.
nemo_skills_ref = "589294c"
print(f"Installing nemo_skills (pinned to {nemo_skills_ref})...")
pip_result = subprocess.run(
[
"uv",
"pip",
"install",
"--python",
f"{_nemo_venv_dir}/venv/bin/python",
f"git+https://github.com/NVIDIA/NeMo-Skills.git@{nemo_skills_ref}",
"mcp<2",
"typer<0.27",
],
capture_output=True,
text=True,
timeout=300,
)
if pip_result.returncode != 0:
raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}")
print("NeMo Skills installed successfully")
return _get_nemo_venv()
def _ensure_nemo_data_prepared(
venv_python: str, env: dict, dataset: str
) -> Tuple[bool, Optional[str]]:
"""Prepare NeMo Skills dataset data if not already done.
Uses the venv python so data lands inside the venv's nemo_skills package.
"""
import subprocess
if dataset in _nemo_data_prepared:
return True, None
print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...")
result = subprocess.run(
[venv_python, "-m", "nemo_skills.dataset.prepare", dataset],
text=True,
timeout=600,
env=env,
)
if result.returncode != 0:
return False, f"Failed to prepare {dataset} data (exit {result.returncode})"
_nemo_data_prepared.add(dataset)
return True, None
def _run_nemo_skills_eval(
model: ModelLaunchSettings,
base_url: str,
dataset: str,
max_tokens: Optional[int] = None,
repeat: Optional[int] = None,
temperature: Optional[float] = None,
top_p: Optional[float] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro.
Uses an isolated uv venv (shared across variants) so nemo_skills dependencies
don't interfere with the system python / sglang server.
Returns:
Tuple of (success, error_message, metrics_dict)
"""
import subprocess
import tempfile
process = None
try:
# Get or create the shared venv (once per process)
venv_python, env = _get_nemo_venv()
# Prepare dataset (once per process, cached)
ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset)
if not ok:
return False, err, None
process = popen_launch_server(
model.model_path,
base_url,
other_args=model.extra_args,
timeout=model.launch_timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
env=model.env,
)
port = int(base_url.split(":")[-1])
server_address = f"http://127.0.0.1:{port}/v1"
repeat_val = repeat or 1
max_tokens_val = max_tokens or 32768
benchmark_spec = f"{dataset}:{repeat_val}"
# Build ns eval command using venv python
# Note: nemo_skills.pipeline.eval requires the "eval" subcommand
output_dir = tempfile.mkdtemp(prefix="ns_eval_output_")
cmd = [
venv_python,
"-m",
"nemo_skills.pipeline.eval",
"eval",
f"--benchmarks={benchmark_spec}",
"--server_type=sglang",
f"--model={model.model_path}",
f"--server_address={server_address}",
f"--output_dir={output_dir}",
f"++inference.tokens_to_generate={max_tokens_val}",
]
if temperature is not None:
cmd.append(f"++inference.temperature={temperature}")
if top_p is not None:
cmd.append(f"++inference.top_p={top_p}")
# Add VLM-specific config
if dataset in ("mmmu-pro", "mmmu_pro"):
cmd.append("++prompt_config=vlm/mmmu-pro")
cmd.append("++max_concurrent_requests=512")
cmd.append("++max_samples=500")
print(f"Running: {' '.join(cmd)}")
eval_result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=7200,
env=env,
)
print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)")
if eval_result.stderr:
print(eval_result.stderr[-1000:])
if eval_result.returncode != 0:
return (
False,
f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}",
None,
)
# Parse results
summarize_result = subprocess.run(
[
venv_python,
"-m",
"nemo_skills.pipeline.summarize_results",
f"{output_dir}/eval-results",
],
capture_output=True,
text=True,
timeout=60,
env=env,
)
output = summarize_result.stdout + "\n" + eval_result.stdout
print(f"Summary: {summarize_result.stdout[:1000]}")
# Parse accuracy from output (format varies, look for common patterns)
import re
score = None
for line in output.split("\n"):
match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE)
if match:
score = float(match.group(1))
if score is None:
# Try to find it in eval-results directory
import glob
import json
for result_file in glob.glob(
f"{output_dir}/eval-results/**/*.json", recursive=True
):
try:
with open(result_file) as f:
data = json.load(f)
if isinstance(data, dict):
score = (
data.get("accuracy")
or data.get("score")
or data.get("mean_score")
)
if score is not None:
break
except (json.JSONDecodeError, KeyError):
continue
if score is None:
# Last resort: compute accuracy directly from JSONL output
import glob
import json
for jsonl_file in sorted(
glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True)
):
correct = 0
total = 0
try:
with open(jsonl_file) as f:
for line in f:
line = line.strip()
if not line:
continue
entry = json.loads(line)
expected = entry.get("expected_answer", "")
generation = entry.get("generation", "")
# Extract "Answer: X" from the end of generation
answer_match = re.search(
r"Answer:\s*([A-J])", generation, re.IGNORECASE
)
if answer_match:
predicted = answer_match.group(1).upper()
if predicted == expected.upper():
correct += 1
total += 1
except (json.JSONDecodeError, KeyError, OSError):
continue
if total > 0:
score = correct / total
print(
f"Computed accuracy from {jsonl_file}: "
f"{correct}/{total} = {score:.4f}"
)
break
if score is None:
return False, "Could not parse accuracy from ns eval output", None
dump_metric(
f"{dataset}_score",
score,
labels={"model": model.model_path, "eval": dataset, "api": "nemo-skills"},
)
return True, None, {"score": score}
except subprocess.TimeoutExpired:
return False, "NeMo Skills eval timed out", None
except Exception as e:
return False, f"NeMo Skills eval exception: {str(e)}", None
finally:
if process:
kill_process_tree(process.pid)
def run_accuracy_test(
model: ModelLaunchSettings,
params: AccuracyTestParams,
@@ -472,35 +190,23 @@ def run_accuracy_test(
print(f" Baseline: {params.baseline_accuracy}")
print(f"{'='*60}\n")
# Run evaluation based on dataset type
# - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
# - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.)
if params.dataset in ("mmmu-pro", "mmmu_pro"):
success, error, metrics = _run_nemo_skills_eval(
model=model,
base_url=base_url,
dataset="mmmu-pro",
max_tokens=params.max_tokens,
repeat=params.repeat or 1,
temperature=params.temperature,
top_p=params.top_p,
)
else:
success, error, metrics = _run_simple_eval(
model=model,
base_url=base_url,
dataset=params.dataset,
num_examples=params.num_examples,
num_threads=params.num_threads,
max_tokens=params.max_tokens,
return_latency=params.return_latency,
thinking_mode=params.thinking_mode,
temperature=params.temperature,
top_p=params.top_p,
top_k=params.top_k,
repeat=params.repeat,
api=params.api,
)
success, error, metrics = _run_simple_eval(
model=model,
base_url=base_url,
dataset=params.dataset,
num_examples=params.num_examples,
num_threads=params.num_threads,
max_tokens=params.max_tokens,
return_latency=params.return_latency,
thinking_mode=params.thinking_mode,
temperature=params.temperature,
top_p=params.top_p,
top_k=params.top_k,
repeat=params.repeat,
api=params.api,
seed=params.seed,
sgl_eval_thinking=params.sgl_eval_thinking,
)
if not success:
print(f"✗ Accuracy test failed for {model.model_path}: {error}")
python/sglang/test/kits/eval_accuracy_kit.py
+3 -3
View File
@@ -231,9 +231,9 @@ class GSM8KMixin:
class MMLUMixin:
"""Mixin for MMLU evaluation.
Backend is selectable via ``mmlu_backend`` (default ``"run_eval"``; or
``"sgl_eval"``: sgl-eval multichoice grader, skipped if sgl-eval is not
installed).
Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it
in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch
picks the call mechanism, not the grader.
Required attributes on the test class:
base_url: str
python/sglang/test/run_eval.py
+15 -4
View File
@@ -136,6 +136,13 @@ def _run_sgl_eval(eval_name, args) -> dict:
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
if getattr(args, "top_p", None) is not None:
cmd += ["--top-p", str(args.top_p)]
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
if getattr(args, "seed", None) is not None:
cmd += ["--seed", str(args.seed)]
if getattr(args, "repeat", None) is not None:
cmd += ["--n-repeats", str(args.repeat)]
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
@@ -242,10 +249,10 @@ def run_eval(args):
)
if args.eval_name == "mmlu":
from sglang.test.simple_eval_mmlu import MMLUEval
filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
eval_obj = MMLUEval(filename, args.num_examples, args.num_threads)
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
return _run_sgl_eval("mmlu", args)
elif args.eval_name == "math":
from sglang.test.simple_eval_math import MathEval
@@ -301,6 +308,10 @@ def run_eval(args):
args.num_threads,
response_answer_regex=getattr(args, "response_answer_regex", None),
)
elif args.eval_name == "mmmu_pro_vision":
# sgl-eval owns this benchmark's dataset, prompt and grader; there is no
# simple_eval implementation to fall back to.
return _run_sgl_eval("mmmu_pro_vision", args)
elif args.eval_name == "aime25":
from sglang.test.simple_eval_aime25 import AIME25Eval
scripts/ci/amd/amd_ci_install_dependency.sh
+4
View File
@@ -134,6 +134,10 @@ else
install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]"
fi
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
source "$(dirname "${BASH_SOURCE[0]}")/../utils/sgl_eval_ref.sh"
install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache "$SGL_EVAL_SPEC"
if [[ -n "${SKIP_TT_DEPS}" ]]; then
echo "Didn't build lmms_eval, human-eval, and others"
else
scripts/ci/cuda/ci_install_dependency.sh
+3 -7
View File
@@ -682,12 +682,8 @@ stabilize_flashinfer_jit_paths() {
install_extra_deps() {
MOONCAKE_VERSION="0.3.12.post1"
NIXL_VERSION="1.3.0"
# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see
# the note there). The nightly GSM8K eval shells out to the sgl-eval CLI and
# fails without it. Bumping the SHA can change zero-shot \boxed{} grading, so
# re-baseline MODEL_SCORE_THRESHOLDS in
# test/registered/eval/test_text_models_gsm8k_eval.py first.
SGL_EVAL_REF="b2a2703c42cae379bbcb8b7ff092df6601a61694"
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
if [ "$CU_MAJOR" = "13" ]; then
MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
MOONCAKE_STALE_PKG="mooncake-transfer-engine"
@@ -723,7 +719,7 @@ install_extra_deps() {
--no-deps --force-reinstall $PIP_INSTALL_SUFFIX
fi
$PIP_CMD install "sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" $PIP_INSTALL_SUFFIX
$PIP_CMD install "$SGL_EVAL_SPEC" $PIP_INSTALL_SUFFIX
if [ "$IS_BLACKWELL" != "1" ]; then
git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
scripts/ci/npu/npu_ci_install_dependency.sh
+5
View File
@@ -84,3 +84,8 @@ rm -rf cann-custom-ops
### Install SGLang
rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml
${UV_PIP_INSTALL} -v -e "python[dev_npu]"
### Install sgl-eval
# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
${UV_PIP_INSTALL} "$SGL_EVAL_SPEC"
scripts/ci/utils/sgl_eval_ref.sh
+13
View File
@@ -0,0 +1,13 @@
# Single source of truth for the sgl-eval commit every CI variant installs.
# Meant to be sourced, not executed -- each variant then installs
# "$SGL_EVAL_SPEC" with its own pip invocation, since those differ (uv pip on
# CUDA/CPU, `docker exec ... pip` on AMD, `python3 -m pip` on NPU).
#
# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see the
# note there). Every eval that shells out to the `sgl-eval` CLI fails without
# it, and a bump moves scoring for all of them at once -- so re-baseline
# MODEL_SCORE_THRESHOLDS in
# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds
# of run_eval's other callers, before changing this.
SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d"
SGL_EVAL_SPEC="sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"
test/registered/attention/test_torch_native_attention_backend.py
+2 -2
View File
@@ -38,12 +38,12 @@ class TestTorchNativeAttnBackend(CustomTestCase):
base_url=base_url,
model=model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
self.assertGreaterEqual(metrics["score"], 0.64)
finally:
kill_process_tree(process.pid)
test/registered/attention/test_triton_sliding_window.py
+4 -6
View File
@@ -10,7 +10,6 @@ from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
)
@@ -53,17 +52,16 @@ class TestSlidingWindowAttentionTriton(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=200,
num_examples=256,
num_threads=32,
)
metrics = run_eval(args)
print(f"MMLU metrics with sliding window: {metrics}")
if is_in_amd_ci():
self.assertGreaterEqual(metrics["score"], 0.55)
else:
self.assertGreaterEqual(metrics["score"], 0.60)
# gemma-3-4b-it scores 0.59 over 256 questions under sgl-eval's grader,
# minus the 0.05 margin the other eval thresholds use.
self.assertGreaterEqual(metrics["score"], 0.54)
def _test_short_context_generation(self):
response = requests.post(
test/registered/backends/test_torch_compile.py
+2 -2
View File
@@ -20,8 +20,8 @@ register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
class TestTorchCompile(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_score_threshold = 0.64
mmlu_num_examples = 256
mmlu_num_threads = 32
@classmethod
test/registered/disaggregation/test_disaggregation_decode_offload.py
+3 -3
View File
@@ -139,7 +139,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
base_url=f"http://{self.base_host}:{self.lb_port}",
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
@@ -166,8 +166,8 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
metrics2 = run_eval(args)
# Assert score is above a minimum threshold for both rounds
self.assertGreater(metrics1["score"], 0.65)
self.assertGreater(metrics2["score"], 0.65)
self.assertGreater(metrics1["score"], 0.64)
self.assertGreater(metrics2["score"], 0.64)
# Score should be consistent: round 2 should be >= round 1, or at least within a 0.05 margin if slightly lower
self.assertGreaterEqual(metrics2["score"], metrics1["score"] - 0.05)
test/registered/gb300/test_kimi_k25_nvfp4.py
+10 -1
View File
@@ -63,8 +63,17 @@ class TestKimiK25Nvfp4(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="Kimi-K2.5-NVFP4",
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
accuracy_params=AccuracyTestParams(
dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
dataset="mmmu_pro_vision",
baseline_accuracy=0.69,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
result_dir="performance_results_gb300",
test/registered/gb300/test_qwen35_fp8.py
+10 -1
View File
@@ -61,8 +61,17 @@ class TestQwen35Fp8(unittest.TestCase):
run_combined_tests(
models=variants,
test_name="Qwen3.5-397B-FP8",
# Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
# its `:1` suffix means temperature 0.7, not greedy -- so the baseline
# carries over unchanged. Do not "simplify" these away.
accuracy_params=AccuracyTestParams(
dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768
dataset="mmmu_pro_vision",
baseline_accuracy=0.76,
repeat=1,
max_tokens=32768,
temperature=0.7,
seed=0,
sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
result_dir="performance_results_gb300",
test/registered/hicache/test_hicache_storage.py
+2 -2
View File
@@ -21,8 +21,8 @@ _is_hip = is_hip()
class TestHiCache(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_score_threshold = 0.64
mmlu_num_examples = 256
mmlu_num_threads = 32
@classmethod
test/registered/hicache/test_hicache_variants.py
+8 -8
View File
@@ -64,8 +64,8 @@ class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):
"--hicache-size",
100 if not _is_hip else 200,
]
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_score_threshold = 0.64
mmlu_num_examples = 256
mmlu_num_threads = 32
@@ -77,8 +77,8 @@ class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin):
"--trust-remote-code",
"--enable-hierarchical-cache",
] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2])
mmlu_score_threshold = 0.5
mmlu_num_examples = 64
mmlu_score_threshold = 0.54
mmlu_num_examples = 256
mmlu_num_threads = 32
mgsm_en_score_threshold = 0.8
@@ -110,8 +110,8 @@ class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin):
"--chunked-prefill-size",
1024,
]
mmlu_score_threshold = 0.72
mmlu_num_examples = 64
mmlu_score_threshold = 0.64
mmlu_num_examples = 256
mmlu_num_threads = 32
mmlu_accept_length_thres = 2.26
@@ -127,8 +127,8 @@ class TestHiCachePage(HiCacheBaseServer, MMLUMixin):
"--hicache-write-policy",
"write_back",
]
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_score_threshold = 0.64
mmlu_num_examples = 256
mmlu_num_threads = 32
test/registered/models_e2e/test_transformers_models.py
+2 -2
View File
@@ -36,7 +36,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--model-impl", "transformers"],
)
cls.mmlu_lower_bound = 0.63
cls.mmlu_lower_bound = 0.64
cls.gsm8k_lower_bound = 0.65
@classmethod
@@ -48,7 +48,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
metrics = run_eval(args)
test/registered/models_e2e/test_zaya.py
-127
View File
@@ -1,127 +0,0 @@
"""End-to-end server test for Zyphra ZAYA1 (hybrid CCA attention + MoE).
This test boots a real ``Zyphra/ZAYA1-base`` SGLang server via
``popen_launch_server``, sends a handful of completions through the HTTP API,
and finishes with a small MMLU sanity slice.
The test is gated behind ``RUN_ZAYA_E2E=1`` so the registered suite does not
have to download the full ZAYA1-base checkpoint (≈17 GB) on every run; the CI
job that owns this test sets the variable explicitly.
"""
import os
import unittest
from types import SimpleNamespace
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
# ZAYA1-base is a heavyweight launch (≈120 transformer layers with MoE), so
# the estimated time is set generously to keep the CI scheduler from preempting
# the job before the server finishes warming up.
register_cuda_ci(est_time=420, stage="extra-a", runner_config="1-gpu-large")
register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-large-amd")
_MODEL_PATH = os.environ.get("ZAYA_MODEL_PATH", "Zyphra/ZAYA1-base")
def _zaya_enabled() -> bool:
return os.environ.get("RUN_ZAYA_E2E", "0") == "1"
@unittest.skipUnless(
_zaya_enabled(),
"Set RUN_ZAYA_E2E=1 to enable the ZAYA1 end-to-end server test "
"(requires downloading the model weights).",
)
class TestZayaServer(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = _MODEL_PATH
cls.base_url = DEFAULT_URL_FOR_TEST
other_args = [
"--mem-fraction-static",
"0.5",
"--max-running-requests",
"8",
]
if is_hip():
other_args += ["--attention-backend", "triton"]
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args,
)
@classmethod
def tearDownClass(cls):
if getattr(cls, "process", None) is not None:
kill_process_tree(cls.process.pid)
def test_generation_basic(self):
"""Send three prompts through the ``/generate`` endpoint and require
non-empty completions for each."""
import requests
prompts = [
"The capital of France is",
"1 + 2 + 3 + 4 + 5 =",
"Write a haiku about silicon:",
]
for prompt in prompts:
resp = requests.post(
f"{self.base_url}/generate",
json={
"text": prompt,
"sampling_params": {
"temperature": 0.0,
"max_new_tokens": 16,
},
},
timeout=60,
)
self.assertEqual(resp.status_code, 200, resp.text)
data = resp.json()
self.assertIn("text", data, data)
self.assertGreater(len(data["text"].strip()), 0, data)
def test_mmlu_sanity(self):
"""32-example MMLU sanity slice.
ZAYA1-base is a pretrained (non instruction-tuned) checkpoint that
emits long ``<think>…</think>`` reasoning blocks before settling on a
final letter, so ``max_tokens`` must be large enough for the evaluator
to see the chosen answer. The threshold sits just above chance: it is
a regression sanity check rather than a production-quality gate. An
instruction-tuned ZAYA1 checkpoint scores meaningfully higher and
should raise this bound when wired in.
"""
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=32,
num_threads=8,
max_tokens=1024,
)
metrics = run_eval(args)
self.assertGreaterEqual(
metrics["score"],
0.30,
f"MMLU sanity below threshold: {metrics}",
)
if __name__ == "__main__":
unittest.main()
test/registered/moe/test_torch_compile_moe.py
+3 -2
View File
@@ -41,12 +41,13 @@ class TestTorchCompileMoe(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.50)
# 0.48 measured, minus the 0.05 margin the other eval thresholds use.
self.assertGreaterEqual(metrics["score"], 0.43)
def run_decode(self, max_new_tokens):
response = requests.post(
test/registered/quant/test_autoround_quantization.py
+4 -2
View File
@@ -26,9 +26,11 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large")
MMLU_NUM_EXAMPLES = 32
MMLU_NUM_EXAMPLES = 256
MMLU_NUM_THREADS = 32
MMLU_SCORE_THRESHOLD = 22 / MMLU_NUM_EXAMPLES
# The unquantized model scores 0.68-0.70 over 256 questions; int8 gives up a
# point or two, then the 0.05 margin the other eval thresholds use.
MMLU_SCORE_THRESHOLD = 0.63
class TestAutoRoundQuantization(CustomTestCase):
test/registered/quant/test_awq.py
+2 -2
View File
@@ -38,7 +38,7 @@ class TestAWQ(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
@@ -72,7 +72,7 @@ class TestAWQMarlinBfloat16(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
test/registered/sampling/test_pytorch_sampling_backend.py
+2 -2
View File
@@ -40,13 +40,13 @@ class TestPyTorchSamplingBackend(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
self.assertGreaterEqual(metrics["score"], 0.64)
@unittest.skipIf(
is_in_amd_ci(),
test/registered/scheduler/test_retract_decode.py
+2 -2
View File
@@ -51,12 +51,12 @@ class TestRetractDecode(CustomTestCase):
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=64,
num_examples=256,
num_threads=32,
)
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
self.assertGreaterEqual(metrics["score"], 0.64)
time.sleep(1) # wait for mem check
assert self.process.poll() is None, "Server crashed during test"
test/registered/tokenizer/test_multi_tokenizer.py
+2 -2
View File
@@ -27,8 +27,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
"""One server covering both worker pools: multi-tokenizer and
multi-detokenizer (the flags are orthogonal)."""
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_score_threshold = 0.64
mmlu_num_examples = 256
mmlu_num_threads = 32
@classmethod
test/registered/unit/bench/test_simple_eval_gsm8k.py
+50
View File
@@ -129,6 +129,56 @@ class TestRunSglEval(CustomTestCase):
self.assertNotIn("--num-examples", captured["cmd"])
def _capture_cmd(self, eval_name="gsm8k", **overrides):
captured = {}
def fake_run(cmd, **kwargs):
captured["cmd"] = cmd
out_dir = Path(cmd[cmd.index("--out-dir") + 1])
_write_fake_metrics(
out_dir,
eval_name,
{
"model": "test-model",
"latency_seconds": 1.0,
"output_throughput_tps": 1.0,
"aggregate": {"score": 0.5},
},
)
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
with tempfile.TemporaryDirectory() as td:
args = self._args(td, **overrides)
with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
_run_sgl_eval(eval_name, args)
return captured["cmd"]
def test_omits_sampling_flags_when_unset(self):
"""Unset top_p / seed / repeat must not reach the CLI -- sgl-eval's own
defaults differ from a forced value (seed unset != seed 0)."""
cmd = self._capture_cmd()
for flag in ("--top-p", "--seed", "--n-repeats"):
self.assertNotIn(flag, cmd)
def test_forwards_sampling_flags_when_set(self):
cmd = self._capture_cmd(top_p=0.95, seed=0, repeat=1)
for flag, value in (("--top-p", "0.95"), ("--seed", "0"), ("--n-repeats", "1")):
self.assertIn(flag, cmd)
self.assertEqual(cmd[cmd.index(flag) + 1], value)
def test_thinking_auto_detected_from_model_name(self):
self.assertIn(
"--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8")
)
def test_explicit_thinking_false_suppresses_auto_detect(self):
"""A caller matching a harness that sent no chat_template_kwargs has to be
able to turn the model-name heuristic off."""
cmd = self._capture_cmd(
model="Qwen/Qwen3.5-397B-A17B-FP8", sgl_eval_thinking=False
)
self.assertNotIn("--thinking", cmd)
def test_raises_on_nonzero_exit(self):
def fake_run(cmd, **kwargs):
return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")