From 50cc1aa241d75614a2e805c740d7c2bc43e3b627 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 12 Aug 2026 19:33:21 -0700 Subject: [PATCH] [CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477) --- .github/workflows/_pr-test-stage-cpu.yml | 2 + python/sglang/test/accuracy_test_runner.py | 352 ++---------------- python/sglang/test/kits/eval_accuracy_kit.py | 6 +- python/sglang/test/run_eval.py | 19 +- scripts/ci/amd/amd_ci_install_dependency.sh | 4 + scripts/ci/cuda/ci_install_dependency.sh | 10 +- scripts/ci/npu/npu_ci_install_dependency.sh | 5 + scripts/ci/utils/sgl_eval_ref.sh | 13 + .../test_torch_native_attention_backend.py | 4 +- .../attention/test_triton_sliding_window.py | 10 +- .../registered/backends/test_torch_compile.py | 4 +- .../test_disaggregation_decode_offload.py | 6 +- test/registered/gb300/test_kimi_k25_nvfp4.py | 11 +- test/registered/gb300/test_qwen35_fp8.py | 11 +- .../hicache/test_hicache_storage.py | 4 +- .../hicache/test_hicache_variants.py | 16 +- .../models_e2e/test_transformers_models.py | 4 +- test/registered/models_e2e/test_zaya.py | 127 ------- test/registered/moe/test_torch_compile_moe.py | 5 +- .../quant/test_autoround_quantization.py | 6 +- test/registered/quant/test_awq.py | 4 +- .../sampling/test_pytorch_sampling_backend.py | 4 +- .../scheduler/test_retract_decode.py | 4 +- .../tokenizer/test_multi_tokenizer.py | 4 +- .../unit/bench/test_simple_eval_gsm8k.py | 50 +++ 25 files changed, 182 insertions(+), 503 deletions(-) create mode 100644 scripts/ci/utils/sgl_eval_ref.sh delete mode 100644 test/registered/models_e2e/test_zaya.py diff --git a/.github/workflows/_pr-test-stage-cpu.yml b/.github/workflows/_pr-test-stage-cpu.yml index 1b726c0c4..cdb5242b3 100644 --- a/.github/workflows/_pr-test-stage-cpu.yml +++ b/.github/workflows/_pr-test-stage-cpu.yml @@ -119,6 +119,8 @@ jobs: UV_SYSTEM_PYTHON: "1" run: | uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow + source scripts/ci/utils/sgl_eval_ref.sh + uv pip install "$SGL_EVAL_SPEC" --index-strategy unsafe-best-match # Hosted runners are ephemeral, so models are re-fetched every run and the # Hub occasionally returns 429s. Persist the HF cache in GitHub's cache diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py index 45fc1a46b..9322e9b92 100644 --- a/python/sglang/test/accuracy_test_runner.py +++ b/python/sglang/test/accuracy_test_runner.py @@ -8,7 +8,6 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, ModelLaunchSettings, - dump_metric, popen_launch_server, write_github_step_summary, ) @@ -31,6 +30,10 @@ class AccuracyTestParams: top_k: Optional[int] = None repeat: Optional[int] = None api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval + seed: Optional[int] = None # pin for reproducibility when temperature > 0 + # sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead + # of letting _run_sgl_eval infer it from the model name. + sgl_eval_thinking: Optional[bool] = None @dataclass @@ -89,6 +92,8 @@ def _run_simple_eval( top_k: Optional[int] = None, repeat: Optional[int] = None, api: Optional[str] = None, + seed: Optional[int] = None, + sgl_eval_thinking: Optional[bool] = None, ) -> Tuple[bool, Optional[str], Optional[dict]]: """Run evaluation using simple_eval backend (run_eval.py). @@ -137,6 +142,12 @@ def _run_simple_eval( if repeat is not None: args.repeat = repeat + if seed is not None: + args.seed = seed + + if sgl_eval_thinking is not None: + args.sgl_eval_thinking = sgl_eval_thinking + result = run_eval(args) # Handle result format (run_eval can return metrics or (metrics, latency)) @@ -156,299 +167,6 @@ def _run_simple_eval( kill_process_tree(process.pid) -# Cached uv venv for NeMo Skills (persists across variants within a process). -_nemo_venv_dir: Optional[str] = None -_nemo_data_prepared: set = set() - - -def _get_nemo_venv() -> Tuple[str, dict]: - """Get or create a uv venv with nemo_skills installed. - - Returns (venv_python_path, env_dict) reusable across calls. - """ - import os - import subprocess - import tempfile - - global _nemo_venv_dir - - if _nemo_venv_dir is not None: - venv_python = f"{_nemo_venv_dir}/venv/bin/python" - env = { - **dict(os.environ), - "NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1", - "OPENAI_API_KEY": "dummy", - "VIRTUAL_ENV": f"{_nemo_venv_dir}/venv", - "PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""), - } - return venv_python, env - - _nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_") - print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...") - - # Create venv - result = subprocess.run( - ["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"], - capture_output=True, - text=True, - ) - if result.returncode != 0: - subprocess.run( - ["uv", "venv", f"{_nemo_venv_dir}/venv"], - capture_output=True, - text=True, - ) - - # Install nemo_skills. - # Pinned: NeMo-Skills main after PR #1433 pins litellm==1.83.14 (httpx==0.28.1), - # which is unsatisfiable against nemo-run's transitive leptonai dep. - nemo_skills_ref = "589294c" - print(f"Installing nemo_skills (pinned to {nemo_skills_ref})...") - pip_result = subprocess.run( - [ - "uv", - "pip", - "install", - "--python", - f"{_nemo_venv_dir}/venv/bin/python", - f"git+https://github.com/NVIDIA/NeMo-Skills.git@{nemo_skills_ref}", - "mcp<2", - "typer<0.27", - ], - capture_output=True, - text=True, - timeout=300, - ) - if pip_result.returncode != 0: - raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}") - - print("NeMo Skills installed successfully") - return _get_nemo_venv() - - -def _ensure_nemo_data_prepared( - venv_python: str, env: dict, dataset: str -) -> Tuple[bool, Optional[str]]: - """Prepare NeMo Skills dataset data if not already done. - - Uses the venv python so data lands inside the venv's nemo_skills package. - """ - import subprocess - - if dataset in _nemo_data_prepared: - return True, None - - print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...") - result = subprocess.run( - [venv_python, "-m", "nemo_skills.dataset.prepare", dataset], - text=True, - timeout=600, - env=env, - ) - if result.returncode != 0: - return False, f"Failed to prepare {dataset} data (exit {result.returncode})" - - _nemo_data_prepared.add(dataset) - return True, None - - -def _run_nemo_skills_eval( - model: ModelLaunchSettings, - base_url: str, - dataset: str, - max_tokens: Optional[int] = None, - repeat: Optional[int] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, -) -> Tuple[bool, Optional[str], Optional[dict]]: - """Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro. - - Uses an isolated uv venv (shared across variants) so nemo_skills dependencies - don't interfere with the system python / sglang server. - - Returns: - Tuple of (success, error_message, metrics_dict) - """ - import subprocess - import tempfile - - process = None - try: - # Get or create the shared venv (once per process) - venv_python, env = _get_nemo_venv() - - # Prepare dataset (once per process, cached) - ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset) - if not ok: - return False, err, None - - process = popen_launch_server( - model.model_path, - base_url, - other_args=model.extra_args, - timeout=model.launch_timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - env=model.env, - ) - - port = int(base_url.split(":")[-1]) - server_address = f"http://127.0.0.1:{port}/v1" - repeat_val = repeat or 1 - max_tokens_val = max_tokens or 32768 - benchmark_spec = f"{dataset}:{repeat_val}" - - # Build ns eval command using venv python - # Note: nemo_skills.pipeline.eval requires the "eval" subcommand - output_dir = tempfile.mkdtemp(prefix="ns_eval_output_") - cmd = [ - venv_python, - "-m", - "nemo_skills.pipeline.eval", - "eval", - f"--benchmarks={benchmark_spec}", - "--server_type=sglang", - f"--model={model.model_path}", - f"--server_address={server_address}", - f"--output_dir={output_dir}", - f"++inference.tokens_to_generate={max_tokens_val}", - ] - - if temperature is not None: - cmd.append(f"++inference.temperature={temperature}") - if top_p is not None: - cmd.append(f"++inference.top_p={top_p}") - - # Add VLM-specific config - if dataset in ("mmmu-pro", "mmmu_pro"): - cmd.append("++prompt_config=vlm/mmmu-pro") - cmd.append("++max_concurrent_requests=512") - cmd.append("++max_samples=500") - - print(f"Running: {' '.join(cmd)}") - eval_result = subprocess.run( - cmd, - capture_output=True, - text=True, - timeout=7200, - env=env, - ) - - print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)") - if eval_result.stderr: - print(eval_result.stderr[-1000:]) - - if eval_result.returncode != 0: - return ( - False, - f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}", - None, - ) - - # Parse results - summarize_result = subprocess.run( - [ - venv_python, - "-m", - "nemo_skills.pipeline.summarize_results", - f"{output_dir}/eval-results", - ], - capture_output=True, - text=True, - timeout=60, - env=env, - ) - - output = summarize_result.stdout + "\n" + eval_result.stdout - print(f"Summary: {summarize_result.stdout[:1000]}") - - # Parse accuracy from output (format varies, look for common patterns) - import re - - score = None - for line in output.split("\n"): - match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE) - if match: - score = float(match.group(1)) - - if score is None: - # Try to find it in eval-results directory - import glob - import json - - for result_file in glob.glob( - f"{output_dir}/eval-results/**/*.json", recursive=True - ): - try: - with open(result_file) as f: - data = json.load(f) - if isinstance(data, dict): - score = ( - data.get("accuracy") - or data.get("score") - or data.get("mean_score") - ) - if score is not None: - break - except (json.JSONDecodeError, KeyError): - continue - - if score is None: - # Last resort: compute accuracy directly from JSONL output - import glob - import json - - for jsonl_file in sorted( - glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True) - ): - correct = 0 - total = 0 - try: - with open(jsonl_file) as f: - for line in f: - line = line.strip() - if not line: - continue - entry = json.loads(line) - expected = entry.get("expected_answer", "") - generation = entry.get("generation", "") - # Extract "Answer: X" from the end of generation - answer_match = re.search( - r"Answer:\s*([A-J])", generation, re.IGNORECASE - ) - if answer_match: - predicted = answer_match.group(1).upper() - if predicted == expected.upper(): - correct += 1 - total += 1 - except (json.JSONDecodeError, KeyError, OSError): - continue - if total > 0: - score = correct / total - print( - f"Computed accuracy from {jsonl_file}: " - f"{correct}/{total} = {score:.4f}" - ) - break - - if score is None: - return False, "Could not parse accuracy from ns eval output", None - - dump_metric( - f"{dataset}_score", - score, - labels={"model": model.model_path, "eval": dataset, "api": "nemo-skills"}, - ) - - return True, None, {"score": score} - - except subprocess.TimeoutExpired: - return False, "NeMo Skills eval timed out", None - except Exception as e: - return False, f"NeMo Skills eval exception: {str(e)}", None - finally: - if process: - kill_process_tree(process.pid) - - def run_accuracy_test( model: ModelLaunchSettings, params: AccuracyTestParams, @@ -472,35 +190,23 @@ def run_accuracy_test( print(f" Baseline: {params.baseline_accuracy}") print(f"{'='*60}\n") - # Run evaluation based on dataset type - # - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval) - # - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.) - if params.dataset in ("mmmu-pro", "mmmu_pro"): - success, error, metrics = _run_nemo_skills_eval( - model=model, - base_url=base_url, - dataset="mmmu-pro", - max_tokens=params.max_tokens, - repeat=params.repeat or 1, - temperature=params.temperature, - top_p=params.top_p, - ) - else: - success, error, metrics = _run_simple_eval( - model=model, - base_url=base_url, - dataset=params.dataset, - num_examples=params.num_examples, - num_threads=params.num_threads, - max_tokens=params.max_tokens, - return_latency=params.return_latency, - thinking_mode=params.thinking_mode, - temperature=params.temperature, - top_p=params.top_p, - top_k=params.top_k, - repeat=params.repeat, - api=params.api, - ) + success, error, metrics = _run_simple_eval( + model=model, + base_url=base_url, + dataset=params.dataset, + num_examples=params.num_examples, + num_threads=params.num_threads, + max_tokens=params.max_tokens, + return_latency=params.return_latency, + thinking_mode=params.thinking_mode, + temperature=params.temperature, + top_p=params.top_p, + top_k=params.top_k, + repeat=params.repeat, + api=params.api, + seed=params.seed, + sgl_eval_thinking=params.sgl_eval_thinking, + ) if not success: print(f"✗ Accuracy test failed for {model.model_path}: {error}") diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index 9a59ca92d..b865dba9d 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -231,9 +231,9 @@ class GSM8KMixin: class MMLUMixin: """Mixin for MMLU evaluation. - Backend is selectable via ``mmlu_backend`` (default ``"run_eval"``; or - ``"sgl_eval"``: sgl-eval multichoice grader, skipped if sgl-eval is not - installed). + Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it + in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch + picks the call mechanism, not the grader. Required attributes on the test class: base_url: str diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index 4b8cc109f..f3b0ceadc 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -136,6 +136,13 @@ def _run_sgl_eval(eval_name, args) -> dict: cmd += ["--model", args.model] if getattr(args, "num_examples", None) is not None: cmd += ["--num-examples", str(args.num_examples)] + if getattr(args, "top_p", None) is not None: + cmd += ["--top-p", str(args.top_p)] + # Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it. + if getattr(args, "seed", None) is not None: + cmd += ["--seed", str(args.seed)] + if getattr(args, "repeat", None) is not None: + cmd += ["--n-repeats", str(args.repeat)] # Bound generation length so long-reasoning models don't stall the eval. if getattr(args, "max_tokens", None) is not None: cmd += ["--max-tokens", str(args.max_tokens)] @@ -242,10 +249,10 @@ def run_eval(args): ) if args.eval_name == "mmlu": - from sglang.test.simple_eval_mmlu import MMLUEval - - filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv" - eval_obj = MMLUEval(filename, args.num_examples, args.num_threads) + # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a + # caller's threshold has to be measured against it, not inherited. + # `simple_eval_mmlu` stays: the ascend eval imports its subject2category. + return _run_sgl_eval("mmlu", args) elif args.eval_name == "math": from sglang.test.simple_eval_math import MathEval @@ -301,6 +308,10 @@ def run_eval(args): args.num_threads, response_answer_regex=getattr(args, "response_answer_regex", None), ) + elif args.eval_name == "mmmu_pro_vision": + # sgl-eval owns this benchmark's dataset, prompt and grader; there is no + # simple_eval implementation to fall back to. + return _run_sgl_eval("mmmu_pro_vision", args) elif args.eval_name == "aime25": from sglang.test.simple_eval_aime25 import AIME25Eval diff --git a/scripts/ci/amd/amd_ci_install_dependency.sh b/scripts/ci/amd/amd_ci_install_dependency.sh index 7ca5189f9..08725d5d8 100755 --- a/scripts/ci/amd/amd_ci_install_dependency.sh +++ b/scripts/ci/amd/amd_ci_install_dependency.sh @@ -134,6 +134,10 @@ else install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]" fi +# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh +source "$(dirname "${BASH_SOURCE[0]}")/../utils/sgl_eval_ref.sh" +install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache "$SGL_EVAL_SPEC" + if [[ -n "${SKIP_TT_DEPS}" ]]; then echo "Didn't build lmms_eval, human-eval, and others" else diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 6db690d75..08df97533 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -682,12 +682,8 @@ stabilize_flashinfer_jit_paths() { install_extra_deps() { MOONCAKE_VERSION="0.3.12.post1" NIXL_VERSION="1.3.0" - # sgl-eval is git-only and cannot be declared in python/pyproject.toml (see - # the note there). The nightly GSM8K eval shells out to the sgl-eval CLI and - # fails without it. Bumping the SHA can change zero-shot \boxed{} grading, so - # re-baseline MODEL_SCORE_THRESHOLDS in - # test/registered/eval/test_text_models_gsm8k_eval.py first. - SGL_EVAL_REF="b2a2703c42cae379bbcb8b7ff092df6601a61694" + # shellcheck source=scripts/ci/utils/sgl_eval_ref.sh + source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh" if [ "$CU_MAJOR" = "13" ]; then MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}" MOONCAKE_STALE_PKG="mooncake-transfer-engine" @@ -723,7 +719,7 @@ install_extra_deps() { --no-deps --force-reinstall $PIP_INSTALL_SUFFIX fi - $PIP_CMD install "sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" $PIP_INSTALL_SUFFIX + $PIP_CMD install "$SGL_EVAL_SPEC" $PIP_INSTALL_SUFFIX if [ "$IS_BLACKWELL" != "1" ]; then git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git diff --git a/scripts/ci/npu/npu_ci_install_dependency.sh b/scripts/ci/npu/npu_ci_install_dependency.sh index a776d2f25..83023face 100755 --- a/scripts/ci/npu/npu_ci_install_dependency.sh +++ b/scripts/ci/npu/npu_ci_install_dependency.sh @@ -84,3 +84,8 @@ rm -rf cann-custom-ops ### Install SGLang rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml ${UV_PIP_INSTALL} -v -e "python[dev_npu]" + +### Install sgl-eval +# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh +source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh" +${UV_PIP_INSTALL} "$SGL_EVAL_SPEC" diff --git a/scripts/ci/utils/sgl_eval_ref.sh b/scripts/ci/utils/sgl_eval_ref.sh new file mode 100644 index 000000000..c00447fbe --- /dev/null +++ b/scripts/ci/utils/sgl_eval_ref.sh @@ -0,0 +1,13 @@ +# Single source of truth for the sgl-eval commit every CI variant installs. +# Meant to be sourced, not executed -- each variant then installs +# "$SGL_EVAL_SPEC" with its own pip invocation, since those differ (uv pip on +# CUDA/CPU, `docker exec ... pip` on AMD, `python3 -m pip` on NPU). +# +# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see the +# note there). Every eval that shells out to the `sgl-eval` CLI fails without +# it, and a bump moves scoring for all of them at once -- so re-baseline +# MODEL_SCORE_THRESHOLDS in +# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds +# of run_eval's other callers, before changing this. +SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d" +SGL_EVAL_SPEC="sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" diff --git a/test/registered/attention/test_torch_native_attention_backend.py b/test/registered/attention/test_torch_native_attention_backend.py index 35310bd7f..194442dcf 100644 --- a/test/registered/attention/test_torch_native_attention_backend.py +++ b/test/registered/attention/test_torch_native_attention_backend.py @@ -38,12 +38,12 @@ class TestTorchNativeAttnBackend(CustomTestCase): base_url=base_url, model=model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) + self.assertGreaterEqual(metrics["score"], 0.64) finally: kill_process_tree(process.pid) diff --git a/test/registered/attention/test_triton_sliding_window.py b/test/registered/attention/test_triton_sliding_window.py index 999d6d190..86eb0e569 100644 --- a/test/registered/attention/test_triton_sliding_window.py +++ b/test/registered/attention/test_triton_sliding_window.py @@ -10,7 +10,6 @@ from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_amd_ci, is_in_ci, popen_launch_server, ) @@ -53,17 +52,16 @@ class TestSlidingWindowAttentionTriton(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=200, + num_examples=256, num_threads=32, ) metrics = run_eval(args) print(f"MMLU metrics with sliding window: {metrics}") - if is_in_amd_ci(): - self.assertGreaterEqual(metrics["score"], 0.55) - else: - self.assertGreaterEqual(metrics["score"], 0.60) + # gemma-3-4b-it scores 0.59 over 256 questions under sgl-eval's grader, + # minus the 0.05 margin the other eval thresholds use. + self.assertGreaterEqual(metrics["score"], 0.54) def _test_short_context_generation(self): response = requests.post( diff --git a/test/registered/backends/test_torch_compile.py b/test/registered/backends/test_torch_compile.py index cdde49ab7..550f5cdd5 100644 --- a/test/registered/backends/test_torch_compile.py +++ b/test/registered/backends/test_torch_compile.py @@ -20,8 +20,8 @@ register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd") class TestTorchCompile(CustomTestCase, MMLUMixin): - mmlu_score_threshold = 0.65 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.64 + mmlu_num_examples = 256 mmlu_num_threads = 32 @classmethod diff --git a/test/registered/disaggregation/test_disaggregation_decode_offload.py b/test/registered/disaggregation/test_disaggregation_decode_offload.py index 058154021..76221105b 100644 --- a/test/registered/disaggregation/test_disaggregation_decode_offload.py +++ b/test/registered/disaggregation/test_disaggregation_decode_offload.py @@ -139,7 +139,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): base_url=f"http://{self.base_host}:{self.lb_port}", model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) @@ -166,8 +166,8 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase): metrics2 = run_eval(args) # Assert score is above a minimum threshold for both rounds - self.assertGreater(metrics1["score"], 0.65) - self.assertGreater(metrics2["score"], 0.65) + self.assertGreater(metrics1["score"], 0.64) + self.assertGreater(metrics2["score"], 0.64) # Score should be consistent: round 2 should be >= round 1, or at least within a 0.05 margin if slightly lower self.assertGreaterEqual(metrics2["score"], metrics1["score"] - 0.05) diff --git a/test/registered/gb300/test_kimi_k25_nvfp4.py b/test/registered/gb300/test_kimi_k25_nvfp4.py index d28d62115..d213a2b52 100644 --- a/test/registered/gb300/test_kimi_k25_nvfp4.py +++ b/test/registered/gb300/test_kimi_k25_nvfp4.py @@ -63,8 +63,17 @@ class TestKimiK25Nvfp4(unittest.TestCase): run_combined_tests( models=variants, test_name="Kimi-K2.5-NVFP4", + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. accuracy_params=AccuracyTestParams( - dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768 + dataset="mmmu_pro_vision", + baseline_accuracy=0.69, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, ), performance_params=PerformanceTestParams( result_dir="performance_results_gb300", diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py index 1a48a4f68..f2551aacc 100644 --- a/test/registered/gb300/test_qwen35_fp8.py +++ b/test/registered/gb300/test_qwen35_fp8.py @@ -61,8 +61,17 @@ class TestQwen35Fp8(unittest.TestCase): run_combined_tests( models=variants, test_name="Qwen3.5-397B-FP8", + # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly -- + # its `:1` suffix means temperature 0.7, not greedy -- so the baseline + # carries over unchanged. Do not "simplify" these away. accuracy_params=AccuracyTestParams( - dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768 + dataset="mmmu_pro_vision", + baseline_accuracy=0.76, + repeat=1, + max_tokens=32768, + temperature=0.7, + seed=0, + sgl_eval_thinking=False, ), performance_params=PerformanceTestParams( result_dir="performance_results_gb300", diff --git a/test/registered/hicache/test_hicache_storage.py b/test/registered/hicache/test_hicache_storage.py index 9f9c23a42..e684130e8 100644 --- a/test/registered/hicache/test_hicache_storage.py +++ b/test/registered/hicache/test_hicache_storage.py @@ -21,8 +21,8 @@ _is_hip = is_hip() class TestHiCache(CustomTestCase, MMLUMixin): - mmlu_score_threshold = 0.65 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.64 + mmlu_num_examples = 256 mmlu_num_threads = 32 @classmethod diff --git a/test/registered/hicache/test_hicache_variants.py b/test/registered/hicache/test_hicache_variants.py index 36e2c4148..419ef37f0 100644 --- a/test/registered/hicache/test_hicache_variants.py +++ b/test/registered/hicache/test_hicache_variants.py @@ -64,8 +64,8 @@ class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin): "--hicache-size", 100 if not _is_hip else 200, ] - mmlu_score_threshold = 0.65 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.64 + mmlu_num_examples = 256 mmlu_num_threads = 32 @@ -77,8 +77,8 @@ class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin): "--trust-remote-code", "--enable-hierarchical-cache", ] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2]) - mmlu_score_threshold = 0.5 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.54 + mmlu_num_examples = 256 mmlu_num_threads = 32 mgsm_en_score_threshold = 0.8 @@ -110,8 +110,8 @@ class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin): "--chunked-prefill-size", 1024, ] - mmlu_score_threshold = 0.72 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.64 + mmlu_num_examples = 256 mmlu_num_threads = 32 mmlu_accept_length_thres = 2.26 @@ -127,8 +127,8 @@ class TestHiCachePage(HiCacheBaseServer, MMLUMixin): "--hicache-write-policy", "write_back", ] - mmlu_score_threshold = 0.65 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.64 + mmlu_num_examples = 256 mmlu_num_threads = 32 diff --git a/test/registered/models_e2e/test_transformers_models.py b/test/registered/models_e2e/test_transformers_models.py index 01e6a24d9..1168c827f 100644 --- a/test/registered/models_e2e/test_transformers_models.py +++ b/test/registered/models_e2e/test_transformers_models.py @@ -36,7 +36,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase): timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=["--model-impl", "transformers"], ) - cls.mmlu_lower_bound = 0.63 + cls.mmlu_lower_bound = 0.64 cls.gsm8k_lower_bound = 0.65 @classmethod @@ -48,7 +48,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) metrics = run_eval(args) diff --git a/test/registered/models_e2e/test_zaya.py b/test/registered/models_e2e/test_zaya.py deleted file mode 100644 index a07dbf285..000000000 --- a/test/registered/models_e2e/test_zaya.py +++ /dev/null @@ -1,127 +0,0 @@ -"""End-to-end server test for Zyphra ZAYA1 (hybrid CCA attention + MoE). - -This test boots a real ``Zyphra/ZAYA1-base`` SGLang server via -``popen_launch_server``, sends a handful of completions through the HTTP API, -and finishes with a small MMLU sanity slice. - -The test is gated behind ``RUN_ZAYA_E2E=1`` so the registered suite does not -have to download the full ZAYA1-base checkpoint (≈17 GB) on every run; the CI -job that owns this test sets the variable explicitly. -""" - -import os -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import is_hip, kill_process_tree -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -# ZAYA1-base is a heavyweight launch (≈120 transformer layers with MoE), so -# the estimated time is set generously to keep the CI scheduler from preempting -# the job before the server finishes warming up. -register_cuda_ci(est_time=420, stage="extra-a", runner_config="1-gpu-large") -register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-large-amd") - - -_MODEL_PATH = os.environ.get("ZAYA_MODEL_PATH", "Zyphra/ZAYA1-base") - - -def _zaya_enabled() -> bool: - return os.environ.get("RUN_ZAYA_E2E", "0") == "1" - - -@unittest.skipUnless( - _zaya_enabled(), - "Set RUN_ZAYA_E2E=1 to enable the ZAYA1 end-to-end server test " - "(requires downloading the model weights).", -) -class TestZayaServer(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = _MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - - other_args = [ - "--mem-fraction-static", - "0.5", - "--max-running-requests", - "8", - ] - if is_hip(): - other_args += ["--attention-backend", "triton"] - - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - if getattr(cls, "process", None) is not None: - kill_process_tree(cls.process.pid) - - def test_generation_basic(self): - """Send three prompts through the ``/generate`` endpoint and require - non-empty completions for each.""" - import requests - - prompts = [ - "The capital of France is", - "1 + 2 + 3 + 4 + 5 =", - "Write a haiku about silicon:", - ] - for prompt in prompts: - resp = requests.post( - f"{self.base_url}/generate", - json={ - "text": prompt, - "sampling_params": { - "temperature": 0.0, - "max_new_tokens": 16, - }, - }, - timeout=60, - ) - self.assertEqual(resp.status_code, 200, resp.text) - data = resp.json() - self.assertIn("text", data, data) - self.assertGreater(len(data["text"].strip()), 0, data) - - def test_mmlu_sanity(self): - """32-example MMLU sanity slice. - - ZAYA1-base is a pretrained (non instruction-tuned) checkpoint that - emits long ``…`` reasoning blocks before settling on a - final letter, so ``max_tokens`` must be large enough for the evaluator - to see the chosen answer. The threshold sits just above chance: it is - a regression sanity check rather than a production-quality gate. An - instruction-tuned ZAYA1 checkpoint scores meaningfully higher and - should raise this bound when wired in. - """ - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=32, - num_threads=8, - max_tokens=1024, - ) - metrics = run_eval(args) - self.assertGreaterEqual( - metrics["score"], - 0.30, - f"MMLU sanity below threshold: {metrics}", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/moe/test_torch_compile_moe.py b/test/registered/moe/test_torch_compile_moe.py index 9e0816555..113e3ddce 100644 --- a/test/registered/moe/test_torch_compile_moe.py +++ b/test/registered/moe/test_torch_compile_moe.py @@ -41,12 +41,13 @@ class TestTorchCompileMoe(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.50) + # 0.48 measured, minus the 0.05 margin the other eval thresholds use. + self.assertGreaterEqual(metrics["score"], 0.43) def run_decode(self, max_new_tokens): response = requests.post( diff --git a/test/registered/quant/test_autoround_quantization.py b/test/registered/quant/test_autoround_quantization.py index 5d76d4746..b2354adef 100644 --- a/test/registered/quant/test_autoround_quantization.py +++ b/test/registered/quant/test_autoround_quantization.py @@ -26,9 +26,11 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large") -MMLU_NUM_EXAMPLES = 32 +MMLU_NUM_EXAMPLES = 256 MMLU_NUM_THREADS = 32 -MMLU_SCORE_THRESHOLD = 22 / MMLU_NUM_EXAMPLES +# The unquantized model scores 0.68-0.70 over 256 questions; int8 gives up a +# point or two, then the 0.05 margin the other eval thresholds use. +MMLU_SCORE_THRESHOLD = 0.63 class TestAutoRoundQuantization(CustomTestCase): diff --git a/test/registered/quant/test_awq.py b/test/registered/quant/test_awq.py index abaadd7ba..f29e6e088 100644 --- a/test/registered/quant/test_awq.py +++ b/test/registered/quant/test_awq.py @@ -38,7 +38,7 @@ class TestAWQ(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) @@ -72,7 +72,7 @@ class TestAWQMarlinBfloat16(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) diff --git a/test/registered/sampling/test_pytorch_sampling_backend.py b/test/registered/sampling/test_pytorch_sampling_backend.py index 4322df298..ba4019db2 100644 --- a/test/registered/sampling/test_pytorch_sampling_backend.py +++ b/test/registered/sampling/test_pytorch_sampling_backend.py @@ -40,13 +40,13 @@ class TestPyTorchSamplingBackend(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, temperature=0.1, ) metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) + self.assertGreaterEqual(metrics["score"], 0.64) @unittest.skipIf( is_in_amd_ci(), diff --git a/test/registered/scheduler/test_retract_decode.py b/test/registered/scheduler/test_retract_decode.py index 8eafe6afd..81c598c05 100644 --- a/test/registered/scheduler/test_retract_decode.py +++ b/test/registered/scheduler/test_retract_decode.py @@ -51,12 +51,12 @@ class TestRetractDecode(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="mmlu", - num_examples=64, + num_examples=256, num_threads=32, ) metrics = run_eval(args) - self.assertGreaterEqual(metrics["score"], 0.65) + self.assertGreaterEqual(metrics["score"], 0.64) time.sleep(1) # wait for mem check assert self.process.poll() is None, "Server crashed during test" diff --git a/test/registered/tokenizer/test_multi_tokenizer.py b/test/registered/tokenizer/test_multi_tokenizer.py index 72466abb6..0789360ce 100644 --- a/test/registered/tokenizer/test_multi_tokenizer.py +++ b/test/registered/tokenizer/test_multi_tokenizer.py @@ -27,8 +27,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin): """One server covering both worker pools: multi-tokenizer and multi-detokenizer (the flags are orthogonal).""" - mmlu_score_threshold = 0.65 - mmlu_num_examples = 64 + mmlu_score_threshold = 0.64 + mmlu_num_examples = 256 mmlu_num_threads = 32 @classmethod diff --git a/test/registered/unit/bench/test_simple_eval_gsm8k.py b/test/registered/unit/bench/test_simple_eval_gsm8k.py index 7d80e28c4..6c4fda7ec 100644 --- a/test/registered/unit/bench/test_simple_eval_gsm8k.py +++ b/test/registered/unit/bench/test_simple_eval_gsm8k.py @@ -129,6 +129,56 @@ class TestRunSglEval(CustomTestCase): self.assertNotIn("--num-examples", captured["cmd"]) + def _capture_cmd(self, eval_name="gsm8k", **overrides): + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + out_dir = Path(cmd[cmd.index("--out-dir") + 1]) + _write_fake_metrics( + out_dir, + eval_name, + { + "model": "test-model", + "latency_seconds": 1.0, + "output_throughput_tps": 1.0, + "aggregate": {"score": 0.5}, + }, + ) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + with tempfile.TemporaryDirectory() as td: + args = self._args(td, **overrides) + with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): + _run_sgl_eval(eval_name, args) + return captured["cmd"] + + def test_omits_sampling_flags_when_unset(self): + """Unset top_p / seed / repeat must not reach the CLI -- sgl-eval's own + defaults differ from a forced value (seed unset != seed 0).""" + cmd = self._capture_cmd() + for flag in ("--top-p", "--seed", "--n-repeats"): + self.assertNotIn(flag, cmd) + + def test_forwards_sampling_flags_when_set(self): + cmd = self._capture_cmd(top_p=0.95, seed=0, repeat=1) + for flag, value in (("--top-p", "0.95"), ("--seed", "0"), ("--n-repeats", "1")): + self.assertIn(flag, cmd) + self.assertEqual(cmd[cmd.index(flag) + 1], value) + + def test_thinking_auto_detected_from_model_name(self): + self.assertIn( + "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8") + ) + + def test_explicit_thinking_false_suppresses_auto_detect(self): + """A caller matching a harness that sent no chat_template_kwargs has to be + able to turn the model-name heuristic off.""" + cmd = self._capture_cmd( + model="Qwen/Qwen3.5-397B-A17B-FP8", sgl_eval_thinking=False + ) + self.assertNotIn("--thinking", cmd) + def test_raises_on_nonzero_exit(self): def fake_run(cmd, **kwargs): return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")