[CI] Route mmlu and GB300 MMMU-Pro evals through sgl-eval (#34477)

.github/workflows/_pr-test-stage-cpu.yml

@@ -119,6 +119,8 @@ jobs:
           UV_SYSTEM_PYTHON: "1"
         run: |
           uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow
+          source scripts/ci/utils/sgl_eval_ref.sh
+          uv pip install "$SGL_EVAL_SPEC" --index-strategy unsafe-best-match
 
       # Hosted runners are ephemeral, so models are re-fetched every run and the
       # Hub occasionally returns 429s. Persist the HF cache in GitHub's cache

python/sglang/test/accuracy_test_runner.py

@@ -8,7 +8,6 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     ModelLaunchSettings,
-    dump_metric,
     popen_launch_server,
     write_github_step_summary,
 )
@@ -31,6 +30,10 @@ class AccuracyTestParams:
     top_k: Optional[int] = None
     repeat: Optional[int] = None
     api: Optional[str] = None  # "chat" or "completion"; defaults to "chat" in run_eval
+    seed: Optional[int] = None  # pin for reproducibility when temperature > 0
+    # sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead
+    # of letting _run_sgl_eval infer it from the model name.
+    sgl_eval_thinking: Optional[bool] = None
 
 
 @dataclass
@@ -89,6 +92,8 @@ def _run_simple_eval(
     top_k: Optional[int] = None,
     repeat: Optional[int] = None,
     api: Optional[str] = None,
+    seed: Optional[int] = None,
+    sgl_eval_thinking: Optional[bool] = None,
 ) -> Tuple[bool, Optional[str], Optional[dict]]:
     """Run evaluation using simple_eval backend (run_eval.py).
 
@@ -137,6 +142,12 @@ def _run_simple_eval(
         if repeat is not None:
             args.repeat = repeat
 
+        if seed is not None:
+            args.seed = seed
+
+        if sgl_eval_thinking is not None:
+            args.sgl_eval_thinking = sgl_eval_thinking
+
         result = run_eval(args)
 
         # Handle result format (run_eval can return metrics or (metrics, latency))
@@ -156,299 +167,6 @@ def _run_simple_eval(
             kill_process_tree(process.pid)
 
 
-# Cached uv venv for NeMo Skills (persists across variants within a process).
-_nemo_venv_dir: Optional[str] = None
-_nemo_data_prepared: set = set()
-
-
-def _get_nemo_venv() -> Tuple[str, dict]:
-    """Get or create a uv venv with nemo_skills installed.
-
-    Returns (venv_python_path, env_dict) reusable across calls.
-    """
-    import os
-    import subprocess
-    import tempfile
-
-    global _nemo_venv_dir
-
-    if _nemo_venv_dir is not None:
-        venv_python = f"{_nemo_venv_dir}/venv/bin/python"
-        env = {
-            **dict(os.environ),
-            "NEMO_SKILLS_DISABLE_UNCOMMITTED_CHANGES_CHECK": "1",
-            "OPENAI_API_KEY": "dummy",
-            "VIRTUAL_ENV": f"{_nemo_venv_dir}/venv",
-            "PATH": f"{_nemo_venv_dir}/venv/bin:" + os.environ.get("PATH", ""),
-        }
-        return venv_python, env
-
-    _nemo_venv_dir = tempfile.mkdtemp(prefix="nemo_skills_")
-    print(f"Creating NeMo Skills venv in {_nemo_venv_dir}...")
-
-    # Create venv
-    result = subprocess.run(
-        ["uv", "venv", f"{_nemo_venv_dir}/venv", "--python", "3.12"],
-        capture_output=True,
-        text=True,
-    )
-    if result.returncode != 0:
-        subprocess.run(
-            ["uv", "venv", f"{_nemo_venv_dir}/venv"],
-            capture_output=True,
-            text=True,
-        )
-
-    # Install nemo_skills.
-    # Pinned: NeMo-Skills main after PR #1433 pins litellm==1.83.14 (httpx==0.28.1),
-    # which is unsatisfiable against nemo-run's transitive leptonai dep.
-    nemo_skills_ref = "589294c"
-    print(f"Installing nemo_skills (pinned to {nemo_skills_ref})...")
-    pip_result = subprocess.run(
-        [
-            "uv",
-            "pip",
-            "install",
-            "--python",
-            f"{_nemo_venv_dir}/venv/bin/python",
-            f"git+https://github.com/NVIDIA/NeMo-Skills.git@{nemo_skills_ref}",
-            "mcp<2",
-            "typer<0.27",
-        ],
-        capture_output=True,
-        text=True,
-        timeout=300,
-    )
-    if pip_result.returncode != 0:
-        raise RuntimeError(f"Failed to install nemo_skills: {pip_result.stderr[-500:]}")
-
-    print("NeMo Skills installed successfully")
-    return _get_nemo_venv()
-
-
-def _ensure_nemo_data_prepared(
-    venv_python: str, env: dict, dataset: str
-) -> Tuple[bool, Optional[str]]:
-    """Prepare NeMo Skills dataset data if not already done.
-
-    Uses the venv python so data lands inside the venv's nemo_skills package.
-    """
-    import subprocess
-
-    if dataset in _nemo_data_prepared:
-        return True, None
-
-    print(f"Preparing {dataset} data (this may take a few minutes for VLM datasets)...")
-    result = subprocess.run(
-        [venv_python, "-m", "nemo_skills.dataset.prepare", dataset],
-        text=True,
-        timeout=600,
-        env=env,
-    )
-    if result.returncode != 0:
-        return False, f"Failed to prepare {dataset} data (exit {result.returncode})"
-
-    _nemo_data_prepared.add(dataset)
-    return True, None
-
-
-def _run_nemo_skills_eval(
-    model: ModelLaunchSettings,
-    base_url: str,
-    dataset: str,
-    max_tokens: Optional[int] = None,
-    repeat: Optional[int] = None,
-    temperature: Optional[float] = None,
-    top_p: Optional[float] = None,
-) -> Tuple[bool, Optional[str], Optional[dict]]:
-    """Run evaluation using NeMo Skills (ns eval) for benchmarks like mmmu-pro.
-
-    Uses an isolated uv venv (shared across variants) so nemo_skills dependencies
-    don't interfere with the system python / sglang server.
-
-    Returns:
-        Tuple of (success, error_message, metrics_dict)
-    """
-    import subprocess
-    import tempfile
-
-    process = None
-    try:
-        # Get or create the shared venv (once per process)
-        venv_python, env = _get_nemo_venv()
-
-        # Prepare dataset (once per process, cached)
-        ok, err = _ensure_nemo_data_prepared(venv_python, env, dataset)
-        if not ok:
-            return False, err, None
-
-        process = popen_launch_server(
-            model.model_path,
-            base_url,
-            other_args=model.extra_args,
-            timeout=model.launch_timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            env=model.env,
-        )
-
-        port = int(base_url.split(":")[-1])
-        server_address = f"http://127.0.0.1:{port}/v1"
-        repeat_val = repeat or 1
-        max_tokens_val = max_tokens or 32768
-        benchmark_spec = f"{dataset}:{repeat_val}"
-
-        # Build ns eval command using venv python
-        # Note: nemo_skills.pipeline.eval requires the "eval" subcommand
-        output_dir = tempfile.mkdtemp(prefix="ns_eval_output_")
-        cmd = [
-            venv_python,
-            "-m",
-            "nemo_skills.pipeline.eval",
-            "eval",
-            f"--benchmarks={benchmark_spec}",
-            "--server_type=sglang",
-            f"--model={model.model_path}",
-            f"--server_address={server_address}",
-            f"--output_dir={output_dir}",
-            f"++inference.tokens_to_generate={max_tokens_val}",
-        ]
-
-        if temperature is not None:
-            cmd.append(f"++inference.temperature={temperature}")
-        if top_p is not None:
-            cmd.append(f"++inference.top_p={top_p}")
-
-        # Add VLM-specific config
-        if dataset in ("mmmu-pro", "mmmu_pro"):
-            cmd.append("++prompt_config=vlm/mmmu-pro")
-            cmd.append("++max_concurrent_requests=512")
-            cmd.append("++max_samples=500")
-
-        print(f"Running: {' '.join(cmd)}")
-        eval_result = subprocess.run(
-            cmd,
-            capture_output=True,
-            text=True,
-            timeout=7200,
-            env=env,
-        )
-
-        print(eval_result.stdout[-2000:] if eval_result.stdout else "(no stdout)")
-        if eval_result.stderr:
-            print(eval_result.stderr[-1000:])
-
-        if eval_result.returncode != 0:
-            return (
-                False,
-                f"ns eval failed (exit {eval_result.returncode}): {eval_result.stderr[-500:]}",
-                None,
-            )
-
-        # Parse results
-        summarize_result = subprocess.run(
-            [
-                venv_python,
-                "-m",
-                "nemo_skills.pipeline.summarize_results",
-                f"{output_dir}/eval-results",
-            ],
-            capture_output=True,
-            text=True,
-            timeout=60,
-            env=env,
-        )
-
-        output = summarize_result.stdout + "\n" + eval_result.stdout
-        print(f"Summary: {summarize_result.stdout[:1000]}")
-
-        # Parse accuracy from output (format varies, look for common patterns)
-        import re
-
-        score = None
-        for line in output.split("\n"):
-            match = re.search(r"(?:accuracy|score)[:\s]+([0-9.]+)", line, re.IGNORECASE)
-            if match:
-                score = float(match.group(1))
-
-        if score is None:
-            # Try to find it in eval-results directory
-            import glob
-            import json
-
-            for result_file in glob.glob(
-                f"{output_dir}/eval-results/**/*.json", recursive=True
-            ):
-                try:
-                    with open(result_file) as f:
-                        data = json.load(f)
-                    if isinstance(data, dict):
-                        score = (
-                            data.get("accuracy")
-                            or data.get("score")
-                            or data.get("mean_score")
-                        )
-                        if score is not None:
-                            break
-                except (json.JSONDecodeError, KeyError):
-                    continue
-
-        if score is None:
-            # Last resort: compute accuracy directly from JSONL output
-            import glob
-            import json
-
-            for jsonl_file in sorted(
-                glob.glob(f"{output_dir}/eval-results/**/*.jsonl*", recursive=True)
-            ):
-                correct = 0
-                total = 0
-                try:
-                    with open(jsonl_file) as f:
-                        for line in f:
-                            line = line.strip()
-                            if not line:
-                                continue
-                            entry = json.loads(line)
-                            expected = entry.get("expected_answer", "")
-                            generation = entry.get("generation", "")
-                            # Extract "Answer: X" from the end of generation
-                            answer_match = re.search(
-                                r"Answer:\s*([A-J])", generation, re.IGNORECASE
-                            )
-                            if answer_match:
-                                predicted = answer_match.group(1).upper()
-                                if predicted == expected.upper():
-                                    correct += 1
-                            total += 1
-                except (json.JSONDecodeError, KeyError, OSError):
-                    continue
-                if total > 0:
-                    score = correct / total
-                    print(
-                        f"Computed accuracy from {jsonl_file}: "
-                        f"{correct}/{total} = {score:.4f}"
-                    )
-                    break
-
-        if score is None:
-            return False, "Could not parse accuracy from ns eval output", None
-
-        dump_metric(
-            f"{dataset}_score",
-            score,
-            labels={"model": model.model_path, "eval": dataset, "api": "nemo-skills"},
-        )
-
-        return True, None, {"score": score}
-
-    except subprocess.TimeoutExpired:
-        return False, "NeMo Skills eval timed out", None
-    except Exception as e:
-        return False, f"NeMo Skills eval exception: {str(e)}", None
-    finally:
-        if process:
-            kill_process_tree(process.pid)
-
-
 def run_accuracy_test(
     model: ModelLaunchSettings,
     params: AccuracyTestParams,
@@ -472,35 +190,23 @@ def run_accuracy_test(
     print(f"  Baseline: {params.baseline_accuracy}")
     print(f"{'='*60}\n")
 
-    # Run evaluation based on dataset type
+    success, error, metrics = _run_simple_eval(
-    # - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
+        model=model,
-    # - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.)
+        base_url=base_url,
-    if params.dataset in ("mmmu-pro", "mmmu_pro"):
+        dataset=params.dataset,
-        success, error, metrics = _run_nemo_skills_eval(
+        num_examples=params.num_examples,
-            model=model,
+        num_threads=params.num_threads,
-            base_url=base_url,
+        max_tokens=params.max_tokens,
-            dataset="mmmu-pro",
+        return_latency=params.return_latency,
-            max_tokens=params.max_tokens,
+        thinking_mode=params.thinking_mode,
-            repeat=params.repeat or 1,
+        temperature=params.temperature,
-            temperature=params.temperature,
+        top_p=params.top_p,
-            top_p=params.top_p,
+        top_k=params.top_k,
-        )
+        repeat=params.repeat,
-    else:
+        api=params.api,
-        success, error, metrics = _run_simple_eval(
+        seed=params.seed,
-            model=model,
+        sgl_eval_thinking=params.sgl_eval_thinking,
-            base_url=base_url,
+    )
-            dataset=params.dataset,
-            num_examples=params.num_examples,
-            num_threads=params.num_threads,
-            max_tokens=params.max_tokens,
-            return_latency=params.return_latency,
-            thinking_mode=params.thinking_mode,
-            temperature=params.temperature,
-            top_p=params.top_p,
-            top_k=params.top_k,
-            repeat=params.repeat,
-            api=params.api,
-        )
 
     if not success:
         print(f"✗ Accuracy test failed for {model.model_path}: {error}")

python/sglang/test/kits/eval_accuracy_kit.py

@@ -231,9 +231,9 @@ class GSM8KMixin:
 class MMLUMixin:
     """Mixin for MMLU evaluation.
 
-    Backend is selectable via ``mmlu_backend`` (default ``"run_eval"``; or
+    Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it
-    ``"sgl_eval"``: sgl-eval multichoice grader, skipped if sgl-eval is not
+    in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch
-    installed).
+    picks the call mechanism, not the grader.
 
     Required attributes on the test class:
         base_url: str

python/sglang/test/run_eval.py

@@ -136,6 +136,13 @@ def _run_sgl_eval(eval_name, args) -> dict:
         cmd += ["--model", args.model]
     if getattr(args, "num_examples", None) is not None:
         cmd += ["--num-examples", str(args.num_examples)]
+    if getattr(args, "top_p", None) is not None:
+        cmd += ["--top-p", str(args.top_p)]
+    # Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
+    if getattr(args, "seed", None) is not None:
+        cmd += ["--seed", str(args.seed)]
+    if getattr(args, "repeat", None) is not None:
+        cmd += ["--n-repeats", str(args.repeat)]
     # Bound generation length so long-reasoning models don't stall the eval.
     if getattr(args, "max_tokens", None) is not None:
         cmd += ["--max-tokens", str(args.max_tokens)]
@@ -242,10 +249,10 @@ def run_eval(args):
     )
 
     if args.eval_name == "mmlu":
-        from sglang.test.simple_eval_mmlu import MMLUEval
+        # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
-
+        # caller's threshold has to be measured against it, not inherited.
-        filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
+        # `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
-        eval_obj = MMLUEval(filename, args.num_examples, args.num_threads)
+        return _run_sgl_eval("mmlu", args)
     elif args.eval_name == "math":
         from sglang.test.simple_eval_math import MathEval
 
@@ -301,6 +308,10 @@ def run_eval(args):
             args.num_threads,
             response_answer_regex=getattr(args, "response_answer_regex", None),
         )
+    elif args.eval_name == "mmmu_pro_vision":
+        # sgl-eval owns this benchmark's dataset, prompt and grader; there is no
+        # simple_eval implementation to fall back to.
+        return _run_sgl_eval("mmmu_pro_vision", args)
     elif args.eval_name == "aime25":
         from sglang.test.simple_eval_aime25 import AIME25Eval
 

scripts/ci/amd/amd_ci_install_dependency.sh

@@ -134,6 +134,10 @@ else
   install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache -e "python[${EXTRAS}]"
 fi
 
+# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
+source "$(dirname "${BASH_SOURCE[0]}")/../utils/sgl_eval_ref.sh"
+install_with_retry docker exec ci_sglang pip install --cache-dir=/sgl-data/pip-cache "$SGL_EVAL_SPEC"
+
 if [[ -n "${SKIP_TT_DEPS}" ]]; then
   echo "Didn't build lmms_eval, human-eval, and others"
 else

scripts/ci/cuda/ci_install_dependency.sh

@@ -682,12 +682,8 @@ stabilize_flashinfer_jit_paths() {
 install_extra_deps() {
     MOONCAKE_VERSION="0.3.12.post1"
     NIXL_VERSION="1.3.0"
-    # sgl-eval is git-only and cannot be declared in python/pyproject.toml (see
+    # shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
-    # the note there). The nightly GSM8K eval shells out to the sgl-eval CLI and
+    source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
-    # fails without it. Bumping the SHA can change zero-shot \boxed{} grading, so
-    # re-baseline MODEL_SCORE_THRESHOLDS in
-    # test/registered/eval/test_text_models_gsm8k_eval.py first.
-    SGL_EVAL_REF="b2a2703c42cae379bbcb8b7ff092df6601a61694"
     if [ "$CU_MAJOR" = "13" ]; then
         MOONCAKE_PKG="mooncake-transfer-engine-cuda13==${MOONCAKE_VERSION}"
         MOONCAKE_STALE_PKG="mooncake-transfer-engine"
@@ -723,7 +719,7 @@ install_extra_deps() {
             --no-deps --force-reinstall $PIP_INSTALL_SUFFIX
     fi
 
-    $PIP_CMD install "sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}" $PIP_INSTALL_SUFFIX
+    $PIP_CMD install "$SGL_EVAL_SPEC" $PIP_INSTALL_SUFFIX
 
     if [ "$IS_BLACKWELL" != "1" ]; then
         git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git

scripts/ci/npu/npu_ci_install_dependency.sh

@@ -84,3 +84,8 @@ rm -rf cann-custom-ops
 ### Install SGLang
 rm -rf python/pyproject.toml && mv python/pyproject_npu.toml python/pyproject.toml
 ${UV_PIP_INSTALL} -v -e "python[dev_npu]"
+
+### Install sgl-eval
+# shellcheck source=scripts/ci/utils/sgl_eval_ref.sh
+source "${SCRIPT_DIR}/../utils/sgl_eval_ref.sh"
+${UV_PIP_INSTALL} "$SGL_EVAL_SPEC"

scripts/ci/utils/sgl_eval_ref.sh

@@ -0,0 +1,13 @@
+# Single source of truth for the sgl-eval commit every CI variant installs.
+# Meant to be sourced, not executed -- each variant then installs
+# "$SGL_EVAL_SPEC" with its own pip invocation, since those differ (uv pip on
+# CUDA/CPU, `docker exec ... pip` on AMD, `python3 -m pip` on NPU).
+#
+# sgl-eval is git-only and cannot be declared in python/pyproject.toml (see the
+# note there). Every eval that shells out to the `sgl-eval` CLI fails without
+# it, and a bump moves scoring for all of them at once -- so re-baseline
+# MODEL_SCORE_THRESHOLDS in
+# test/registered/eval/test_text_models_gsm8k_eval.py, and the mmlu thresholds
+# of run_eval's other callers, before changing this.
+SGL_EVAL_REF="6690895609dcbc5df1e7b00dd57c9502b868ec4d"
+SGL_EVAL_SPEC="sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@${SGL_EVAL_REF}"

test/registered/attention/test_torch_native_attention_backend.py

@@ -38,12 +38,12 @@ class TestTorchNativeAttnBackend(CustomTestCase):
                 base_url=base_url,
                 model=model,
                 eval_name="mmlu",
-                num_examples=64,
+                num_examples=256,
                 num_threads=32,
             )
 
             metrics = run_eval(args)
-            self.assertGreaterEqual(metrics["score"], 0.65)
+            self.assertGreaterEqual(metrics["score"], 0.64)
         finally:
             kill_process_tree(process.pid)
 

test/registered/attention/test_triton_sliding_window.py

@@ -10,7 +10,6 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
-    is_in_amd_ci,
     is_in_ci,
     popen_launch_server,
 )
@@ -53,17 +52,16 @@ class TestSlidingWindowAttentionTriton(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=200,
+            num_examples=256,
             num_threads=32,
         )
 
         metrics = run_eval(args)
         print(f"MMLU metrics with sliding window: {metrics}")
 
-        if is_in_amd_ci():
+        # gemma-3-4b-it scores 0.59 over 256 questions under sgl-eval's grader,
-            self.assertGreaterEqual(metrics["score"], 0.55)
+        # minus the 0.05 margin the other eval thresholds use.
-        else:
+        self.assertGreaterEqual(metrics["score"], 0.54)
-            self.assertGreaterEqual(metrics["score"], 0.60)
 
     def _test_short_context_generation(self):
         response = requests.post(

test/registered/backends/test_torch_compile.py

@@ -20,8 +20,8 @@ register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-small-amd")
 
 
 class TestTorchCompile(CustomTestCase, MMLUMixin):
-    mmlu_score_threshold = 0.65
+    mmlu_score_threshold = 0.64
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
     @classmethod

test/registered/disaggregation/test_disaggregation_decode_offload.py

@@ -139,7 +139,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
             base_url=f"http://{self.base_host}:{self.lb_port}",
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
@@ -166,8 +166,8 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
         metrics2 = run_eval(args)
 
         # Assert score is above a minimum threshold for both rounds
-        self.assertGreater(metrics1["score"], 0.65)
+        self.assertGreater(metrics1["score"], 0.64)
-        self.assertGreater(metrics2["score"], 0.65)
+        self.assertGreater(metrics2["score"], 0.64)
 
         # Score should be consistent: round 2 should be >= round 1, or at least within a 0.05 margin if slightly lower
         self.assertGreaterEqual(metrics2["score"], metrics1["score"] - 0.05)

test/registered/gb300/test_kimi_k25_nvfp4.py

@@ -63,8 +63,17 @@ class TestKimiK25Nvfp4(unittest.TestCase):
         run_combined_tests(
             models=variants,
             test_name="Kimi-K2.5-NVFP4",
+            # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
+            # its `:1` suffix means temperature 0.7, not greedy -- so the baseline
+            # carries over unchanged. Do not "simplify" these away.
             accuracy_params=AccuracyTestParams(
-                dataset="mmmu-pro", baseline_accuracy=0.69, repeat=1, max_tokens=32768
+                dataset="mmmu_pro_vision",
+                baseline_accuracy=0.69,
+                repeat=1,
+                max_tokens=32768,
+                temperature=0.7,
+                seed=0,
+                sgl_eval_thinking=False,
             ),
             performance_params=PerformanceTestParams(
                 result_dir="performance_results_gb300",

test/registered/gb300/test_qwen35_fp8.py

@@ -61,8 +61,17 @@ class TestQwen35Fp8(unittest.TestCase):
         run_combined_tests(
             models=variants,
             test_name="Qwen3.5-397B-FP8",
+            # Pinned to what `ns eval --benchmarks=mmmu-pro:1` sent implicitly --
+            # its `:1` suffix means temperature 0.7, not greedy -- so the baseline
+            # carries over unchanged. Do not "simplify" these away.
             accuracy_params=AccuracyTestParams(
-                dataset="mmmu-pro", baseline_accuracy=0.76, repeat=1, max_tokens=32768
+                dataset="mmmu_pro_vision",
+                baseline_accuracy=0.76,
+                repeat=1,
+                max_tokens=32768,
+                temperature=0.7,
+                seed=0,
+                sgl_eval_thinking=False,
             ),
             performance_params=PerformanceTestParams(
                 result_dir="performance_results_gb300",

test/registered/hicache/test_hicache_storage.py

@@ -21,8 +21,8 @@ _is_hip = is_hip()
 
 
 class TestHiCache(CustomTestCase, MMLUMixin):
-    mmlu_score_threshold = 0.65
+    mmlu_score_threshold = 0.64
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
     @classmethod

test/registered/hicache/test_hicache_variants.py

@@ -64,8 +64,8 @@ class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):
         "--hicache-size",
         100 if not _is_hip else 200,
     ]
-    mmlu_score_threshold = 0.65
+    mmlu_score_threshold = 0.64
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
 
@@ -77,8 +77,8 @@ class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin):
         "--trust-remote-code",
         "--enable-hierarchical-cache",
     ] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2])
-    mmlu_score_threshold = 0.5
+    mmlu_score_threshold = 0.54
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
     mgsm_en_score_threshold = 0.8
 
@@ -110,8 +110,8 @@ class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin):
         "--chunked-prefill-size",
         1024,
     ]
-    mmlu_score_threshold = 0.72
+    mmlu_score_threshold = 0.64
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
     mmlu_accept_length_thres = 2.26
 
@@ -127,8 +127,8 @@ class TestHiCachePage(HiCacheBaseServer, MMLUMixin):
         "--hicache-write-policy",
         "write_back",
     ]
-    mmlu_score_threshold = 0.65
+    mmlu_score_threshold = 0.64
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
 

test/registered/models_e2e/test_transformers_models.py

@@ -36,7 +36,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
             timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
             other_args=["--model-impl", "transformers"],
         )
-        cls.mmlu_lower_bound = 0.63
+        cls.mmlu_lower_bound = 0.64
         cls.gsm8k_lower_bound = 0.65
 
     @classmethod
@@ -48,7 +48,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
         metrics = run_eval(args)

test/registered/models_e2e/test_zaya.py

@@ -1,127 +0,0 @@
-"""End-to-end server test for Zyphra ZAYA1 (hybrid CCA attention + MoE).
-
-This test boots a real ``Zyphra/ZAYA1-base`` SGLang server via
-``popen_launch_server``, sends a handful of completions through the HTTP API,
-and finishes with a small MMLU sanity slice.
-
-The test is gated behind ``RUN_ZAYA_E2E=1`` so the registered suite does not
-have to download the full ZAYA1-base checkpoint (≈17 GB) on every run; the CI
-job that owns this test sets the variable explicitly.
-"""
-
-import os
-import unittest
-from types import SimpleNamespace
-
-from sglang.srt.utils import is_hip, kill_process_tree
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    popen_launch_server,
-)
-
-# ZAYA1-base is a heavyweight launch (≈120 transformer layers with MoE), so
-# the estimated time is set generously to keep the CI scheduler from preempting
-# the job before the server finishes warming up.
-register_cuda_ci(est_time=420, stage="extra-a", runner_config="1-gpu-large")
-register_amd_ci(est_time=420, suite="stage-b-test-1-gpu-large-amd")
-
-
-_MODEL_PATH = os.environ.get("ZAYA_MODEL_PATH", "Zyphra/ZAYA1-base")
-
-
-def _zaya_enabled() -> bool:
-    return os.environ.get("RUN_ZAYA_E2E", "0") == "1"
-
-
-@unittest.skipUnless(
-    _zaya_enabled(),
-    "Set RUN_ZAYA_E2E=1 to enable the ZAYA1 end-to-end server test "
-    "(requires downloading the model weights).",
-)
-class TestZayaServer(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.model = _MODEL_PATH
-        cls.base_url = DEFAULT_URL_FOR_TEST
-
-        other_args = [
-            "--mem-fraction-static",
-            "0.5",
-            "--max-running-requests",
-            "8",
-        ]
-        if is_hip():
-            other_args += ["--attention-backend", "triton"]
-
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        if getattr(cls, "process", None) is not None:
-            kill_process_tree(cls.process.pid)
-
-    def test_generation_basic(self):
-        """Send three prompts through the ``/generate`` endpoint and require
-        non-empty completions for each."""
-        import requests
-
-        prompts = [
-            "The capital of France is",
-            "1 + 2 + 3 + 4 + 5 =",
-            "Write a haiku about silicon:",
-        ]
-        for prompt in prompts:
-            resp = requests.post(
-                f"{self.base_url}/generate",
-                json={
-                    "text": prompt,
-                    "sampling_params": {
-                        "temperature": 0.0,
-                        "max_new_tokens": 16,
-                    },
-                },
-                timeout=60,
-            )
-            self.assertEqual(resp.status_code, 200, resp.text)
-            data = resp.json()
-            self.assertIn("text", data, data)
-            self.assertGreater(len(data["text"].strip()), 0, data)
-
-    def test_mmlu_sanity(self):
-        """32-example MMLU sanity slice.
-
-        ZAYA1-base is a pretrained (non instruction-tuned) checkpoint that
-        emits long ``<think>…</think>`` reasoning blocks before settling on a
-        final letter, so ``max_tokens`` must be large enough for the evaluator
-        to see the chosen answer. The threshold sits just above chance: it is
-        a regression sanity check rather than a production-quality gate. An
-        instruction-tuned ZAYA1 checkpoint scores meaningfully higher and
-        should raise this bound when wired in.
-        """
-        args = SimpleNamespace(
-            base_url=self.base_url,
-            model=self.model,
-            eval_name="mmlu",
-            num_examples=32,
-            num_threads=8,
-            max_tokens=1024,
-        )
-        metrics = run_eval(args)
-        self.assertGreaterEqual(
-            metrics["score"],
-            0.30,
-            f"MMLU sanity below threshold: {metrics}",
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/moe/test_torch_compile_moe.py

@@ -41,12 +41,13 @@ class TestTorchCompileMoe(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
         metrics = run_eval(args)
-        self.assertGreaterEqual(metrics["score"], 0.50)
+        # 0.48 measured, minus the 0.05 margin the other eval thresholds use.
+        self.assertGreaterEqual(metrics["score"], 0.43)
 
     def run_decode(self, max_new_tokens):
         response = requests.post(

test/registered/quant/test_autoround_quantization.py

@@ -26,9 +26,11 @@ from sglang.test.test_utils import (
 
 register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large")
 
-MMLU_NUM_EXAMPLES = 32
+MMLU_NUM_EXAMPLES = 256
 MMLU_NUM_THREADS = 32
-MMLU_SCORE_THRESHOLD = 22 / MMLU_NUM_EXAMPLES
+# The unquantized model scores 0.68-0.70 over 256 questions; int8 gives up a
+# point or two, then the 0.05 margin the other eval thresholds use.
+MMLU_SCORE_THRESHOLD = 0.63
 
 
 class TestAutoRoundQuantization(CustomTestCase):

test/registered/quant/test_awq.py

@@ -38,7 +38,7 @@ class TestAWQ(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
@@ -72,7 +72,7 @@ class TestAWQMarlinBfloat16(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 

test/registered/sampling/test_pytorch_sampling_backend.py

@@ -40,13 +40,13 @@ class TestPyTorchSamplingBackend(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
             temperature=0.1,
         )
 
         metrics = run_eval(args)
-        self.assertGreaterEqual(metrics["score"], 0.65)
+        self.assertGreaterEqual(metrics["score"], 0.64)
 
     @unittest.skipIf(
         is_in_amd_ci(),

test/registered/scheduler/test_retract_decode.py

@@ -51,12 +51,12 @@ class TestRetractDecode(CustomTestCase):
             base_url=self.base_url,
             model=self.model,
             eval_name="mmlu",
-            num_examples=64,
+            num_examples=256,
             num_threads=32,
         )
 
         metrics = run_eval(args)
-        self.assertGreaterEqual(metrics["score"], 0.65)
+        self.assertGreaterEqual(metrics["score"], 0.64)
         time.sleep(1)  # wait for mem check
 
         assert self.process.poll() is None, "Server crashed during test"

test/registered/tokenizer/test_multi_tokenizer.py

@@ -27,8 +27,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
     """One server covering both worker pools: multi-tokenizer and
     multi-detokenizer (the flags are orthogonal)."""
 
-    mmlu_score_threshold = 0.65
+    mmlu_score_threshold = 0.64
-    mmlu_num_examples = 64
+    mmlu_num_examples = 256
     mmlu_num_threads = 32
 
     @classmethod

test/registered/unit/bench/test_simple_eval_gsm8k.py

@@ -129,6 +129,56 @@ class TestRunSglEval(CustomTestCase):
 
         self.assertNotIn("--num-examples", captured["cmd"])
 
+    def _capture_cmd(self, eval_name="gsm8k", **overrides):
+        captured = {}
+
+        def fake_run(cmd, **kwargs):
+            captured["cmd"] = cmd
+            out_dir = Path(cmd[cmd.index("--out-dir") + 1])
+            _write_fake_metrics(
+                out_dir,
+                eval_name,
+                {
+                    "model": "test-model",
+                    "latency_seconds": 1.0,
+                    "output_throughput_tps": 1.0,
+                    "aggregate": {"score": 0.5},
+                },
+            )
+            return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
+
+        with tempfile.TemporaryDirectory() as td:
+            args = self._args(td, **overrides)
+            with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
+                _run_sgl_eval(eval_name, args)
+        return captured["cmd"]
+
+    def test_omits_sampling_flags_when_unset(self):
+        """Unset top_p / seed / repeat must not reach the CLI -- sgl-eval's own
+        defaults differ from a forced value (seed unset != seed 0)."""
+        cmd = self._capture_cmd()
+        for flag in ("--top-p", "--seed", "--n-repeats"):
+            self.assertNotIn(flag, cmd)
+
+    def test_forwards_sampling_flags_when_set(self):
+        cmd = self._capture_cmd(top_p=0.95, seed=0, repeat=1)
+        for flag, value in (("--top-p", "0.95"), ("--seed", "0"), ("--n-repeats", "1")):
+            self.assertIn(flag, cmd)
+            self.assertEqual(cmd[cmd.index(flag) + 1], value)
+
+    def test_thinking_auto_detected_from_model_name(self):
+        self.assertIn(
+            "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8")
+        )
+
+    def test_explicit_thinking_false_suppresses_auto_detect(self):
+        """A caller matching a harness that sent no chat_template_kwargs has to be
+        able to turn the model-name heuristic off."""
+        cmd = self._capture_cmd(
+            model="Qwen/Qwen3.5-397B-A17B-FP8", sgl_eval_thinking=False
+        )
+        self.assertNotIn("--thinking", cmd)
+
     def test_raises_on_nonzero_exit(self):
         def fake_run(cmd, **kwargs):
             return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")