[Test] Route all sgl-eval benchmarks through run_sgl_eval and deprecate run_eval (#41280)

This commit is contained in:
Liangsheng Yin
2026-09-25 15:48:38 -07:00
committed by GitHub
parent 27e883a20d
commit 516ab77619
44 changed files with 321 additions and 1378 deletions
+6 -31
View File
@@ -4,6 +4,7 @@ from typing import List, Optional, Tuple
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -31,9 +32,6 @@ class AccuracyTestParams:
repeat: Optional[int] = None
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
seed: Optional[int] = None # pin for reproducibility when temperature > 0
# sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead
# of letting _run_sgl_eval infer it from the model name.
sgl_eval_thinking: Optional[bool] = None
num_shots: Optional[int] = None # few-shot count; None = run_eval's default
@@ -56,13 +54,7 @@ def write_accuracy_github_summary(
dataset: str,
results: List[AccuracyTestResult],
) -> None:
"""Write accuracy test results to GitHub step summary.
Args:
test_name: Name of the test
dataset: Dataset name used for evaluation
results: List of AccuracyTestResult objects
"""
"""Write accuracy test results to GitHub step summary."""
summary = f"#### {test_name} - Accuracy ({dataset})\n"
summary += "| config | status | score | baseline | error |\n"
summary += "| ------ | ------ | ----- | -------- | ----- |\n"
@@ -95,13 +87,8 @@ def _run_simple_eval(
repeat: Optional[int] = None,
api: Optional[str] = None,
seed: Optional[int] = None,
sgl_eval_thinking: Optional[bool] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using simple_eval backend (run_eval.py).
Returns:
Tuple of (success, error_message, metrics_dict)
"""
"""Run ``dataset`` through run_sgl_eval (sgl-eval benchmarks) or run_eval."""
process = None
try:
process = popen_launch_server(
@@ -150,10 +137,8 @@ def _run_simple_eval(
if seed is not None:
args.seed = seed
if sgl_eval_thinking is not None:
args.sgl_eval_thinking = sgl_eval_thinking
result = run_eval(args)
evaluate = run_sgl_eval if dataset in SGL_EVAL_BENCHMARKS else run_eval
result = evaluate(args)
# Handle result format (run_eval can return metrics or (metrics, latency))
if return_latency and isinstance(result, tuple):
@@ -177,16 +162,7 @@ def run_accuracy_test(
params: AccuracyTestParams,
base_url: Optional[str] = None,
) -> AccuracyTestResult:
"""Run accuracy test for a single model.
Args:
model: ModelLaunchSettings with model config
params: AccuracyTestParams with dataset, baseline, and optional settings
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
Returns:
AccuracyTestResult with test outcome
"""
"""Run accuracy test for a single model."""
base_url = base_url or DEFAULT_URL_FOR_TEST
print(f"\n{'=' * 60}")
@@ -211,7 +187,6 @@ def run_accuracy_test(
repeat=params.repeat,
api=params.api,
seed=params.seed,
sgl_eval_thinking=params.sgl_eval_thinking,
)
if not success:
+12 -104
View File
@@ -1,14 +1,6 @@
"""
Common utilities for testing and benchmarking on NPU.
"""Common utilities for testing and benchmarking on NPU.
This file contains the following weight path categories:
- LLM model weights path
- VLM model weights path
- Embedding model weights path
- Rerank model weights path
- Reward model weights path
Please remember to sort by variable name within each section.
Keep the weight-path constants sorted by variable name within each section.
"""
import asyncio
@@ -26,7 +18,7 @@ import requests
from sglang.benchmark.serving import run_benchmark
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -395,15 +387,7 @@ CONFIG_YAML_PATH = (
class ModelTestConfig(NamedTuple):
"""
Configuration for model testing.
Attributes:
model_path: Path to the model weights directory
mmlu_score: Weight for MMLU benchmark score
gsm8k_accuracy: Weight for GSM8K benchmark score
mmmu_accuracy: Weight for MMMU benchmark score
"""
"""Model weights path and the expected MMLU / GSM8K / MMMU scores."""
model_path: str
mmlu_score: Optional[float] = None
@@ -436,16 +420,7 @@ DEFAULT_WEIGHTS_FOR_TEST = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_FOR_TEST
def run_command(cmd, shell=True):
"""Execute system command and return stdout
parameter:
cmd: command to execute
shell:
True, Execute command in shell
False, Commands are invoked directly without shell parsing
return:
The result of executing the command
"""
"""Run ``cmd`` and return its stdout."""
try:
result = subprocess.run(
cmd, shell=shell, capture_output=True, text=True, check=True
@@ -556,37 +531,7 @@ def run_bench_serving(
lora_name: Optional[str] = None,
timeout_for_server_launch=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
):
"""Start the service and obtain the inference results.
Parameters:
model: Model name
num_prompts: Total number of test requests
request_rate: Request rate
other_server_args: Additional configuration when starting the service
dataset_name: Data set name
dataset_path: Dataset path
tokenizer: tokenizer
random_input_len: The length of the randomly generated input prompt
random_output_len: The length of the randomly generated output prompt
sharegpt_context_len: Sharegpt dataset context length
disable_stream: Disable streaming output
disable_ignore_eos: Should eos_token be ignored?
need_warmup: Preheating required
seed: random seed
device: Device type
gsp_num_groups: Grouped Sequence Parallelism
gsp_prompts_per_group: Number of parallel prompts within each group
gsp_system_prompt_len: GSP system prompts length
gsp_question_len: GSP question length
gsp_output_len: GSP output length
max_concurrency: Maximum number of concurrent requests
background_task: Background tasks
lora_name: LoRA fine-tuning model path
timeout_for_server_launch: Raise the service timeout period
Returns:
res: Number of requests successfully completed
"""
"""Launch the server, run bench_serving against it, and return the result."""
if device == "auto":
device = auto_config_device()
@@ -658,17 +603,7 @@ def run_bench_serving(
# hook factory
def create_attention_monitor_hook_factory(config):
"""
Factory function to create a forward hook for monitoring self-attention layer states.
This hook records input/output statistics during model forward propagation.
Args:
config (dict): Configuration dictionary containing hook parameters
layer_index (int): Index of the target attention layer to monitor
Returns:
function: Forward hook function to be registered on the target module
"""
"""Return a forward hook that logs self-attention states of ``config["layer_index"]``."""
# Get target layer index from config, default to 0 if not specified
layer_index = config.get("layer_index", 0)
@@ -681,18 +616,7 @@ def create_attention_monitor_hook_factory(config):
)
def attention_monitor_hook(module, inputs, output):
"""
Forward hook function that monitors and logs the internal states of a self-attention layer.
Executed automatically during the forward pass of the module it is registered to.
Args:
module (torch.nn.Module): The module this hook is attached to
inputs (tuple): Input tensors passed to the module's forward method
output (torch.Tensor): Output tensor returned by the module's forward method
Returns:
torch.Tensor: Unmodified output tensor to preserve model computation flow
"""
"""Log the attention layer's input/output statistics; returns ``output`` unchanged."""
# Record current timestamp for time-series tracking
timestamp = time.time()
@@ -832,7 +756,7 @@ def run_mmlu_test(
)
try:
metrics = run_eval(args)
metrics = run_sgl_eval(args)
assert metrics["score"] >= 0.65, f"{metrics=}"
finally:
pass
@@ -856,26 +780,10 @@ def send_concurrent_requests(
temperature: float = 0.0,
request_timeout: int = 60,
) -> list:
"""Send multiple concurrent HTTP POST requests to the /generate endpoint.
"""Send ``num_requests`` concurrent POSTs to /generate; returns one result dict per request.
Uses threading (NOT asyncio + blocking calls) to achieve true concurrency.
asyncio.gather() combined with synchronous requests.post() does not produce
real parallelism; threading is required for concurrent blocking I/O.
Parameters:
base_url: Server base URL, e.g. "http://127.0.0.1:30000"
num_requests: Total number of requests to send
num_concurrent: Maximum in-flight requests at any given time (semaphore)
input_text: Text prompt sent to every request
max_new_tokens: Maximum new tokens to generate per request
temperature: Sampling temperature (0 = greedy / deterministic)
request_timeout: Per-request HTTP timeout in seconds; raises on exceed
Returns:
Unsorted list of result dicts, one per request, each with:
task_id (int) -- zero-based request index
status_code (int)-- HTTP status code, or -1 on exception
text (str) -- response body, or exception message on failure
Uses threads: asyncio.gather() over synchronous requests.post() gives no real
parallelism for blocking I/O.
"""
results: list = []
+2 -2
View File
@@ -2,7 +2,7 @@ import subprocess
from types import SimpleNamespace
from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import run_sgl_eval
class TestMMLU:
@@ -28,7 +28,7 @@ class TestMMLU:
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
metrics = run_sgl_eval(args)
model_metrics["accuracy"] = metrics["score"]
model_metrics["latency"] = metrics.get("latency", "-")
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
+4 -6
View File
@@ -7,7 +7,7 @@ from typing import Dict, List, Literal, Optional
import requests
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -66,8 +66,7 @@ class BaseTestGptOss(CustomTestCase):
try:
self._check_streaming_responses_api_request(model)
# run multiple tests in parallel since we are mostly bound by the longest generate sequence
# instead of the number of questions
# Parallel: bound by the longest generation, not the question count.
with ThreadPoolExecutor(max_workers=4) as executor:
list(
executor.map(
@@ -131,8 +130,7 @@ class BaseTestGptOss(CustomTestCase):
num_threads=198,
# sgl-eval's gpqa defaults to n_repeats=8.
repeat=1,
# TODO 4k is still not enough, we need e.g. 64k token, but that is super slow
# otherwise a lot of questions are not answered
# Too small (many answers truncate); ~64k would fit but is too slow.
max_tokens=4096,
# Arbitrary; non-zero so a tier is not scored on one greedy path.
temperature=0.1,
@@ -142,7 +140,7 @@ class BaseTestGptOss(CustomTestCase):
setup = f"model={model} reasoning_effort={reasoning_effort} expected_score={expected_score}"
print(f"Evaluation start: {setup}")
metrics = run_eval(args)
metrics = run_sgl_eval(args)
print(f"Evaluation end: {setup} {metrics=}")
self.assertGreaterEqual(metrics["score"], expected_score)
+36 -217
View File
@@ -4,6 +4,7 @@ from typing import Optional
import requests
from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval
from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
_THRESHOLD_NOT_SET = float("nan")
@@ -51,14 +52,8 @@ def _run_accuracy_eval(
):
"""Shared driver for the accuracy mixins below.
Runs ``run_eval`` for ``eval_name`` against the test class's server
(``base_url`` / ``model``), records a CI step summary, asserts the score
meets ``score_threshold``, and checks the speculative accept length.
``eval_overrides`` (e.g. ``api``, ``max_tokens``, ``temperature``,
``top_p``, ``num_shots``) are forwarded to ``run_eval`` only when not
``None``, so the common case stays identical to ``run_eval``'s defaults.
Returns the metrics dict.
``eval_overrides`` are forwarded only when not ``None``, so unset knobs keep
the evaluator's defaults.
"""
assert score_threshold == score_threshold, (
f"{type(test_case).__name__} must set the {eval_name} score threshold"
@@ -74,7 +69,8 @@ def _run_accuracy_eval(
)
kwargs.update({k: v for k, v in eval_overrides.items() if v is not None})
metrics = run_eval(SimpleNamespace(**kwargs))
evaluate = run_sgl_eval if eval_name in SGL_EVAL_BENCHMARKS else run_eval
metrics = evaluate(SimpleNamespace(**kwargs))
print(f"{eval_name} {metrics=}")
_finalize_eval(
test_case,
@@ -92,80 +88,53 @@ def _run_sgl_eval(
*,
eval_name: str,
score_threshold: float,
metric: str = "score",
n_repeats: int = 1,
num_examples: Optional[int] = None,
num_threads: int = 512,
thinking: bool = True,
chat_template_kwargs: Optional[dict] = None,
reasoning_effort: Optional[str] = None,
max_tokens: Optional[int] = None,
temperature: Optional[float] = None,
top_p: Optional[float] = None,
accept_length_thres: Optional[float] = None,
summary_label: Optional[str] = None,
):
"""Shared sgl-eval driver for the reasoning mixins and the ``sgl_eval`` backend.
"""Shared sgl-eval driver for the MMLU sanity gate and the GSM8K ``sgl_eval`` backend.
Runs ``eval_name`` via the sgl-eval Python API (``registry.get`` ->
``EvalSpec.run``) against the test class's server, records a CI step summary,
asserts the score meets ``score_threshold``, and checks the speculative accept
length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking":
True}`` so the server separates reasoning from the final answer. Skips the test
if sgl-eval is not installed. Returns the RunResult.
``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": True}``
so the server separates reasoning from the final answer.
"""
assert score_threshold == score_threshold, (
f"{type(test_case).__name__} must set the {eval_name} score threshold"
)
try:
from sgl_eval.registry import get as get_eval_spec
from sgl_eval.sampler import ChatCompletionSampler
from sgl_eval.types import GenConfig
import sgl_eval # noqa: F401
except ImportError:
test_case.skipTest("sgl-eval not installed; pip install 'sglang[test]'")
base_url = test_case.base_url.rstrip("/")
if not base_url.endswith("/v1"):
base_url += "/v1"
sampler = ChatCompletionSampler(
base_url=base_url, model=getattr(test_case, "model", None), api_key="EMPTY"
)
gen_kwargs = dict(
args = SimpleNamespace(
eval_name=eval_name,
base_url=test_case.base_url,
model=getattr(test_case, "model", None),
num_examples=num_examples,
num_threads=num_threads,
max_tokens=max_tokens,
reasoning_effort=reasoning_effort,
temperature=temperature,
chat_template_kwargs=(
chat_template_kwargs
if chat_template_kwargs is not None
else ({"thinking": True} if thinking else None)
),
)
if temperature is not None:
gen_kwargs["temperature"] = temperature
if top_p is not None:
gen_kwargs["top_p"] = top_p
result = get_eval_spec(eval_name).run(
sampler=sampler,
gen=GenConfig(**gen_kwargs),
n_repeats=n_repeats,
num_examples=num_examples,
num_threads=num_threads,
predictions_writer=None,
load_examples=None,
)
score = result.aggregate[metric]
print(f"{eval_name} sgl-eval {metric}={score:.4f}")
metrics = run_sgl_eval(args)
score = metrics["score"]
print(f"{eval_name} sgl-eval score={score:.4f}")
_finalize_eval(
test_case,
eval_name=eval_name,
score=score,
score_threshold=score_threshold,
accept_length_thres=accept_length_thres,
summary_label=summary_label,
)
return result
return metrics
class MMLUSanityMixin:
@@ -192,18 +161,9 @@ class MMLUSanityMixin:
class GSM8KMixin:
"""Mixin for GSM8K evaluation.
Backend is selectable via ``gsm8k_backend`` (default ``"run_eval"``: OpenAI
completion API, 5-shot; or ``"sgl_eval"``: sgl-eval chat + boxed/sympy grader,
skipped if sgl-eval is not installed). The canonical threshold/count knobs are
``gsm8k_score_threshold`` / ``gsm8k_num_examples``; the legacy
``gsm8k_accuracy_thres`` / ``gsm8k_num_questions`` are still honored.
Required attributes on the test class:
base_url: str
gsm8k_score_threshold: float
Optional attributes:
model: str (if not set, auto-detected from server)
``"run_eval"`` backend: OpenAI completion API, 5-shot; ``"sgl_eval"``: sgl-eval
chat + boxed/sympy grader. The legacy ``gsm8k_accuracy_thres`` /
``gsm8k_num_questions`` are honored when the canonical knobs are unset.
"""
gsm8k_score_threshold: float = _THRESHOLD_NOT_SET
@@ -216,7 +176,6 @@ class GSM8KMixin:
gsm8k_backend: str = "run_eval" # "run_eval" | "sgl_eval"
gsm8k_thinking: bool = False # sgl_eval backend
gsm8k_max_tokens: Optional[int] = None # sgl_eval backend
gsm8k_n_repeats: int = 1 # sgl_eval backend
# None keeps run_eval's greedy default; set both to route the run through
# the sampling path.
gsm8k_temperature: Optional[float] = None
@@ -237,7 +196,6 @@ class GSM8KMixin:
self,
eval_name="gsm8k",
score_threshold=threshold,
n_repeats=self.gsm8k_n_repeats,
num_examples=num_examples,
num_threads=self.gsm8k_num_threads,
thinking=self.gsm8k_thinking,
@@ -261,60 +219,29 @@ class GSM8KMixin:
class MMLUMixin:
"""Mixin for MMLU evaluation.
Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it
in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch
picks the call mechanism, not the grader.
Required attributes on the test class:
base_url: str
model: str
mmlu_score_threshold: float
"""
"""Mixin for MMLU evaluation via sgl-eval (2048-token cap, no thinking)."""
mmlu_score_threshold: float = _THRESHOLD_NOT_SET
mmlu_accept_length_thres: Optional[float] = None
mmlu_num_examples: int = 5000
mmlu_num_threads: int = 1024
mmlu_backend: str = "run_eval" # "run_eval" | "sgl_eval"
mmlu_thinking: bool = False # sgl_eval backend
mmlu_n_repeats: int = 1 # sgl_eval backend
def test_mmlu(self):
if self.mmlu_backend == "sgl_eval":
_run_sgl_eval(
self,
eval_name="mmlu",
score_threshold=self.mmlu_score_threshold,
n_repeats=self.mmlu_n_repeats,
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
thinking=self.mmlu_thinking,
accept_length_thres=self.mmlu_accept_length_thres,
)
else:
_run_accuracy_eval(
self,
eval_name="mmlu",
score_threshold=self.mmlu_score_threshold,
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
accept_length_thres=self.mmlu_accept_length_thres,
)
_run_accuracy_eval(
self,
eval_name="mmlu",
score_threshold=self.mmlu_score_threshold,
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
accept_length_thres=self.mmlu_accept_length_thres,
)
class MMMUProMixin:
"""Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval.
The model preset supplies the endpoint model and all generation settings.
Leaving those values to sgl-eval is important for reasoning models whose
recommended token budget and sampling settings differ from run_eval defaults.
Required attributes on the test class:
base_url: str
mmmu_pro_score_threshold: float
mmmu_pro_load_preset_from_model_id: str
The model preset supplies the endpoint model and all generation settings;
reasoning models' token budget and sampling differ from run_eval defaults.
"""
mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET
@@ -339,110 +266,8 @@ class MMMUProMixin:
)
class GPQAMixin:
"""Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
installed). ``gpqa_thinking`` defaults to True, which
enables per-request thinking so the server separates reasoning from the final
answer.
Required attributes on the test class:
base_url: str
model: str
gpqa_score_threshold: float
Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
gpqa_reasoning_effort="max", gpqa_max_tokens=200000, gpqa_temperature=1.0,
gpqa_top_p=1.0. GPQA-Diamond is 198 questions; raise gpqa_n_repeats (e.g. 16)
for a stable number.
"""
gpqa_score_threshold: float = _THRESHOLD_NOT_SET
gpqa_accept_length_thres: Optional[float] = None
gpqa_num_examples: Optional[int] = None
gpqa_num_threads: int = 1024
gpqa_n_repeats: int = 1
gpqa_thinking: bool = True
gpqa_reasoning_effort: Optional[str] = None
gpqa_max_tokens: Optional[int] = None
gpqa_temperature: Optional[float] = None
gpqa_top_p: Optional[float] = None
def test_gpqa(self):
_run_sgl_eval(
self,
eval_name="gpqa",
score_threshold=self.gpqa_score_threshold,
n_repeats=self.gpqa_n_repeats,
num_examples=self.gpqa_num_examples,
num_threads=self.gpqa_num_threads,
thinking=self.gpqa_thinking,
reasoning_effort=self.gpqa_reasoning_effort,
max_tokens=self.gpqa_max_tokens,
temperature=self.gpqa_temperature,
top_p=self.gpqa_top_p,
accept_length_thres=self.gpqa_accept_length_thres,
)
class AIME25Mixin:
"""Mixin for AIME 2025 evaluation (competition math, integer answers).
Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
installed). ``aime25_thinking`` defaults to True, which
enables per-request thinking so the server separates reasoning from the final
answer.
Required attributes on the test class:
base_url: str
model: str
aime25_score_threshold: float
Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
aime25_reasoning_effort="max", aime25_max_tokens=200000, aime25_temperature=1.0,
aime25_top_p=1.0. AIME25 has only 30 problems, so it is high variance; raise
aime25_n_repeats (e.g. 16) for a stable number.
"""
aime25_score_threshold: float = _THRESHOLD_NOT_SET
aime25_accept_length_thres: Optional[float] = None
aime25_num_examples: Optional[int] = None
aime25_num_threads: int = 1024
aime25_n_repeats: int = 1
aime25_thinking: bool = True
aime25_reasoning_effort: Optional[str] = None
aime25_max_tokens: Optional[int] = None
aime25_temperature: Optional[float] = None
aime25_top_p: Optional[float] = None
def test_aime25(self):
_run_sgl_eval(
self,
eval_name="aime25",
score_threshold=self.aime25_score_threshold,
n_repeats=self.aime25_n_repeats,
num_examples=self.aime25_num_examples,
num_threads=self.aime25_num_threads,
thinking=self.aime25_thinking,
reasoning_effort=self.aime25_reasoning_effort,
max_tokens=self.aime25_max_tokens,
temperature=self.aime25_temperature,
top_p=self.aime25_top_p,
accept_length_thres=self.aime25_accept_length_thres,
)
class HumanEvalMixin:
"""Mixin for HumanEval evaluation.
Required attributes on the test class:
base_url: str
model: str
humaneval_score_threshold: float
"""
"""Mixin for HumanEval evaluation."""
humaneval_score_threshold: float = _THRESHOLD_NOT_SET
humaneval_score_threshold_amd: Optional[float] = None
@@ -464,13 +289,7 @@ class HumanEvalMixin:
class MGSMEnMixin:
"""Mixin for MGSM English evaluation.
Required attributes on the test class:
base_url: str
model: str
mgsm_en_score_threshold: float
"""
"""Mixin for MGSM English evaluation."""
mgsm_en_score_threshold: float = _THRESHOLD_NOT_SET
mgsm_en_num_examples: Optional[int] = None
+22 -158
View File
@@ -1,18 +1,17 @@
"""
Usage:
python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10
python3 -m sglang.test.run_eval --port 30000 --eval-name gsm8k --num-examples 10
"""
import argparse
import json
import os
import statistics
import subprocess
import time
import uuid
from pathlib import Path
import warnings
from sglang.test.sgl_eval_utils import (
SGL_EVAL_BENCHMARKS,
THINKING_MODE_CHOICES,
get_thinking_kwargs,
parse_json_object,
@@ -89,125 +88,6 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
return result, latency, sampler
def _run_sgl_eval(eval_name, args) -> dict:
# Returns a metrics dict (score, latency, output_throughput) so the
# existing write_results_to_json + threshold gate keep working.
from sglang.test.test_utils import dump_metric
base_url = (
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
out_parent = Path(
getattr(args, "sgl_eval_out_dir", None)
or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex)
).expanduser()
out_parent.mkdir(parents=True, exist_ok=True)
model_preset_id = getattr(args, "load_preset_from_model_id", None)
cmd = [
"sgl-eval",
"run",
eval_name,
"--base-url",
base_url,
"--out-dir",
str(out_parent),
]
if model_preset_id:
cmd += ["--load-preset-from-model-id", model_preset_id]
if getattr(args, "model", None):
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
if getattr(args, "num_threads", None) is not None:
cmd += ["--num-threads", str(args.num_threads)]
if getattr(args, "temperature", None) is not None:
cmd += ["--temperature", str(args.temperature)]
elif not model_preset_id:
cmd += ["--temperature", "0.0"]
if getattr(args, "top_p", None) is not None:
cmd += ["--top-p", str(args.top_p)]
elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False):
cmd += ["--top-p", "1.0"]
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
if getattr(args, "seed", None) is not None:
cmd += ["--seed", str(args.seed)]
# gpt-oss grades one score per effort tier, so dropping this collapses every
# tier onto the served model's default.
if getattr(args, "reasoning_effort", None) is not None:
cmd += ["--reasoning-effort", str(args.reasoning_effort)]
if getattr(args, "repeat", None) is not None:
cmd += ["--n-repeats", str(args.repeat)]
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
elif not model_preset_id:
cmd += ["--max-tokens", "2048"]
# Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
# without --thinking their message.content is empty and sgl-eval scores 0.
sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None)
if sgl_eval_thinking is None:
if not model_preset_id:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif sgl_eval_thinking:
cmd += ["--thinking"]
try:
completed = subprocess.run(
cmd,
text=True,
capture_output=True,
check=False,
timeout=getattr(args, "sgl_eval_timeout", None),
)
except subprocess.TimeoutExpired as e:
raise TimeoutError(
f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n"
f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}"
) from e
if completed.returncode != 0:
raise RuntimeError(
f"sgl-eval failed with exit code {completed.returncode}: "
f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
)
metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json"))
if len(metrics_files) != 1:
raise FileNotFoundError(
f"Expected exactly one metrics.json under {out_parent}, "
f"found {len(metrics_files)}"
)
payload = json.loads(metrics_files[0].read_text())
aggregate = payload.get("aggregate")
if not isinstance(aggregate, dict) or "score" not in aggregate:
raise KeyError(f"{metrics_files[0]} missing aggregate.score")
metrics = dict(aggregate)
metrics["latency"] = payload.get("latency_seconds", 0.0)
metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0)
metrics["sgl_eval_metrics_path"] = str(metrics_files[0])
model = payload.get("model") or getattr(args, "model", None)
dump_metric(
f"{eval_name}_score",
metrics["score"],
labels={"model": model, "eval": eval_name},
)
dump_metric(
f"{eval_name}_latency",
metrics["latency"],
labels={"model": model, "eval": eval_name},
)
print(f"Score: {metrics['score']:.3f}")
print(f"Total latency: {metrics['latency']:.3f} s")
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
print(f"sgl-eval metrics: {metrics_files[0]}")
return metrics
def print_accept_length_summary(samplers: list) -> None:
accept_lengths = [
m["spec_accept_length"]
@@ -236,6 +116,23 @@ def run_eval(args):
# Lazy import to avoid circular dependency with test_utils
from sglang.test.test_utils import dump_metric
warnings.warn(
"sglang.test.run_eval is deprecated; its legacy scorers will be removed. "
"Use sglang.test.sgl_eval_utils.run_sgl_eval or `sgl-eval run` instead.",
DeprecationWarning,
stacklevel=2,
)
if (
args.eval_name in SGL_EVAL_BENCHMARKS
or args.eval_name == "mmmu-pro"
or getattr(args, "api", None) == "sgl_eval"
or getattr(args, "load_preset_from_model_id", None)
):
raise ValueError(
f"{args.eval_name} is scored by sgl-eval; call "
"sglang.test.sgl_eval_utils.run_sgl_eval instead of run_eval."
)
set_ulimit()
if "OPENAI_API_KEY" not in os.environ:
@@ -245,18 +142,10 @@ def run_eval(args):
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
if args.eval_name == "mmlu":
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
return _run_sgl_eval("mmlu", args)
elif args.eval_name == "mgsm_en":
if args.eval_name == "mgsm_en":
from sglang.test.simple_eval_mgsm import MGSMEval
eval_obj = MGSMEval(args.num_examples, args.num_threads, languages=["en"])
elif args.eval_name == "gpqa":
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
return _run_sgl_eval("gpqa", args)
elif args.eval_name == "humaneval":
from sglang.test.simple_eval_humaneval import HumanEval
@@ -270,25 +159,7 @@ def run_eval(args):
args.num_threads,
response_answer_regex=getattr(args, "response_answer_regex", None),
)
elif args.eval_name in ("mmmu_pro", "mmmu-pro"):
# Canonical sgl-eval name for MMMU-Pro's standard 10-option split.
return _run_sgl_eval("mmmu_pro", args)
elif args.eval_name == "mmmu_pro_vision":
# sgl-eval owns this benchmark's dataset, prompt and grader; there is no
# simple_eval implementation to fall back to.
return _run_sgl_eval("mmmu_pro_vision", args)
elif args.eval_name == "aime25":
return _run_sgl_eval("aime25", args)
elif args.eval_name == "aime26":
return _run_sgl_eval("aime26", args)
elif args.eval_name == "gsm8k":
if getattr(args, "api", None) == "sgl_eval":
# Only the nightly correctness eval opts into sgl-eval (zero-shot
# chat, \boxed{}, math_verify). Every other gsm8k caller — spec
# decoding perf/accuracy, disaggregation, quant, model e2e — uses
# the 5-shot completion last-number scorer and relies on
# max_tokens/throughput behavior sgl-eval cannot provide.
return _run_sgl_eval("gsm8k", args)
from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval
eval_obj = GSM8KEval(
@@ -428,16 +299,10 @@ if __name__ == "__main__":
type=str,
help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
)
parser.add_argument(
"--load-preset-from-model-id",
type=str,
default=None,
help="Load repository-maintained sgl-eval generation defaults for this model ID.",
)
parser.add_argument(
"--repeat", type=int, default=1, help="repeat the evaluation n times"
)
parser.add_argument("--eval-name", type=str, default="mmlu")
parser.add_argument("--eval-name", type=str, default="gsm8k")
parser.add_argument(
"--api",
type=str,
@@ -498,6 +363,5 @@ if __name__ == "__main__":
)
args = parser.parse_args()
args._sgl_eval_from_cli = True
run_eval(args)
+39 -6
View File
@@ -6,13 +6,20 @@ import multiprocessing
import os
import threading
import uuid
import warnings
from concurrent.futures import ProcessPoolExecutor
from dataclasses import replace
from itertools import islice
from pathlib import Path
from types import SimpleNamespace
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
# Benchmarks scored only by sgl-eval; run_eval does not serve them.
SGL_EVAL_BENCHMARKS = frozenset(
{"mmlu", "gpqa", "mmmu_pro", "mmmu_pro_vision", "aime25", "aime26"}
)
def get_thinking_kwargs(args):
thinking_mode = getattr(args, "thinking_mode", None)
@@ -85,9 +92,35 @@ def run_sgl_eval(args):
from sglang.test.test_utils import dump_metric
spec = get(args.eval_name)
overrides = {"max_tokens": 2048}
model = getattr(args, "model", None)
default_gen = spec.default_gen
model_preset_id = getattr(args, "load_preset_from_model_id", None)
if model_preset_id:
from sgl_eval.model_preset import load_model_preset
from sgl_eval.preset import apply_to_gen
model_preset = load_model_preset(model_preset_id)
unset = dict.fromkeys(
("thinking", "temperature", "top_p", "max_tokens", "reasoning_effort")
)
default_gen = apply_to_gen(
default_gen, None, SimpleNamespace(**unset), model_preset
)
model = model or model_preset.model
overrides = {}
# Omitted max_tokens keeps the 2048 CI cap; an explicit None defers to the server.
if hasattr(args, "max_tokens"):
if args.max_tokens is None:
warnings.warn(
f"sgl-eval {spec.name}: max_tokens=None leaves the output length "
"uncapped; generation stops only at the server limit.",
stacklevel=2,
)
overrides["max_tokens"] = args.max_tokens
elif not model_preset_id:
overrides["max_tokens"] = 2048
for key in (
"max_tokens",
"temperature",
"top_p",
"min_p",
@@ -107,12 +140,12 @@ def run_sgl_eval(args):
chat_kwargs.setdefault("enable_thinking", thinking)
if chat_kwargs:
overrides["chat_template_kwargs"] = {
**(spec.default_gen.chat_template_kwargs or {}),
**(default_gen.chat_template_kwargs or {}),
**chat_kwargs,
}
if getattr(args, "top_k", None) is not None:
overrides["extra_body"] = {"top_k": args.top_k}
gen = replace(spec.default_gen, **overrides)
gen = replace(default_gen, **overrides)
generation = {
key: getattr(gen, key)
for key in ("max_tokens", "temperature", "top_p", "min_p", "seed")
@@ -120,7 +153,7 @@ def run_sgl_eval(args):
print(f"sgl-eval {spec.name} generation: {generation}", flush=True)
sampler = ChatCompletionSampler(
base_url=api_base_url(args),
model=getattr(args, "model", None),
model=model,
api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"),
)
out_parent = Path(
@@ -129,7 +162,7 @@ def run_sgl_eval(args):
).expanduser()
out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}"
out_dir.mkdir(parents=True, exist_ok=True)
repeats = getattr(args, "repeat", 1)
repeats = getattr(args, "repeat", None) or spec.default_n_repeats
writer = PredictionsWriter(out_dir, repeats, spec.pred_schema)
try:
result = spec.run(
+25 -111
View File
@@ -48,7 +48,7 @@ from sglang.srt.utils import (
retry,
)
from sglang.srt.utils.network import is_port_available
from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.utils import normalize_base_url
# General test models
@@ -399,24 +399,11 @@ def start_subprocess_fail_fast_watcher(
def _try_enable_offline_mode_if_cache_complete(
model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None
) -> Optional[str]:
"""
CI helper: Check if model cache is complete and enable offline mode.
"""Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the
per-run marker path, or None if offline mode was not enabled.
Uses per-run validation markers that are NOT shared across runners.
Each runner independently validates its cache using lightweight checks
before enabling offline mode.
IMPORTANT: Even if a per-run marker exists, this function ALWAYS validates
the current launch's requirements (e.g., hf_quant_config.json for modelopt).
The marker is only a hint that this snapshot was validated earlier in the run.
Args:
model_name_or_path: Model identifier or path
env: Environment dict to modify (will add HF_HUB_OFFLINE=1 if validation passes)
other_args: Launch command arguments (used to detect quantization requirement)
Returns:
Per-run marker path if offline mode was enabled, None otherwise
Markers are per-run and not shared across runners. A marker is only a hint;
the current launch's requirements (e.g. hf_quant_config.json) are revalidated.
"""
from sglang.srt.model_loader.ci_weight_validation import (
_get_per_run_marker_path,
@@ -453,8 +440,7 @@ def _try_enable_offline_mode_if_cache_complete(
except Exception:
return None
# Detect if quantization requires hf_quant_config.json
# Do this BEFORE checking marker to ensure current launch requirements are known
# Detect before the marker check so the current launch's requirements are known.
requires_hf_quant_config = False
for i, arg in enumerate(other_args):
if arg == "--quantization" and i + 1 < len(other_args):
@@ -466,9 +452,8 @@ def _try_enable_offline_mode_if_cache_complete(
# Check per-run marker (fast hint - snapshot validated earlier in this run)
per_run_marker = _read_per_run_marker(snapshot_dir)
if per_run_marker is not None:
# Marker exists, but STILL validate for current launch requirements
# This prevents a test without --quantization from enabling offline
# for a later test with --quantization that needs hf_quant_config.json
# Still validate: a marker from a launch without --quantization must not
# enable offline mode for a later launch that needs hf_quant_config.json.
is_valid = validate_cache_lightweight(snapshot_dir, requires_hf_quant_config)
if not is_valid:
@@ -522,17 +507,7 @@ def _try_enable_offline_mode_if_cache_complete(
def _create_clean_subprocess_env(env: dict) -> dict:
"""Create a clean subprocess environment without internal CI keys.
Removes all keys starting with '_CI_OFFLINE_' or 'CI_OFFLINE' to prevent
leaking implementation details to the server subprocess.
Args:
env: Source environment dict
Returns:
Clean copy of environment dict
"""
"""Create a clean subprocess environment without internal CI keys."""
child_env = env.copy()
keys_to_remove = [
k for k in child_env if k.startswith(("_CI_OFFLINE_", "CI_OFFLINE_"))
@@ -592,14 +567,7 @@ def _launch_server_process(
) -> subprocess.Popen:
"""Launch server subprocess with clean environment.
Args:
command: Command list for subprocess
env: Environment dict (will be cleaned before use)
return_stdout_stderr: Optional tuple of (stdout_file, stderr_file) for output capture
model: Model name for logging
Returns:
Started subprocess.Popen object
`return_stdout_stderr` is an optional (stdout_file, stderr_file) pair.
"""
child_env = _create_clean_subprocess_env(env)
@@ -619,17 +587,7 @@ def _wait_for_server_health(
api_key: Optional[str],
timeout_duration: float,
) -> Tuple[bool, Optional[str]]:
"""Wait for server health check to pass.
Args:
proc: Server subprocess
base_url: Base URL for health check
api_key: Optional API key for authorization
timeout_duration: Maximum wait time in seconds
Returns:
Tuple of (success, error_message)
"""
"""Wait for server health check to pass; return (success, error_message)."""
start_time = time.perf_counter()
with requests.Session() as session:
while time.perf_counter() - start_time < timeout_duration:
@@ -686,20 +644,8 @@ def popen_launch_server(
):
"""Launch a server process with automatic device detection and offline/online retry.
Args:
model: Model path or identifier
base_url: Base URL for the server
timeout: Timeout for server startup
api_key: Optional API key for authentication
other_args: Additional command line arguments
env: Environment dict for subprocess
return_stdout_stderr: Optional tuple for output capture
device: Device type ("auto", "cuda", "rocm" or "cpu")
pd_separated: Whether to use PD separated mode
num_replicas: Number of replicas for mixed PD mode
Returns:
Started subprocess.Popen object
`device` is one of "auto", "cuda", "rocm", "cpu";
`num_replicas` applies to mixed PD mode.
"""
other_args = other_args or []
@@ -897,9 +843,7 @@ def popen_launch_pd_server(
print(f"command={' '.join(command)}")
# Merge with os.environ so caller-supplied env adds to (not replaces)
# PATH / PYTHONPATH / HF_HOME / etc. When env is None, Popen inherits
# parent's environment automatically.
# Merge so caller-supplied env adds to, not replaces, PATH / PYTHONPATH / etc.
if env is not None:
env = {**os.environ, **env}
@@ -1018,9 +962,8 @@ def run_bench_serving(
other_args=other_server_args,
)
# Resolve tokenizer to local snapshot path when available, so the benchmark
# client's AutoTokenizer.from_pretrained uses the local path directly instead
# of calling the HF Hub API (which can stall for minutes in CI).
# Prefer the local snapshot so the client's AutoTokenizer skips the HF Hub API,
# which can stall for minutes in CI.
bench_tokenizer = tokenizer
if bench_tokenizer is None:
try:
@@ -1090,19 +1033,7 @@ async def _run_api_benchmark_requests(
num_requests: int,
response_validator: Callable[[dict], bool],
):
"""
Helper function to run API benchmark requests and collect metrics.
Args:
base_url: The base URL of the server
endpoint: The API endpoint to test (e.g., "/v1/score", "/v1/embeddings")
test_requests: List of request payloads to send
num_requests: Total number of requests expected
response_validator: Function to validate if response contains expected data
Returns:
Dictionary with benchmark metrics
"""
"""Run API benchmark requests and return a dict of metrics."""
start_time = time.monotonic()
successful_requests = 0
total_latency = 0
@@ -1433,12 +1364,7 @@ def run_bench_serving_multi(
def run_bench_one_batch(model, other_args):
"""Launch a offline process with automatic device detection.
Args:
device: Device type ("auto", "cuda", "rocm" or "cpu").
If "auto", will detect available platforms automatically.
"""
"""Launch an offline process with automatic device detection."""
# Auto-detect device if needed
device = auto_config_device()
@@ -1740,7 +1666,7 @@ def run_mmlu_test(
)
try:
metrics = run_eval(args)
metrics = run_sgl_eval(args)
assert metrics["score"] >= 0.65, f"{metrics=}"
finally:
pass
@@ -2340,11 +2266,9 @@ def server_args_variant(server_args, **fields):
def enter_override(test_case, override):
"""Install a scoped context override for the length of one test.
`unittest.TestCase.enterContext` does exactly this in one call, but it is
Python 3.11+ and this package supports 3.10 (`requires-python = ">=3.10"`).
On 3.10 it raises `AttributeError: ... has no attribute 'enterContext'` --
and only there, so a developer on a newer interpreter sees every test pass
while CI does not.
Stands in for `unittest.TestCase.enterContext`, which is 3.11+ while this package
supports 3.10; on 3.10 it raises AttributeError, so tests pass locally on newer
interpreters but fail in CI.
"""
installed = override.install()
test_case.addCleanup(override.restore)
@@ -2366,9 +2290,8 @@ class CustomTestCase(unittest.TestCase):
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
# Wrap the effective setUpClass so that tearDownClass is called
# even when setUpClass fails. Python's unittest skips tearDownClass
# if setUpClass raises, which can leak resources (ports, processes).
# unittest skips tearDownClass when setUpClass raises, leaking ports and
# processes; wrap setUpClass so tearDownClass still runs.
setup = cls.setUpClass
if getattr(setup, "_safe_setup_wrapped", False):
return
@@ -2669,11 +2592,6 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
- stdout: [METRIC] metric_name=value [labels=...]
This function never fails tests - all errors are silently caught.
Args:
metric_name: Metric name (e.g., "gsm8k_accuracy", "cache_hit_rate")
value: Metric value
labels: Optional label dict (e.g., {"backend": "fa3"})
"""
try:
# 1. Capture test context
@@ -2750,11 +2668,7 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
def _get_test_context() -> tuple[str, str]:
"""
Get current test's filename and test_case.
Tries PYTEST_CURRENT_TEST first, falls back to inspect.stack().
"""
"""Get current test's filename and test_case, via PYTEST_CURRENT_TEST or stack."""
# 1. Try parsing PYTEST_CURRENT_TEST
pytest_current = os.getenv("PYTEST_CURRENT_TEST")
if pytest_current: