diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx
index e082532e95..f609d2705a 100644
--- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx
+++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx
@@ -785,15 +785,15 @@ The command evaluates all 57 subjects. Historical results below use the removed
- **Benchmark Command:**
```shell Command
-python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --thinking-mode deepseek-v3
+sgl-eval run gpqa --base-url http://127.0.0.1:30000/v1 --num-examples 198 --max-tokens 128000 --n-repeats 8 --thinking
```
-- **Test Results** (model: `deepseek-ai/DeepSeek-V3.2-Exp`, 8×B200):
+- **Historical Test Results** (model: `deepseek-ai/DeepSeek-V3.2-Exp`, 8×B200; measured with the earlier `sglang.test.run_eval` GPQA command):
- Default (`temperature=0`): mean **0.797** over 8 runs — closely matches the official GPQA-Diamond score of **79.9** for DeepSeek-V3.2-Exp reported in its [model card](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp)
- With `temperature=1.0, top_p=0.95` (as recommended by DeepSeek):
```shell Command
-python3 -m sglang.test.run_eval --port 30000 --eval-name gpqa --num-examples 198 --max-tokens 128000 --repeat 8 --top-p 0.95 --temperature 1.0 --thinking-mode deepseek-v3
+sgl-eval run gpqa --base-url http://127.0.0.1:30000/v1 --num-examples 198 --max-tokens 128000 --n-repeats 8 --top-p 0.95 --temperature 1.0 --thinking
```
```
diff --git a/docs/docs/developer_guide/evaluating_new_models.mdx b/docs/docs/developer_guide/evaluating_new_models.mdx
index e20a2f54f9..33a0b7f143 100644
--- a/docs/docs/developer_guide/evaluating_new_models.mdx
+++ b/docs/docs/developer_guide/evaluating_new_models.mdx
@@ -45,16 +45,15 @@ python benchmark/hellaswag/bench_sglang.py \
**GPQA**
```bash Command
-python -m sglang.test.run_eval \
- --eval-name gpqa \
- --port 30000 \
+sgl-eval run gpqa \
+ --base-url http://127.0.0.1:30000/v1 \
--num-examples 198 \
--max-tokens 120000 \
- --repeat 8
+ --n-repeats 8
```
-For reasoning models, add `--thinking-mode `. Supported values are `deepseek-v3`, `qwen-3`, `glm-45`, and `kimi-k2`. You may skip it if the model has forced thinking enabled.
+For reasoning models, add `--thinking` (DeepSeek-V3 / Kimi-K2 style `thinking` key) or `--chat-template-kwarg enable_thinking=true` (Qwen3 / GLM style). You may skip it if the model has forced thinking enabled.
**HumanEval**
diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py
index fb7c5d6f01..412ca716ac 100644
--- a/python/sglang/test/accuracy_test_runner.py
+++ b/python/sglang/test/accuracy_test_runner.py
@@ -4,6 +4,7 @@ from typing import List, Optional, Tuple
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -31,9 +32,6 @@ class AccuracyTestParams:
repeat: Optional[int] = None
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
seed: Optional[int] = None # pin for reproducibility when temperature > 0
- # sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead
- # of letting _run_sgl_eval infer it from the model name.
- sgl_eval_thinking: Optional[bool] = None
num_shots: Optional[int] = None # few-shot count; None = run_eval's default
@@ -56,13 +54,7 @@ def write_accuracy_github_summary(
dataset: str,
results: List[AccuracyTestResult],
) -> None:
- """Write accuracy test results to GitHub step summary.
-
- Args:
- test_name: Name of the test
- dataset: Dataset name used for evaluation
- results: List of AccuracyTestResult objects
- """
+ """Write accuracy test results to GitHub step summary."""
summary = f"#### {test_name} - Accuracy ({dataset})\n"
summary += "| config | status | score | baseline | error |\n"
summary += "| ------ | ------ | ----- | -------- | ----- |\n"
@@ -95,13 +87,8 @@ def _run_simple_eval(
repeat: Optional[int] = None,
api: Optional[str] = None,
seed: Optional[int] = None,
- sgl_eval_thinking: Optional[bool] = None,
) -> Tuple[bool, Optional[str], Optional[dict]]:
- """Run evaluation using simple_eval backend (run_eval.py).
-
- Returns:
- Tuple of (success, error_message, metrics_dict)
- """
+ """Run ``dataset`` through run_sgl_eval (sgl-eval benchmarks) or run_eval."""
process = None
try:
process = popen_launch_server(
@@ -150,10 +137,8 @@ def _run_simple_eval(
if seed is not None:
args.seed = seed
- if sgl_eval_thinking is not None:
- args.sgl_eval_thinking = sgl_eval_thinking
-
- result = run_eval(args)
+ evaluate = run_sgl_eval if dataset in SGL_EVAL_BENCHMARKS else run_eval
+ result = evaluate(args)
# Handle result format (run_eval can return metrics or (metrics, latency))
if return_latency and isinstance(result, tuple):
@@ -177,16 +162,7 @@ def run_accuracy_test(
params: AccuracyTestParams,
base_url: Optional[str] = None,
) -> AccuracyTestResult:
- """Run accuracy test for a single model.
-
- Args:
- model: ModelLaunchSettings with model config
- params: AccuracyTestParams with dataset, baseline, and optional settings
- base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
-
- Returns:
- AccuracyTestResult with test outcome
- """
+ """Run accuracy test for a single model."""
base_url = base_url or DEFAULT_URL_FOR_TEST
print(f"\n{'=' * 60}")
@@ -211,7 +187,6 @@ def run_accuracy_test(
repeat=params.repeat,
api=params.api,
seed=params.seed,
- sgl_eval_thinking=params.sgl_eval_thinking,
)
if not success:
diff --git a/python/sglang/test/ascend/test_ascend_utils.py b/python/sglang/test/ascend/test_ascend_utils.py
index ddbeb5b7b0..a64473ea34 100644
--- a/python/sglang/test/ascend/test_ascend_utils.py
+++ b/python/sglang/test/ascend/test_ascend_utils.py
@@ -1,14 +1,6 @@
-"""
-Common utilities for testing and benchmarking on NPU.
+"""Common utilities for testing and benchmarking on NPU.
-This file contains the following weight path categories:
-- LLM model weights path
-- VLM model weights path
-- Embedding model weights path
-- Rerank model weights path
-- Reward model weights path
-
-Please remember to sort by variable name within each section.
+Keep the weight-path constants sorted by variable name within each section.
"""
import asyncio
@@ -26,7 +18,7 @@ import requests
from sglang.benchmark.serving import run_benchmark
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -395,15 +387,7 @@ CONFIG_YAML_PATH = (
class ModelTestConfig(NamedTuple):
- """
- Configuration for model testing.
-
- Attributes:
- model_path: Path to the model weights directory
- mmlu_score: Weight for MMLU benchmark score
- gsm8k_accuracy: Weight for GSM8K benchmark score
- mmmu_accuracy: Weight for MMMU benchmark score
- """
+ """Model weights path and the expected MMLU / GSM8K / MMMU scores."""
model_path: str
mmlu_score: Optional[float] = None
@@ -436,16 +420,7 @@ DEFAULT_WEIGHTS_FOR_TEST = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_FOR_TEST
def run_command(cmd, shell=True):
- """Execute system command and return stdout
-
- parameter:
- cmd: command to execute
- shell:
- True, Execute command in shell
- False, Commands are invoked directly without shell parsing
- return:
- The result of executing the command
- """
+ """Run ``cmd`` and return its stdout."""
try:
result = subprocess.run(
cmd, shell=shell, capture_output=True, text=True, check=True
@@ -556,37 +531,7 @@ def run_bench_serving(
lora_name: Optional[str] = None,
timeout_for_server_launch=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
):
- """Start the service and obtain the inference results.
-
- Parameters:
- model: Model name
- num_prompts: Total number of test requests
- request_rate: Request rate
- other_server_args: Additional configuration when starting the service
- dataset_name: Data set name
- dataset_path: Dataset path
- tokenizer: tokenizer
- random_input_len: The length of the randomly generated input prompt
- random_output_len: The length of the randomly generated output prompt
- sharegpt_context_len: Sharegpt dataset context length
- disable_stream: Disable streaming output
- disable_ignore_eos: Should eos_token be ignored?
- need_warmup: Preheating required
- seed: random seed
- device: Device type
- gsp_num_groups: Grouped Sequence Parallelism
- gsp_prompts_per_group: Number of parallel prompts within each group
- gsp_system_prompt_len: GSP system prompts length
- gsp_question_len: GSP question length
- gsp_output_len: GSP output length
- max_concurrency: Maximum number of concurrent requests
- background_task: Background tasks
- lora_name: LoRA fine-tuning model path
- timeout_for_server_launch: Raise the service timeout period
- Returns:
- res: Number of requests successfully completed
-
- """
+ """Launch the server, run bench_serving against it, and return the result."""
if device == "auto":
device = auto_config_device()
@@ -658,17 +603,7 @@ def run_bench_serving(
# hook factory
def create_attention_monitor_hook_factory(config):
- """
- Factory function to create a forward hook for monitoring self-attention layer states.
- This hook records input/output statistics during model forward propagation.
-
- Args:
- config (dict): Configuration dictionary containing hook parameters
- layer_index (int): Index of the target attention layer to monitor
-
- Returns:
- function: Forward hook function to be registered on the target module
- """
+ """Return a forward hook that logs self-attention states of ``config["layer_index"]``."""
# Get target layer index from config, default to 0 if not specified
layer_index = config.get("layer_index", 0)
@@ -681,18 +616,7 @@ def create_attention_monitor_hook_factory(config):
)
def attention_monitor_hook(module, inputs, output):
- """
- Forward hook function that monitors and logs the internal states of a self-attention layer.
- Executed automatically during the forward pass of the module it is registered to.
-
- Args:
- module (torch.nn.Module): The module this hook is attached to
- inputs (tuple): Input tensors passed to the module's forward method
- output (torch.Tensor): Output tensor returned by the module's forward method
-
- Returns:
- torch.Tensor: Unmodified output tensor to preserve model computation flow
- """
+ """Log the attention layer's input/output statistics; returns ``output`` unchanged."""
# Record current timestamp for time-series tracking
timestamp = time.time()
@@ -832,7 +756,7 @@ def run_mmlu_test(
)
try:
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
assert metrics["score"] >= 0.65, f"{metrics=}"
finally:
pass
@@ -856,26 +780,10 @@ def send_concurrent_requests(
temperature: float = 0.0,
request_timeout: int = 60,
) -> list:
- """Send multiple concurrent HTTP POST requests to the /generate endpoint.
+ """Send ``num_requests`` concurrent POSTs to /generate; returns one result dict per request.
- Uses threading (NOT asyncio + blocking calls) to achieve true concurrency.
- asyncio.gather() combined with synchronous requests.post() does not produce
- real parallelism; threading is required for concurrent blocking I/O.
-
- Parameters:
- base_url: Server base URL, e.g. "http://127.0.0.1:30000"
- num_requests: Total number of requests to send
- num_concurrent: Maximum in-flight requests at any given time (semaphore)
- input_text: Text prompt sent to every request
- max_new_tokens: Maximum new tokens to generate per request
- temperature: Sampling temperature (0 = greedy / deterministic)
- request_timeout: Per-request HTTP timeout in seconds; raises on exceed
-
- Returns:
- Unsorted list of result dicts, one per request, each with:
- task_id (int) -- zero-based request index
- status_code (int)-- HTTP status code, or -1 on exception
- text (str) -- response body, or exception message on failure
+ Uses threads: asyncio.gather() over synchronous requests.post() gives no real
+ parallelism for blocking I/O.
"""
results: list = []
diff --git a/python/sglang/test/ascend/test_mmlu.py b/python/sglang/test/ascend/test_mmlu.py
index 8a46b5ba1e..91b28b59dd 100644
--- a/python/sglang/test/ascend/test_mmlu.py
+++ b/python/sglang/test/ascend/test_mmlu.py
@@ -2,7 +2,7 @@ import subprocess
from types import SimpleNamespace
from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
class TestMMLU:
@@ -28,7 +28,7 @@ class TestMMLU:
num_threads=32,
)
print("Starting mmlu test...")
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
model_metrics["accuracy"] = metrics["score"]
model_metrics["latency"] = metrics.get("latency", "-")
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
diff --git a/python/sglang/test/gpt_oss_common.py b/python/sglang/test/gpt_oss_common.py
index 1954330485..6a0a830156 100644
--- a/python/sglang/test/gpt_oss_common.py
+++ b/python/sglang/test/gpt_oss_common.py
@@ -7,7 +7,7 @@ from typing import Dict, List, Literal, Optional
import requests
from sglang.srt.utils import is_hip, kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -66,8 +66,7 @@ class BaseTestGptOss(CustomTestCase):
try:
self._check_streaming_responses_api_request(model)
- # run multiple tests in parallel since we are mostly bound by the longest generate sequence
- # instead of the number of questions
+ # Parallel: bound by the longest generation, not the question count.
with ThreadPoolExecutor(max_workers=4) as executor:
list(
executor.map(
@@ -131,8 +130,7 @@ class BaseTestGptOss(CustomTestCase):
num_threads=198,
# sgl-eval's gpqa defaults to n_repeats=8.
repeat=1,
- # TODO 4k is still not enough, we need e.g. 64k token, but that is super slow
- # otherwise a lot of questions are not answered
+ # Too small (many answers truncate); ~64k would fit but is too slow.
max_tokens=4096,
# Arbitrary; non-zero so a tier is not scored on one greedy path.
temperature=0.1,
@@ -142,7 +140,7 @@ class BaseTestGptOss(CustomTestCase):
setup = f"model={model} reasoning_effort={reasoning_effort} expected_score={expected_score}"
print(f"Evaluation start: {setup}")
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"Evaluation end: {setup} {metrics=}")
self.assertGreaterEqual(metrics["score"], expected_score)
diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py
index 81eedf8b7f..04afac8cbe 100644
--- a/python/sglang/test/kits/eval_accuracy_kit.py
+++ b/python/sglang/test/kits/eval_accuracy_kit.py
@@ -4,6 +4,7 @@ from typing import Optional
import requests
from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval
from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
_THRESHOLD_NOT_SET = float("nan")
@@ -51,14 +52,8 @@ def _run_accuracy_eval(
):
"""Shared driver for the accuracy mixins below.
- Runs ``run_eval`` for ``eval_name`` against the test class's server
- (``base_url`` / ``model``), records a CI step summary, asserts the score
- meets ``score_threshold``, and checks the speculative accept length.
-
- ``eval_overrides`` (e.g. ``api``, ``max_tokens``, ``temperature``,
- ``top_p``, ``num_shots``) are forwarded to ``run_eval`` only when not
- ``None``, so the common case stays identical to ``run_eval``'s defaults.
- Returns the metrics dict.
+ ``eval_overrides`` are forwarded only when not ``None``, so unset knobs keep
+ the evaluator's defaults.
"""
assert score_threshold == score_threshold, (
f"{type(test_case).__name__} must set the {eval_name} score threshold"
@@ -74,7 +69,8 @@ def _run_accuracy_eval(
)
kwargs.update({k: v for k, v in eval_overrides.items() if v is not None})
- metrics = run_eval(SimpleNamespace(**kwargs))
+ evaluate = run_sgl_eval if eval_name in SGL_EVAL_BENCHMARKS else run_eval
+ metrics = evaluate(SimpleNamespace(**kwargs))
print(f"{eval_name} {metrics=}")
_finalize_eval(
test_case,
@@ -92,80 +88,53 @@ def _run_sgl_eval(
*,
eval_name: str,
score_threshold: float,
- metric: str = "score",
- n_repeats: int = 1,
num_examples: Optional[int] = None,
num_threads: int = 512,
thinking: bool = True,
chat_template_kwargs: Optional[dict] = None,
- reasoning_effort: Optional[str] = None,
max_tokens: Optional[int] = None,
temperature: Optional[float] = None,
- top_p: Optional[float] = None,
accept_length_thres: Optional[float] = None,
- summary_label: Optional[str] = None,
):
- """Shared sgl-eval driver for the reasoning mixins and the ``sgl_eval`` backend.
+ """Shared sgl-eval driver for the MMLU sanity gate and the GSM8K ``sgl_eval`` backend.
- Runs ``eval_name`` via the sgl-eval Python API (``registry.get`` ->
- ``EvalSpec.run``) against the test class's server, records a CI step summary,
- asserts the score meets ``score_threshold``, and checks the speculative accept
- length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking":
- True}`` so the server separates reasoning from the final answer. Skips the test
- if sgl-eval is not installed. Returns the RunResult.
+ ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": True}``
+ so the server separates reasoning from the final answer.
"""
assert score_threshold == score_threshold, (
f"{type(test_case).__name__} must set the {eval_name} score threshold"
)
try:
- from sgl_eval.registry import get as get_eval_spec
- from sgl_eval.sampler import ChatCompletionSampler
- from sgl_eval.types import GenConfig
+ import sgl_eval # noqa: F401
except ImportError:
test_case.skipTest("sgl-eval not installed; pip install 'sglang[test]'")
- base_url = test_case.base_url.rstrip("/")
- if not base_url.endswith("/v1"):
- base_url += "/v1"
- sampler = ChatCompletionSampler(
- base_url=base_url, model=getattr(test_case, "model", None), api_key="EMPTY"
- )
-
- gen_kwargs = dict(
+ args = SimpleNamespace(
+ eval_name=eval_name,
+ base_url=test_case.base_url,
+ model=getattr(test_case, "model", None),
+ num_examples=num_examples,
+ num_threads=num_threads,
max_tokens=max_tokens,
- reasoning_effort=reasoning_effort,
+ temperature=temperature,
chat_template_kwargs=(
chat_template_kwargs
if chat_template_kwargs is not None
else ({"thinking": True} if thinking else None)
),
)
- if temperature is not None:
- gen_kwargs["temperature"] = temperature
- if top_p is not None:
- gen_kwargs["top_p"] = top_p
-
- result = get_eval_spec(eval_name).run(
- sampler=sampler,
- gen=GenConfig(**gen_kwargs),
- n_repeats=n_repeats,
- num_examples=num_examples,
- num_threads=num_threads,
- predictions_writer=None,
- load_examples=None,
- )
- score = result.aggregate[metric]
- print(f"{eval_name} sgl-eval {metric}={score:.4f}")
+ metrics = run_sgl_eval(args)
+ score = metrics["score"]
+ print(f"{eval_name} sgl-eval score={score:.4f}")
_finalize_eval(
test_case,
eval_name=eval_name,
score=score,
score_threshold=score_threshold,
accept_length_thres=accept_length_thres,
- summary_label=summary_label,
)
- return result
+ return metrics
class MMLUSanityMixin:
@@ -192,18 +161,9 @@ class MMLUSanityMixin:
class GSM8KMixin:
"""Mixin for GSM8K evaluation.
- Backend is selectable via ``gsm8k_backend`` (default ``"run_eval"``: OpenAI
- completion API, 5-shot; or ``"sgl_eval"``: sgl-eval chat + boxed/sympy grader,
- skipped if sgl-eval is not installed). The canonical threshold/count knobs are
- ``gsm8k_score_threshold`` / ``gsm8k_num_examples``; the legacy
- ``gsm8k_accuracy_thres`` / ``gsm8k_num_questions`` are still honored.
-
- Required attributes on the test class:
- base_url: str
- gsm8k_score_threshold: float
-
- Optional attributes:
- model: str (if not set, auto-detected from server)
+ ``"run_eval"`` backend: OpenAI completion API, 5-shot; ``"sgl_eval"``: sgl-eval
+ chat + boxed/sympy grader. The legacy ``gsm8k_accuracy_thres`` /
+ ``gsm8k_num_questions`` are honored when the canonical knobs are unset.
"""
gsm8k_score_threshold: float = _THRESHOLD_NOT_SET
@@ -216,7 +176,6 @@ class GSM8KMixin:
gsm8k_backend: str = "run_eval" # "run_eval" | "sgl_eval"
gsm8k_thinking: bool = False # sgl_eval backend
gsm8k_max_tokens: Optional[int] = None # sgl_eval backend
- gsm8k_n_repeats: int = 1 # sgl_eval backend
# None keeps run_eval's greedy default; set both to route the run through
# the sampling path.
gsm8k_temperature: Optional[float] = None
@@ -237,7 +196,6 @@ class GSM8KMixin:
self,
eval_name="gsm8k",
score_threshold=threshold,
- n_repeats=self.gsm8k_n_repeats,
num_examples=num_examples,
num_threads=self.gsm8k_num_threads,
thinking=self.gsm8k_thinking,
@@ -261,60 +219,29 @@ class GSM8KMixin:
class MMLUMixin:
- """Mixin for MMLU evaluation.
-
- Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it
- in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch
- picks the call mechanism, not the grader.
-
- Required attributes on the test class:
- base_url: str
- model: str
- mmlu_score_threshold: float
- """
+ """Mixin for MMLU evaluation via sgl-eval (2048-token cap, no thinking)."""
mmlu_score_threshold: float = _THRESHOLD_NOT_SET
mmlu_accept_length_thres: Optional[float] = None
mmlu_num_examples: int = 5000
mmlu_num_threads: int = 1024
- mmlu_backend: str = "run_eval" # "run_eval" | "sgl_eval"
- mmlu_thinking: bool = False # sgl_eval backend
- mmlu_n_repeats: int = 1 # sgl_eval backend
def test_mmlu(self):
- if self.mmlu_backend == "sgl_eval":
- _run_sgl_eval(
- self,
- eval_name="mmlu",
- score_threshold=self.mmlu_score_threshold,
- n_repeats=self.mmlu_n_repeats,
- num_examples=self.mmlu_num_examples,
- num_threads=self.mmlu_num_threads,
- thinking=self.mmlu_thinking,
- accept_length_thres=self.mmlu_accept_length_thres,
- )
- else:
- _run_accuracy_eval(
- self,
- eval_name="mmlu",
- score_threshold=self.mmlu_score_threshold,
- num_examples=self.mmlu_num_examples,
- num_threads=self.mmlu_num_threads,
- accept_length_thres=self.mmlu_accept_length_thres,
- )
+ _run_accuracy_eval(
+ self,
+ eval_name="mmlu",
+ score_threshold=self.mmlu_score_threshold,
+ num_examples=self.mmlu_num_examples,
+ num_threads=self.mmlu_num_threads,
+ accept_length_thres=self.mmlu_accept_length_thres,
+ )
class MMMUProMixin:
"""Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval.
- The model preset supplies the endpoint model and all generation settings.
- Leaving those values to sgl-eval is important for reasoning models whose
- recommended token budget and sampling settings differ from run_eval defaults.
-
- Required attributes on the test class:
- base_url: str
- mmmu_pro_score_threshold: float
- mmmu_pro_load_preset_from_model_id: str
+ The model preset supplies the endpoint model and all generation settings;
+ reasoning models' token budget and sampling differ from run_eval defaults.
"""
mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET
@@ -339,110 +266,8 @@ class MMMUProMixin:
)
-class GPQAMixin:
- """Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
-
- Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
- installed). ``gpqa_thinking`` defaults to True, which
- enables per-request thinking so the server separates reasoning from the final
- answer.
-
- Required attributes on the test class:
- base_url: str
- model: str
- gpqa_score_threshold: float
-
- Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
- for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
- gpqa_reasoning_effort="max", gpqa_max_tokens=200000, gpqa_temperature=1.0,
- gpqa_top_p=1.0. GPQA-Diamond is 198 questions; raise gpqa_n_repeats (e.g. 16)
- for a stable number.
- """
-
- gpqa_score_threshold: float = _THRESHOLD_NOT_SET
- gpqa_accept_length_thres: Optional[float] = None
- gpqa_num_examples: Optional[int] = None
- gpqa_num_threads: int = 1024
- gpqa_n_repeats: int = 1
- gpqa_thinking: bool = True
- gpqa_reasoning_effort: Optional[str] = None
- gpqa_max_tokens: Optional[int] = None
- gpqa_temperature: Optional[float] = None
- gpqa_top_p: Optional[float] = None
-
- def test_gpqa(self):
- _run_sgl_eval(
- self,
- eval_name="gpqa",
- score_threshold=self.gpqa_score_threshold,
- n_repeats=self.gpqa_n_repeats,
- num_examples=self.gpqa_num_examples,
- num_threads=self.gpqa_num_threads,
- thinking=self.gpqa_thinking,
- reasoning_effort=self.gpqa_reasoning_effort,
- max_tokens=self.gpqa_max_tokens,
- temperature=self.gpqa_temperature,
- top_p=self.gpqa_top_p,
- accept_length_thres=self.gpqa_accept_length_thres,
- )
-
-
-class AIME25Mixin:
- """Mixin for AIME 2025 evaluation (competition math, integer answers).
-
- Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
- installed). ``aime25_thinking`` defaults to True, which
- enables per-request thinking so the server separates reasoning from the final
- answer.
-
- Required attributes on the test class:
- base_url: str
- model: str
- aime25_score_threshold: float
-
- Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
- for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
- aime25_reasoning_effort="max", aime25_max_tokens=200000, aime25_temperature=1.0,
- aime25_top_p=1.0. AIME25 has only 30 problems, so it is high variance; raise
- aime25_n_repeats (e.g. 16) for a stable number.
- """
-
- aime25_score_threshold: float = _THRESHOLD_NOT_SET
- aime25_accept_length_thres: Optional[float] = None
- aime25_num_examples: Optional[int] = None
- aime25_num_threads: int = 1024
- aime25_n_repeats: int = 1
- aime25_thinking: bool = True
- aime25_reasoning_effort: Optional[str] = None
- aime25_max_tokens: Optional[int] = None
- aime25_temperature: Optional[float] = None
- aime25_top_p: Optional[float] = None
-
- def test_aime25(self):
- _run_sgl_eval(
- self,
- eval_name="aime25",
- score_threshold=self.aime25_score_threshold,
- n_repeats=self.aime25_n_repeats,
- num_examples=self.aime25_num_examples,
- num_threads=self.aime25_num_threads,
- thinking=self.aime25_thinking,
- reasoning_effort=self.aime25_reasoning_effort,
- max_tokens=self.aime25_max_tokens,
- temperature=self.aime25_temperature,
- top_p=self.aime25_top_p,
- accept_length_thres=self.aime25_accept_length_thres,
- )
-
-
class HumanEvalMixin:
- """Mixin for HumanEval evaluation.
-
- Required attributes on the test class:
- base_url: str
- model: str
- humaneval_score_threshold: float
- """
+ """Mixin for HumanEval evaluation."""
humaneval_score_threshold: float = _THRESHOLD_NOT_SET
humaneval_score_threshold_amd: Optional[float] = None
@@ -464,13 +289,7 @@ class HumanEvalMixin:
class MGSMEnMixin:
- """Mixin for MGSM English evaluation.
-
- Required attributes on the test class:
- base_url: str
- model: str
- mgsm_en_score_threshold: float
- """
+ """Mixin for MGSM English evaluation."""
mgsm_en_score_threshold: float = _THRESHOLD_NOT_SET
mgsm_en_num_examples: Optional[int] = None
diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py
index 5b74d93fdd..37d17d4036 100644
--- a/python/sglang/test/run_eval.py
+++ b/python/sglang/test/run_eval.py
@@ -1,18 +1,17 @@
"""
Usage:
-python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10
+python3 -m sglang.test.run_eval --port 30000 --eval-name gsm8k --num-examples 10
"""
import argparse
import json
import os
import statistics
-import subprocess
import time
-import uuid
-from pathlib import Path
+import warnings
from sglang.test.sgl_eval_utils import (
+ SGL_EVAL_BENCHMARKS,
THINKING_MODE_CHOICES,
get_thinking_kwargs,
parse_json_object,
@@ -89,125 +88,6 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
return result, latency, sampler
-def _run_sgl_eval(eval_name, args) -> dict:
- # Returns a metrics dict (score, latency, output_throughput) so the
- # existing write_results_to_json + threshold gate keep working.
- from sglang.test.test_utils import dump_metric
-
- base_url = (
- f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
- )
- out_parent = Path(
- getattr(args, "sgl_eval_out_dir", None)
- or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex)
- ).expanduser()
- out_parent.mkdir(parents=True, exist_ok=True)
-
- model_preset_id = getattr(args, "load_preset_from_model_id", None)
- cmd = [
- "sgl-eval",
- "run",
- eval_name,
- "--base-url",
- base_url,
- "--out-dir",
- str(out_parent),
- ]
- if model_preset_id:
- cmd += ["--load-preset-from-model-id", model_preset_id]
- if getattr(args, "model", None):
- cmd += ["--model", args.model]
- if getattr(args, "num_examples", None) is not None:
- cmd += ["--num-examples", str(args.num_examples)]
- if getattr(args, "num_threads", None) is not None:
- cmd += ["--num-threads", str(args.num_threads)]
- if getattr(args, "temperature", None) is not None:
- cmd += ["--temperature", str(args.temperature)]
- elif not model_preset_id:
- cmd += ["--temperature", "0.0"]
- if getattr(args, "top_p", None) is not None:
- cmd += ["--top-p", str(args.top_p)]
- elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False):
- cmd += ["--top-p", "1.0"]
- # Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
- if getattr(args, "seed", None) is not None:
- cmd += ["--seed", str(args.seed)]
- # gpt-oss grades one score per effort tier, so dropping this collapses every
- # tier onto the served model's default.
- if getattr(args, "reasoning_effort", None) is not None:
- cmd += ["--reasoning-effort", str(args.reasoning_effort)]
- if getattr(args, "repeat", None) is not None:
- cmd += ["--n-repeats", str(args.repeat)]
- # Bound generation length so long-reasoning models don't stall the eval.
- if getattr(args, "max_tokens", None) is not None:
- cmd += ["--max-tokens", str(args.max_tokens)]
- elif not model_preset_id:
- cmd += ["--max-tokens", "2048"]
- # Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
- # without --thinking their message.content is empty and sgl-eval scores 0.
- sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None)
- if sgl_eval_thinking is None:
- if not model_preset_id:
- model_l = (getattr(args, "model", None) or "").lower()
- if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
- cmd += ["--thinking"]
- elif sgl_eval_thinking:
- cmd += ["--thinking"]
-
- try:
- completed = subprocess.run(
- cmd,
- text=True,
- capture_output=True,
- check=False,
- timeout=getattr(args, "sgl_eval_timeout", None),
- )
- except subprocess.TimeoutExpired as e:
- raise TimeoutError(
- f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n"
- f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}"
- ) from e
-
- if completed.returncode != 0:
- raise RuntimeError(
- f"sgl-eval failed with exit code {completed.returncode}: "
- f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
- )
-
- metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json"))
- if len(metrics_files) != 1:
- raise FileNotFoundError(
- f"Expected exactly one metrics.json under {out_parent}, "
- f"found {len(metrics_files)}"
- )
- payload = json.loads(metrics_files[0].read_text())
- aggregate = payload.get("aggregate")
- if not isinstance(aggregate, dict) or "score" not in aggregate:
- raise KeyError(f"{metrics_files[0]} missing aggregate.score")
-
- metrics = dict(aggregate)
- metrics["latency"] = payload.get("latency_seconds", 0.0)
- metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0)
- metrics["sgl_eval_metrics_path"] = str(metrics_files[0])
-
- model = payload.get("model") or getattr(args, "model", None)
- dump_metric(
- f"{eval_name}_score",
- metrics["score"],
- labels={"model": model, "eval": eval_name},
- )
- dump_metric(
- f"{eval_name}_latency",
- metrics["latency"],
- labels={"model": model, "eval": eval_name},
- )
- print(f"Score: {metrics['score']:.3f}")
- print(f"Total latency: {metrics['latency']:.3f} s")
- print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
- print(f"sgl-eval metrics: {metrics_files[0]}")
- return metrics
-
-
def print_accept_length_summary(samplers: list) -> None:
accept_lengths = [
m["spec_accept_length"]
@@ -236,6 +116,23 @@ def run_eval(args):
# Lazy import to avoid circular dependency with test_utils
from sglang.test.test_utils import dump_metric
+ warnings.warn(
+ "sglang.test.run_eval is deprecated; its legacy scorers will be removed. "
+ "Use sglang.test.sgl_eval_utils.run_sgl_eval or `sgl-eval run` instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ if (
+ args.eval_name in SGL_EVAL_BENCHMARKS
+ or args.eval_name == "mmmu-pro"
+ or getattr(args, "api", None) == "sgl_eval"
+ or getattr(args, "load_preset_from_model_id", None)
+ ):
+ raise ValueError(
+ f"{args.eval_name} is scored by sgl-eval; call "
+ "sglang.test.sgl_eval_utils.run_sgl_eval instead of run_eval."
+ )
+
set_ulimit()
if "OPENAI_API_KEY" not in os.environ:
@@ -245,18 +142,10 @@ def run_eval(args):
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
- if args.eval_name == "mmlu":
- # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
- # caller's threshold has to be measured against it, not inherited.
- return _run_sgl_eval("mmlu", args)
- elif args.eval_name == "mgsm_en":
+ if args.eval_name == "mgsm_en":
from sglang.test.simple_eval_mgsm import MGSMEval
eval_obj = MGSMEval(args.num_examples, args.num_threads, languages=["en"])
- elif args.eval_name == "gpqa":
- # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
- # caller's threshold has to be measured against it, not inherited.
- return _run_sgl_eval("gpqa", args)
elif args.eval_name == "humaneval":
from sglang.test.simple_eval_humaneval import HumanEval
@@ -270,25 +159,7 @@ def run_eval(args):
args.num_threads,
response_answer_regex=getattr(args, "response_answer_regex", None),
)
- elif args.eval_name in ("mmmu_pro", "mmmu-pro"):
- # Canonical sgl-eval name for MMMU-Pro's standard 10-option split.
- return _run_sgl_eval("mmmu_pro", args)
- elif args.eval_name == "mmmu_pro_vision":
- # sgl-eval owns this benchmark's dataset, prompt and grader; there is no
- # simple_eval implementation to fall back to.
- return _run_sgl_eval("mmmu_pro_vision", args)
- elif args.eval_name == "aime25":
- return _run_sgl_eval("aime25", args)
- elif args.eval_name == "aime26":
- return _run_sgl_eval("aime26", args)
elif args.eval_name == "gsm8k":
- if getattr(args, "api", None) == "sgl_eval":
- # Only the nightly correctness eval opts into sgl-eval (zero-shot
- # chat, \boxed{}, math_verify). Every other gsm8k caller — spec
- # decoding perf/accuracy, disaggregation, quant, model e2e — uses
- # the 5-shot completion last-number scorer and relies on
- # max_tokens/throughput behavior sgl-eval cannot provide.
- return _run_sgl_eval("gsm8k", args)
from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval
eval_obj = GSM8KEval(
@@ -428,16 +299,10 @@ if __name__ == "__main__":
type=str,
help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
)
- parser.add_argument(
- "--load-preset-from-model-id",
- type=str,
- default=None,
- help="Load repository-maintained sgl-eval generation defaults for this model ID.",
- )
parser.add_argument(
"--repeat", type=int, default=1, help="repeat the evaluation n times"
)
- parser.add_argument("--eval-name", type=str, default="mmlu")
+ parser.add_argument("--eval-name", type=str, default="gsm8k")
parser.add_argument(
"--api",
type=str,
@@ -498,6 +363,5 @@ if __name__ == "__main__":
)
args = parser.parse_args()
- args._sgl_eval_from_cli = True
run_eval(args)
diff --git a/python/sglang/test/sgl_eval_utils.py b/python/sglang/test/sgl_eval_utils.py
index 01f7c651ce..abc9a95339 100644
--- a/python/sglang/test/sgl_eval_utils.py
+++ b/python/sglang/test/sgl_eval_utils.py
@@ -6,13 +6,20 @@ import multiprocessing
import os
import threading
import uuid
+import warnings
from concurrent.futures import ProcessPoolExecutor
from dataclasses import replace
from itertools import islice
from pathlib import Path
+from types import SimpleNamespace
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
+# Benchmarks scored only by sgl-eval; run_eval does not serve them.
+SGL_EVAL_BENCHMARKS = frozenset(
+ {"mmlu", "gpqa", "mmmu_pro", "mmmu_pro_vision", "aime25", "aime26"}
+)
+
def get_thinking_kwargs(args):
thinking_mode = getattr(args, "thinking_mode", None)
@@ -85,9 +92,35 @@ def run_sgl_eval(args):
from sglang.test.test_utils import dump_metric
spec = get(args.eval_name)
- overrides = {"max_tokens": 2048}
+ model = getattr(args, "model", None)
+ default_gen = spec.default_gen
+ model_preset_id = getattr(args, "load_preset_from_model_id", None)
+ if model_preset_id:
+ from sgl_eval.model_preset import load_model_preset
+ from sgl_eval.preset import apply_to_gen
+
+ model_preset = load_model_preset(model_preset_id)
+ unset = dict.fromkeys(
+ ("thinking", "temperature", "top_p", "max_tokens", "reasoning_effort")
+ )
+ default_gen = apply_to_gen(
+ default_gen, None, SimpleNamespace(**unset), model_preset
+ )
+ model = model or model_preset.model
+
+ overrides = {}
+ # Omitted max_tokens keeps the 2048 CI cap; an explicit None defers to the server.
+ if hasattr(args, "max_tokens"):
+ if args.max_tokens is None:
+ warnings.warn(
+ f"sgl-eval {spec.name}: max_tokens=None leaves the output length "
+ "uncapped; generation stops only at the server limit.",
+ stacklevel=2,
+ )
+ overrides["max_tokens"] = args.max_tokens
+ elif not model_preset_id:
+ overrides["max_tokens"] = 2048
for key in (
- "max_tokens",
"temperature",
"top_p",
"min_p",
@@ -107,12 +140,12 @@ def run_sgl_eval(args):
chat_kwargs.setdefault("enable_thinking", thinking)
if chat_kwargs:
overrides["chat_template_kwargs"] = {
- **(spec.default_gen.chat_template_kwargs or {}),
+ **(default_gen.chat_template_kwargs or {}),
**chat_kwargs,
}
if getattr(args, "top_k", None) is not None:
overrides["extra_body"] = {"top_k": args.top_k}
- gen = replace(spec.default_gen, **overrides)
+ gen = replace(default_gen, **overrides)
generation = {
key: getattr(gen, key)
for key in ("max_tokens", "temperature", "top_p", "min_p", "seed")
@@ -120,7 +153,7 @@ def run_sgl_eval(args):
print(f"sgl-eval {spec.name} generation: {generation}", flush=True)
sampler = ChatCompletionSampler(
base_url=api_base_url(args),
- model=getattr(args, "model", None),
+ model=model,
api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"),
)
out_parent = Path(
@@ -129,7 +162,7 @@ def run_sgl_eval(args):
).expanduser()
out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}"
out_dir.mkdir(parents=True, exist_ok=True)
- repeats = getattr(args, "repeat", 1)
+ repeats = getattr(args, "repeat", None) or spec.default_n_repeats
writer = PredictionsWriter(out_dir, repeats, spec.pred_schema)
try:
result = spec.run(
diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py
index 521f330cb4..1d745a118e 100644
--- a/python/sglang/test/test_utils.py
+++ b/python/sglang/test/test_utils.py
@@ -48,7 +48,7 @@ from sglang.srt.utils import (
retry,
)
from sglang.srt.utils.network import is_port_available
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.utils import normalize_base_url
# General test models
@@ -399,24 +399,11 @@ def start_subprocess_fail_fast_watcher(
def _try_enable_offline_mode_if_cache_complete(
model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None
) -> Optional[str]:
- """
- CI helper: Check if model cache is complete and enable offline mode.
+ """Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the
+ per-run marker path, or None if offline mode was not enabled.
- Uses per-run validation markers that are NOT shared across runners.
- Each runner independently validates its cache using lightweight checks
- before enabling offline mode.
-
- IMPORTANT: Even if a per-run marker exists, this function ALWAYS validates
- the current launch's requirements (e.g., hf_quant_config.json for modelopt).
- The marker is only a hint that this snapshot was validated earlier in the run.
-
- Args:
- model_name_or_path: Model identifier or path
- env: Environment dict to modify (will add HF_HUB_OFFLINE=1 if validation passes)
- other_args: Launch command arguments (used to detect quantization requirement)
-
- Returns:
- Per-run marker path if offline mode was enabled, None otherwise
+ Markers are per-run and not shared across runners. A marker is only a hint;
+ the current launch's requirements (e.g. hf_quant_config.json) are revalidated.
"""
from sglang.srt.model_loader.ci_weight_validation import (
_get_per_run_marker_path,
@@ -453,8 +440,7 @@ def _try_enable_offline_mode_if_cache_complete(
except Exception:
return None
- # Detect if quantization requires hf_quant_config.json
- # Do this BEFORE checking marker to ensure current launch requirements are known
+ # Detect before the marker check so the current launch's requirements are known.
requires_hf_quant_config = False
for i, arg in enumerate(other_args):
if arg == "--quantization" and i + 1 < len(other_args):
@@ -466,9 +452,8 @@ def _try_enable_offline_mode_if_cache_complete(
# Check per-run marker (fast hint - snapshot validated earlier in this run)
per_run_marker = _read_per_run_marker(snapshot_dir)
if per_run_marker is not None:
- # Marker exists, but STILL validate for current launch requirements
- # This prevents a test without --quantization from enabling offline
- # for a later test with --quantization that needs hf_quant_config.json
+ # Still validate: a marker from a launch without --quantization must not
+ # enable offline mode for a later launch that needs hf_quant_config.json.
is_valid = validate_cache_lightweight(snapshot_dir, requires_hf_quant_config)
if not is_valid:
@@ -522,17 +507,7 @@ def _try_enable_offline_mode_if_cache_complete(
def _create_clean_subprocess_env(env: dict) -> dict:
- """Create a clean subprocess environment without internal CI keys.
-
- Removes all keys starting with '_CI_OFFLINE_' or 'CI_OFFLINE' to prevent
- leaking implementation details to the server subprocess.
-
- Args:
- env: Source environment dict
-
- Returns:
- Clean copy of environment dict
- """
+ """Create a clean subprocess environment without internal CI keys."""
child_env = env.copy()
keys_to_remove = [
k for k in child_env if k.startswith(("_CI_OFFLINE_", "CI_OFFLINE_"))
@@ -592,14 +567,7 @@ def _launch_server_process(
) -> subprocess.Popen:
"""Launch server subprocess with clean environment.
- Args:
- command: Command list for subprocess
- env: Environment dict (will be cleaned before use)
- return_stdout_stderr: Optional tuple of (stdout_file, stderr_file) for output capture
- model: Model name for logging
-
- Returns:
- Started subprocess.Popen object
+ `return_stdout_stderr` is an optional (stdout_file, stderr_file) pair.
"""
child_env = _create_clean_subprocess_env(env)
@@ -619,17 +587,7 @@ def _wait_for_server_health(
api_key: Optional[str],
timeout_duration: float,
) -> Tuple[bool, Optional[str]]:
- """Wait for server health check to pass.
-
- Args:
- proc: Server subprocess
- base_url: Base URL for health check
- api_key: Optional API key for authorization
- timeout_duration: Maximum wait time in seconds
-
- Returns:
- Tuple of (success, error_message)
- """
+ """Wait for server health check to pass; return (success, error_message)."""
start_time = time.perf_counter()
with requests.Session() as session:
while time.perf_counter() - start_time < timeout_duration:
@@ -686,20 +644,8 @@ def popen_launch_server(
):
"""Launch a server process with automatic device detection and offline/online retry.
- Args:
- model: Model path or identifier
- base_url: Base URL for the server
- timeout: Timeout for server startup
- api_key: Optional API key for authentication
- other_args: Additional command line arguments
- env: Environment dict for subprocess
- return_stdout_stderr: Optional tuple for output capture
- device: Device type ("auto", "cuda", "rocm" or "cpu")
- pd_separated: Whether to use PD separated mode
- num_replicas: Number of replicas for mixed PD mode
-
- Returns:
- Started subprocess.Popen object
+ `device` is one of "auto", "cuda", "rocm", "cpu";
+ `num_replicas` applies to mixed PD mode.
"""
other_args = other_args or []
@@ -897,9 +843,7 @@ def popen_launch_pd_server(
print(f"command={' '.join(command)}")
- # Merge with os.environ so caller-supplied env adds to (not replaces)
- # PATH / PYTHONPATH / HF_HOME / etc. When env is None, Popen inherits
- # parent's environment automatically.
+ # Merge so caller-supplied env adds to, not replaces, PATH / PYTHONPATH / etc.
if env is not None:
env = {**os.environ, **env}
@@ -1018,9 +962,8 @@ def run_bench_serving(
other_args=other_server_args,
)
- # Resolve tokenizer to local snapshot path when available, so the benchmark
- # client's AutoTokenizer.from_pretrained uses the local path directly instead
- # of calling the HF Hub API (which can stall for minutes in CI).
+ # Prefer the local snapshot so the client's AutoTokenizer skips the HF Hub API,
+ # which can stall for minutes in CI.
bench_tokenizer = tokenizer
if bench_tokenizer is None:
try:
@@ -1090,19 +1033,7 @@ async def _run_api_benchmark_requests(
num_requests: int,
response_validator: Callable[[dict], bool],
):
- """
- Helper function to run API benchmark requests and collect metrics.
-
- Args:
- base_url: The base URL of the server
- endpoint: The API endpoint to test (e.g., "/v1/score", "/v1/embeddings")
- test_requests: List of request payloads to send
- num_requests: Total number of requests expected
- response_validator: Function to validate if response contains expected data
-
- Returns:
- Dictionary with benchmark metrics
- """
+ """Run API benchmark requests and return a dict of metrics."""
start_time = time.monotonic()
successful_requests = 0
total_latency = 0
@@ -1433,12 +1364,7 @@ def run_bench_serving_multi(
def run_bench_one_batch(model, other_args):
- """Launch a offline process with automatic device detection.
-
- Args:
- device: Device type ("auto", "cuda", "rocm" or "cpu").
- If "auto", will detect available platforms automatically.
- """
+ """Launch an offline process with automatic device detection."""
# Auto-detect device if needed
device = auto_config_device()
@@ -1740,7 +1666,7 @@ def run_mmlu_test(
)
try:
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
assert metrics["score"] >= 0.65, f"{metrics=}"
finally:
pass
@@ -2340,11 +2266,9 @@ def server_args_variant(server_args, **fields):
def enter_override(test_case, override):
"""Install a scoped context override for the length of one test.
- `unittest.TestCase.enterContext` does exactly this in one call, but it is
- Python 3.11+ and this package supports 3.10 (`requires-python = ">=3.10"`).
- On 3.10 it raises `AttributeError: ... has no attribute 'enterContext'` --
- and only there, so a developer on a newer interpreter sees every test pass
- while CI does not.
+ Stands in for `unittest.TestCase.enterContext`, which is 3.11+ while this package
+ supports 3.10; on 3.10 it raises AttributeError, so tests pass locally on newer
+ interpreters but fail in CI.
"""
installed = override.install()
test_case.addCleanup(override.restore)
@@ -2366,9 +2290,8 @@ class CustomTestCase(unittest.TestCase):
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
- # Wrap the effective setUpClass so that tearDownClass is called
- # even when setUpClass fails. Python's unittest skips tearDownClass
- # if setUpClass raises, which can leak resources (ports, processes).
+ # unittest skips tearDownClass when setUpClass raises, leaking ports and
+ # processes; wrap setUpClass so tearDownClass still runs.
setup = cls.setUpClass
if getattr(setup, "_safe_setup_wrapped", False):
return
@@ -2669,11 +2592,6 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
- stdout: [METRIC] metric_name=value [labels=...]
This function never fails tests - all errors are silently caught.
-
- Args:
- metric_name: Metric name (e.g., "gsm8k_accuracy", "cache_hit_rate")
- value: Metric value
- labels: Optional label dict (e.g., {"backend": "fa3"})
"""
try:
# 1. Capture test context
@@ -2750,11 +2668,7 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
def _get_test_context() -> tuple[str, str]:
- """
- Get current test's filename and test_case.
-
- Tries PYTEST_CURRENT_TEST first, falls back to inspect.stack().
- """
+ """Get current test's filename and test_case, via PYTEST_CURRENT_TEST or stack."""
# 1. Try parsing PYTEST_CURRENT_TEST
pytest_current = os.getenv("PYTEST_CURRENT_TEST")
if pytest_current:
diff --git a/test/manual/dsv4/_common.py b/test/manual/dsv4/_common.py
index 19aefd08a0..40b357bf0d 100644
--- a/test/manual/dsv4/_common.py
+++ b/test/manual/dsv4/_common.py
@@ -1,90 +1,65 @@
-"""Shared fixture for DeepSeek-V4 cookbook launch-command tests.
+"""Shared fixture for DeepSeek-V4 cookbook launch-command tests (manual, not CI).
-Each sibling ``test__.py`` declares ONE
-``hardware x model_size`` cell from the cookbook (e.g. B200 x Flash)
-and contains one ``CustomTestCase`` subclass per recipe
-(Low-Latency / Balanced / Max-Throughput / CP, where supported).
-
-Each subclass launches the server with the cookbook's exact flags and
-runs two sgl-eval evaluations (https://github.com/sgl-project/sgl-eval):
-- ``test_smoke_gsm8k`` — short, cheap GSM8K pass to verify the server
- can produce coherent math answers at all (sanity gate).
-- ``test_aime25`` — full AIME25 accuracy run (heavy; 16 repeats default).
+Each sibling ``test__.py`` covers one cookbook cell
+(e.g. B200 x Flash) with one ``CustomTestCase`` subclass per recipe. Each
+launches the server with the cookbook's exact flags and runs
+``test_smoke_gsm8k`` (sanity gate) and ``test_aime25`` (full accuracy run)
+through sgl-eval, which must be installed.
Cookbook reference:
https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4
-These are MANUAL tests (not CI). ``sgl-eval`` must be on PATH.
-
-Per-variant defaults (set on the Flash/Pro intermediate base classes):
- Flash recipes -> AIME25 score threshold 0.93
- Pro recipes -> AIME25 score threshold 0.95
-GSM8K sanity threshold (0.93) is shared across Flash and Pro.
-
-AIME25 knobs (env vars):
- DSV4_AIME25_NUM_REPEATS (default 16 -> --n-repeats)
- DSV4_AIME25_TEMPERATURE (default 1.0 -> --temperature)
- DSV4_AIME25_TOP_P (default 1.0 -> --top-p)
- DSV4_AIME25_MAX_TOKENS (default 65536 -> --max-tokens)
- DSV4_AIME25_NUM_THREADS (default 512 -> --num-threads)
- DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate")
+AIME25 knobs (env vars); the score threshold is 0.93 for Flash, 0.95 for Pro:
+ DSV4_AIME25_NUM_REPEATS (default 16)
+ DSV4_AIME25_TEMPERATURE (default 1.0)
+ DSV4_AIME25_TOP_P (default 1.0)
+ DSV4_AIME25_MAX_TOKENS (default 65536)
+ DSV4_AIME25_NUM_THREADS (default 512)
+ DSV4_AIME25_SCORE_METRIC (default "score"; sgl-eval aggregate key)
DSV4_AIME25_SCORE_THRESHOLD (default 0; >0 overrides per-variant default)
GSM8K sanity knobs (env vars):
- DSV4_GSM8K_NUM_EXAMPLES (default 50 -> --num-examples)
- DSV4_GSM8K_N_REPEATS (default 1 -> --n-repeats)
- DSV4_GSM8K_TEMPERATURE (default 0.6 -> --temperature)
- DSV4_GSM8K_TOP_P (default 0.95 -> --top-p)
- DSV4_GSM8K_MAX_TOKENS (default 8192 -> --max-tokens)
- DSV4_GSM8K_NUM_THREADS (default 64 -> --num-threads)
- DSV4_GSM8K_SCORE_METRIC (default "score"; sgl-eval JSON key under "aggregate")
+ DSV4_GSM8K_NUM_EXAMPLES (default 50)
+ DSV4_GSM8K_N_REPEATS (default 1)
+ DSV4_GSM8K_TEMPERATURE (default 0.6)
+ DSV4_GSM8K_TOP_P (default 0.95)
+ DSV4_GSM8K_MAX_TOKENS (default 8192)
+ DSV4_GSM8K_NUM_THREADS (default 64)
+ DSV4_GSM8K_SCORE_METRIC (default "score"; sgl-eval aggregate key)
DSV4_GSM8K_SCORE_THRESHOLD (default 0.93; set to 0 to skip the assertion)
Shared knobs:
- DSV4_SGL_EVAL_OUT_DIR (default /tmp/sgl-eval-out -> --out-dir)
- DSV4_SGL_EVAL_BIN (default "sgl-eval"; override path to the CLI)
- DSV4_SERVER_LAUNCH_TIMEOUT (default 3600s; the sglang 600s default is
- too short for DSV4 model load + DeepGEMM
- warmup. 1800s is also tight for the heavier
- recipes (DP-attn + DeepEP); 3600s is the
- safe default. Bump again for first-run
- model downloads if needed.)
+ DSV4_SGL_EVAL_OUT_DIR (default /tmp/sgl-eval-out)
+ DSV4_SERVER_LAUNCH_TIMEOUT (default 3600s; bump for first-run downloads)
-Multi-node knobs (only consumed by multi-node test classes; if either
-is unset, those classes ``SkipTest``):
+Multi-node knobs (multi-node classes ``SkipTest`` if either is unset):
DSV4_NODE_RANK (per-node rank for --node-rank)
DSV4_DIST_INIT_ADDR (e.g. 10.0.0.1:20000 for --dist-init-addr)
-Always-on env (set by the base class for every recipe; per-recipe EXTRA_ENV
-wins on key conflict):
- SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 skip the slow DeepGEMM warmup grid
+Every recipe also gets SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 (skips the slow
+DeepGEMM warmup grid); per-recipe EXTRA_ENV wins on key conflict.
"""
-import json
import os
-import shutil
-import subprocess
import unittest
-from pathlib import Path
+from types import SimpleNamespace
from typing import ClassVar, Dict, List, Optional
from sglang.srt.utils import kill_process_tree
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
-SGL_EVAL_BIN = os.environ.get("DSV4_SGL_EVAL_BIN", "sgl-eval")
SGL_EVAL_OUT_DIR = os.environ.get("DSV4_SGL_EVAL_OUT_DIR", "/tmp/sgl-eval-out")
-# DSV4 server launch needs more than the 600s sglang default: model load alone
-# can take 5+ min and DeepGEMM warmup another ~5 min. First-run model download
-# adds ~10-30 min on top. 1800s covers steady-state; bump via env for downloads.
+# Model load (5+ min) plus DeepGEMM warmup (~5 min) exceed sglang's 600s default,
+# and 1800s is tight for DP-attn + DeepEP recipes; downloads add ~10-30 min.
SERVER_LAUNCH_TIMEOUT = int(os.environ.get("DSV4_SERVER_LAUNCH_TIMEOUT", "3600"))
-# Defaults applied to every recipe's EXTRA_ENV. Per-recipe EXTRA_ENV wins on key
-# conflict.
+# Defaults for every recipe; per-recipe EXTRA_ENV wins on key conflict.
BASE_ENV: Dict[str, str] = {
# Skip the slow exhaustive DeepGEMM warmup grid; covers the shapes DSV4
# actually hits and shaves several minutes off server startup.
@@ -108,8 +83,8 @@ GSM8K_NUM_THREADS = int(os.environ.get("DSV4_GSM8K_NUM_THREADS", "64"))
GSM8K_SCORE_METRIC = os.environ.get("DSV4_GSM8K_SCORE_METRIC", "score")
GSM8K_SCORE_THRESHOLD = float(os.environ.get("DSV4_GSM8K_SCORE_THRESHOLD", "0.93"))
-# DeepEP "large SMS" config — appears as `--deepep-config '{...}'` in every
-# DeepEP recipe except multi-node ones (where it is gated off in the JSX).
+# Cookbook `--deepep-config` of every DeepEP recipe except multi-node,
+# where the JSX gates it off.
DEEPEP_LARGE_SMS_CONFIG = (
'{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
)
@@ -118,9 +93,7 @@ DEEPEP_LARGE_SMS_CONFIG = (
def multinode_args(nnodes: int) -> List[str]:
"""Return CLI args for a multi-node launch, or skip the test.
- Reads DSV4_NODE_RANK and DSV4_DIST_INIT_ADDR from the env. Raises
- ``unittest.SkipTest`` when either is missing — call from inside
- ``setUpClass`` so the whole class skips cleanly.
+ Call from ``setUpClass`` so the whole class skips cleanly.
"""
rank = os.environ.get("DSV4_NODE_RANK")
addr = os.environ.get("DSV4_DIST_INIT_ADDR")
@@ -141,9 +114,6 @@ def multinode_args(nnodes: int) -> List[str]:
class DSV4Aime25TestBase(CustomTestCase):
"""Subclass via ``DSV4FlashAime25TestBase`` or ``DSV4ProAime25TestBase``,
not directly. Per-recipe subclasses set MODEL / OTHER_ARGS / EXTRA_ENV.
-
- SCORE_THRESHOLD is set by the Flash/Pro intermediate base classes:
- Flash 0.93, Pro 0.95.
"""
MODEL: ClassVar[str] = ""
@@ -220,52 +190,18 @@ class DSV4Aime25TestBase(CustomTestCase):
metric,
threshold,
):
- if shutil.which(SGL_EVAL_BIN) is None:
- self.skipTest(f"{SGL_EVAL_BIN!r} not found on PATH")
-
- out_dir = Path(SGL_EVAL_OUT_DIR)
- out_dir.mkdir(parents=True, exist_ok=True)
- glob_pattern = f"sgl_eval_{eval_name}_*.json"
- before = set(out_dir.glob(glob_pattern))
-
- cmd = [
- SGL_EVAL_BIN,
- "run",
- eval_name,
- "--base-url",
- f"{self.base_url}/v1",
- "--n-repeats",
- str(n_repeats),
- "--temperature",
- str(temperature),
- "--top-p",
- str(top_p),
- "--max-tokens",
- str(max_tokens),
- "--num-threads",
- str(num_threads),
- "--out-dir",
- str(out_dir),
- ]
- if num_examples is not None:
- cmd += ["--num-examples", str(num_examples)]
-
- print(f"[{type(self).__name__}] + {' '.join(cmd)}", flush=True)
- subprocess.run(cmd, check=True)
-
- new = sorted(set(out_dir.glob(glob_pattern)) - before)
- if not new:
- self.fail(f"sgl-eval produced no new {eval_name} JSON in {out_dir}")
- result_path = new[-1]
- with open(result_path) as f:
- result = json.load(f)
- print(
- f"[{type(self).__name__}] sgl-eval {eval_name} result "
- f"({result_path.name}): {json.dumps(result, indent=2)}",
- flush=True,
+ args = SimpleNamespace(
+ eval_name=eval_name,
+ base_url=self.base_url,
+ repeat=n_repeats,
+ temperature=temperature,
+ top_p=top_p,
+ max_tokens=max_tokens,
+ num_threads=num_threads,
+ num_examples=num_examples,
+ sgl_eval_out_dir=SGL_EVAL_OUT_DIR,
)
-
- score = self._extract_score(result, metric)
+ score = run_sgl_eval(args)[metric]
if threshold > 0:
self.assertGreaterEqual(
score,
@@ -273,30 +209,6 @@ class DSV4Aime25TestBase(CustomTestCase):
f"{eval_name} {metric}={score} below threshold {threshold}",
)
- @staticmethod
- def _extract_score(result, metric):
- """Find ``metric`` (e.g. "pass@1") anywhere in the sgl-eval JSON tree."""
-
- def walk(o):
- if isinstance(o, dict):
- if metric in o and isinstance(o[metric], (int, float)):
- return float(o[metric])
- for v in o.values():
- s = walk(v)
- if s is not None:
- return s
- elif isinstance(o, list):
- for v in o:
- s = walk(v)
- if s is not None:
- return s
- return None
-
- score = walk(result)
- if score is None:
- raise AssertionError(f"metric {metric!r} not found in sgl-eval result JSON")
- return score
-
class DSV4FlashAime25TestBase(DSV4Aime25TestBase):
"""Base for DeepSeek-V4-Flash recipes: AIME25 threshold 0.93."""
diff --git a/test/manual/ep/test_eplb.py b/test/manual/ep/test_eplb.py
index 5dd97364ee..f0c5cce2e4 100755
--- a/test/manual/ep/test_eplb.py
+++ b/test/manual/ep/test_eplb.py
@@ -8,7 +8,7 @@ from types import SimpleNamespace
import sglang as sgl
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -96,7 +96,7 @@ class _BaseTestDynamicEPLB(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.5)
@@ -123,8 +123,6 @@ class TestStaticEPLB(CustomTestCase):
tp_size=2,
dp_size=2,
log_level="info",
- # TODO pr-chain: enable later
- # enable_expert_distribution_metrics=True,
)
engine_kwargs.update(get_a2a_backend_kwargs())
diff --git a/test/manual/ep/test_moe_deepep.py b/test/manual/ep/test_moe_deepep.py
index 495a3aefff..ce7074b817 100644
--- a/test/manual/ep/test_moe_deepep.py
+++ b/test/manual/ep/test_moe_deepep.py
@@ -4,7 +4,7 @@ from types import SimpleNamespace
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -46,7 +46,7 @@ class TestPureTP(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.5)
@@ -108,7 +108,7 @@ class TestDPAttn(unittest.TestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.5)
diff --git a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py
index 0b990f3b35..9693e7f530 100644
--- a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py
+++ b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py
@@ -1,13 +1,7 @@
-"""
-Usage:
-python -m unittest test_moe_deepep_eval_accuracy_large.TestMoEDeepEPEvalAccuracyLarge.test_mmlu
-"""
-
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST,
@@ -65,7 +59,7 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"Eval accuracy of MMLU: {metrics=}")
self.assertGreater(metrics["score"], 0.87)
diff --git a/test/manual/quant/test_autoround.py b/test/manual/quant/test_autoround.py
index 9d879330d2..cd7dba0fd8 100644
--- a/test/manual/quant/test_autoround.py
+++ b/test/manual/quant/test_autoround.py
@@ -1,13 +1,8 @@
-"""
-Usage:
-python3 -m unittest test_autoround.TestAutoRound.test_mmlu
-"""
-
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -48,7 +43,7 @@ class TestAutoRound(CustomTestCase):
num_threads=32,
device=device,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
if "Llama" in model:
self.assertGreaterEqual(metrics["score"], 0.6)
else:
diff --git a/test/manual/quant/test_awq_archived.py b/test/manual/quant/test_awq_archived.py
index 64a42acf4b..2e558006ef 100644
--- a/test/manual/quant/test_awq_archived.py
+++ b/test/manual/quant/test_awq_archived.py
@@ -1,15 +1,10 @@
-"""Archived test classes split out of test/registered/quant/test_awq.py.
-
-Originally registered with `register_cuda_ci(...)`. Moved here as part of
-the per-commit pruning effort to keep the code reachable manually.
-Run with `python3 test/manual/quant/test_awq_archived.py`.
-"""
+"""Archived test classes split out of test/registered/quant/test_awq.py; not in CI."""
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -21,10 +16,6 @@ from sglang.test.test_utils import (
@unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs")
class TestAWQMarlinFloat16(CustomTestCase):
- """
- Verify that the model can be loaded with float16 dtype and awq_marlin quantization
- """
-
@classmethod
def setUpClass(cls):
cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ"
@@ -49,7 +40,7 @@ class TestAWQMarlinFloat16(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.85)
diff --git a/test/manual/quant/test_eval_fp8_accuracy.py b/test/manual/quant/test_eval_fp8_accuracy.py
index d6b8b9eea1..f69af9dc5f 100644
--- a/test/manual/quant/test_eval_fp8_accuracy.py
+++ b/test/manual/quant/test_eval_fp8_accuracy.py
@@ -2,7 +2,7 @@ import unittest
from types import SimpleNamespace
from sglang.srt.utils import is_hip, kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8,
DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8,
@@ -37,7 +37,7 @@ class TestEvalFP8Accuracy(CustomTestCase):
temperature=0.1,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
if is_hip():
# Another threshold for AMD because fp8 dtype is difference
self.assertGreaterEqual(metrics["score"], 0.60)
@@ -67,7 +67,7 @@ class TestEvalFP8DynamicQuantAccuracy(CustomTestCase):
temperature=0.1,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], expected_score)
finally:
kill_process_tree(process.pid)
@@ -86,7 +86,6 @@ class TestEvalFP8DynamicQuantAccuracy(CustomTestCase):
model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8,
other_args=["--quantization", "w8a8_fp8"],
# inference will use sgl kernel w/ online quant override
- # we observed that the accuracy is higher then offline only
expected_score=0.64,
)
@@ -95,7 +94,6 @@ class TestEvalFP8DynamicQuantAccuracy(CustomTestCase):
self._run_test(
model=DEFAULT_MODEL_NAME_FOR_TEST,
# inference will use sgl kernel w/ online quantization only
- # we observed that the accuracy is higher then offline only
other_args=["--quantization", "w8a8_fp8"],
expected_score=0.64,
)
diff --git a/test/manual/quant/test_fp8_kvcache.py b/test/manual/quant/test_fp8_kvcache.py
index 669926f5db..9ba9aed00a 100644
--- a/test/manual/quant/test_fp8_kvcache.py
+++ b/test/manual/quant/test_fp8_kvcache.py
@@ -4,6 +4,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN,
@@ -71,7 +72,7 @@ class TestFp8KvcacheLlama(TestFp8KvcacheBase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
@@ -106,7 +107,7 @@ class TestFp8KvcacheQwen(TestFp8KvcacheBase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.3)
diff --git a/test/manual/spec/eagle/test_eagle3_basic.py b/test/manual/spec/eagle/test_eagle3_basic.py
index b0c1956d28..ec7f735db3 100644
--- a/test/manual/spec/eagle/test_eagle3_basic.py
+++ b/test/manual/spec/eagle/test_eagle3_basic.py
@@ -4,8 +4,8 @@ from types import SimpleNamespace
import requests
from sglang.srt.utils import is_hip
-from sglang.test.run_eval import run_eval
from sglang.test.server_fixtures.eagle_fixture import EagleServerBase
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE3,
DEFAULT_TARGET_MODEL_EAGLE3,
@@ -44,7 +44,7 @@ class TestEagle3Basic(EagleServerBase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.72)
server_info = requests.get(self.base_url + "/server_info").json()
diff --git a/test/manual/test_modelopt.py b/test/manual/test_modelopt.py
index 69a90e3851..7951fdc0c3 100644
--- a/test/manual/test_modelopt.py
+++ b/test/manual/test_modelopt.py
@@ -4,7 +4,7 @@ from types import SimpleNamespace
import torch
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8,
DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8_REVISION,
@@ -37,7 +37,7 @@ class TestEvalFP8ModelOptQuantAccuracy(CustomTestCase):
temperature=0.1,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], expected_score)
finally:
kill_process_tree(process.pid)
diff --git a/test/manual/test_two_batch_overlap.py b/test/manual/test_two_batch_overlap.py
index 410872166a..2786aa6f90 100644
--- a/test/manual/test_two_batch_overlap.py
+++ b/test/manual/test_two_batch_overlap.py
@@ -10,7 +10,7 @@ from sglang.srt.batch_overlap.two_batch_overlap import (
from sglang.srt.environ import envs
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST,
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -71,7 +71,7 @@ class TestTwoBatchOverlap(unittest.TestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.5)
diff --git a/test/manual/test_wave_attention_backend.py b/test/manual/test_wave_attention_backend.py
index 5feab45955..dd38ad48dc 100644
--- a/test/manual/test_wave_attention_backend.py
+++ b/test/manual/test_wave_attention_backend.py
@@ -1,13 +1,8 @@
-"""
-Usage:
-python3 -m unittest test_wave_attention_backend.TestWaveAttnBackend.test_mmlu
-"""
-
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -51,7 +46,7 @@ class TestWaveAttnBackend(unittest.TestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
finally:
kill_process_tree(process.pid)
diff --git a/test/registered/accuracy/models/test_text_models_gsm8k_eval.py b/test/registered/accuracy/models/test_text_models_gsm8k_eval.py
index 359ffe6351..0a2c9d37aa 100644
--- a/test/registered/accuracy/models/test_text_models_gsm8k_eval.py
+++ b/test/registered/accuracy/models/test_text_models_gsm8k_eval.py
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
@@ -82,12 +82,11 @@ class TestNightlyGsm8KEval(unittest.TestCase):
base_url=self.base_url,
model=model_setup.model_path,
eval_name="gsm8k",
- api="sgl_eval",
num_examples=None,
num_threads=1024,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(
f"{'=' * 42}\n{model_setup.model_path} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
)
diff --git a/test/registered/attention/test_torch_native_attention_backend.py b/test/registered/attention/test_torch_native_attention_backend.py
index 4e9f8ad304..f97c374183 100644
--- a/test/registered/attention/test_torch_native_attention_backend.py
+++ b/test/registered/attention/test_torch_native_attention_backend.py
@@ -1,14 +1,9 @@
-"""
-Usage:
-python3 -m unittest test_torch_native_attention_backend.TestTorchNativeAttnBackend.test_mmlu
-"""
-
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -42,7 +37,7 @@ class TestTorchNativeAttnBackend(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.64)
finally:
kill_process_tree(process.pid)
diff --git a/test/registered/attention/test_triton_sliding_window.py b/test/registered/attention/test_triton_sliding_window.py
index 24f88fcd1e..77eff242ed 100644
--- a/test/registered/attention/test_triton_sliding_window.py
+++ b/test/registered/attention/test_triton_sliding_window.py
@@ -5,7 +5,7 @@ import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -59,7 +59,7 @@ class TestSlidingWindowAttentionTriton(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"MMLU metrics with sliding window: {metrics}")
# gemma-3-4b-it scores 0.59 over 256 questions under sgl-eval's grader,
diff --git a/test/registered/cpu/quant/test_autoround.py b/test/registered/cpu/quant/test_autoround.py
index 7e954af714..f428044c9d 100644
--- a/test/registered/cpu/quant/test_autoround.py
+++ b/test/registered/cpu/quant/test_autoround.py
@@ -1,12 +1,5 @@
-"""
-Usage:
-SGLANG_USE_CPU_ENGINE=1 python3 -m unittest test_autoround
-
-CPU accuracy test for AutoRound INT4 checkpoints. Covers both AutoRound packing
-formats (auto_round:auto_gptq / auto_round:auto_awq) by launching a server and
-running an MMLU eval. AutoRound INT4 CPU inference uses the Intel AMX backend,
-so the test is skipped on AMD CPUs and other non-AMX CPU hosts.
-"""
+"""CPU accuracy test for AutoRound INT4 checkpoints in both packing formats
+(auto_round:auto_gptq / auto_round:auto_awq)."""
import os
import unittest
@@ -15,7 +8,7 @@ from types import SimpleNamespace
from sglang.srt.layers.quantization.auto_round import AutoRoundConfig
from sglang.srt.utils import cpu_has_amx_support, kill_process_tree
from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_AUTOROUND_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -89,7 +82,7 @@ class TestAutoRoundCPU(CustomTestCase):
num_threads=32,
device=device,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.25)
finally:
kill_process_tree(process.pid)
diff --git a/test/registered/cpu/test_cpu_graph.py b/test/registered/cpu/test_cpu_graph.py
index c9511f3b71..f3bdef6d48 100644
--- a/test/registered/cpu/test_cpu_graph.py
+++ b/test/registered/cpu/test_cpu_graph.py
@@ -1,8 +1,3 @@
-"""
-Usage:
-python3 -m unittest test_cpu_graph.TestCPUGraph.test_mmlu_torch_compile_cpu
-"""
-
import copy
import os
import unittest
@@ -10,7 +5,7 @@ from types import SimpleNamespace
from sglang.srt.utils import get_cpu_ids_by_node, kill_process_tree
from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -79,7 +74,7 @@ class TestCPUGraph(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
if is_in_ci():
self.assertGreater(metrics["score"], 0.45)
finally:
diff --git a/test/registered/cpu/test_intel_amx_attention_backend_a.py b/test/registered/cpu/test_intel_amx_attention_backend_a.py
index c9f0ea2ce6..23a27c8269 100644
--- a/test/registered/cpu/test_intel_amx_attention_backend_a.py
+++ b/test/registered/cpu/test_intel_amx_attention_backend_a.py
@@ -1,15 +1,10 @@
-"""
-Usage:
-python3 -m unittest test_intel_amx_attention_backend.TestIntelAMXAttnBackend.test_latency_default_model
-"""
-
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -65,7 +60,7 @@ class TestIntelAMXAttnBackend(CustomTestCase):
num_examples=64,
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
if is_in_ci():
self.assertGreater(metrics["score"], 0.45)
finally:
diff --git a/test/registered/disaggregation/test_disaggregation_decode_offload.py b/test/registered/disaggregation/test_disaggregation_decode_offload.py
index f4dfd02198..3091583089 100644
--- a/test/registered/disaggregation/test_disaggregation_decode_offload.py
+++ b/test/registered/disaggregation/test_disaggregation_decode_offload.py
@@ -4,10 +4,10 @@ import unittest
from types import SimpleNamespace
from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.run_eval import run_eval
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
)
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -15,8 +15,7 @@ from sglang.test.test_utils import (
terminate_and_kill_process_tree,
)
-# Registering the test for CUDA CI with appropriate parameters
-# Increasing estimated time since we run evaluation twice
+# est_time covers two eval rounds.
register_cuda_ci(
est_time=600,
stage="base-b",
@@ -25,10 +24,7 @@ register_cuda_ci(
class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
- """
- Test class for verifying KV cache offloading on the decode side in a
- prefill-decode disaggregation setup.
- """
+ """Decode-side KV cache offloading in a PD disaggregation setup."""
@classmethod
def setUpClass(cls):
@@ -128,13 +124,8 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
)
def test_mmlu_double_eval(self):
- """
- Run two rounds of MMLU evaluation:
- 1. First round: Decode node offloads KV cache back to disk (HiCache).
- 2. Restart All Nodes to clear memory cache.
- 3. Second round: Prefill node loads KV cache from disk (HiCache).
- Verify that both rounds produce consistent scores.
- """
+ """Round 1 offloads decode KV to HiCache disk; after restarting all nodes,
+ round 2 loads it on prefill. Both rounds must score consistently."""
args = SimpleNamespace(
base_url=f"http://{self.base_host}:{self.lb_port}",
model=self.model,
@@ -143,7 +134,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
num_threads=32,
)
- metrics1 = run_eval(args)
+ metrics1 = run_sgl_eval(args)
# Ensure all offloads are committed to disk
import time
@@ -160,7 +151,7 @@ class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
self.wait_server_ready(self.prefill_url + "/health")
self.wait_server_ready(self.decode_url + "/health")
- metrics2 = run_eval(args)
+ metrics2 = run_sgl_eval(args)
# Assert score is above a minimum threshold for both rounds
self.assertGreater(metrics1["score"], 0.64)
diff --git a/test/registered/e2e/models/test_transformers_models.py b/test/registered/e2e/models/test_transformers_models.py
index 826cf6e528..107a0b6a6f 100644
--- a/test/registered/e2e/models/test_transformers_models.py
+++ b/test/registered/e2e/models/test_transformers_models.py
@@ -10,7 +10,6 @@ import torch
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
@@ -52,7 +51,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
num_examples=256,
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound)
def test_gsm8k(self):
diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py
index 589ff28010..9554584c04 100644
--- a/test/registered/gb300/test_kimi_k25_nvfp4_dp.py
+++ b/test/registered/gb300/test_kimi_k25_nvfp4_dp.py
@@ -62,7 +62,6 @@ class TestKimiK25Nvfp4Dp(CustomTestCase):
max_tokens=32768,
temperature=0.7,
seed=0,
- sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[16],
diff --git a/test/registered/gb300/test_kimi_k25_nvfp4_tp.py b/test/registered/gb300/test_kimi_k25_nvfp4_tp.py
index 36c49c4502..4974ec169c 100644
--- a/test/registered/gb300/test_kimi_k25_nvfp4_tp.py
+++ b/test/registered/gb300/test_kimi_k25_nvfp4_tp.py
@@ -60,7 +60,6 @@ class TestKimiK25Nvfp4Tp(CustomTestCase):
max_tokens=32768,
temperature=0.7,
seed=0,
- sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 8],
diff --git a/test/registered/gb300/test_qwen35_fp8_dp.py b/test/registered/gb300/test_qwen35_fp8_dp.py
index e7c5e31da6..380c713f8a 100644
--- a/test/registered/gb300/test_qwen35_fp8_dp.py
+++ b/test/registered/gb300/test_qwen35_fp8_dp.py
@@ -59,7 +59,6 @@ class TestQwen35Fp8Dp(CustomTestCase):
max_tokens=32768,
temperature=0.7,
seed=0,
- sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[16],
diff --git a/test/registered/gb300/test_qwen35_fp8_tp.py b/test/registered/gb300/test_qwen35_fp8_tp.py
index 2e6f17b9d8..310e91737c 100644
--- a/test/registered/gb300/test_qwen35_fp8_tp.py
+++ b/test/registered/gb300/test_qwen35_fp8_tp.py
@@ -57,7 +57,6 @@ class TestQwen35Fp8Tp(CustomTestCase):
max_tokens=32768,
temperature=0.7,
seed=0,
- sgl_eval_thinking=False,
),
performance_params=PerformanceTestParams(
batch_sizes=[1, 8],
diff --git a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py
index 3b36d5eb4e..946891b48b 100644
--- a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py
+++ b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST,
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -49,7 +49,7 @@ class Test00(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -86,7 +86,7 @@ class Test01(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -123,7 +123,7 @@ class Test02(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -160,7 +160,7 @@ class Test03(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -199,7 +199,7 @@ class Test04(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -238,7 +238,7 @@ class Test05(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -276,7 +276,7 @@ class Test06(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -314,7 +314,7 @@ class Test07(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -354,7 +354,7 @@ class Test08(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -394,7 +394,7 @@ class Test09(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -432,7 +432,7 @@ class Test10(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -473,7 +473,7 @@ class Test11(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -514,7 +514,7 @@ class Test12(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -554,7 +554,7 @@ class Test13(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -597,7 +597,7 @@ class Test14(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -640,7 +640,7 @@ class Test15(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -682,7 +682,7 @@ class Test16(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -724,7 +724,7 @@ class Test17(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -768,7 +768,7 @@ class Test18(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -812,7 +812,7 @@ class Test19(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -848,7 +848,7 @@ class Test20(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -887,7 +887,7 @@ class Test21(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -926,7 +926,7 @@ class Test22(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -964,7 +964,7 @@ class Test23(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1005,7 +1005,7 @@ class Test24(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1046,7 +1046,7 @@ class Test25(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1086,7 +1086,7 @@ class Test26(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1126,7 +1126,7 @@ class Test27(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1168,7 +1168,7 @@ class Test28(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1210,7 +1210,7 @@ class Test29(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1254,7 +1254,7 @@ class Test30(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1301,7 +1301,7 @@ class Test31(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1348,7 +1348,7 @@ class Test32(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1394,7 +1394,7 @@ class Test33(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1443,7 +1443,7 @@ class Test34(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1492,7 +1492,7 @@ class Test35(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1540,7 +1540,7 @@ class Test36(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1588,7 +1588,7 @@ class Test37(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1638,7 +1638,7 @@ class Test38(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1688,7 +1688,7 @@ class Test39(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1738,7 +1738,7 @@ class Test40(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1791,7 +1791,7 @@ class Test41(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1844,7 +1844,7 @@ class Test42(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1896,7 +1896,7 @@ class Test43(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -1951,7 +1951,7 @@ class Test44(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2006,7 +2006,7 @@ class Test45(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2060,7 +2060,7 @@ class Test46(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2114,7 +2114,7 @@ class Test47(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2170,7 +2170,7 @@ class Test48(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2226,7 +2226,7 @@ class Test49(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2272,7 +2272,7 @@ class Test50(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2321,7 +2321,7 @@ class Test51(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2370,7 +2370,7 @@ class Test52(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2418,7 +2418,7 @@ class Test53(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2469,7 +2469,7 @@ class Test54(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2520,7 +2520,7 @@ class Test55(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2570,7 +2570,7 @@ class Test56(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2620,7 +2620,7 @@ class Test57(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2672,7 +2672,7 @@ class Test58(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
@@ -2724,7 +2724,7 @@ class Test59(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.48)
diff --git a/test/registered/moe/test_torch_compile_moe.py b/test/registered/moe/test_torch_compile_moe.py
index de6308adbc..a1d3fa99d7 100644
--- a/test/registered/moe/test_torch_compile_moe.py
+++ b/test/registered/moe/test_torch_compile_moe.py
@@ -6,7 +6,7 @@ import requests
from sglang.srt.utils import is_cuda, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -45,7 +45,7 @@ class TestTorchCompileMoe(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
# 0.48 measured, minus the 0.05 margin the other eval thresholds use.
self.assertGreaterEqual(metrics["score"], 0.43)
diff --git a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py
index df93ae543e..e876a758b1 100644
--- a/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py
+++ b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py
@@ -6,7 +6,7 @@ import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.npu_eval_accuracy_kit import _is_pr_pipeline, run_npu_pr_smoke
from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -54,7 +54,7 @@ class TestAscendSamplingBackend(CustomTestCase):
temperature=0.1,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
def test_greedy(self):
diff --git a/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py
index f5d28eae58..e885e3334f 100644
--- a/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py
+++ b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py
@@ -4,7 +4,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ascend.test_ascend_utils import LLAMA_3_1_8B_INSTRUCT_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -17,7 +17,7 @@ register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True)
class TestNoChunkedPrefill(CustomTestCase):
- """Testcase: Verify Llama-3.1-8B-Instruct accuracy ≥ 0.65 and serving normal with chunked prefill disabled.
+ """Testcase: Verify Llama-3.1-8B-Instruct accuracy >= 0.65 and serving normal with chunked prefill disabled.
[Test Category] Parameter
[Test Target] --chunked-prefill-size
@@ -56,7 +56,7 @@ class TestNoChunkedPrefill(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.65)
def test_no_chunked_prefill_without_radix_cache(self):
diff --git a/test/registered/quant/test_autoround_quantization.py b/test/registered/quant/test_autoround_quantization.py
index 7ea13b35f6..2d122cbd03 100644
--- a/test/registered/quant/test_autoround_quantization.py
+++ b/test/registered/quant/test_autoround_quantization.py
@@ -1,8 +1,3 @@
-"""
-Usage:
-python3 -m unittest test_autoround_quantization
-"""
-
import os
import shutil
import tempfile
@@ -15,7 +10,7 @@ from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.model_loader.loader import get_model_loader
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -62,7 +57,7 @@ class TestAutoRoundQuantization(CustomTestCase):
num_threads=MMLU_NUM_THREADS,
device="auto",
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], MMLU_SCORE_THRESHOLD)
finally:
kill_process_tree(process.pid)
diff --git a/test/registered/quant/test_awq.py b/test/registered/quant/test_awq.py
index 4ef1bfa471..f07e62ad0b 100644
--- a/test/registered/quant/test_awq.py
+++ b/test/registered/quant/test_awq.py
@@ -3,7 +3,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -42,16 +42,12 @@ class TestAWQ(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.64)
@unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs")
class TestAWQMarlinBfloat16(CustomTestCase):
- """
- Verify that the model can be loaded with bfloat16 dtype and awq_marlin quantization
- """
-
@classmethod
def setUpClass(cls):
cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ"
@@ -76,7 +72,7 @@ class TestAWQMarlinBfloat16(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreater(metrics["score"], 0.80)
diff --git a/test/registered/sampling/test_pytorch_sampling_backend.py b/test/registered/sampling/test_pytorch_sampling_backend.py
index 3973412b47..0f5a61fe88 100644
--- a/test/registered/sampling/test_pytorch_sampling_backend.py
+++ b/test/registered/sampling/test_pytorch_sampling_backend.py
@@ -5,7 +5,7 @@ import requests
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -49,7 +49,7 @@ class TestPyTorchSamplingBackend(CustomTestCase):
temperature=0.1,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.64)
@unittest.skipIf(
diff --git a/test/registered/scheduler/test_retract_decode.py b/test/registered/scheduler/test_retract_decode.py
index 541671c6d7..d6bba96525 100644
--- a/test/registered/scheduler/test_retract_decode.py
+++ b/test/registered/scheduler/test_retract_decode.py
@@ -7,7 +7,7 @@ import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -55,7 +55,7 @@ class TestRetractDecode(CustomTestCase):
num_threads=32,
)
- metrics = run_eval(args)
+ metrics = run_sgl_eval(args)
self.assertGreaterEqual(metrics["score"], 0.64)
time.sleep(1) # wait for mem check
diff --git a/test/registered/unit/bench/test_simple_eval_gsm8k.py b/test/registered/unit/bench/test_simple_eval_gsm8k.py
deleted file mode 100644
index 8314691de3..0000000000
--- a/test/registered/unit/bench/test_simple_eval_gsm8k.py
+++ /dev/null
@@ -1,269 +0,0 @@
-import json
-import subprocess
-import tempfile
-import unittest
-from pathlib import Path
-from types import SimpleNamespace
-from unittest.mock import patch
-
-from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.run_eval import _run_sgl_eval, run_eval
-from sglang.test.test_utils import CustomTestCase
-
-register_cpu_ci(est_time=6, suite="stage-a-test-cpu-intel")
-
-
-def _write_fake_metrics(out_parent: Path, eval_name: str, payload: dict) -> None:
- run_dir = out_parent / f"sgl_eval_{eval_name}_20260101-000000"
- run_dir.mkdir(parents=True, exist_ok=True)
- (run_dir / "metrics.json").write_text(json.dumps(payload))
-
-
-class TestRunSglEval(CustomTestCase):
- """sgl-eval is a black box, so these mock subprocess.run and assert the shim
- builds the CLI and parses metrics.json's aggregate.score (not top-level)."""
-
- def _args(self, out_dir: str, **overrides):
- defaults = dict(
- base_url="http://127.0.0.1:30000",
- model="test-model",
- num_examples=7,
- num_threads=8,
- temperature=0.0,
- sgl_eval_out_dir=out_dir,
- )
- defaults.update(overrides)
- return SimpleNamespace(**defaults)
-
- def _fake_run_factory(self, out_dir: Path, eval_name: str, payload: dict):
- def fake_run(cmd, **kwargs):
- _write_fake_metrics(out_dir, eval_name, payload)
- return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
-
- return fake_run
-
- def test_parses_aggregate_score_and_maps_latency(self):
- payload = {
- "name": "gsm8k",
- "model": "test-model",
- "num_examples": 7,
- "n_repeats": 1,
- "latency_seconds": 12.5,
- "output_throughput_tps": 34.0,
- "aggregate": {"score": 0.75, "no_answer": 0.1},
- }
- with tempfile.TemporaryDirectory() as td:
- out_dir = Path(td)
- args = self._args(td)
- with patch(
- "sglang.test.run_eval.subprocess.run",
- side_effect=self._fake_run_factory(out_dir, "gsm8k", payload),
- ):
- metrics = _run_sgl_eval("gsm8k", args)
-
- self.assertAlmostEqual(metrics["score"], 0.75)
- self.assertAlmostEqual(metrics["latency"], 12.5)
- self.assertAlmostEqual(metrics["output_throughput"], 34.0)
- self.assertEqual(metrics["no_answer"], 0.1)
- self.assertTrue(metrics["sgl_eval_metrics_path"].endswith("metrics.json"))
-
- def test_builds_cli_with_required_flags(self):
- captured = {}
-
- def fake_run(cmd, **kwargs):
- captured["cmd"] = cmd
- out_dir = Path(captured["cmd"][captured["cmd"].index("--out-dir") + 1])
- _write_fake_metrics(
- out_dir,
- "gsm8k",
- {
- "name": "gsm8k",
- "model": "test-model",
- "latency_seconds": 1.0,
- "output_throughput_tps": 1.0,
- "aggregate": {"score": 0.5},
- },
- )
- return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
-
- with tempfile.TemporaryDirectory() as td:
- args = self._args(td)
- with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
- _run_sgl_eval("gsm8k", args)
-
- cmd = captured["cmd"]
- self.assertEqual(cmd[0:3], ["sgl-eval", "run", "gsm8k"])
- self.assertIn("--base-url", cmd)
- self.assertIn("http://127.0.0.1:30000/v1", cmd)
- self.assertIn("--model", cmd)
- self.assertIn("test-model", cmd)
- self.assertIn("--num-threads", cmd)
- self.assertIn("8", cmd)
- self.assertIn("--temperature", cmd)
- self.assertIn("0.0", cmd)
- self.assertIn("--num-examples", cmd)
- self.assertIn("7", cmd)
-
- def test_omits_num_examples_when_none(self):
- captured = {}
-
- def fake_run(cmd, **kwargs):
- captured["cmd"] = cmd
- out_dir = Path(cmd[cmd.index("--out-dir") + 1])
- _write_fake_metrics(
- out_dir,
- "gsm8k",
- {
- "model": "test-model",
- "latency_seconds": 1.0,
- "output_throughput_tps": 1.0,
- "aggregate": {"score": 0.5},
- },
- )
- return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
-
- with tempfile.TemporaryDirectory() as td:
- args = self._args(td, num_examples=None)
- with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
- _run_sgl_eval("gsm8k", args)
-
- self.assertNotIn("--num-examples", captured["cmd"])
-
- def _capture_cmd(self, eval_name="gsm8k", **overrides):
- captured = {}
-
- def fake_run(cmd, **kwargs):
- captured["cmd"] = cmd
- out_dir = Path(cmd[cmd.index("--out-dir") + 1])
- _write_fake_metrics(
- out_dir,
- eval_name,
- {
- "model": "test-model",
- "latency_seconds": 1.0,
- "output_throughput_tps": 1.0,
- "aggregate": {"score": 0.5},
- },
- )
- return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
-
- with tempfile.TemporaryDirectory() as td:
- args = self._args(td, **overrides)
- with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
- _run_sgl_eval(eval_name, args)
- return captured["cmd"]
-
- def test_omits_sampling_flags_when_unset(self):
- """Unset top_p / seed / repeat must not reach the CLI -- sgl-eval's own
- defaults differ from a forced value (seed unset != seed 0)."""
- cmd = self._capture_cmd()
- for flag in ("--top-p", "--seed", "--n-repeats"):
- self.assertNotIn(flag, cmd)
-
- def test_forwards_sampling_flags_when_set(self):
- cmd = self._capture_cmd(top_p=0.95, seed=0, repeat=1)
- for flag, value in (("--top-p", "0.95"), ("--seed", "0"), ("--n-repeats", "1")):
- self.assertIn(flag, cmd)
- self.assertEqual(cmd[cmd.index(flag) + 1], value)
-
- def test_model_preset_owns_model_and_sampling_defaults(self):
- cmd = self._capture_cmd(
- eval_name="mmmu_pro",
- model=None,
- num_examples=300,
- num_threads=None,
- temperature=None,
- load_preset_from_model_id="moonshotai/Kimi-K3",
- )
-
- self.assertEqual(cmd[:3], ["sgl-eval", "run", "mmmu_pro"])
- self.assertIn("--load-preset-from-model-id", cmd)
- self.assertEqual(
- cmd[cmd.index("--load-preset-from-model-id") + 1],
- "moonshotai/Kimi-K3",
- )
- self.assertEqual(cmd[cmd.index("--num-examples") + 1], "300")
- for flag in (
- "--model",
- "--num-threads",
- "--temperature",
- "--top-p",
- "--max-tokens",
- "--thinking",
- ):
- self.assertNotIn(flag, cmd)
-
- def test_non_preset_cli_keeps_legacy_top_p_default(self):
- cmd = self._capture_cmd(top_p=None, _sgl_eval_from_cli=True)
-
- self.assertIn("--top-p", cmd)
- self.assertEqual(cmd[cmd.index("--top-p") + 1], "1.0")
-
- @patch("sglang.test.run_eval._run_sgl_eval", return_value={"score": 0.8})
- def test_run_eval_dispatches_hyphenated_mmmu_pro_name(self, mock_sgl_eval):
- args = SimpleNamespace(
- base_url="http://127.0.0.1:30000",
- eval_name="mmmu-pro",
- )
-
- try:
- result = run_eval(args)
- except ValueError as exc:
- self.fail(f"mmmu-pro must dispatch to sgl-eval: {exc}")
- self.assertEqual(result, {"score": 0.8})
- mock_sgl_eval.assert_called_once_with("mmmu_pro", args)
-
- def test_thinking_auto_detected_from_model_name(self):
- self.assertIn(
- "--thinking", self._capture_cmd(model="Qwen/Qwen3.5-397B-A17B-FP8")
- )
-
- def test_explicit_thinking_false_suppresses_auto_detect(self):
- """A caller matching a harness that sent no chat_template_kwargs has to be
- able to turn the model-name heuristic off."""
- cmd = self._capture_cmd(
- model="Qwen/Qwen3.5-397B-A17B-FP8", sgl_eval_thinking=False
- )
- self.assertNotIn("--thinking", cmd)
-
- def test_raises_on_nonzero_exit(self):
- def fake_run(cmd, **kwargs):
- return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")
-
- with tempfile.TemporaryDirectory() as td:
- args = self._args(td)
- with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
- with self.assertRaises(RuntimeError) as cm:
- _run_sgl_eval("gsm8k", args)
- self.assertIn("exit code 2", str(cm.exception))
-
- def test_raises_when_metrics_json_missing(self):
- def fake_run(cmd, **kwargs):
- return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
-
- with tempfile.TemporaryDirectory() as td:
- args = self._args(td)
- with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
- with self.assertRaises(FileNotFoundError):
- _run_sgl_eval("gsm8k", args)
-
- def test_raises_when_aggregate_score_missing(self):
- payload = {
- "name": "gsm8k",
- "latency_seconds": 1.0,
- "output_throughput_tps": 1.0,
- "aggregate": {"no_answer": 0.5}, # no score key
- }
- with tempfile.TemporaryDirectory() as td:
- out_dir = Path(td)
- args = self._args(td)
- with patch(
- "sglang.test.run_eval.subprocess.run",
- side_effect=self._fake_run_factory(out_dir, "gsm8k", payload),
- ):
- with self.assertRaises(KeyError):
- _run_sgl_eval("gsm8k", args)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py b/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py
deleted file mode 100644
index 9154663716..0000000000
--- a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py
+++ /dev/null
@@ -1,136 +0,0 @@
-"""Unit tests for sgl-eval-backed accuracy mixin dispatch.
-
-Hermetic (no server, no real sgl-eval install). These guard the behavior that
-existing consumers rely on -- not the sgl-eval happy path, which the live
-accuracy runs already cover:
-
- 1. The default GSM8K backend stays on ``run_eval`` (OpenAI completion API);
- the ~47 existing GSM8K consumers must never be silently rerouted.
- 2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail
- gate when the canonical ``gsm8k_score_threshold`` is unset.
- 3. MMMU-Pro delegates model and sampling selection to a built-in model preset.
- 4. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent,
- so CI without the optional dependency stays green.
-"""
-
-import sys
-import unittest
-from unittest.mock import MagicMock, patch
-
-import requests
-
-from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.kits import eval_accuracy_kit as kit
-from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin, MMMUProMixin
-from sglang.test.test_utils import CustomTestCase
-
-register_cpu_ci(est_time=12, suite="base-a-test-cpu")
-
-
-def _fake_get(url, *args, **kwargs):
- # flush_cache must succeed (GSM8K calls it unguarded); /server_info is probed
- # by _check_accept_length, which swallows RequestException.
- if str(url).endswith("/flush_cache"):
- return MagicMock()
- raise requests.RequestException()
-
-
-def _make_host(mixin, method):
- """Build a throwaway mixin host bound to ``method``.
-
- Created dynamically (never bound at module scope) so it is collected by
- neither runner: CI executes this file via ``python3 `` ->
- ``unittest.main()``, whose loader ignores pytest's ``__test__`` flag, and
- pytest only collects module-level ``Test*`` classes. The host runs only when
- a test below instantiates and drives it directly.
- """
- return type(f"_{mixin.__name__}Host", (mixin, CustomTestCase), {})(method)
-
-
-class TestEvalKitBackendDispatch(CustomTestCase):
- def _run_gsm8k_default(self, score, **attrs):
- """Run GSM8K on the default (run_eval) backend with run_eval faked to
- return ``score``; returns the SimpleNamespace args run_eval received."""
- captured = {}
-
- def fake_run_eval(args):
- captured["args"] = args
- return {"score": score}
-
- host = _make_host(GSM8KMixin, "test_gsm8k")
- host.base_url = "http://127.0.0.1:0"
- host.model = "m"
- for k, v in attrs.items():
- setattr(host, k, v)
- with (
- patch.object(kit, "run_eval", side_effect=fake_run_eval),
- patch.object(kit.requests, "get", side_effect=_fake_get),
- ):
- host.test_gsm8k()
- return captured["args"]
-
- def test_default_backend_uses_run_eval_completion(self):
- # The default path that all existing GSM8K consumers rely on must stay on
- # run_eval's OpenAI completion API -- it must not touch sgl-eval.
- args = self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.5)
- self.assertEqual(args.eval_name, "gsm8k")
- self.assertEqual(args.api, "completion")
-
- def test_legacy_accuracy_thres_alias_gates_score(self):
- # Canonical gsm8k_score_threshold left unset (NaN) -> the legacy
- # gsm8k_accuracy_thres must still be the pass/fail gate.
- self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.90) # above -> passes
- with self.assertRaises(AssertionError):
- self._run_gsm8k_default(0.80, gsm8k_accuracy_thres=0.90) # below -> fails
-
- def test_sgl_eval_path_skips_when_not_installed(self):
- # GPQA/AIME25 and the sgl_eval backend must skip -- not error -- when
- # sgl-eval is not installed. None in sys.modules makes the import raise.
- host = _make_host(GPQAMixin, "test_gpqa")
- host.base_url = "http://127.0.0.1:0"
- host.model = "m"
- host.gpqa_score_threshold = 0.5
- absent = {
- k: None for k in ("sgl_eval.registry", "sgl_eval.sampler", "sgl_eval.types")
- }
- with patch.dict(sys.modules, absent):
- with self.assertRaises(unittest.SkipTest):
- host.test_gpqa()
-
- def _run_mmmu_pro(self, score):
- captured = {}
-
- def fake_run_eval(args):
- captured["args"] = args
- return {"score": score}
-
- host = _make_host(MMMUProMixin, "test_mmmu_pro")
- host.base_url = "http://127.0.0.1:0"
- host.model = "deployment-model"
- host.mmmu_pro_score_threshold = 0.75
- host.mmmu_pro_load_preset_from_model_id = "moonshotai/Kimi-K3"
- with (
- patch.object(kit, "run_eval", side_effect=fake_run_eval),
- patch.object(kit.requests, "get", side_effect=_fake_get),
- ):
- host.test_mmmu_pro()
- return captured["args"]
-
- def test_mmmu_pro_uses_kimi_preset_and_300_examples(self):
- args = self._run_mmmu_pro(0.80)
-
- self.assertEqual(args.eval_name, "mmmu_pro")
- self.assertEqual(args.load_preset_from_model_id, "moonshotai/Kimi-K3")
- self.assertEqual(args.num_examples, 300)
- self.assertIsNone(args.num_threads)
- self.assertIsNone(args.model)
- for attr in ("temperature", "top_p", "max_tokens", "reasoning_effort"):
- self.assertFalse(hasattr(args, attr))
-
- def test_mmmu_pro_score_threshold_gates_result(self):
- with self.assertRaises(AssertionError):
- self._run_mmmu_pro(0.74)
-
-
-if __name__ == "__main__":
- unittest.main()