[Test] Route all sgl-eval benchmarks through run_sgl_eval and deprecate run_eval (#41280)
This commit is contained in:
@@ -4,6 +4,7 @@ from typing import List, Optional, Tuple
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -31,9 +32,6 @@ class AccuracyTestParams:
|
||||
repeat: Optional[int] = None
|
||||
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
|
||||
seed: Optional[int] = None # pin for reproducibility when temperature > 0
|
||||
# sgl-eval-backed datasets only: force chat_template_kwargs.thinking instead
|
||||
# of letting _run_sgl_eval infer it from the model name.
|
||||
sgl_eval_thinking: Optional[bool] = None
|
||||
num_shots: Optional[int] = None # few-shot count; None = run_eval's default
|
||||
|
||||
|
||||
@@ -56,13 +54,7 @@ def write_accuracy_github_summary(
|
||||
dataset: str,
|
||||
results: List[AccuracyTestResult],
|
||||
) -> None:
|
||||
"""Write accuracy test results to GitHub step summary.
|
||||
|
||||
Args:
|
||||
test_name: Name of the test
|
||||
dataset: Dataset name used for evaluation
|
||||
results: List of AccuracyTestResult objects
|
||||
"""
|
||||
"""Write accuracy test results to GitHub step summary."""
|
||||
summary = f"#### {test_name} - Accuracy ({dataset})\n"
|
||||
summary += "| config | status | score | baseline | error |\n"
|
||||
summary += "| ------ | ------ | ----- | -------- | ----- |\n"
|
||||
@@ -95,13 +87,8 @@ def _run_simple_eval(
|
||||
repeat: Optional[int] = None,
|
||||
api: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
sgl_eval_thinking: Optional[bool] = None,
|
||||
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
||||
"""Run evaluation using simple_eval backend (run_eval.py).
|
||||
|
||||
Returns:
|
||||
Tuple of (success, error_message, metrics_dict)
|
||||
"""
|
||||
"""Run ``dataset`` through run_sgl_eval (sgl-eval benchmarks) or run_eval."""
|
||||
process = None
|
||||
try:
|
||||
process = popen_launch_server(
|
||||
@@ -150,10 +137,8 @@ def _run_simple_eval(
|
||||
if seed is not None:
|
||||
args.seed = seed
|
||||
|
||||
if sgl_eval_thinking is not None:
|
||||
args.sgl_eval_thinking = sgl_eval_thinking
|
||||
|
||||
result = run_eval(args)
|
||||
evaluate = run_sgl_eval if dataset in SGL_EVAL_BENCHMARKS else run_eval
|
||||
result = evaluate(args)
|
||||
|
||||
# Handle result format (run_eval can return metrics or (metrics, latency))
|
||||
if return_latency and isinstance(result, tuple):
|
||||
@@ -177,16 +162,7 @@ def run_accuracy_test(
|
||||
params: AccuracyTestParams,
|
||||
base_url: Optional[str] = None,
|
||||
) -> AccuracyTestResult:
|
||||
"""Run accuracy test for a single model.
|
||||
|
||||
Args:
|
||||
model: ModelLaunchSettings with model config
|
||||
params: AccuracyTestParams with dataset, baseline, and optional settings
|
||||
base_url: Server base URL (default: DEFAULT_URL_FOR_TEST)
|
||||
|
||||
Returns:
|
||||
AccuracyTestResult with test outcome
|
||||
"""
|
||||
"""Run accuracy test for a single model."""
|
||||
base_url = base_url or DEFAULT_URL_FOR_TEST
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
@@ -211,7 +187,6 @@ def run_accuracy_test(
|
||||
repeat=params.repeat,
|
||||
api=params.api,
|
||||
seed=params.seed,
|
||||
sgl_eval_thinking=params.sgl_eval_thinking,
|
||||
)
|
||||
|
||||
if not success:
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
"""
|
||||
Common utilities for testing and benchmarking on NPU.
|
||||
"""Common utilities for testing and benchmarking on NPU.
|
||||
|
||||
This file contains the following weight path categories:
|
||||
- LLM model weights path
|
||||
- VLM model weights path
|
||||
- Embedding model weights path
|
||||
- Rerank model weights path
|
||||
- Reward model weights path
|
||||
|
||||
Please remember to sort by variable name within each section.
|
||||
Keep the weight-path constants sorted by variable name within each section.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -26,7 +18,7 @@ import requests
|
||||
|
||||
from sglang.benchmark.serving import run_benchmark
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.sgl_eval_utils import run_sgl_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -395,15 +387,7 @@ CONFIG_YAML_PATH = (
|
||||
|
||||
|
||||
class ModelTestConfig(NamedTuple):
|
||||
"""
|
||||
Configuration for model testing.
|
||||
|
||||
Attributes:
|
||||
model_path: Path to the model weights directory
|
||||
mmlu_score: Weight for MMLU benchmark score
|
||||
gsm8k_accuracy: Weight for GSM8K benchmark score
|
||||
mmmu_accuracy: Weight for MMMU benchmark score
|
||||
"""
|
||||
"""Model weights path and the expected MMLU / GSM8K / MMMU scores."""
|
||||
|
||||
model_path: str
|
||||
mmlu_score: Optional[float] = None
|
||||
@@ -436,16 +420,7 @@ DEFAULT_WEIGHTS_FOR_TEST = LLAMA_3_2_1B_INSTRUCT_WEIGHTS_FOR_TEST
|
||||
|
||||
|
||||
def run_command(cmd, shell=True):
|
||||
"""Execute system command and return stdout
|
||||
|
||||
parameter:
|
||||
cmd: command to execute
|
||||
shell:
|
||||
True, Execute command in shell
|
||||
False, Commands are invoked directly without shell parsing
|
||||
return:
|
||||
The result of executing the command
|
||||
"""
|
||||
"""Run ``cmd`` and return its stdout."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, shell=shell, capture_output=True, text=True, check=True
|
||||
@@ -556,37 +531,7 @@ def run_bench_serving(
|
||||
lora_name: Optional[str] = None,
|
||||
timeout_for_server_launch=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
):
|
||||
"""Start the service and obtain the inference results.
|
||||
|
||||
Parameters:
|
||||
model: Model name
|
||||
num_prompts: Total number of test requests
|
||||
request_rate: Request rate
|
||||
other_server_args: Additional configuration when starting the service
|
||||
dataset_name: Data set name
|
||||
dataset_path: Dataset path
|
||||
tokenizer: tokenizer
|
||||
random_input_len: The length of the randomly generated input prompt
|
||||
random_output_len: The length of the randomly generated output prompt
|
||||
sharegpt_context_len: Sharegpt dataset context length
|
||||
disable_stream: Disable streaming output
|
||||
disable_ignore_eos: Should eos_token be ignored?
|
||||
need_warmup: Preheating required
|
||||
seed: random seed
|
||||
device: Device type
|
||||
gsp_num_groups: Grouped Sequence Parallelism
|
||||
gsp_prompts_per_group: Number of parallel prompts within each group
|
||||
gsp_system_prompt_len: GSP system prompts length
|
||||
gsp_question_len: GSP question length
|
||||
gsp_output_len: GSP output length
|
||||
max_concurrency: Maximum number of concurrent requests
|
||||
background_task: Background tasks
|
||||
lora_name: LoRA fine-tuning model path
|
||||
timeout_for_server_launch: Raise the service timeout period
|
||||
Returns:
|
||||
res: Number of requests successfully completed
|
||||
|
||||
"""
|
||||
"""Launch the server, run bench_serving against it, and return the result."""
|
||||
|
||||
if device == "auto":
|
||||
device = auto_config_device()
|
||||
@@ -658,17 +603,7 @@ def run_bench_serving(
|
||||
|
||||
# hook factory
|
||||
def create_attention_monitor_hook_factory(config):
|
||||
"""
|
||||
Factory function to create a forward hook for monitoring self-attention layer states.
|
||||
This hook records input/output statistics during model forward propagation.
|
||||
|
||||
Args:
|
||||
config (dict): Configuration dictionary containing hook parameters
|
||||
layer_index (int): Index of the target attention layer to monitor
|
||||
|
||||
Returns:
|
||||
function: Forward hook function to be registered on the target module
|
||||
"""
|
||||
"""Return a forward hook that logs self-attention states of ``config["layer_index"]``."""
|
||||
# Get target layer index from config, default to 0 if not specified
|
||||
layer_index = config.get("layer_index", 0)
|
||||
|
||||
@@ -681,18 +616,7 @@ def create_attention_monitor_hook_factory(config):
|
||||
)
|
||||
|
||||
def attention_monitor_hook(module, inputs, output):
|
||||
"""
|
||||
Forward hook function that monitors and logs the internal states of a self-attention layer.
|
||||
Executed automatically during the forward pass of the module it is registered to.
|
||||
|
||||
Args:
|
||||
module (torch.nn.Module): The module this hook is attached to
|
||||
inputs (tuple): Input tensors passed to the module's forward method
|
||||
output (torch.Tensor): Output tensor returned by the module's forward method
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Unmodified output tensor to preserve model computation flow
|
||||
"""
|
||||
"""Log the attention layer's input/output statistics; returns ``output`` unchanged."""
|
||||
# Record current timestamp for time-series tracking
|
||||
timestamp = time.time()
|
||||
|
||||
@@ -832,7 +756,7 @@ def run_mmlu_test(
|
||||
)
|
||||
|
||||
try:
|
||||
metrics = run_eval(args)
|
||||
metrics = run_sgl_eval(args)
|
||||
assert metrics["score"] >= 0.65, f"{metrics=}"
|
||||
finally:
|
||||
pass
|
||||
@@ -856,26 +780,10 @@ def send_concurrent_requests(
|
||||
temperature: float = 0.0,
|
||||
request_timeout: int = 60,
|
||||
) -> list:
|
||||
"""Send multiple concurrent HTTP POST requests to the /generate endpoint.
|
||||
"""Send ``num_requests`` concurrent POSTs to /generate; returns one result dict per request.
|
||||
|
||||
Uses threading (NOT asyncio + blocking calls) to achieve true concurrency.
|
||||
asyncio.gather() combined with synchronous requests.post() does not produce
|
||||
real parallelism; threading is required for concurrent blocking I/O.
|
||||
|
||||
Parameters:
|
||||
base_url: Server base URL, e.g. "http://127.0.0.1:30000"
|
||||
num_requests: Total number of requests to send
|
||||
num_concurrent: Maximum in-flight requests at any given time (semaphore)
|
||||
input_text: Text prompt sent to every request
|
||||
max_new_tokens: Maximum new tokens to generate per request
|
||||
temperature: Sampling temperature (0 = greedy / deterministic)
|
||||
request_timeout: Per-request HTTP timeout in seconds; raises on exceed
|
||||
|
||||
Returns:
|
||||
Unsorted list of result dicts, one per request, each with:
|
||||
task_id (int) -- zero-based request index
|
||||
status_code (int)-- HTTP status code, or -1 on exception
|
||||
text (str) -- response body, or exception message on failure
|
||||
Uses threads: asyncio.gather() over synchronous requests.post() gives no real
|
||||
parallelism for blocking I/O.
|
||||
"""
|
||||
|
||||
results: list = []
|
||||
|
||||
@@ -2,7 +2,7 @@ import subprocess
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.sgl_eval_utils import run_sgl_eval
|
||||
|
||||
|
||||
class TestMMLU:
|
||||
@@ -28,7 +28,7 @@ class TestMMLU:
|
||||
num_threads=32,
|
||||
)
|
||||
print("Starting mmlu test...")
|
||||
metrics = run_eval(args)
|
||||
metrics = run_sgl_eval(args)
|
||||
model_metrics["accuracy"] = metrics["score"]
|
||||
model_metrics["latency"] = metrics.get("latency", "-")
|
||||
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Dict, List, Literal, Optional
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import is_hip, kill_process_tree
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.sgl_eval_utils import run_sgl_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
@@ -66,8 +66,7 @@ class BaseTestGptOss(CustomTestCase):
|
||||
try:
|
||||
self._check_streaming_responses_api_request(model)
|
||||
|
||||
# run multiple tests in parallel since we are mostly bound by the longest generate sequence
|
||||
# instead of the number of questions
|
||||
# Parallel: bound by the longest generation, not the question count.
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
list(
|
||||
executor.map(
|
||||
@@ -131,8 +130,7 @@ class BaseTestGptOss(CustomTestCase):
|
||||
num_threads=198,
|
||||
# sgl-eval's gpqa defaults to n_repeats=8.
|
||||
repeat=1,
|
||||
# TODO 4k is still not enough, we need e.g. 64k token, but that is super slow
|
||||
# otherwise a lot of questions are not answered
|
||||
# Too small (many answers truncate); ~64k would fit but is too slow.
|
||||
max_tokens=4096,
|
||||
# Arbitrary; non-zero so a tier is not scored on one greedy path.
|
||||
temperature=0.1,
|
||||
@@ -142,7 +140,7 @@ class BaseTestGptOss(CustomTestCase):
|
||||
setup = f"model={model} reasoning_effort={reasoning_effort} expected_score={expected_score}"
|
||||
|
||||
print(f"Evaluation start: {setup}")
|
||||
metrics = run_eval(args)
|
||||
metrics = run_sgl_eval(args)
|
||||
print(f"Evaluation end: {setup} {metrics=}")
|
||||
self.assertGreaterEqual(metrics["score"], expected_score)
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Optional
|
||||
import requests
|
||||
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.sgl_eval_utils import SGL_EVAL_BENCHMARKS, run_sgl_eval
|
||||
from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
|
||||
|
||||
_THRESHOLD_NOT_SET = float("nan")
|
||||
@@ -51,14 +52,8 @@ def _run_accuracy_eval(
|
||||
):
|
||||
"""Shared driver for the accuracy mixins below.
|
||||
|
||||
Runs ``run_eval`` for ``eval_name`` against the test class's server
|
||||
(``base_url`` / ``model``), records a CI step summary, asserts the score
|
||||
meets ``score_threshold``, and checks the speculative accept length.
|
||||
|
||||
``eval_overrides`` (e.g. ``api``, ``max_tokens``, ``temperature``,
|
||||
``top_p``, ``num_shots``) are forwarded to ``run_eval`` only when not
|
||||
``None``, so the common case stays identical to ``run_eval``'s defaults.
|
||||
Returns the metrics dict.
|
||||
``eval_overrides`` are forwarded only when not ``None``, so unset knobs keep
|
||||
the evaluator's defaults.
|
||||
"""
|
||||
assert score_threshold == score_threshold, (
|
||||
f"{type(test_case).__name__} must set the {eval_name} score threshold"
|
||||
@@ -74,7 +69,8 @@ def _run_accuracy_eval(
|
||||
)
|
||||
kwargs.update({k: v for k, v in eval_overrides.items() if v is not None})
|
||||
|
||||
metrics = run_eval(SimpleNamespace(**kwargs))
|
||||
evaluate = run_sgl_eval if eval_name in SGL_EVAL_BENCHMARKS else run_eval
|
||||
metrics = evaluate(SimpleNamespace(**kwargs))
|
||||
print(f"{eval_name} {metrics=}")
|
||||
_finalize_eval(
|
||||
test_case,
|
||||
@@ -92,80 +88,53 @@ def _run_sgl_eval(
|
||||
*,
|
||||
eval_name: str,
|
||||
score_threshold: float,
|
||||
metric: str = "score",
|
||||
n_repeats: int = 1,
|
||||
num_examples: Optional[int] = None,
|
||||
num_threads: int = 512,
|
||||
thinking: bool = True,
|
||||
chat_template_kwargs: Optional[dict] = None,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
max_tokens: Optional[int] = None,
|
||||
temperature: Optional[float] = None,
|
||||
top_p: Optional[float] = None,
|
||||
accept_length_thres: Optional[float] = None,
|
||||
summary_label: Optional[str] = None,
|
||||
):
|
||||
"""Shared sgl-eval driver for the reasoning mixins and the ``sgl_eval`` backend.
|
||||
"""Shared sgl-eval driver for the MMLU sanity gate and the GSM8K ``sgl_eval`` backend.
|
||||
|
||||
Runs ``eval_name`` via the sgl-eval Python API (``registry.get`` ->
|
||||
``EvalSpec.run``) against the test class's server, records a CI step summary,
|
||||
asserts the score meets ``score_threshold``, and checks the speculative accept
|
||||
length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking":
|
||||
True}`` so the server separates reasoning from the final answer. Skips the test
|
||||
if sgl-eval is not installed. Returns the RunResult.
|
||||
``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": True}``
|
||||
so the server separates reasoning from the final answer.
|
||||
"""
|
||||
assert score_threshold == score_threshold, (
|
||||
f"{type(test_case).__name__} must set the {eval_name} score threshold"
|
||||
)
|
||||
|
||||
try:
|
||||
from sgl_eval.registry import get as get_eval_spec
|
||||
from sgl_eval.sampler import ChatCompletionSampler
|
||||
from sgl_eval.types import GenConfig
|
||||
import sgl_eval # noqa: F401
|
||||
except ImportError:
|
||||
test_case.skipTest("sgl-eval not installed; pip install 'sglang[test]'")
|
||||
|
||||
base_url = test_case.base_url.rstrip("/")
|
||||
if not base_url.endswith("/v1"):
|
||||
base_url += "/v1"
|
||||
sampler = ChatCompletionSampler(
|
||||
base_url=base_url, model=getattr(test_case, "model", None), api_key="EMPTY"
|
||||
)
|
||||
|
||||
gen_kwargs = dict(
|
||||
args = SimpleNamespace(
|
||||
eval_name=eval_name,
|
||||
base_url=test_case.base_url,
|
||||
model=getattr(test_case, "model", None),
|
||||
num_examples=num_examples,
|
||||
num_threads=num_threads,
|
||||
max_tokens=max_tokens,
|
||||
reasoning_effort=reasoning_effort,
|
||||
temperature=temperature,
|
||||
chat_template_kwargs=(
|
||||
chat_template_kwargs
|
||||
if chat_template_kwargs is not None
|
||||
else ({"thinking": True} if thinking else None)
|
||||
),
|
||||
)
|
||||
if temperature is not None:
|
||||
gen_kwargs["temperature"] = temperature
|
||||
if top_p is not None:
|
||||
gen_kwargs["top_p"] = top_p
|
||||
|
||||
result = get_eval_spec(eval_name).run(
|
||||
sampler=sampler,
|
||||
gen=GenConfig(**gen_kwargs),
|
||||
n_repeats=n_repeats,
|
||||
num_examples=num_examples,
|
||||
num_threads=num_threads,
|
||||
predictions_writer=None,
|
||||
load_examples=None,
|
||||
)
|
||||
score = result.aggregate[metric]
|
||||
print(f"{eval_name} sgl-eval {metric}={score:.4f}")
|
||||
metrics = run_sgl_eval(args)
|
||||
score = metrics["score"]
|
||||
print(f"{eval_name} sgl-eval score={score:.4f}")
|
||||
_finalize_eval(
|
||||
test_case,
|
||||
eval_name=eval_name,
|
||||
score=score,
|
||||
score_threshold=score_threshold,
|
||||
accept_length_thres=accept_length_thres,
|
||||
summary_label=summary_label,
|
||||
)
|
||||
return result
|
||||
return metrics
|
||||
|
||||
|
||||
class MMLUSanityMixin:
|
||||
@@ -192,18 +161,9 @@ class MMLUSanityMixin:
|
||||
class GSM8KMixin:
|
||||
"""Mixin for GSM8K evaluation.
|
||||
|
||||
Backend is selectable via ``gsm8k_backend`` (default ``"run_eval"``: OpenAI
|
||||
completion API, 5-shot; or ``"sgl_eval"``: sgl-eval chat + boxed/sympy grader,
|
||||
skipped if sgl-eval is not installed). The canonical threshold/count knobs are
|
||||
``gsm8k_score_threshold`` / ``gsm8k_num_examples``; the legacy
|
||||
``gsm8k_accuracy_thres`` / ``gsm8k_num_questions`` are still honored.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
gsm8k_score_threshold: float
|
||||
|
||||
Optional attributes:
|
||||
model: str (if not set, auto-detected from server)
|
||||
``"run_eval"`` backend: OpenAI completion API, 5-shot; ``"sgl_eval"``: sgl-eval
|
||||
chat + boxed/sympy grader. The legacy ``gsm8k_accuracy_thres`` /
|
||||
``gsm8k_num_questions`` are honored when the canonical knobs are unset.
|
||||
"""
|
||||
|
||||
gsm8k_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
@@ -216,7 +176,6 @@ class GSM8KMixin:
|
||||
gsm8k_backend: str = "run_eval" # "run_eval" | "sgl_eval"
|
||||
gsm8k_thinking: bool = False # sgl_eval backend
|
||||
gsm8k_max_tokens: Optional[int] = None # sgl_eval backend
|
||||
gsm8k_n_repeats: int = 1 # sgl_eval backend
|
||||
# None keeps run_eval's greedy default; set both to route the run through
|
||||
# the sampling path.
|
||||
gsm8k_temperature: Optional[float] = None
|
||||
@@ -237,7 +196,6 @@ class GSM8KMixin:
|
||||
self,
|
||||
eval_name="gsm8k",
|
||||
score_threshold=threshold,
|
||||
n_repeats=self.gsm8k_n_repeats,
|
||||
num_examples=num_examples,
|
||||
num_threads=self.gsm8k_num_threads,
|
||||
thinking=self.gsm8k_thinking,
|
||||
@@ -261,60 +219,29 @@ class GSM8KMixin:
|
||||
|
||||
|
||||
class MMLUMixin:
|
||||
"""Mixin for MMLU evaluation.
|
||||
|
||||
Both ``mmlu_backend`` values score through sgl-eval -- ``"sgl_eval"`` calls it
|
||||
in-process, ``"run_eval"`` reaches the same CLI via ``run_eval``. The switch
|
||||
picks the call mechanism, not the grader.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
model: str
|
||||
mmlu_score_threshold: float
|
||||
"""
|
||||
"""Mixin for MMLU evaluation via sgl-eval (2048-token cap, no thinking)."""
|
||||
|
||||
mmlu_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
mmlu_accept_length_thres: Optional[float] = None
|
||||
mmlu_num_examples: int = 5000
|
||||
mmlu_num_threads: int = 1024
|
||||
mmlu_backend: str = "run_eval" # "run_eval" | "sgl_eval"
|
||||
mmlu_thinking: bool = False # sgl_eval backend
|
||||
mmlu_n_repeats: int = 1 # sgl_eval backend
|
||||
|
||||
def test_mmlu(self):
|
||||
if self.mmlu_backend == "sgl_eval":
|
||||
_run_sgl_eval(
|
||||
self,
|
||||
eval_name="mmlu",
|
||||
score_threshold=self.mmlu_score_threshold,
|
||||
n_repeats=self.mmlu_n_repeats,
|
||||
num_examples=self.mmlu_num_examples,
|
||||
num_threads=self.mmlu_num_threads,
|
||||
thinking=self.mmlu_thinking,
|
||||
accept_length_thres=self.mmlu_accept_length_thres,
|
||||
)
|
||||
else:
|
||||
_run_accuracy_eval(
|
||||
self,
|
||||
eval_name="mmlu",
|
||||
score_threshold=self.mmlu_score_threshold,
|
||||
num_examples=self.mmlu_num_examples,
|
||||
num_threads=self.mmlu_num_threads,
|
||||
accept_length_thres=self.mmlu_accept_length_thres,
|
||||
)
|
||||
_run_accuracy_eval(
|
||||
self,
|
||||
eval_name="mmlu",
|
||||
score_threshold=self.mmlu_score_threshold,
|
||||
num_examples=self.mmlu_num_examples,
|
||||
num_threads=self.mmlu_num_threads,
|
||||
accept_length_thres=self.mmlu_accept_length_thres,
|
||||
)
|
||||
|
||||
|
||||
class MMMUProMixin:
|
||||
"""Mixin for the standard 10-option MMMU-Pro evaluation via sgl-eval.
|
||||
|
||||
The model preset supplies the endpoint model and all generation settings.
|
||||
Leaving those values to sgl-eval is important for reasoning models whose
|
||||
recommended token budget and sampling settings differ from run_eval defaults.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
mmmu_pro_score_threshold: float
|
||||
mmmu_pro_load_preset_from_model_id: str
|
||||
The model preset supplies the endpoint model and all generation settings;
|
||||
reasoning models' token budget and sampling differ from run_eval defaults.
|
||||
"""
|
||||
|
||||
mmmu_pro_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
@@ -339,110 +266,8 @@ class MMMUProMixin:
|
||||
)
|
||||
|
||||
|
||||
class GPQAMixin:
|
||||
"""Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
|
||||
|
||||
Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
|
||||
installed). ``gpqa_thinking`` defaults to True, which
|
||||
enables per-request thinking so the server separates reasoning from the final
|
||||
answer.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
model: str
|
||||
gpqa_score_threshold: float
|
||||
|
||||
Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
|
||||
for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
|
||||
gpqa_reasoning_effort="max", gpqa_max_tokens=200000, gpqa_temperature=1.0,
|
||||
gpqa_top_p=1.0. GPQA-Diamond is 198 questions; raise gpqa_n_repeats (e.g. 16)
|
||||
for a stable number.
|
||||
"""
|
||||
|
||||
gpqa_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
gpqa_accept_length_thres: Optional[float] = None
|
||||
gpqa_num_examples: Optional[int] = None
|
||||
gpqa_num_threads: int = 1024
|
||||
gpqa_n_repeats: int = 1
|
||||
gpqa_thinking: bool = True
|
||||
gpqa_reasoning_effort: Optional[str] = None
|
||||
gpqa_max_tokens: Optional[int] = None
|
||||
gpqa_temperature: Optional[float] = None
|
||||
gpqa_top_p: Optional[float] = None
|
||||
|
||||
def test_gpqa(self):
|
||||
_run_sgl_eval(
|
||||
self,
|
||||
eval_name="gpqa",
|
||||
score_threshold=self.gpqa_score_threshold,
|
||||
n_repeats=self.gpqa_n_repeats,
|
||||
num_examples=self.gpqa_num_examples,
|
||||
num_threads=self.gpqa_num_threads,
|
||||
thinking=self.gpqa_thinking,
|
||||
reasoning_effort=self.gpqa_reasoning_effort,
|
||||
max_tokens=self.gpqa_max_tokens,
|
||||
temperature=self.gpqa_temperature,
|
||||
top_p=self.gpqa_top_p,
|
||||
accept_length_thres=self.gpqa_accept_length_thres,
|
||||
)
|
||||
|
||||
|
||||
class AIME25Mixin:
|
||||
"""Mixin for AIME 2025 evaluation (competition math, integer answers).
|
||||
|
||||
Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
|
||||
installed). ``aime25_thinking`` defaults to True, which
|
||||
enables per-request thinking so the server separates reasoning from the final
|
||||
answer.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
model: str
|
||||
aime25_score_threshold: float
|
||||
|
||||
Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
|
||||
for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
|
||||
aime25_reasoning_effort="max", aime25_max_tokens=200000, aime25_temperature=1.0,
|
||||
aime25_top_p=1.0. AIME25 has only 30 problems, so it is high variance; raise
|
||||
aime25_n_repeats (e.g. 16) for a stable number.
|
||||
"""
|
||||
|
||||
aime25_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
aime25_accept_length_thres: Optional[float] = None
|
||||
aime25_num_examples: Optional[int] = None
|
||||
aime25_num_threads: int = 1024
|
||||
aime25_n_repeats: int = 1
|
||||
aime25_thinking: bool = True
|
||||
aime25_reasoning_effort: Optional[str] = None
|
||||
aime25_max_tokens: Optional[int] = None
|
||||
aime25_temperature: Optional[float] = None
|
||||
aime25_top_p: Optional[float] = None
|
||||
|
||||
def test_aime25(self):
|
||||
_run_sgl_eval(
|
||||
self,
|
||||
eval_name="aime25",
|
||||
score_threshold=self.aime25_score_threshold,
|
||||
n_repeats=self.aime25_n_repeats,
|
||||
num_examples=self.aime25_num_examples,
|
||||
num_threads=self.aime25_num_threads,
|
||||
thinking=self.aime25_thinking,
|
||||
reasoning_effort=self.aime25_reasoning_effort,
|
||||
max_tokens=self.aime25_max_tokens,
|
||||
temperature=self.aime25_temperature,
|
||||
top_p=self.aime25_top_p,
|
||||
accept_length_thres=self.aime25_accept_length_thres,
|
||||
)
|
||||
|
||||
|
||||
class HumanEvalMixin:
|
||||
"""Mixin for HumanEval evaluation.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
model: str
|
||||
humaneval_score_threshold: float
|
||||
"""
|
||||
"""Mixin for HumanEval evaluation."""
|
||||
|
||||
humaneval_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
humaneval_score_threshold_amd: Optional[float] = None
|
||||
@@ -464,13 +289,7 @@ class HumanEvalMixin:
|
||||
|
||||
|
||||
class MGSMEnMixin:
|
||||
"""Mixin for MGSM English evaluation.
|
||||
|
||||
Required attributes on the test class:
|
||||
base_url: str
|
||||
model: str
|
||||
mgsm_en_score_threshold: float
|
||||
"""
|
||||
"""Mixin for MGSM English evaluation."""
|
||||
|
||||
mgsm_en_score_threshold: float = _THRESHOLD_NOT_SET
|
||||
mgsm_en_num_examples: Optional[int] = None
|
||||
|
||||
+22
-158
@@ -1,18 +1,17 @@
|
||||
"""
|
||||
Usage:
|
||||
python3 -m sglang.test.run_eval --port 30000 --eval-name mmlu --num-examples 10
|
||||
python3 -m sglang.test.run_eval --port 30000 --eval-name gsm8k --num-examples 10
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import warnings
|
||||
|
||||
from sglang.test.sgl_eval_utils import (
|
||||
SGL_EVAL_BENCHMARKS,
|
||||
THINKING_MODE_CHOICES,
|
||||
get_thinking_kwargs,
|
||||
parse_json_object,
|
||||
@@ -89,125 +88,6 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
|
||||
return result, latency, sampler
|
||||
|
||||
|
||||
def _run_sgl_eval(eval_name, args) -> dict:
|
||||
# Returns a metrics dict (score, latency, output_throughput) so the
|
||||
# existing write_results_to_json + threshold gate keep working.
|
||||
from sglang.test.test_utils import dump_metric
|
||||
|
||||
base_url = (
|
||||
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
|
||||
)
|
||||
out_parent = Path(
|
||||
getattr(args, "sgl_eval_out_dir", None)
|
||||
or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex)
|
||||
).expanduser()
|
||||
out_parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
model_preset_id = getattr(args, "load_preset_from_model_id", None)
|
||||
cmd = [
|
||||
"sgl-eval",
|
||||
"run",
|
||||
eval_name,
|
||||
"--base-url",
|
||||
base_url,
|
||||
"--out-dir",
|
||||
str(out_parent),
|
||||
]
|
||||
if model_preset_id:
|
||||
cmd += ["--load-preset-from-model-id", model_preset_id]
|
||||
if getattr(args, "model", None):
|
||||
cmd += ["--model", args.model]
|
||||
if getattr(args, "num_examples", None) is not None:
|
||||
cmd += ["--num-examples", str(args.num_examples)]
|
||||
if getattr(args, "num_threads", None) is not None:
|
||||
cmd += ["--num-threads", str(args.num_threads)]
|
||||
if getattr(args, "temperature", None) is not None:
|
||||
cmd += ["--temperature", str(args.temperature)]
|
||||
elif not model_preset_id:
|
||||
cmd += ["--temperature", "0.0"]
|
||||
if getattr(args, "top_p", None) is not None:
|
||||
cmd += ["--top-p", str(args.top_p)]
|
||||
elif not model_preset_id and getattr(args, "_sgl_eval_from_cli", False):
|
||||
cmd += ["--top-p", "1.0"]
|
||||
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
|
||||
if getattr(args, "seed", None) is not None:
|
||||
cmd += ["--seed", str(args.seed)]
|
||||
# gpt-oss grades one score per effort tier, so dropping this collapses every
|
||||
# tier onto the served model's default.
|
||||
if getattr(args, "reasoning_effort", None) is not None:
|
||||
cmd += ["--reasoning-effort", str(args.reasoning_effort)]
|
||||
if getattr(args, "repeat", None) is not None:
|
||||
cmd += ["--n-repeats", str(args.repeat)]
|
||||
# Bound generation length so long-reasoning models don't stall the eval.
|
||||
if getattr(args, "max_tokens", None) is not None:
|
||||
cmd += ["--max-tokens", str(args.max_tokens)]
|
||||
elif not model_preset_id:
|
||||
cmd += ["--max-tokens", "2048"]
|
||||
# Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
|
||||
# without --thinking their message.content is empty and sgl-eval scores 0.
|
||||
sgl_eval_thinking = getattr(args, "sgl_eval_thinking", None)
|
||||
if sgl_eval_thinking is None:
|
||||
if not model_preset_id:
|
||||
model_l = (getattr(args, "model", None) or "").lower()
|
||||
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
|
||||
cmd += ["--thinking"]
|
||||
elif sgl_eval_thinking:
|
||||
cmd += ["--thinking"]
|
||||
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
cmd,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=getattr(args, "sgl_eval_timeout", None),
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
raise TimeoutError(
|
||||
f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n"
|
||||
f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}"
|
||||
) from e
|
||||
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"sgl-eval failed with exit code {completed.returncode}: "
|
||||
f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
|
||||
)
|
||||
|
||||
metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json"))
|
||||
if len(metrics_files) != 1:
|
||||
raise FileNotFoundError(
|
||||
f"Expected exactly one metrics.json under {out_parent}, "
|
||||
f"found {len(metrics_files)}"
|
||||
)
|
||||
payload = json.loads(metrics_files[0].read_text())
|
||||
aggregate = payload.get("aggregate")
|
||||
if not isinstance(aggregate, dict) or "score" not in aggregate:
|
||||
raise KeyError(f"{metrics_files[0]} missing aggregate.score")
|
||||
|
||||
metrics = dict(aggregate)
|
||||
metrics["latency"] = payload.get("latency_seconds", 0.0)
|
||||
metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0)
|
||||
metrics["sgl_eval_metrics_path"] = str(metrics_files[0])
|
||||
|
||||
model = payload.get("model") or getattr(args, "model", None)
|
||||
dump_metric(
|
||||
f"{eval_name}_score",
|
||||
metrics["score"],
|
||||
labels={"model": model, "eval": eval_name},
|
||||
)
|
||||
dump_metric(
|
||||
f"{eval_name}_latency",
|
||||
metrics["latency"],
|
||||
labels={"model": model, "eval": eval_name},
|
||||
)
|
||||
print(f"Score: {metrics['score']:.3f}")
|
||||
print(f"Total latency: {metrics['latency']:.3f} s")
|
||||
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
|
||||
print(f"sgl-eval metrics: {metrics_files[0]}")
|
||||
return metrics
|
||||
|
||||
|
||||
def print_accept_length_summary(samplers: list) -> None:
|
||||
accept_lengths = [
|
||||
m["spec_accept_length"]
|
||||
@@ -236,6 +116,23 @@ def run_eval(args):
|
||||
# Lazy import to avoid circular dependency with test_utils
|
||||
from sglang.test.test_utils import dump_metric
|
||||
|
||||
warnings.warn(
|
||||
"sglang.test.run_eval is deprecated; its legacy scorers will be removed. "
|
||||
"Use sglang.test.sgl_eval_utils.run_sgl_eval or `sgl-eval run` instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
if (
|
||||
args.eval_name in SGL_EVAL_BENCHMARKS
|
||||
or args.eval_name == "mmmu-pro"
|
||||
or getattr(args, "api", None) == "sgl_eval"
|
||||
or getattr(args, "load_preset_from_model_id", None)
|
||||
):
|
||||
raise ValueError(
|
||||
f"{args.eval_name} is scored by sgl-eval; call "
|
||||
"sglang.test.sgl_eval_utils.run_sgl_eval instead of run_eval."
|
||||
)
|
||||
|
||||
set_ulimit()
|
||||
|
||||
if "OPENAI_API_KEY" not in os.environ:
|
||||
@@ -245,18 +142,10 @@ def run_eval(args):
|
||||
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
|
||||
)
|
||||
|
||||
if args.eval_name == "mmlu":
|
||||
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
|
||||
# caller's threshold has to be measured against it, not inherited.
|
||||
return _run_sgl_eval("mmlu", args)
|
||||
elif args.eval_name == "mgsm_en":
|
||||
if args.eval_name == "mgsm_en":
|
||||
from sglang.test.simple_eval_mgsm import MGSMEval
|
||||
|
||||
eval_obj = MGSMEval(args.num_examples, args.num_threads, languages=["en"])
|
||||
elif args.eval_name == "gpqa":
|
||||
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
|
||||
# caller's threshold has to be measured against it, not inherited.
|
||||
return _run_sgl_eval("gpqa", args)
|
||||
elif args.eval_name == "humaneval":
|
||||
from sglang.test.simple_eval_humaneval import HumanEval
|
||||
|
||||
@@ -270,25 +159,7 @@ def run_eval(args):
|
||||
args.num_threads,
|
||||
response_answer_regex=getattr(args, "response_answer_regex", None),
|
||||
)
|
||||
elif args.eval_name in ("mmmu_pro", "mmmu-pro"):
|
||||
# Canonical sgl-eval name for MMMU-Pro's standard 10-option split.
|
||||
return _run_sgl_eval("mmmu_pro", args)
|
||||
elif args.eval_name == "mmmu_pro_vision":
|
||||
# sgl-eval owns this benchmark's dataset, prompt and grader; there is no
|
||||
# simple_eval implementation to fall back to.
|
||||
return _run_sgl_eval("mmmu_pro_vision", args)
|
||||
elif args.eval_name == "aime25":
|
||||
return _run_sgl_eval("aime25", args)
|
||||
elif args.eval_name == "aime26":
|
||||
return _run_sgl_eval("aime26", args)
|
||||
elif args.eval_name == "gsm8k":
|
||||
if getattr(args, "api", None) == "sgl_eval":
|
||||
# Only the nightly correctness eval opts into sgl-eval (zero-shot
|
||||
# chat, \boxed{}, math_verify). Every other gsm8k caller — spec
|
||||
# decoding perf/accuracy, disaggregation, quant, model e2e — uses
|
||||
# the 5-shot completion last-number scorer and relies on
|
||||
# max_tokens/throughput behavior sgl-eval cannot provide.
|
||||
return _run_sgl_eval("gsm8k", args)
|
||||
from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval
|
||||
|
||||
eval_obj = GSM8KEval(
|
||||
@@ -428,16 +299,10 @@ if __name__ == "__main__":
|
||||
type=str,
|
||||
help="Name or path of the model. If not set, the default model will request /v1/models for conf.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--load-preset-from-model-id",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Load repository-maintained sgl-eval generation defaults for this model ID.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repeat", type=int, default=1, help="repeat the evaluation n times"
|
||||
)
|
||||
parser.add_argument("--eval-name", type=str, default="mmlu")
|
||||
parser.add_argument("--eval-name", type=str, default="gsm8k")
|
||||
parser.add_argument(
|
||||
"--api",
|
||||
type=str,
|
||||
@@ -498,6 +363,5 @@ if __name__ == "__main__":
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
args._sgl_eval_from_cli = True
|
||||
|
||||
run_eval(args)
|
||||
|
||||
@@ -6,13 +6,20 @@ import multiprocessing
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
import warnings
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import replace
|
||||
from itertools import islice
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
|
||||
|
||||
# Benchmarks scored only by sgl-eval; run_eval does not serve them.
|
||||
SGL_EVAL_BENCHMARKS = frozenset(
|
||||
{"mmlu", "gpqa", "mmmu_pro", "mmmu_pro_vision", "aime25", "aime26"}
|
||||
)
|
||||
|
||||
|
||||
def get_thinking_kwargs(args):
|
||||
thinking_mode = getattr(args, "thinking_mode", None)
|
||||
@@ -85,9 +92,35 @@ def run_sgl_eval(args):
|
||||
from sglang.test.test_utils import dump_metric
|
||||
|
||||
spec = get(args.eval_name)
|
||||
overrides = {"max_tokens": 2048}
|
||||
model = getattr(args, "model", None)
|
||||
default_gen = spec.default_gen
|
||||
model_preset_id = getattr(args, "load_preset_from_model_id", None)
|
||||
if model_preset_id:
|
||||
from sgl_eval.model_preset import load_model_preset
|
||||
from sgl_eval.preset import apply_to_gen
|
||||
|
||||
model_preset = load_model_preset(model_preset_id)
|
||||
unset = dict.fromkeys(
|
||||
("thinking", "temperature", "top_p", "max_tokens", "reasoning_effort")
|
||||
)
|
||||
default_gen = apply_to_gen(
|
||||
default_gen, None, SimpleNamespace(**unset), model_preset
|
||||
)
|
||||
model = model or model_preset.model
|
||||
|
||||
overrides = {}
|
||||
# Omitted max_tokens keeps the 2048 CI cap; an explicit None defers to the server.
|
||||
if hasattr(args, "max_tokens"):
|
||||
if args.max_tokens is None:
|
||||
warnings.warn(
|
||||
f"sgl-eval {spec.name}: max_tokens=None leaves the output length "
|
||||
"uncapped; generation stops only at the server limit.",
|
||||
stacklevel=2,
|
||||
)
|
||||
overrides["max_tokens"] = args.max_tokens
|
||||
elif not model_preset_id:
|
||||
overrides["max_tokens"] = 2048
|
||||
for key in (
|
||||
"max_tokens",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"min_p",
|
||||
@@ -107,12 +140,12 @@ def run_sgl_eval(args):
|
||||
chat_kwargs.setdefault("enable_thinking", thinking)
|
||||
if chat_kwargs:
|
||||
overrides["chat_template_kwargs"] = {
|
||||
**(spec.default_gen.chat_template_kwargs or {}),
|
||||
**(default_gen.chat_template_kwargs or {}),
|
||||
**chat_kwargs,
|
||||
}
|
||||
if getattr(args, "top_k", None) is not None:
|
||||
overrides["extra_body"] = {"top_k": args.top_k}
|
||||
gen = replace(spec.default_gen, **overrides)
|
||||
gen = replace(default_gen, **overrides)
|
||||
generation = {
|
||||
key: getattr(gen, key)
|
||||
for key in ("max_tokens", "temperature", "top_p", "min_p", "seed")
|
||||
@@ -120,7 +153,7 @@ def run_sgl_eval(args):
|
||||
print(f"sgl-eval {spec.name} generation: {generation}", flush=True)
|
||||
sampler = ChatCompletionSampler(
|
||||
base_url=api_base_url(args),
|
||||
model=getattr(args, "model", None),
|
||||
model=model,
|
||||
api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"),
|
||||
)
|
||||
out_parent = Path(
|
||||
@@ -129,7 +162,7 @@ def run_sgl_eval(args):
|
||||
).expanduser()
|
||||
out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
repeats = getattr(args, "repeat", 1)
|
||||
repeats = getattr(args, "repeat", None) or spec.default_n_repeats
|
||||
writer = PredictionsWriter(out_dir, repeats, spec.pred_schema)
|
||||
try:
|
||||
result = spec.run(
|
||||
|
||||
@@ -48,7 +48,7 @@ from sglang.srt.utils import (
|
||||
retry,
|
||||
)
|
||||
from sglang.srt.utils.network import is_port_available
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.sgl_eval_utils import run_sgl_eval
|
||||
from sglang.utils import normalize_base_url
|
||||
|
||||
# General test models
|
||||
@@ -399,24 +399,11 @@ def start_subprocess_fail_fast_watcher(
|
||||
def _try_enable_offline_mode_if_cache_complete(
|
||||
model_name_or_path: str, env: dict, other_args: Optional[list[str]] = None
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
CI helper: Check if model cache is complete and enable offline mode.
|
||||
"""Set HF_HUB_OFFLINE=1 in `env` if the model cache validates; return the
|
||||
per-run marker path, or None if offline mode was not enabled.
|
||||
|
||||
Uses per-run validation markers that are NOT shared across runners.
|
||||
Each runner independently validates its cache using lightweight checks
|
||||
before enabling offline mode.
|
||||
|
||||
IMPORTANT: Even if a per-run marker exists, this function ALWAYS validates
|
||||
the current launch's requirements (e.g., hf_quant_config.json for modelopt).
|
||||
The marker is only a hint that this snapshot was validated earlier in the run.
|
||||
|
||||
Args:
|
||||
model_name_or_path: Model identifier or path
|
||||
env: Environment dict to modify (will add HF_HUB_OFFLINE=1 if validation passes)
|
||||
other_args: Launch command arguments (used to detect quantization requirement)
|
||||
|
||||
Returns:
|
||||
Per-run marker path if offline mode was enabled, None otherwise
|
||||
Markers are per-run and not shared across runners. A marker is only a hint;
|
||||
the current launch's requirements (e.g. hf_quant_config.json) are revalidated.
|
||||
"""
|
||||
from sglang.srt.model_loader.ci_weight_validation import (
|
||||
_get_per_run_marker_path,
|
||||
@@ -453,8 +440,7 @@ def _try_enable_offline_mode_if_cache_complete(
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Detect if quantization requires hf_quant_config.json
|
||||
# Do this BEFORE checking marker to ensure current launch requirements are known
|
||||
# Detect before the marker check so the current launch's requirements are known.
|
||||
requires_hf_quant_config = False
|
||||
for i, arg in enumerate(other_args):
|
||||
if arg == "--quantization" and i + 1 < len(other_args):
|
||||
@@ -466,9 +452,8 @@ def _try_enable_offline_mode_if_cache_complete(
|
||||
# Check per-run marker (fast hint - snapshot validated earlier in this run)
|
||||
per_run_marker = _read_per_run_marker(snapshot_dir)
|
||||
if per_run_marker is not None:
|
||||
# Marker exists, but STILL validate for current launch requirements
|
||||
# This prevents a test without --quantization from enabling offline
|
||||
# for a later test with --quantization that needs hf_quant_config.json
|
||||
# Still validate: a marker from a launch without --quantization must not
|
||||
# enable offline mode for a later launch that needs hf_quant_config.json.
|
||||
is_valid = validate_cache_lightweight(snapshot_dir, requires_hf_quant_config)
|
||||
|
||||
if not is_valid:
|
||||
@@ -522,17 +507,7 @@ def _try_enable_offline_mode_if_cache_complete(
|
||||
|
||||
|
||||
def _create_clean_subprocess_env(env: dict) -> dict:
|
||||
"""Create a clean subprocess environment without internal CI keys.
|
||||
|
||||
Removes all keys starting with '_CI_OFFLINE_' or 'CI_OFFLINE' to prevent
|
||||
leaking implementation details to the server subprocess.
|
||||
|
||||
Args:
|
||||
env: Source environment dict
|
||||
|
||||
Returns:
|
||||
Clean copy of environment dict
|
||||
"""
|
||||
"""Create a clean subprocess environment without internal CI keys."""
|
||||
child_env = env.copy()
|
||||
keys_to_remove = [
|
||||
k for k in child_env if k.startswith(("_CI_OFFLINE_", "CI_OFFLINE_"))
|
||||
@@ -592,14 +567,7 @@ def _launch_server_process(
|
||||
) -> subprocess.Popen:
|
||||
"""Launch server subprocess with clean environment.
|
||||
|
||||
Args:
|
||||
command: Command list for subprocess
|
||||
env: Environment dict (will be cleaned before use)
|
||||
return_stdout_stderr: Optional tuple of (stdout_file, stderr_file) for output capture
|
||||
model: Model name for logging
|
||||
|
||||
Returns:
|
||||
Started subprocess.Popen object
|
||||
`return_stdout_stderr` is an optional (stdout_file, stderr_file) pair.
|
||||
"""
|
||||
child_env = _create_clean_subprocess_env(env)
|
||||
|
||||
@@ -619,17 +587,7 @@ def _wait_for_server_health(
|
||||
api_key: Optional[str],
|
||||
timeout_duration: float,
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""Wait for server health check to pass.
|
||||
|
||||
Args:
|
||||
proc: Server subprocess
|
||||
base_url: Base URL for health check
|
||||
api_key: Optional API key for authorization
|
||||
timeout_duration: Maximum wait time in seconds
|
||||
|
||||
Returns:
|
||||
Tuple of (success, error_message)
|
||||
"""
|
||||
"""Wait for server health check to pass; return (success, error_message)."""
|
||||
start_time = time.perf_counter()
|
||||
with requests.Session() as session:
|
||||
while time.perf_counter() - start_time < timeout_duration:
|
||||
@@ -686,20 +644,8 @@ def popen_launch_server(
|
||||
):
|
||||
"""Launch a server process with automatic device detection and offline/online retry.
|
||||
|
||||
Args:
|
||||
model: Model path or identifier
|
||||
base_url: Base URL for the server
|
||||
timeout: Timeout for server startup
|
||||
api_key: Optional API key for authentication
|
||||
other_args: Additional command line arguments
|
||||
env: Environment dict for subprocess
|
||||
return_stdout_stderr: Optional tuple for output capture
|
||||
device: Device type ("auto", "cuda", "rocm" or "cpu")
|
||||
pd_separated: Whether to use PD separated mode
|
||||
num_replicas: Number of replicas for mixed PD mode
|
||||
|
||||
Returns:
|
||||
Started subprocess.Popen object
|
||||
`device` is one of "auto", "cuda", "rocm", "cpu";
|
||||
`num_replicas` applies to mixed PD mode.
|
||||
"""
|
||||
other_args = other_args or []
|
||||
|
||||
@@ -897,9 +843,7 @@ def popen_launch_pd_server(
|
||||
|
||||
print(f"command={' '.join(command)}")
|
||||
|
||||
# Merge with os.environ so caller-supplied env adds to (not replaces)
|
||||
# PATH / PYTHONPATH / HF_HOME / etc. When env is None, Popen inherits
|
||||
# parent's environment automatically.
|
||||
# Merge so caller-supplied env adds to, not replaces, PATH / PYTHONPATH / etc.
|
||||
if env is not None:
|
||||
env = {**os.environ, **env}
|
||||
|
||||
@@ -1018,9 +962,8 @@ def run_bench_serving(
|
||||
other_args=other_server_args,
|
||||
)
|
||||
|
||||
# Resolve tokenizer to local snapshot path when available, so the benchmark
|
||||
# client's AutoTokenizer.from_pretrained uses the local path directly instead
|
||||
# of calling the HF Hub API (which can stall for minutes in CI).
|
||||
# Prefer the local snapshot so the client's AutoTokenizer skips the HF Hub API,
|
||||
# which can stall for minutes in CI.
|
||||
bench_tokenizer = tokenizer
|
||||
if bench_tokenizer is None:
|
||||
try:
|
||||
@@ -1090,19 +1033,7 @@ async def _run_api_benchmark_requests(
|
||||
num_requests: int,
|
||||
response_validator: Callable[[dict], bool],
|
||||
):
|
||||
"""
|
||||
Helper function to run API benchmark requests and collect metrics.
|
||||
|
||||
Args:
|
||||
base_url: The base URL of the server
|
||||
endpoint: The API endpoint to test (e.g., "/v1/score", "/v1/embeddings")
|
||||
test_requests: List of request payloads to send
|
||||
num_requests: Total number of requests expected
|
||||
response_validator: Function to validate if response contains expected data
|
||||
|
||||
Returns:
|
||||
Dictionary with benchmark metrics
|
||||
"""
|
||||
"""Run API benchmark requests and return a dict of metrics."""
|
||||
start_time = time.monotonic()
|
||||
successful_requests = 0
|
||||
total_latency = 0
|
||||
@@ -1433,12 +1364,7 @@ def run_bench_serving_multi(
|
||||
|
||||
|
||||
def run_bench_one_batch(model, other_args):
|
||||
"""Launch a offline process with automatic device detection.
|
||||
|
||||
Args:
|
||||
device: Device type ("auto", "cuda", "rocm" or "cpu").
|
||||
If "auto", will detect available platforms automatically.
|
||||
"""
|
||||
"""Launch an offline process with automatic device detection."""
|
||||
# Auto-detect device if needed
|
||||
|
||||
device = auto_config_device()
|
||||
@@ -1740,7 +1666,7 @@ def run_mmlu_test(
|
||||
)
|
||||
|
||||
try:
|
||||
metrics = run_eval(args)
|
||||
metrics = run_sgl_eval(args)
|
||||
assert metrics["score"] >= 0.65, f"{metrics=}"
|
||||
finally:
|
||||
pass
|
||||
@@ -2340,11 +2266,9 @@ def server_args_variant(server_args, **fields):
|
||||
def enter_override(test_case, override):
|
||||
"""Install a scoped context override for the length of one test.
|
||||
|
||||
`unittest.TestCase.enterContext` does exactly this in one call, but it is
|
||||
Python 3.11+ and this package supports 3.10 (`requires-python = ">=3.10"`).
|
||||
On 3.10 it raises `AttributeError: ... has no attribute 'enterContext'` --
|
||||
and only there, so a developer on a newer interpreter sees every test pass
|
||||
while CI does not.
|
||||
Stands in for `unittest.TestCase.enterContext`, which is 3.11+ while this package
|
||||
supports 3.10; on 3.10 it raises AttributeError, so tests pass locally on newer
|
||||
interpreters but fail in CI.
|
||||
"""
|
||||
installed = override.install()
|
||||
test_case.addCleanup(override.restore)
|
||||
@@ -2366,9 +2290,8 @@ class CustomTestCase(unittest.TestCase):
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
super().__init_subclass__(**kwargs)
|
||||
|
||||
# Wrap the effective setUpClass so that tearDownClass is called
|
||||
# even when setUpClass fails. Python's unittest skips tearDownClass
|
||||
# if setUpClass raises, which can leak resources (ports, processes).
|
||||
# unittest skips tearDownClass when setUpClass raises, leaking ports and
|
||||
# processes; wrap setUpClass so tearDownClass still runs.
|
||||
setup = cls.setUpClass
|
||||
if getattr(setup, "_safe_setup_wrapped", False):
|
||||
return
|
||||
@@ -2669,11 +2592,6 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
|
||||
- stdout: [METRIC] metric_name=value [labels=...]
|
||||
|
||||
This function never fails tests - all errors are silently caught.
|
||||
|
||||
Args:
|
||||
metric_name: Metric name (e.g., "gsm8k_accuracy", "cache_hit_rate")
|
||||
value: Metric value
|
||||
labels: Optional label dict (e.g., {"backend": "fa3"})
|
||||
"""
|
||||
try:
|
||||
# 1. Capture test context
|
||||
@@ -2750,11 +2668,7 @@ def dump_metric(metric_name: str, value: Any, labels: Optional[dict] = None):
|
||||
|
||||
|
||||
def _get_test_context() -> tuple[str, str]:
|
||||
"""
|
||||
Get current test's filename and test_case.
|
||||
|
||||
Tries PYTEST_CURRENT_TEST first, falls back to inspect.stack().
|
||||
"""
|
||||
"""Get current test's filename and test_case, via PYTEST_CURRENT_TEST or stack."""
|
||||
# 1. Try parsing PYTEST_CURRENT_TEST
|
||||
pytest_current = os.getenv("PYTEST_CURRENT_TEST")
|
||||
if pytest_current:
|
||||
|
||||
Reference in New Issue
Block a user