minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Eval accuracy gpqa aime25 mixins (#27102)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Jimmy Shong
2026-06-14 00:49:12 -07:00
committed by GitHub
co-authored by github-actions[bot]
parent f293ddf3ce
commit 54acffc864
2 changed files with 419 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/kits/eval_accuracy_kit.py
+319 -79
View File
@@ -10,7 +10,7 @@ _THRESHOLD_NOT_SET = float("nan")
def _check_accept_length(test_case, base_url, threshold=None):
"""Print accept length; optionally assert it exceeds threshold."""
"""Print speculative accept length; optionally assert it exceeds threshold."""
try:
server_info = requests.get(base_url + "/server_info").json()
val = server_info["internal_states"][0]["avg_spec_accept_length"]
@@ -21,54 +21,220 @@ def _check_accept_length(test_case, base_url, threshold=None):
test_case.assertGreater(val, threshold)
def _finalize_eval(
test_case,
*,
eval_name: str,
score: float,
score_threshold: float,
accept_length_thres: Optional[float] = None,
summary_label: Optional[str] = None,
):
"""Shared driver tail: CI step summary, accept-length check, threshold assert."""
if is_in_ci():
label = summary_label or f"test_{eval_name}"
write_github_step_summary(f"### {label}\n{eval_name}_score={score:.4f}\n")
_check_accept_length(test_case, test_case.base_url, accept_length_thres)
test_case.assertGreaterEqual(score, score_threshold)
def _run_accuracy_eval(
test_case,
*,
eval_name: str,
score_threshold: float,
num_examples: Optional[int],
num_threads: int,
accept_length_thres: Optional[float] = None,
summary_label: Optional[str] = None,
**eval_overrides,
):
"""Shared driver for the accuracy mixins below.
Runs ``run_eval`` for ``eval_name`` against the test class's server
(``base_url`` / ``model``), records a CI step summary, asserts the score
meets ``score_threshold``, and checks the speculative accept length.
``eval_overrides`` (e.g. ``api``, ``max_tokens``, ``temperature``,
``top_p``, ``num_shots``) are forwarded to ``run_eval`` only when not
``None``, so the common case stays identical to ``run_eval``'s defaults.
Returns the metrics dict.
"""
assert (
score_threshold == score_threshold
), f"{type(test_case).__name__} must set the {eval_name} score threshold"
kwargs = dict(
base_url=test_case.base_url,
model=getattr(test_case, "model", None),
eval_name=eval_name,
num_examples=num_examples,
num_threads=num_threads,
)
kwargs.update({k: v for k, v in eval_overrides.items() if v is not None})
metrics = run_eval(SimpleNamespace(**kwargs))
print(f"{eval_name} {metrics=}")
_finalize_eval(
test_case,
eval_name=eval_name,
score=metrics["score"],
score_threshold=score_threshold,
accept_length_thres=accept_length_thres,
summary_label=summary_label,
)
return metrics
def _run_sgl_eval(
test_case,
*,
eval_name: str,
score_threshold: float,
metric: str = "score",
n_repeats: int = 1,
num_examples: Optional[int] = None,
num_threads: int = 512,
thinking: bool = True,
reasoning_effort: Optional[str] = None,
max_tokens: Optional[int] = None,
temperature: Optional[float] = None,
top_p: Optional[float] = None,
accept_length_thres: Optional[float] = None,
summary_label: Optional[str] = None,
):
"""Shared sgl-eval driver for the reasoning mixins and the ``sgl_eval`` backend.
Runs ``eval_name`` via the sgl-eval Python API (``registry.get`` ->
``EvalSpec.run``) against the test class's server, records a CI step summary,
asserts the score meets ``score_threshold``, and checks the speculative accept
length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking":
True}`` so the server separates reasoning from the final answer. Skips the test
if sgl-eval (git-only) is not installed. Returns the RunResult.
"""
assert (
score_threshold == score_threshold
), f"{type(test_case).__name__} must set the {eval_name} score threshold"
try:
from sgl_eval.registry import get as get_eval_spec
from sgl_eval.sampler import ChatCompletionSampler
from sgl_eval.types import GenConfig
except ImportError:
test_case.skipTest(
"sgl-eval not installed; pip install "
"'sgl-eval @ git+https://github.com/sgl-project/sgl-eval'"
)
base_url = test_case.base_url.rstrip("/")
if not base_url.endswith("/v1"):
base_url += "/v1"
sampler = ChatCompletionSampler(
base_url=base_url, model=getattr(test_case, "model", None), api_key="EMPTY"
)
gen_kwargs = dict(
max_tokens=max_tokens,
reasoning_effort=reasoning_effort,
chat_template_kwargs={"thinking": True} if thinking else None,
)
if temperature is not None:
gen_kwargs["temperature"] = temperature
if top_p is not None:
gen_kwargs["top_p"] = top_p
result = get_eval_spec(eval_name).run(
sampler=sampler,
gen=GenConfig(**gen_kwargs),
n_repeats=n_repeats,
num_examples=num_examples,
num_threads=num_threads,
predictions_writer=None,
load_examples=None,
)
score = result.aggregate[metric]
print(f"{eval_name} sgl-eval {metric}={score:.4f}")
_finalize_eval(
test_case,
eval_name=eval_name,
score=score,
score_threshold=score_threshold,
accept_length_thres=accept_length_thres,
summary_label=summary_label,
)
return result
class GSM8KMixin:
"""Mixin for GSM8K evaluation via OpenAI Chat API.
"""Mixin for GSM8K evaluation.
Backend is selectable via ``gsm8k_backend`` (default ``"run_eval"``: OpenAI
completion API, 5-shot; or ``"sgl_eval"``: sgl-eval chat + boxed/sympy grader,
skipped if sgl-eval is not installed). The canonical threshold/count knobs are
``gsm8k_score_threshold`` / ``gsm8k_num_examples``; the legacy
``gsm8k_accuracy_thres`` / ``gsm8k_num_questions`` are still honored.
Required attributes on the test class:
base_url: str
gsm8k_accuracy_thres: float
gsm8k_score_threshold: float
Optional attributes:
model: str (if not set, auto-detected from server)
"""
gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET
gsm8k_score_threshold: float = _THRESHOLD_NOT_SET
gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET # legacy alias
gsm8k_num_examples: Optional[int] = None
gsm8k_num_questions: int = 200 # legacy alias
gsm8k_accept_length_thres: Optional[float] = None
gsm8k_num_questions: int = 200
gsm8k_num_threads: int = 128
gsm8k_num_shots: int = 5
gsm8k_num_shots: int = 5 # run_eval backend only
gsm8k_backend: str = "run_eval" # "run_eval" | "sgl_eval"
gsm8k_thinking: bool = False # sgl_eval backend
gsm8k_n_repeats: int = 1 # sgl_eval backend
def test_gsm8k(self):
assert (
self.gsm8k_accuracy_thres == self.gsm8k_accuracy_thres
), f"{type(self).__name__} must set gsm8k_accuracy_thres"
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=self.gsm8k_num_questions,
num_threads=self.gsm8k_num_threads,
num_shots=self.gsm8k_num_shots,
threshold = self.gsm8k_score_threshold
if threshold != threshold: # canonical unset (NaN) -> legacy alias
threshold = self.gsm8k_accuracy_thres
num_examples = (
self.gsm8k_num_examples
if self.gsm8k_num_examples is not None
else self.gsm8k_num_questions
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(f"### test_gsm8k\n{metrics['score']=:.4f}\n")
self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
_check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres)
if self.gsm8k_backend == "sgl_eval":
_run_sgl_eval(
self,
eval_name="gsm8k",
score_threshold=threshold,
n_repeats=self.gsm8k_n_repeats,
num_examples=num_examples,
num_threads=self.gsm8k_num_threads,
thinking=self.gsm8k_thinking,
accept_length_thres=self.gsm8k_accept_length_thres,
)
else:
_run_accuracy_eval(
self,
eval_name="gsm8k",
score_threshold=threshold,
num_examples=num_examples,
num_threads=self.gsm8k_num_threads,
accept_length_thres=self.gsm8k_accept_length_thres,
api="completion",
max_tokens=512,
num_shots=self.gsm8k_num_shots,
)
class MMLUMixin:
"""Mixin for MMLU evaluation.
Backend is selectable via ``mmlu_backend`` (default ``"run_eval"``; or
``"sgl_eval"``: sgl-eval multichoice grader, skipped if sgl-eval is not
installed).
Required attributes on the test class:
base_url: str
model: str
@@ -79,28 +245,127 @@ class MMLUMixin:
mmlu_accept_length_thres: Optional[float] = None
mmlu_num_examples: int = 5000
mmlu_num_threads: int = 1024
mmlu_backend: str = "run_eval" # "run_eval" | "sgl_eval"
mmlu_thinking: bool = False # sgl_eval backend
mmlu_n_repeats: int = 1 # sgl_eval backend
def test_mmlu(self):
assert (
self.mmlu_score_threshold == self.mmlu_score_threshold
), f"{type(self).__name__} must set mmlu_score_threshold"
if self.mmlu_backend == "sgl_eval":
_run_sgl_eval(
self,
eval_name="mmlu",
score_threshold=self.mmlu_score_threshold,
n_repeats=self.mmlu_n_repeats,
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
thinking=self.mmlu_thinking,
accept_length_thres=self.mmlu_accept_length_thres,
)
else:
_run_accuracy_eval(
self,
eval_name="mmlu",
score_threshold=self.mmlu_score_threshold,
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
accept_length_thres=self.mmlu_accept_length_thres,
)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=self.mmlu_num_examples,
num_threads=self.mmlu_num_threads,
class GPQAMixin:
"""Mixin for GPQA-Diamond evaluation (graduate-level multiple choice).
Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
installed). ``gpqa_thinking`` defaults to True, which
enables per-request thinking so the server separates reasoning from the final
answer.
Required attributes on the test class:
base_url: str
model: str
gpqa_score_threshold: float
Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
gpqa_reasoning_effort="max", gpqa_max_tokens=200000, gpqa_temperature=1.0,
gpqa_top_p=1.0. GPQA-Diamond is 198 questions; raise gpqa_n_repeats (e.g. 16)
for a stable number.
"""
gpqa_score_threshold: float = _THRESHOLD_NOT_SET
gpqa_accept_length_thres: Optional[float] = None
gpqa_num_examples: Optional[int] = None
gpqa_num_threads: int = 1024
gpqa_n_repeats: int = 1
gpqa_thinking: bool = True
gpqa_reasoning_effort: Optional[str] = None
gpqa_max_tokens: Optional[int] = None
gpqa_temperature: Optional[float] = None
gpqa_top_p: Optional[float] = None
def test_gpqa(self):
_run_sgl_eval(
self,
eval_name="gpqa",
score_threshold=self.gpqa_score_threshold,
n_repeats=self.gpqa_n_repeats,
num_examples=self.gpqa_num_examples,
num_threads=self.gpqa_num_threads,
thinking=self.gpqa_thinking,
reasoning_effort=self.gpqa_reasoning_effort,
max_tokens=self.gpqa_max_tokens,
temperature=self.gpqa_temperature,
top_p=self.gpqa_top_p,
accept_length_thres=self.gpqa_accept_length_thres,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_mmlu\n{metrics['score']=:.4f}\n")
class AIME25Mixin:
"""Mixin for AIME 2025 evaluation (competition math, integer answers).
self.assertGreaterEqual(metrics["score"], self.mmlu_score_threshold)
Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not
installed). ``aime25_thinking`` defaults to True, which
enables per-request thinking so the server separates reasoning from the final
answer.
_check_accept_length(self, self.base_url, self.mmlu_accept_length_thres)
Required attributes on the test class:
base_url: str
model: str
aime25_score_threshold: float
Optional sampling knobs (default to sgl-eval's defaults when unset). Set these
for reasoning models -- e.g. DeepSeek-V4 Think-Max wants
aime25_reasoning_effort="max", aime25_max_tokens=200000, aime25_temperature=1.0,
aime25_top_p=1.0. AIME25 has only 30 problems, so it is high variance; raise
aime25_n_repeats (e.g. 16) for a stable number.
"""
aime25_score_threshold: float = _THRESHOLD_NOT_SET
aime25_accept_length_thres: Optional[float] = None
aime25_num_examples: Optional[int] = None
aime25_num_threads: int = 1024
aime25_n_repeats: int = 1
aime25_thinking: bool = True
aime25_reasoning_effort: Optional[str] = None
aime25_max_tokens: Optional[int] = None
aime25_temperature: Optional[float] = None
aime25_top_p: Optional[float] = None
def test_aime25(self):
_run_sgl_eval(
self,
eval_name="aime25",
score_threshold=self.aime25_score_threshold,
n_repeats=self.aime25_n_repeats,
num_examples=self.aime25_num_examples,
num_threads=self.aime25_num_threads,
thinking=self.aime25_thinking,
reasoning_effort=self.aime25_reasoning_effort,
max_tokens=self.aime25_max_tokens,
temperature=self.aime25_temperature,
top_p=self.aime25_top_p,
accept_length_thres=self.aime25_accept_length_thres,
)
class HumanEvalMixin:
@@ -117,30 +382,18 @@ class HumanEvalMixin:
humaneval_num_threads: int = 1024
def test_human_eval(self):
assert (
self.humaneval_score_threshold == self.humaneval_score_threshold
), f"{type(self).__name__} must set humaneval_score_threshold"
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="humaneval",
num_examples=None,
num_threads=self.humaneval_num_threads,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_human_eval\n{metrics['score']=:.4f}\n")
threshold = self.humaneval_score_threshold
if is_in_amd_ci() and self.humaneval_score_threshold_amd is not None:
threshold = self.humaneval_score_threshold_amd
self.assertGreaterEqual(metrics["score"], threshold)
_check_accept_length(self, self.base_url)
_run_accuracy_eval(
self,
eval_name="humaneval",
score_threshold=threshold,
num_examples=None,
num_threads=self.humaneval_num_threads,
summary_label="test_human_eval",
)
class MGSMEnMixin:
@@ -157,23 +410,10 @@ class MGSMEnMixin:
mgsm_en_num_threads: int = 1024
def test_mgsm_en(self):
assert (
self.mgsm_en_score_threshold == self.mgsm_en_score_threshold
), f"{type(self).__name__} must set mgsm_en_score_threshold"
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
_run_accuracy_eval(
self,
eval_name="mgsm_en",
score_threshold=self.mgsm_en_score_threshold,
num_examples=self.mgsm_en_num_examples,
num_threads=self.mgsm_en_num_threads,
)
metrics = run_eval(args)
if is_in_ci():
write_github_step_summary(f"### test_mgsm_en\n{metrics['score']=:.4f}\n")
self.assertGreaterEqual(metrics["score"], self.mgsm_en_score_threshold)
_check_accept_length(self, self.base_url)
test/registered/unit/test_eval_accuracy_kit_sgl_eval.py
+100
View File
@@ -0,0 +1,100 @@
"""Unit tests for the GSM8K backend dispatch + sgl-eval skip in eval_accuracy_kit.
Hermetic (no server, no real sgl-eval install). These guard the behavior that
existing consumers rely on -- not the sgl-eval happy path, which the live
accuracy runs already cover:
1. The default GSM8K backend stays on ``run_eval`` (OpenAI completion API);
the ~47 existing GSM8K consumers must never be silently rerouted.
2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail
gate when the canonical ``gsm8k_score_threshold`` is unset.
3. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent,
so CI without the optional dependency stays green.
"""
import sys
import unittest
from unittest.mock import MagicMock, patch
import requests
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.kits import eval_accuracy_kit as kit
from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _fake_get(url, *args, **kwargs):
# flush_cache must succeed (GSM8K calls it unguarded); /server_info is probed
# by _check_accept_length, which swallows RequestException.
if str(url).endswith("/flush_cache"):
return MagicMock()
raise requests.RequestException()
def _make_host(mixin, method):
"""Build a throwaway mixin host bound to ``method``.
Created dynamically (never bound at module scope) so it is collected by
neither runner: CI executes this file via ``python3 <file>`` ->
``unittest.main()``, whose loader ignores pytest's ``__test__`` flag, and
pytest only collects module-level ``Test*`` classes. The host runs only when
a test below instantiates and drives it directly.
"""
return type(f"_{mixin.__name__}Host", (mixin, CustomTestCase), {})(method)
class TestEvalKitBackendDispatch(CustomTestCase):
def _run_gsm8k_default(self, score, **attrs):
"""Run GSM8K on the default (run_eval) backend with run_eval faked to
return ``score``; returns the SimpleNamespace args run_eval received."""
captured = {}
def fake_run_eval(args):
captured["args"] = args
return {"score": score}
host = _make_host(GSM8KMixin, "test_gsm8k")
host.base_url = "http://127.0.0.1:0"
host.model = "m"
for k, v in attrs.items():
setattr(host, k, v)
with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object(
kit.requests, "get", side_effect=_fake_get
):
host.test_gsm8k()
return captured["args"]
def test_default_backend_uses_run_eval_completion(self):
# The default path that all existing GSM8K consumers rely on must stay on
# run_eval's OpenAI completion API -- it must not touch sgl-eval.
args = self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.5)
self.assertEqual(args.eval_name, "gsm8k")
self.assertEqual(args.api, "completion")
def test_legacy_accuracy_thres_alias_gates_score(self):
# Canonical gsm8k_score_threshold left unset (NaN) -> the legacy
# gsm8k_accuracy_thres must still be the pass/fail gate.
self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.90) # above -> passes
with self.assertRaises(AssertionError):
self._run_gsm8k_default(0.80, gsm8k_accuracy_thres=0.90) # below -> fails
def test_sgl_eval_path_skips_when_not_installed(self):
# GPQA/AIME25 and the sgl_eval backend must skip -- not error -- when
# sgl-eval is not installed. None in sys.modules makes the import raise.
host = _make_host(GPQAMixin, "test_gpqa")
host.base_url = "http://127.0.0.1:0"
host.model = "m"
host.gpqa_score_threshold = 0.5
absent = {
k: None for k in ("sgl_eval.registry", "sgl_eval.sampler", "sgl_eval.types")
}
with patch.dict(sys.modules, absent):
with self.assertRaises(unittest.SkipTest):
host.test_gpqa()
if __name__ == "__main__":
unittest.main()