diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index 22ac7bde2..9a59ca92d 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -10,7 +10,7 @@ _THRESHOLD_NOT_SET = float("nan") def _check_accept_length(test_case, base_url, threshold=None): - """Print accept length; optionally assert it exceeds threshold.""" + """Print speculative accept length; optionally assert it exceeds threshold.""" try: server_info = requests.get(base_url + "/server_info").json() val = server_info["internal_states"][0]["avg_spec_accept_length"] @@ -21,54 +21,220 @@ def _check_accept_length(test_case, base_url, threshold=None): test_case.assertGreater(val, threshold) +def _finalize_eval( + test_case, + *, + eval_name: str, + score: float, + score_threshold: float, + accept_length_thres: Optional[float] = None, + summary_label: Optional[str] = None, +): + """Shared driver tail: CI step summary, accept-length check, threshold assert.""" + if is_in_ci(): + label = summary_label or f"test_{eval_name}" + write_github_step_summary(f"### {label}\n{eval_name}_score={score:.4f}\n") + _check_accept_length(test_case, test_case.base_url, accept_length_thres) + test_case.assertGreaterEqual(score, score_threshold) + + +def _run_accuracy_eval( + test_case, + *, + eval_name: str, + score_threshold: float, + num_examples: Optional[int], + num_threads: int, + accept_length_thres: Optional[float] = None, + summary_label: Optional[str] = None, + **eval_overrides, +): + """Shared driver for the accuracy mixins below. + + Runs ``run_eval`` for ``eval_name`` against the test class's server + (``base_url`` / ``model``), records a CI step summary, asserts the score + meets ``score_threshold``, and checks the speculative accept length. + + ``eval_overrides`` (e.g. ``api``, ``max_tokens``, ``temperature``, + ``top_p``, ``num_shots``) are forwarded to ``run_eval`` only when not + ``None``, so the common case stays identical to ``run_eval``'s defaults. + Returns the metrics dict. + """ + assert ( + score_threshold == score_threshold + ), f"{type(test_case).__name__} must set the {eval_name} score threshold" + + kwargs = dict( + base_url=test_case.base_url, + model=getattr(test_case, "model", None), + eval_name=eval_name, + num_examples=num_examples, + num_threads=num_threads, + ) + kwargs.update({k: v for k, v in eval_overrides.items() if v is not None}) + + metrics = run_eval(SimpleNamespace(**kwargs)) + print(f"{eval_name} {metrics=}") + _finalize_eval( + test_case, + eval_name=eval_name, + score=metrics["score"], + score_threshold=score_threshold, + accept_length_thres=accept_length_thres, + summary_label=summary_label, + ) + return metrics + + +def _run_sgl_eval( + test_case, + *, + eval_name: str, + score_threshold: float, + metric: str = "score", + n_repeats: int = 1, + num_examples: Optional[int] = None, + num_threads: int = 512, + thinking: bool = True, + reasoning_effort: Optional[str] = None, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + accept_length_thres: Optional[float] = None, + summary_label: Optional[str] = None, +): + """Shared sgl-eval driver for the reasoning mixins and the ``sgl_eval`` backend. + + Runs ``eval_name`` via the sgl-eval Python API (``registry.get`` -> + ``EvalSpec.run``) against the test class's server, records a CI step summary, + asserts the score meets ``score_threshold``, and checks the speculative accept + length. ``thinking=True`` sends per-request ``chat_template_kwargs={"thinking": + True}`` so the server separates reasoning from the final answer. Skips the test + if sgl-eval (git-only) is not installed. Returns the RunResult. + """ + assert ( + score_threshold == score_threshold + ), f"{type(test_case).__name__} must set the {eval_name} score threshold" + + try: + from sgl_eval.registry import get as get_eval_spec + from sgl_eval.sampler import ChatCompletionSampler + from sgl_eval.types import GenConfig + except ImportError: + test_case.skipTest( + "sgl-eval not installed; pip install " + "'sgl-eval @ git+https://github.com/sgl-project/sgl-eval'" + ) + + base_url = test_case.base_url.rstrip("/") + if not base_url.endswith("/v1"): + base_url += "/v1" + sampler = ChatCompletionSampler( + base_url=base_url, model=getattr(test_case, "model", None), api_key="EMPTY" + ) + + gen_kwargs = dict( + max_tokens=max_tokens, + reasoning_effort=reasoning_effort, + chat_template_kwargs={"thinking": True} if thinking else None, + ) + if temperature is not None: + gen_kwargs["temperature"] = temperature + if top_p is not None: + gen_kwargs["top_p"] = top_p + + result = get_eval_spec(eval_name).run( + sampler=sampler, + gen=GenConfig(**gen_kwargs), + n_repeats=n_repeats, + num_examples=num_examples, + num_threads=num_threads, + predictions_writer=None, + load_examples=None, + ) + score = result.aggregate[metric] + print(f"{eval_name} sgl-eval {metric}={score:.4f}") + _finalize_eval( + test_case, + eval_name=eval_name, + score=score, + score_threshold=score_threshold, + accept_length_thres=accept_length_thres, + summary_label=summary_label, + ) + return result + + class GSM8KMixin: - """Mixin for GSM8K evaluation via OpenAI Chat API. + """Mixin for GSM8K evaluation. + + Backend is selectable via ``gsm8k_backend`` (default ``"run_eval"``: OpenAI + completion API, 5-shot; or ``"sgl_eval"``: sgl-eval chat + boxed/sympy grader, + skipped if sgl-eval is not installed). The canonical threshold/count knobs are + ``gsm8k_score_threshold`` / ``gsm8k_num_examples``; the legacy + ``gsm8k_accuracy_thres`` / ``gsm8k_num_questions`` are still honored. Required attributes on the test class: base_url: str - gsm8k_accuracy_thres: float + gsm8k_score_threshold: float Optional attributes: model: str (if not set, auto-detected from server) """ - gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET + gsm8k_score_threshold: float = _THRESHOLD_NOT_SET + gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET # legacy alias + gsm8k_num_examples: Optional[int] = None + gsm8k_num_questions: int = 200 # legacy alias gsm8k_accept_length_thres: Optional[float] = None - gsm8k_num_questions: int = 200 gsm8k_num_threads: int = 128 - gsm8k_num_shots: int = 5 + gsm8k_num_shots: int = 5 # run_eval backend only + gsm8k_backend: str = "run_eval" # "run_eval" | "sgl_eval" + gsm8k_thinking: bool = False # sgl_eval backend + gsm8k_n_repeats: int = 1 # sgl_eval backend def test_gsm8k(self): - assert ( - self.gsm8k_accuracy_thres == self.gsm8k_accuracy_thres - ), f"{type(self).__name__} must set gsm8k_accuracy_thres" - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=self.gsm8k_num_questions, - num_threads=self.gsm8k_num_threads, - num_shots=self.gsm8k_num_shots, + threshold = self.gsm8k_score_threshold + if threshold != threshold: # canonical unset (NaN) -> legacy alias + threshold = self.gsm8k_accuracy_thres + num_examples = ( + self.gsm8k_num_examples + if self.gsm8k_num_examples is not None + else self.gsm8k_num_questions ) - metrics = run_eval(args) - print(f"{metrics=}") - - if is_in_ci(): - write_github_step_summary(f"### test_gsm8k\n{metrics['score']=:.4f}\n") - - self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres) - - _check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres) + if self.gsm8k_backend == "sgl_eval": + _run_sgl_eval( + self, + eval_name="gsm8k", + score_threshold=threshold, + n_repeats=self.gsm8k_n_repeats, + num_examples=num_examples, + num_threads=self.gsm8k_num_threads, + thinking=self.gsm8k_thinking, + accept_length_thres=self.gsm8k_accept_length_thres, + ) + else: + _run_accuracy_eval( + self, + eval_name="gsm8k", + score_threshold=threshold, + num_examples=num_examples, + num_threads=self.gsm8k_num_threads, + accept_length_thres=self.gsm8k_accept_length_thres, + api="completion", + max_tokens=512, + num_shots=self.gsm8k_num_shots, + ) class MMLUMixin: """Mixin for MMLU evaluation. + Backend is selectable via ``mmlu_backend`` (default ``"run_eval"``; or + ``"sgl_eval"``: sgl-eval multichoice grader, skipped if sgl-eval is not + installed). + Required attributes on the test class: base_url: str model: str @@ -79,28 +245,127 @@ class MMLUMixin: mmlu_accept_length_thres: Optional[float] = None mmlu_num_examples: int = 5000 mmlu_num_threads: int = 1024 + mmlu_backend: str = "run_eval" # "run_eval" | "sgl_eval" + mmlu_thinking: bool = False # sgl_eval backend + mmlu_n_repeats: int = 1 # sgl_eval backend def test_mmlu(self): - assert ( - self.mmlu_score_threshold == self.mmlu_score_threshold - ), f"{type(self).__name__} must set mmlu_score_threshold" + if self.mmlu_backend == "sgl_eval": + _run_sgl_eval( + self, + eval_name="mmlu", + score_threshold=self.mmlu_score_threshold, + n_repeats=self.mmlu_n_repeats, + num_examples=self.mmlu_num_examples, + num_threads=self.mmlu_num_threads, + thinking=self.mmlu_thinking, + accept_length_thres=self.mmlu_accept_length_thres, + ) + else: + _run_accuracy_eval( + self, + eval_name="mmlu", + score_threshold=self.mmlu_score_threshold, + num_examples=self.mmlu_num_examples, + num_threads=self.mmlu_num_threads, + accept_length_thres=self.mmlu_accept_length_thres, + ) - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=self.mmlu_num_examples, - num_threads=self.mmlu_num_threads, + +class GPQAMixin: + """Mixin for GPQA-Diamond evaluation (graduate-level multiple choice). + + Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not + installed). ``gpqa_thinking`` defaults to True, which + enables per-request thinking so the server separates reasoning from the final + answer. + + Required attributes on the test class: + base_url: str + model: str + gpqa_score_threshold: float + + Optional sampling knobs (default to sgl-eval's defaults when unset). Set these + for reasoning models -- e.g. DeepSeek-V4 Think-Max wants + gpqa_reasoning_effort="max", gpqa_max_tokens=200000, gpqa_temperature=1.0, + gpqa_top_p=1.0. GPQA-Diamond is 198 questions; raise gpqa_n_repeats (e.g. 16) + for a stable number. + """ + + gpqa_score_threshold: float = _THRESHOLD_NOT_SET + gpqa_accept_length_thres: Optional[float] = None + gpqa_num_examples: Optional[int] = None + gpqa_num_threads: int = 1024 + gpqa_n_repeats: int = 1 + gpqa_thinking: bool = True + gpqa_reasoning_effort: Optional[str] = None + gpqa_max_tokens: Optional[int] = None + gpqa_temperature: Optional[float] = None + gpqa_top_p: Optional[float] = None + + def test_gpqa(self): + _run_sgl_eval( + self, + eval_name="gpqa", + score_threshold=self.gpqa_score_threshold, + n_repeats=self.gpqa_n_repeats, + num_examples=self.gpqa_num_examples, + num_threads=self.gpqa_num_threads, + thinking=self.gpqa_thinking, + reasoning_effort=self.gpqa_reasoning_effort, + max_tokens=self.gpqa_max_tokens, + temperature=self.gpqa_temperature, + top_p=self.gpqa_top_p, + accept_length_thres=self.gpqa_accept_length_thres, ) - metrics = run_eval(args) - if is_in_ci(): - write_github_step_summary(f"### test_mmlu\n{metrics['score']=:.4f}\n") +class AIME25Mixin: + """Mixin for AIME 2025 evaluation (competition math, integer answers). - self.assertGreaterEqual(metrics["score"], self.mmlu_score_threshold) + Runs via the sgl-eval Python API (the test is skipped if sgl-eval is not + installed). ``aime25_thinking`` defaults to True, which + enables per-request thinking so the server separates reasoning from the final + answer. - _check_accept_length(self, self.base_url, self.mmlu_accept_length_thres) + Required attributes on the test class: + base_url: str + model: str + aime25_score_threshold: float + + Optional sampling knobs (default to sgl-eval's defaults when unset). Set these + for reasoning models -- e.g. DeepSeek-V4 Think-Max wants + aime25_reasoning_effort="max", aime25_max_tokens=200000, aime25_temperature=1.0, + aime25_top_p=1.0. AIME25 has only 30 problems, so it is high variance; raise + aime25_n_repeats (e.g. 16) for a stable number. + """ + + aime25_score_threshold: float = _THRESHOLD_NOT_SET + aime25_accept_length_thres: Optional[float] = None + aime25_num_examples: Optional[int] = None + aime25_num_threads: int = 1024 + aime25_n_repeats: int = 1 + aime25_thinking: bool = True + aime25_reasoning_effort: Optional[str] = None + aime25_max_tokens: Optional[int] = None + aime25_temperature: Optional[float] = None + aime25_top_p: Optional[float] = None + + def test_aime25(self): + _run_sgl_eval( + self, + eval_name="aime25", + score_threshold=self.aime25_score_threshold, + n_repeats=self.aime25_n_repeats, + num_examples=self.aime25_num_examples, + num_threads=self.aime25_num_threads, + thinking=self.aime25_thinking, + reasoning_effort=self.aime25_reasoning_effort, + max_tokens=self.aime25_max_tokens, + temperature=self.aime25_temperature, + top_p=self.aime25_top_p, + accept_length_thres=self.aime25_accept_length_thres, + ) class HumanEvalMixin: @@ -117,30 +382,18 @@ class HumanEvalMixin: humaneval_num_threads: int = 1024 def test_human_eval(self): - assert ( - self.humaneval_score_threshold == self.humaneval_score_threshold - ), f"{type(self).__name__} must set humaneval_score_threshold" - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="humaneval", - num_examples=None, - num_threads=self.humaneval_num_threads, - ) - - metrics = run_eval(args) - - if is_in_ci(): - write_github_step_summary(f"### test_human_eval\n{metrics['score']=:.4f}\n") - threshold = self.humaneval_score_threshold if is_in_amd_ci() and self.humaneval_score_threshold_amd is not None: threshold = self.humaneval_score_threshold_amd - self.assertGreaterEqual(metrics["score"], threshold) - - _check_accept_length(self, self.base_url) + _run_accuracy_eval( + self, + eval_name="humaneval", + score_threshold=threshold, + num_examples=None, + num_threads=self.humaneval_num_threads, + summary_label="test_human_eval", + ) class MGSMEnMixin: @@ -157,23 +410,10 @@ class MGSMEnMixin: mgsm_en_num_threads: int = 1024 def test_mgsm_en(self): - assert ( - self.mgsm_en_score_threshold == self.mgsm_en_score_threshold - ), f"{type(self).__name__} must set mgsm_en_score_threshold" - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, + _run_accuracy_eval( + self, eval_name="mgsm_en", + score_threshold=self.mgsm_en_score_threshold, num_examples=self.mgsm_en_num_examples, num_threads=self.mgsm_en_num_threads, ) - - metrics = run_eval(args) - - if is_in_ci(): - write_github_step_summary(f"### test_mgsm_en\n{metrics['score']=:.4f}\n") - - self.assertGreaterEqual(metrics["score"], self.mgsm_en_score_threshold) - - _check_accept_length(self, self.base_url) diff --git a/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py b/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py new file mode 100644 index 000000000..70fc3a20d --- /dev/null +++ b/test/registered/unit/test_eval_accuracy_kit_sgl_eval.py @@ -0,0 +1,100 @@ +"""Unit tests for the GSM8K backend dispatch + sgl-eval skip in eval_accuracy_kit. + +Hermetic (no server, no real sgl-eval install). These guard the behavior that +existing consumers rely on -- not the sgl-eval happy path, which the live +accuracy runs already cover: + + 1. The default GSM8K backend stays on ``run_eval`` (OpenAI completion API); + the ~47 existing GSM8K consumers must never be silently rerouted. + 2. The legacy ``gsm8k_accuracy_thres`` alias is still honored as the pass/fail + gate when the canonical ``gsm8k_score_threshold`` is unset. + 3. The sgl-eval reasoning path skips (does not error) when sgl-eval is absent, + so CI without the optional dependency stays green. +""" + +import sys +import unittest +from unittest.mock import MagicMock, patch + +import requests + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.kits import eval_accuracy_kit as kit +from sglang.test.kits.eval_accuracy_kit import GPQAMixin, GSM8KMixin +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _fake_get(url, *args, **kwargs): + # flush_cache must succeed (GSM8K calls it unguarded); /server_info is probed + # by _check_accept_length, which swallows RequestException. + if str(url).endswith("/flush_cache"): + return MagicMock() + raise requests.RequestException() + + +def _make_host(mixin, method): + """Build a throwaway mixin host bound to ``method``. + + Created dynamically (never bound at module scope) so it is collected by + neither runner: CI executes this file via ``python3 `` -> + ``unittest.main()``, whose loader ignores pytest's ``__test__`` flag, and + pytest only collects module-level ``Test*`` classes. The host runs only when + a test below instantiates and drives it directly. + """ + return type(f"_{mixin.__name__}Host", (mixin, CustomTestCase), {})(method) + + +class TestEvalKitBackendDispatch(CustomTestCase): + def _run_gsm8k_default(self, score, **attrs): + """Run GSM8K on the default (run_eval) backend with run_eval faked to + return ``score``; returns the SimpleNamespace args run_eval received.""" + captured = {} + + def fake_run_eval(args): + captured["args"] = args + return {"score": score} + + host = _make_host(GSM8KMixin, "test_gsm8k") + host.base_url = "http://127.0.0.1:0" + host.model = "m" + for k, v in attrs.items(): + setattr(host, k, v) + with patch.object(kit, "run_eval", side_effect=fake_run_eval), patch.object( + kit.requests, "get", side_effect=_fake_get + ): + host.test_gsm8k() + return captured["args"] + + def test_default_backend_uses_run_eval_completion(self): + # The default path that all existing GSM8K consumers rely on must stay on + # run_eval's OpenAI completion API -- it must not touch sgl-eval. + args = self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.5) + self.assertEqual(args.eval_name, "gsm8k") + self.assertEqual(args.api, "completion") + + def test_legacy_accuracy_thres_alias_gates_score(self): + # Canonical gsm8k_score_threshold left unset (NaN) -> the legacy + # gsm8k_accuracy_thres must still be the pass/fail gate. + self._run_gsm8k_default(0.95, gsm8k_accuracy_thres=0.90) # above -> passes + with self.assertRaises(AssertionError): + self._run_gsm8k_default(0.80, gsm8k_accuracy_thres=0.90) # below -> fails + + def test_sgl_eval_path_skips_when_not_installed(self): + # GPQA/AIME25 and the sgl_eval backend must skip -- not error -- when + # sgl-eval is not installed. None in sys.modules makes the import raise. + host = _make_host(GPQAMixin, "test_gpqa") + host.base_url = "http://127.0.0.1:0" + host.model = "m" + host.gpqa_score_threshold = 0.5 + absent = { + k: None for k in ("sgl_eval.registry", "sgl_eval.sampler", "sgl_eval.types") + } + with patch.dict(sys.modules, absent): + with self.assertRaises(unittest.SkipTest): + host.test_gpqa() + + +if __name__ == "__main__": + unittest.main()