minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Unify GSM8K eval path to Chat API for regression CI readiness (#21667)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-04-01 17:12:19 -07:00
committed by GitHub
parent 1081a25983
commit d7256eb69a
79 changed files with 1349 additions and 1359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/accuracy_test_runner.py
+1 -63
View File
@@ -432,56 +432,6 @@ def _run_nemo_skills_eval(
kill_process_tree(process.pid)
def _run_few_shot_eval(
model: ModelLaunchSettings,
base_url: str,
num_questions: Optional[int] = None,
num_shots: int = 8,
max_tokens: int = 512,
) -> Tuple[bool, Optional[str], Optional[dict]]:
"""Run evaluation using few_shot backend (few_shot_gsm8k.py).
Returns:
Tuple of (success, error_message, metrics_dict)
"""
from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval
process = None
try:
process = popen_launch_server(
model.model_path,
base_url,
other_args=model.extra_args,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
env=model.env,
)
args = SimpleNamespace(
num_shots=num_shots,
data_path=None,
num_questions=num_questions or 200,
max_new_tokens=max_tokens,
parallel=128,
host="http://127.0.0.1",
port=int(base_url.split(":")[-1]),
)
metrics = run_few_shot_eval(args)
# Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score")
if "accuracy" in metrics and "score" not in metrics:
metrics["score"] = metrics["accuracy"]
return True, None, metrics
except Exception as e:
return False, f"Few-shot evaluation exception: {str(e)}", None
finally:
if process:
kill_process_tree(process.pid)
def run_accuracy_test(
model: ModelLaunchSettings,
params: AccuracyTestParams,
@@ -507,12 +457,7 @@ def run_accuracy_test(
# Run evaluation based on dataset type
# - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
# - few_shot_eval: gsm8k (default, backward compatible)
# - simple_eval: everything else (gpqa, mmmu, etc.)
has_extended_params = any(
getattr(params, field) is not None
for field in ("thinking_mode", "temperature", "top_p", "top_k", "repeat")
)
# - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.)
if params.dataset in ("mmmu-pro", "mmmu_pro"):
success, error, metrics = _run_nemo_skills_eval(
model=model,
@@ -523,13 +468,6 @@ def run_accuracy_test(
temperature=params.temperature,
top_p=params.top_p,
)
elif params.dataset == "gsm8k" and not has_extended_params:
success, error, metrics = _run_few_shot_eval(
model=model,
base_url=base_url,
num_questions=params.num_examples,
max_tokens=params.max_tokens or 512,
)
else:
success, error, metrics = _run_simple_eval(
model=model,
python/sglang/test/few_shot_gsm8k.py
+12
View File
@@ -1,6 +1,11 @@
"""
Run few-shot GSM-8K evaluation.
.. deprecated::
This module is deprecated. Use ``sglang.test.run_eval`` with
``eval_name="gsm8k"`` instead, which routes through the unified
Chat API evaluation framework with dump_metric support.
Usage:
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
"""
@@ -9,6 +14,7 @@ import argparse
import ast
import re
import time
import warnings
import numpy as np
@@ -50,6 +56,12 @@ def get_answer_value(answer_str):
def run_eval(args):
warnings.warn(
"sglang.test.few_shot_gsm8k is deprecated. "
"Use sglang.test.run_eval with eval_name='gsm8k' instead.",
DeprecationWarning,
stacklevel=2,
)
# Select backend
set_default_backend(RuntimeEndpoint(normalize_base_url(args.host, args.port)))
python/sglang/test/few_shot_gsm8k_engine.py
+14
View File
@@ -1,8 +1,16 @@
"""
.. deprecated::
This module is deprecated. Use ``sglang.test.run_eval`` with
``eval_name="gsm8k"`` instead, which routes through the unified
Chat API evaluation framework with dump_metric support.
"""
import argparse
import ast
import asyncio
import re
import time
import warnings
from typing import Optional
import numpy as np
@@ -49,6 +57,12 @@ async def concurrent_generate(engine, prompts, sampling_param):
def run_eval(args):
warnings.warn(
"sglang.test.few_shot_gsm8k_engine is deprecated. "
"Use sglang.test.run_eval with eval_name='gsm8k' instead.",
DeprecationWarning,
stacklevel=2,
)
# Select backend
engine = sgl.Engine(model_path=args.model_path, log_level="error")
python/sglang/test/kits/eval_accuracy_kit.py
+18 -12
View File
@@ -3,7 +3,6 @@ from typing import Optional
import requests
from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
@@ -19,17 +18,20 @@ def _check_accept_length(test_case, base_url, threshold):
class GSM8KMixin:
"""Mixin for few-shot GSM8K evaluation.
"""Mixin for GSM8K evaluation via OpenAI Chat API.
Required attributes on the test class:
base_url: str
gsm8k_accuracy_thres: float
Optional attributes:
model: str (if not set, auto-detected from server)
"""
gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET
gsm8k_accept_length_thres: Optional[float] = None
gsm8k_num_questions: int = 200
gsm8k_parallel: int = 128
gsm8k_num_threads: int = 128
def test_gsm8k(self):
assert (
@@ -39,17 +41,21 @@ class GSM8KMixin:
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=self.gsm8k_num_questions,
max_new_tokens=512,
parallel=self.gsm8k_parallel,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=self.gsm8k_num_questions,
num_threads=self.gsm8k_num_threads,
)
metrics = run_eval_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres)
if is_in_ci():
write_github_step_summary(f"### test_gsm8k\n{metrics['score']=:.4f}\n")
self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
if self.gsm8k_accept_length_thres is not None:
_check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres)
python/sglang/test/run_eval.py
+8 -3
View File
@@ -62,7 +62,7 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
extra_body[param_name] = value
common_kwargs = dict(
model=args.model,
model=getattr(args, "model", None),
max_tokens=getattr(args, "max_tokens", 2048),
top_p=getattr(args, "top_p", 1.0),
base_url=base_url,
@@ -71,7 +71,12 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
api_mode = getattr(args, "api", "chat")
if api_mode == "completion":
sampler = CompletionSampler(**common_kwargs)
# Default stop tokens for completion API (matches few_shot_gsm8k behavior)
stop = getattr(args, "stop", ["Question", "Assistant:", "<|separator|>"])
sampler = CompletionSampler(
**common_kwargs,
stop=stop,
)
else:
sampler = ChatCompletionSampler(
**common_kwargs,
@@ -143,7 +148,7 @@ def run_eval(args):
categories = args.categories.split(",") if args.categories else None
eval_obj = LongBenchV2Eval(
model=args.model,
model=getattr(args, "model", None),
data_source=data_source,
num_examples=args.num_examples,
num_threads=args.num_threads,
python/sglang/test/server_fixtures/disaggregation_fixture.py
+1
View File
@@ -32,6 +32,7 @@ class PDDisaggregationServerBase(CustomTestCase):
cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}"
cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}"
cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}"
cls.base_url = cls.lb_url
print(
f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} {cls.bootstrap_port=}"
)
python/sglang/test/simple_eval_common.py
+4 -1
View File
@@ -185,6 +185,7 @@ class CompletionSampler(SamplerBase):
temperature: float = 0.0,
top_p: float = 1.0,
max_tokens: int = 2048,
stop: Optional[List[str]] = None,
):
self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
@@ -195,9 +196,10 @@ class CompletionSampler(SamplerBase):
self.temperature = temperature
self.top_p = top_p
self.max_tokens = max_tokens
self.stop = stop
self._completion_tokens: list[int] = []
print(
f"CompletionSampler initialized with {self.model=} {self.temperature=} {self.max_tokens=}"
f"CompletionSampler initialized with {self.model=} {self.temperature=} {self.max_tokens=} {self.stop=}"
)
def _pack_message(self, role: str, content: Any):
@@ -219,6 +221,7 @@ class CompletionSampler(SamplerBase):
temperature=self.temperature,
top_p=self.top_p,
max_tokens=self.max_tokens,
stop=self.stop,
)
if response.usage and response.usage.completion_tokens is not None:
self._completion_tokens.append(response.usage.completion_tokens)