minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Test] Move gpqa and aime25 onto sgl-eval, drop unused eval paths (#36979)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-29 17:36:13 -07:00
committed by GitHub
parent 032fe91bf1
commit 9a489f8d2f
15 changed files with 18 additions and 1841 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/gpt_oss_common.py
+3 -2
View File
@@ -129,11 +129,12 @@ class BaseTestGptOss(CustomTestCase):
num_examples=198,
# use enough threads to allow parallelism
num_threads=198,
# sgl-eval's gpqa defaults to n_repeats=8.
repeat=1,
# TODO 4k is still not enough, we need e.g. 64k token, but that is super slow
# otherwise a lot of questions are not answered
max_tokens=4096,
# simple-evals by default use 0.5 and is better than 0.0 temperature
# but here for reproducibility, we use 0.1
# Arbitrary; non-zero so a tier is not scored on one greedy path.
temperature=0.1,
reasoning_effort=reasoning_effort,
)
python/sglang/test/run_eval.py
+8 -62
View File
@@ -151,6 +151,10 @@ def _run_sgl_eval(eval_name, args) -> dict:
# Unset by default in sgl-eval; only a sampling caller (temperature > 0) needs it.
if getattr(args, "seed", None) is not None:
cmd += ["--seed", str(args.seed)]
# gpt-oss grades one score per effort tier, so dropping this collapses every
# tier onto the served model's default.
if getattr(args, "reasoning_effort", None) is not None:
cmd += ["--reasoning-effort", str(args.reasoning_effort)]
if getattr(args, "repeat", None) is not None:
cmd += ["--n-repeats", str(args.repeat)]
# Bound generation length so long-reasoning models don't stall the eval.
@@ -265,52 +269,18 @@ def run_eval(args):
# caller's threshold has to be measured against it, not inherited.
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
return _run_sgl_eval("mmlu", args)
elif args.eval_name == "math":
from sglang.test.simple_eval_math import MathEval
equality_checker = ChatCompletionSampler(model="gpt-4-turbo")
filename = (
"https://openaipublic.blob.core.windows.net/simple-evals/math_test.csv"
)
eval_obj = MathEval(
filename, equality_checker, args.num_examples, args.num_threads
)
elif args.eval_name == "mgsm":
from sglang.test.simple_eval_mgsm import MGSMEval
eval_obj = MGSMEval(args.num_examples, args.num_threads)
elif args.eval_name == "mgsm_en":
from sglang.test.simple_eval_mgsm import MGSMEval
eval_obj = MGSMEval(args.num_examples, args.num_threads, languages=["en"])
elif args.eval_name == "gpqa":
from sglang.test.simple_eval_gpqa import GPQAEval
filename = (
"https://openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv"
)
eval_obj = GPQAEval(filename, args.num_examples, args.num_threads)
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
return _run_sgl_eval("gpqa", args)
elif args.eval_name == "humaneval":
from sglang.test.simple_eval_humaneval import HumanEval
eval_obj = HumanEval(args.num_examples, args.num_threads)
elif args.eval_name == "longbench_v2":
from sglang.test.simple_eval_longbench_v2 import LongBenchV2Eval
# Default to HuggingFace dataset, can be overridden with --dataset-path
data_source = args.dataset_path
categories = args.categories.split(",") if args.categories else None
eval_obj = LongBenchV2Eval(
model=getattr(args, "model", None),
data_source=data_source,
num_examples=args.num_examples,
num_threads=args.num_threads,
categories=categories,
max_context_length=getattr(args, "max_context_length", None),
min_context_length=getattr(args, "min_context_length", None),
)
elif args.eval_name == "mmmu":
# VLM MMMU evaluation with fixed 100 examples by default
from sglang.test.simple_eval_mmmu_vlm import MMMUVLMEval
@@ -328,9 +298,7 @@ def run_eval(args):
# simple_eval implementation to fall back to.
return _run_sgl_eval("mmmu_pro_vision", args)
elif args.eval_name == "aime25":
from sglang.test.simple_eval_aime25 import AIME25Eval
eval_obj = AIME25Eval(args.num_examples, args.num_threads)
return _run_sgl_eval("aime25", args)
elif args.eval_name == "gsm8k":
if getattr(args, "api", None) == "sgl_eval":
# Only the nightly correctness eval opts into sgl-eval (zero-shot
@@ -524,28 +492,6 @@ if __name__ == "__main__":
)
# LongBench-v2 specific arguments
parser.add_argument(
"--dataset-path",
type=str,
default="THUDM/LongBench-v2",
help="Path to dataset file or HuggingFace dataset name for LongBench-v2",
)
parser.add_argument(
"--categories",
type=str,
default=None,
help="Comma-separated list of categories to evaluate for LongBench-v2",
)
parser.add_argument(
"--max-context-length",
type=int,
help="Maximum context length in characters for LongBench-v2",
)
parser.add_argument(
"--min-context-length",
type=int,
help="Minimum context length in characters for LongBench-v2",
)
parser.add_argument(
"--num-shots",
type=int,
python/sglang/test/simple_eval_aime25.py
-124
View File
@@ -1,124 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
AIME 2025 - American Invitational Mathematics Examination 2025
Dataset: opencompass/AIME2025
https://huggingface.co/datasets/opencompass/AIME2025
The American Invitational Mathematics Examination (AIME) is a challenging
competition math exam. All answers are integers from 000 to 999.
"""
import re
from typing import Optional
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
)
QUERY_TEMPLATE = """
Solve the following AIME (American Invitational Mathematics Examination) problem step by step. The last line of your response should be of the form Answer: $ANSWER (without quotes) where $ANSWER is the answer to the problem.
Note: AIME answers are always integers from 000 to 999 (inclusive). If you get a non-integer answer, you likely made a computational error.
{question}
Remember to put your answer on its own line after "Answer:", and express your answer as an integer from 000 to 999.
""".strip()
def normalize_aime_answer(answer: str) -> Optional[str]:
"""
Normalize AIME answer to standard format.
AIME answers are integers from 000 to 999.
"""
if answer is None:
return None
# Remove whitespace and convert to string
answer = str(answer).strip()
# Try to extract integer from answer
try:
# Handle various formats like "42", "042", "42.0", etc.
num = int(float(answer))
if 0 <= num <= 999:
return str(num)
except (ValueError, TypeError):
pass
return answer
class AIME25Eval(Eval):
def __init__(
self,
num_examples: Optional[int],
num_threads: int,
):
try:
from datasets import load_dataset
except ImportError:
raise ImportError(
"The 'datasets' package is required for AIME25 evaluation. "
"Please install it with: pip install datasets"
)
# Load AIME 2025 dataset from HuggingFace
dataset1 = load_dataset("opencompass/AIME2025", "AIME2025-I", split="test")
dataset2 = load_dataset("opencompass/AIME2025", "AIME2025-II", split="test")
examples1 = [
{"question": row["question"], "answer": str(row["answer"])}
for row in dataset1
]
examples2 = [
{"question": row["question"], "answer": str(row["answer"])}
for row in dataset2
]
examples = examples1 + examples2
if num_examples:
examples = examples[: min(num_examples, len(examples))]
self.examples = examples
self.num_threads = num_threads
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
prompt_messages = [
sampler._pack_message(content=QUERY_TEMPLATE.format(**row), role="user")
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
# Extract answer from response
match = re.search(ANSWER_PATTERN, response_text)
extracted_answer = match.group(1).strip() if match else None
# Normalize both answers for comparison
normalized_extracted = normalize_aime_answer(extracted_answer)
normalized_correct = normalize_aime_answer(row["answer"])
# Score: 1.0 if correct, 0.0 otherwise
score = 1.0 if normalized_extracted == normalized_correct else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(
html=html,
score=score,
convo=convo,
metrics={"chars": len(response_text)},
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
python/sglang/test/simple_eval_gpqa.py
-97
View File
@@ -1,97 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
GPQA: A Graduate-Level Google-Proof Q&A Benchmark
David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, Samuel R. Bowman
https://arxiv.org/abs/2311.12022
"""
import random
import re
from typing import Optional
import pandas
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
format_multichoice_question,
)
class GPQAEval(Eval):
def __init__(
self,
filename: str,
num_examples: Optional[int],
num_threads: int,
n_repeats: int = 1,
):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
rng = random.Random(0)
if num_examples:
assert n_repeats == 1, "n_repeats only supported for num_examples"
examples = rng.sample(examples, num_examples)
examples = examples * n_repeats
examples = [
example | {"permutation": rng.sample(range(4), 4)} for example in examples
]
self.examples = examples
self.n_repeats = n_repeats
self.num_threads = num_threads
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
choices = [
row["Correct Answer"],
row["Incorrect Answer 1"],
row["Incorrect Answer 2"],
row["Incorrect Answer 3"],
]
choices = [choices[i] for i in row["permutation"]]
correct_index = choices.index(row["Correct Answer"])
correct_answer = "ABCD"[correct_index]
choices_dict = dict(
A=choices[0],
B=choices[1],
C=choices[2],
D=choices[3],
Question=row["Question"],
)
prompt_messages = [
sampler._pack_message(
content=format_multichoice_question(choices_dict), role="user"
)
]
response_text = sampler(prompt_messages)
if response_text is None:
response_text = ""
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
extracted_answer = match.group(1) if match else None
score = 1.0 if extracted_answer == correct_answer else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=correct_answer,
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(
html=html,
score=score,
convo=convo,
metrics={"chars": len(response_text)},
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
python/sglang/test/simple_eval_longbench_v2.py
-344
View File
@@ -1,344 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-Context Multitasks
Yushi Bai, Shangqing Tu, Jiajie Zhang, Hao Peng, Xiaozhi Wang, Xin Lv, Shulin Cao, Jiazheng Xu, Lei Hou, Yuxiao Dong, Jie Tang, Juanzi Li
https://arxiv.org/abs/2412.15204
"""
import csv
import json
import os
import re
from typing import Any, Dict, List, Optional
from transformers import AutoTokenizer
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
)
# LongBench-v2 task categories
TASK_CATEGORIES = {
"single_document_qa",
"multi_document_qa",
"long_in_context_learning",
"long_dialogue_history",
"code_repo_understanding",
"long_structured_data",
}
DEFAULT_DATASET = "THUDM/LongBench-v2"
DEFAULT_DATASET_SPLIT = "train"
def format_longbench_v2_question(row: dict) -> str:
"""Format a LongBench-v2 question using the official template."""
context = row.get("context", "")
question = row.get("question", "")
# Handle both standard format (A, B, C, D) and alternative format (choices list)
if "choices" in row:
choices = row["choices"]
choice_A = choices[0] if len(choices) > 0 else ""
choice_B = choices[1] if len(choices) > 1 else ""
choice_C = choices[2] if len(choices) > 2 else ""
choice_D = choices[3] if len(choices) > 3 else ""
else:
choice_A = row.get("A", row.get("choice_A", ""))
choice_B = row.get("B", row.get("choice_B", ""))
choice_C = row.get("C", row.get("choice_C", ""))
choice_D = row.get("D", row.get("choice_D", ""))
# Official LongBench-v2 template
prompt = f"""
Please read the following text and answer the question below.
<text>
{context.strip()}
</text>
What is the correct answer to this question: {question.strip()}
Choices:
(A) {choice_A.strip()}
(B) {choice_B.strip()}
(C) {choice_C.strip()}
(D) {choice_D.strip()}
Format your response as follows: "The correct answer is (insert answer here)"."""
return prompt
def extract_longbench_v2_answer(response: str) -> Optional[str]:
"""Extract answer from model response using official LongBench-v2 method."""
response = response.replace("*", "")
# First try: "The correct answer is (A)"
match = re.search(r"The correct answer is \(([A-D])\)", response, re.IGNORECASE)
if match:
return match.group(1).upper()
# Second try: "The correct answer is A"
match = re.search(r"The correct answer is ([A-D])", response, re.IGNORECASE)
if match:
return match.group(1).upper()
# Fallback: Standard SGLang multichoice pattern
match = re.search(ANSWER_PATTERN_MULTICHOICE, response)
if match:
return match.group(1).upper()
# Generic fallback when model says "answer is A"
match = re.search(r"answer\s+is\s*\(?([A-D])\)?", response, re.IGNORECASE)
if match:
return match.group(1).upper()
return None
class LongBenchV2Eval(Eval):
"""
Evaluation utility for LongBench-v2 dataset.
LongBench-v2 is designed to assess the ability of LLMs to handle long-context problems
requiring deep understanding and reasoning across real-world multitasks.
"""
def __init__(
self,
model: str = None,
data_source: str = DEFAULT_DATASET,
num_examples: Optional[int] = None,
num_threads: int = 1,
n_repeats: int = 1,
categories: Optional[List[str]] = None,
max_context_length: Optional[int] = None,
min_context_length: Optional[int] = None,
):
"""
Initialize LongBench-v2 evaluation.
Args:
data_source: HuggingFace dataset name, local file path (CSV/JSON)
num_examples: Number of examples to evaluate (None for all)
num_threads: Number of threads for parallel processing
n_repeats: Number of times to repeat evaluation for error bars
categories: List of task categories to include (None for all)
max_context_length: Maximum context length in characters
min_context_length: Minimum context length in characters
"""
self.tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
self.min_context_length = min_context_length
self.max_context_length = max_context_length
# Load dataset based on data source type
examples = self._load_dataset(data_source)
# Apply filtering
if categories:
examples = [ex for ex in examples if ex.get("category") in categories]
# Sample examples if specified
if num_examples:
assert n_repeats == 1, "n_repeats only supported when not sampling examples"
examples = examples[: min(num_examples, len(examples))]
# Repeat examples for multiple runs
examples = examples * n_repeats
if not examples:
raise ValueError(
"No examples available for LongBench-v2 evaluation after filtering"
)
self.examples = examples
self.n_repeats = n_repeats
self.num_threads = num_threads
print(f"Loaded {len(self.examples)} examples from LongBench-v2")
if categories:
print(f"Filtered to categories: {categories}")
if min_context_length or max_context_length:
print(
f"Context length filter: {min_context_length}-{max_context_length} characters"
)
def _load_dataset(self, data_source: str) -> List[Dict[str, Any]]:
"""Load dataset from HuggingFace hub or local files."""
if not data_source:
data_source = DEFAULT_DATASET
if os.path.exists(data_source):
raw_examples = self._load_local_file(data_source)
else:
raw_examples = self._load_hf_dataset(data_source)
return [self._normalize_example(example) for example in raw_examples]
def _load_local_file(self, path: str) -> List[Dict[str, Any]]:
"""Load examples from a local CSV/JSON/JSONL file."""
suffix = os.path.splitext(path)[1].lower()
if suffix in {".json", ".jsonl"}:
with open(path, "r", encoding="utf-8") as fh:
if suffix == ".jsonl":
data = [json.loads(line) for line in fh if line.strip()]
else:
data = json.load(fh)
elif suffix == ".csv":
with open(path, "r", encoding="utf-8") as fh:
reader = csv.DictReader(fh)
data = list(reader)
else:
# Try JSON, then CSV as fallback
try:
with open(path, "r", encoding="utf-8") as fh:
data = json.load(fh)
except json.JSONDecodeError:
with open(path, "r", encoding="utf-8") as fh:
reader = csv.DictReader(fh)
data = list(reader)
if isinstance(data, dict):
data = data.get("data", [])
if not isinstance(data, list):
raise ValueError("Expected list of examples from local file")
return data
def _load_hf_dataset(self, identifier: str) -> List[Dict[str, Any]]:
"""Load the dataset from HuggingFace Hub."""
parts = identifier.split(":", maxsplit=1)
dataset_name = parts[0]
split = parts[1] if len(parts) == 2 else DEFAULT_DATASET_SPLIT
try:
from datasets import load_dataset # type: ignore
except ImportError as exc:
raise ImportError(
"Please install the 'datasets' package to load LongBench-v2 from HuggingFace: pip install datasets"
) from exc
dataset = load_dataset(dataset_name, split=split)
return [dict(row) for row in dataset]
def _normalize_example(self, example: Dict[str, Any]) -> Dict[str, Any]:
"""Ensure each example exposes the expected keys."""
normalized = dict(example)
for letter in ["A", "B", "C", "D"]:
choice_key = f"choice_{letter}"
if letter not in normalized and choice_key in normalized:
normalized[letter] = normalized[choice_key]
if "category" not in normalized and "domain" in normalized:
normalized["category"] = normalized["domain"]
answer = normalized.get("answer")
if isinstance(answer, str):
normalized["answer"] = answer.strip().upper()
elif isinstance(answer, int) and 0 <= answer < 4:
normalized["answer"] = ["A", "B", "C", "D"][answer]
return normalized
def _check_context_length(
self,
formatted_question: str,
tokenizer: AutoTokenizer,
min_length: Optional[int],
max_length: Optional[int],
) -> bool:
"""Filter examples by context length measured in characters."""
input_ids = tokenizer.encode(formatted_question)
context_length = len(input_ids)
if min_length is not None and context_length < min_length:
return False
if max_length is not None and context_length > max_length:
return False
return True
def __call__(self, sampler: SamplerBase) -> EvalResult:
"""Run the evaluation."""
def fn(row: dict):
# Format the question using official template
formatted_question = format_longbench_v2_question(row)
if self.min_context_length or self.max_context_length:
if not self._check_context_length(
formatted_question,
self.tokenizer,
self.min_context_length,
self.max_context_length,
):
# Skip this example
return None
prompt_messages = [
sampler._pack_message(content=formatted_question, role="user")
]
# Get model response
response_text = sampler(prompt_messages)
if response_text is None:
response_text = ""
# Extract answer using official method
extracted_answer = extract_longbench_v2_answer(response_text)
# Get correct answer
correct_answer = row.get("answer", "")
if isinstance(correct_answer, str):
correct_answer = correct_answer.strip().upper()
elif isinstance(correct_answer, int) and 0 <= correct_answer < 4:
correct_answer = ["A", "B", "C", "D"][correct_answer]
# Calculate score
score = 1.0 if extracted_answer == correct_answer else 0.0
# Generate HTML report
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=correct_answer,
extracted_answer=extracted_answer,
)
# Build conversation
convo = prompt_messages + [dict(content=response_text, role="assistant")]
# Prepare metrics
metrics = {"chars": len(response_text)}
# Add category-specific metrics
category = row.get("category", row.get("domain", "unknown"))
if category in TASK_CATEGORIES:
metrics[category] = score
difficulty = row.get("difficulty")
if isinstance(difficulty, str) and difficulty:
metrics[f"difficulty_{difficulty.lower()}"] = score
return SingleEvalResult(
html=html,
score=score,
convo=convo,
metrics=metrics,
)
# Run evaluation with progress tracking
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
python/sglang/test/simple_eval_math.py
-77
View File
@@ -1,77 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
Measuring Mathematical Problem Solving With the MATH Dataset
Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt
https://arxiv.org/abs/2103.03874
"""
import random
import re
from typing import Optional
import pandas
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
check_equality,
)
QUERY_TEMPLATE = """
Solve the following math problem step by step. The last line of your response should be of the form Answer: $ANSWER (without quotes) where $ANSWER is the answer to the problem.
{Question}
Remember to put your answer on its own line after "Answer:", and you do not need to use a \\boxed command.
""".strip()
class MathEval(Eval):
def __init__(
self,
filename: str,
equality_checker: SamplerBase,
num_examples: Optional[int],
num_threads: int,
):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
if num_examples:
examples = random.Random(0).sample(examples, num_examples)
self.examples = examples
self.equality_checker = equality_checker
self.num_threads = num_threads
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
prompt_messages = [
sampler._pack_message(content=QUERY_TEMPLATE.format(**row), role="user")
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
match = re.search(ANSWER_PATTERN, response_text)
extracted_answer = match.group(1) if match else None
score = float(
check_equality(self.equality_checker, row["Answer"], extracted_answer)
)
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["Answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(html=html, score=score, convo=convo)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)