[Test] Remove dead eval modules and point GSM8K/MMLU docs to sgl-eval (#41215)

This commit is contained in:
Liangsheng Yin
2026-09-25 12:50:31 -07:00
committed by GitHub
parent ac62ddf164
commit f337f01b80
80 changed files with 394 additions and 2388 deletions
@@ -9,7 +9,7 @@ _DESCRIPTION = """Compare and find differences to benchmark outputs.
Supported inputs:
* The samples jsonl from `lm_eval --log_samples --output_path FOLDER_NAME`
* The output from `gsm8k/bench_sglang.py --raw-result-file FILE_NAME` (or mmlu)
* Legacy benchmark JSONL with prompt_id, prompt, output, and correct fields
"""
-122
View File
@@ -1,122 +0,0 @@
import json
import os
from sglang.test.run_eval import run_eval_once
from sglang.test.simple_eval_common import (
make_report,
set_ulimit,
)
def run_eval(args):
# Lazy import to avoid circular dependency with test_utils
from sglang.test.test_utils import dump_metric
set_ulimit()
if "OPENAI_API_KEY" not in os.environ:
os.environ["OPENAI_API_KEY"] = "EMPTY"
base_url = (
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
if args.eval_name == "mmlu":
from sglang.test.ascend.simple_eval_mmlu import MMLUEval
filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
eval_obj = MMLUEval(
filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0)
)
else:
raise ValueError(f"Invalid eval name: {args.eval_name}")
if getattr(args, "repeat", 1) == 1:
result, latency, sampler = run_eval_once(args, base_url, eval_obj)
metrics = result.metrics | {"score": result.score}
metrics["latency"] = latency
print(f"Total latency: {latency:.3f} s")
print(f"Score: {metrics['score']:.3f}")
# Compute output throughput from accumulated completion tokens
total_completion_tokens = sum(sampler._completion_tokens)
if total_completion_tokens > 0 and latency > 0:
metrics["output_throughput"] = total_completion_tokens / latency
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
# Report metrics to unified collection framework
dump_metric(
f"{args.eval_name}_score",
metrics["score"],
labels={"model": sampler.model, "eval": args.eval_name},
)
dump_metric(
f"{args.eval_name}_latency",
latency,
labels={"model": sampler.model, "eval": args.eval_name},
)
else:
from concurrent.futures import ThreadPoolExecutor
executor = ThreadPoolExecutor(max_workers=args.repeat)
futures = [
executor.submit(run_eval_once, args, base_url, eval_obj)
for _ in range(args.repeat)
]
scores_repeat = []
latencies = []
total_completion_tokens = 0
for f in futures:
result, latency, sampler = f.result()
scores_repeat.append(result.score)
latencies.append(latency)
total_completion_tokens += sum(sampler._completion_tokens)
mean_score = sum(scores_repeat) / len(scores_repeat)
mean_latency = sum(latencies) / len(latencies)
total_latency = sum(latencies)
scores_repeat = [f"{s:.3f}" for s in scores_repeat]
print("=" * 20)
print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
print(f"Scores: {scores_repeat}")
print(f"Mean latency: {mean_latency:.3f} s")
print("=" * 20)
metrics = result.metrics | {"scores": scores_repeat}
metrics = metrics | {"mean_score": mean_score}
metrics["latency"] = mean_latency
if total_completion_tokens > 0 and total_latency > 0:
metrics["output_throughput"] = total_completion_tokens / total_latency
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
# Report metrics to unified collection framework
dump_metric(
f"{args.eval_name}_mean_score",
mean_score,
labels={
"model": sampler.model,
"eval": args.eval_name,
"repeat": args.repeat,
},
)
executor.shutdown()
# Dump reports
file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
report_filename = f"/tmp/{file_stem}.html"
print(f"Writing report to {report_filename}")
with open(report_filename, "w") as fh:
fh.write(make_report(result))
print(metrics)
result_filename = f"/tmp/{file_stem}.json"
with open(result_filename, "w") as f:
f.write(json.dumps(metrics, indent=2))
print(f"Writing results to {result_filename}")
if getattr(args, "return_latency", False):
return metrics, latency
return metrics
@@ -1,125 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
Measuring Massive Multitask Language Understanding
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
https://arxiv.org/abs/2009.03300
"""
import random
import re
from typing import Optional
import pandas
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
format_multichoice_question,
)
from sglang.test.simple_eval_mmlu import subject2category
def format_multichoice_question_example(row):
return QUERY_TEMPLATE_MULTICHOICE.format(**row)
QUERY_TEMPLATE_MULTICHOICE = """
Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
{Question}
A) {A}
B) {B}
C) {C}
D) {D}
""".strip()
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """
Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD.
"""
TEMPLATE_MULTICHOICE_EXAMPLE = """
Example question:
{Question}
A {A}
B {B}
C {C}
D {D}
The last line of your response should be
Answer: {Answer}
""".strip()
class MMLUEval(Eval):
def __init__(
self,
filename: str,
num_examples: Optional[int],
num_threads: int,
num_shots: int,
):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
if num_shots:
example_questions = "".join(
format_multichoice_question_example(row) + "\n\n"
for row in examples[:num_shots]
)
self.template = (
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN
+ example_questions
+ QUERY_TEMPLATE_MULTICHOICE
)
examples = examples[num_shots:]
if num_examples:
examples = random.Random(0).sample(examples, num_examples)
self.examples = examples
self.num_threads = num_threads
self.num_shots = num_shots
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
if self.num_shots:
prompt_messages = [
sampler._pack_message(
content=self.template.format(**row), role="user"
)
]
else:
prompt_messages = [
sampler._pack_message(
content=format_multichoice_question(row), role="user"
)
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
extracted_answer = match.group(1) if match else None
score = 1.0 if extracted_answer == row["Answer"] else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["Answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
category = subject2category.get(row["Subject"], "other")
return SingleEvalResult(
html=html, score=score, metrics={category: score}, convo=convo
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
+1 -1
View File
@@ -15,7 +15,7 @@ class TestMMLU:
"server": getattr(
self, "server_cmd", subprocess.list2cmdline(map(str, self.other_args))
),
"client": "simple_eval_mmlu",
"client": "sgl-eval",
"accuracy_threshold": getattr(self, "accuracy_mmlu", "N/A"),
}
-157
View File
@@ -1,157 +0,0 @@
"""
.. deprecated::
This module is deprecated. Use ``sglang.test.run_eval`` with
``eval_name="gsm8k"`` instead, which routes through the unified
Chat API evaluation framework with dump_metric support.
"""
import argparse
import ast
import asyncio
import re
import time
import warnings
from typing import Optional
import numpy as np
import sglang as sgl
from sglang.srt.utils import get_or_create_event_loop
from sglang.utils import download_and_cache_file, read_jsonl
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
async def concurrent_generate(engine, prompts, sampling_param):
tasks = []
for prompt in prompts:
tasks.append(asyncio.create_task(engine.async_generate(prompt, sampling_param)))
outputs = await asyncio.gather(*tasks)
return outputs
def run_eval(args):
warnings.warn(
"sglang.test.few_shot_gsm8k_engine is deprecated. "
"Use sglang.test.run_eval with eval_name='gsm8k' instead.",
DeprecationWarning,
stacklevel=2,
)
# Select backend
engine = sgl.Engine(model_path=args.model_path, log_level="error")
if args.local_data_path is None:
# Read data
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
filename = download_and_cache_file(url)
else:
filename = args.local_data_path
lines = list(read_jsonl(filename))
# Construct prompts
num_questions = args.num_questions
num_shots = args.num_shots
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
# construct the prompts
prompts = []
for i, arg in enumerate(arguments):
q = arg["question"]
prompt = few_shot_examples + q
prompts.append(prompt)
sampling_param = {
"stop": ["Question", "Assistant:", "<|separator|>"],
"max_new_tokens": 512,
"temperature": 0,
}
# Run requests
tic = time.perf_counter()
loop = get_or_create_event_loop()
outputs = loop.run_until_complete(
concurrent_generate(engine, prompts, sampling_param)
)
# End requests
latency = time.perf_counter() - tic
# Shutdown the engine
engine.shutdown()
# Parse output
preds = []
for output in outputs:
preds.append(get_answer_value(output["text"]))
# Compute accuracy
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
# Compute speed
num_output_tokens = sum(
output["meta_info"]["completion_tokens"] for output in outputs
)
output_throughput = num_output_tokens / latency
# Print results
print(f"Accuracy: {acc:.3f}")
print(f"Invalid: {invalid:.3f}")
print(f"Latency: {latency:.3f} s")
print(f"Output throughput: {output_throughput:.3f} token/s")
return {
"accuracy": acc,
"latency": latency,
"output_throughput": output_throughput,
}
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--model-path", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct"
)
parser.add_argument("--local-data-path", type=Optional[str], default=None)
parser.add_argument("--num-shots", type=int, default=5)
parser.add_argument("--num-questions", type=int, default=200)
args = parser.parse_args()
metrics = run_eval(args)
-1
View File
@@ -267,7 +267,6 @@ def run_eval(args):
if args.eval_name == "mmlu":
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
return _run_sgl_eval("mmlu", args)
elif args.eval_name == "mgsm_en":
from sglang.test.simple_eval_mgsm import MGSMEval
-125
View File
@@ -1,125 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
Measuring Massive Multitask Language Understanding
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
https://arxiv.org/abs/2009.03300
"""
import random
import re
from typing import Optional
import pandas
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
format_multichoice_question,
)
subject2category = {
"abstract_algebra": "stem",
"anatomy": "other",
"astronomy": "stem",
"business_ethics": "other",
"clinical_knowledge": "other",
"college_biology": "stem",
"college_chemistry": "stem",
"college_computer_science": "stem",
"college_mathematics": "stem",
"college_medicine": "other",
"college_physics": "stem",
"computer_security": "stem",
"conceptual_physics": "stem",
"econometrics": "social_sciences",
"electrical_engineering": "stem",
"elementary_mathematics": "stem",
"formal_logic": "humanities",
"global_facts": "other",
"high_school_biology": "stem",
"high_school_chemistry": "stem",
"high_school_computer_science": "stem",
"high_school_european_history": "humanities",
"high_school_geography": "social_sciences",
"high_school_government_and_politics": "social_sciences",
"high_school_macroeconomics": "social_sciences",
"high_school_mathematics": "stem",
"high_school_microeconomics": "social_sciences",
"high_school_physics": "stem",
"high_school_psychology": "social_sciences",
"high_school_statistics": "stem",
"high_school_us_history": "humanities",
"high_school_world_history": "humanities",
"human_aging": "other",
"human_sexuality": "social_sciences",
"international_law": "humanities",
"jurisprudence": "humanities",
"logical_fallacies": "humanities",
"machine_learning": "stem",
"management": "other",
"marketing": "other",
"medical_genetics": "other",
"miscellaneous": "other",
"moral_disputes": "humanities",
"moral_scenarios": "humanities",
"nutrition": "other",
"philosophy": "humanities",
"prehistory": "humanities",
"professional_accounting": "other",
"professional_law": "humanities",
"professional_medicine": "other",
"professional_psychology": "social_sciences",
"public_relations": "social_sciences",
"security_studies": "social_sciences",
"sociology": "social_sciences",
"us_foreign_policy": "social_sciences",
"virology": "other",
"world_religions": "humanities",
}
class MMLUEval(Eval):
def __init__(self, filename: str, num_examples: Optional[int], num_threads: int):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
if num_examples:
examples = random.Random(0).sample(examples, num_examples)
self.examples = examples
self.num_threads = num_threads
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
prompt_messages = [
sampler._pack_message(
content=format_multichoice_question(row), role="user"
)
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
extracted_answer = match.group(1) if match else None
score = 1.0 if extracted_answer == row["Answer"] else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["Answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
category = subject2category.get(row["Subject"], "other")
return SingleEvalResult(
html=html, score=score, metrics={category: score}, convo=convo
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
-32
View File
@@ -2409,38 +2409,6 @@ class CustomTestCase(unittest.TestCase):
)
def dump_bench_raw_result(
path: str,
states,
preds,
labels,
):
if not path:
return
rows = []
for i in range(len(states)):
state = states[i]
output = state["answer"]
prompt = _ensure_remove_suffix(state.text(), output)
rows.append(
dict(
prompt_id=i,
prompt=prompt,
output=output,
correct=bool(preds[i] == labels[i]),
)
)
print(f"BenchRawResultDumper save results to {path}")
Path(path).write_text("\n".join(json.dumps(row) for row in rows))
def _ensure_remove_suffix(text: str, suffix: str):
assert text.endswith(suffix)
return text.removesuffix(suffix)
class ModelLaunchSettings:
def __init__(
self,