[Test] Remove dead eval modules and point GSM8K/MMLU docs to sgl-eval (#41215)
This commit is contained in:
@@ -9,7 +9,7 @@ _DESCRIPTION = """Compare and find differences to benchmark outputs.
|
||||
|
||||
Supported inputs:
|
||||
* The samples jsonl from `lm_eval --log_samples --output_path FOLDER_NAME`
|
||||
* The output from `gsm8k/bench_sglang.py --raw-result-file FILE_NAME` (or mmlu)
|
||||
* Legacy benchmark JSONL with prompt_id, prompt, output, and correct fields
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from sglang.test.run_eval import run_eval_once
|
||||
from sglang.test.simple_eval_common import (
|
||||
make_report,
|
||||
set_ulimit,
|
||||
)
|
||||
|
||||
|
||||
def run_eval(args):
|
||||
# Lazy import to avoid circular dependency with test_utils
|
||||
from sglang.test.test_utils import dump_metric
|
||||
|
||||
set_ulimit()
|
||||
|
||||
if "OPENAI_API_KEY" not in os.environ:
|
||||
os.environ["OPENAI_API_KEY"] = "EMPTY"
|
||||
|
||||
base_url = (
|
||||
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
|
||||
)
|
||||
|
||||
if args.eval_name == "mmlu":
|
||||
from sglang.test.ascend.simple_eval_mmlu import MMLUEval
|
||||
|
||||
filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
|
||||
eval_obj = MMLUEval(
|
||||
filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0)
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid eval name: {args.eval_name}")
|
||||
|
||||
if getattr(args, "repeat", 1) == 1:
|
||||
result, latency, sampler = run_eval_once(args, base_url, eval_obj)
|
||||
metrics = result.metrics | {"score": result.score}
|
||||
metrics["latency"] = latency
|
||||
print(f"Total latency: {latency:.3f} s")
|
||||
print(f"Score: {metrics['score']:.3f}")
|
||||
|
||||
# Compute output throughput from accumulated completion tokens
|
||||
total_completion_tokens = sum(sampler._completion_tokens)
|
||||
if total_completion_tokens > 0 and latency > 0:
|
||||
metrics["output_throughput"] = total_completion_tokens / latency
|
||||
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
|
||||
|
||||
# Report metrics to unified collection framework
|
||||
dump_metric(
|
||||
f"{args.eval_name}_score",
|
||||
metrics["score"],
|
||||
labels={"model": sampler.model, "eval": args.eval_name},
|
||||
)
|
||||
dump_metric(
|
||||
f"{args.eval_name}_latency",
|
||||
latency,
|
||||
labels={"model": sampler.model, "eval": args.eval_name},
|
||||
)
|
||||
else:
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
executor = ThreadPoolExecutor(max_workers=args.repeat)
|
||||
|
||||
futures = [
|
||||
executor.submit(run_eval_once, args, base_url, eval_obj)
|
||||
for _ in range(args.repeat)
|
||||
]
|
||||
|
||||
scores_repeat = []
|
||||
latencies = []
|
||||
total_completion_tokens = 0
|
||||
|
||||
for f in futures:
|
||||
result, latency, sampler = f.result()
|
||||
scores_repeat.append(result.score)
|
||||
latencies.append(latency)
|
||||
total_completion_tokens += sum(sampler._completion_tokens)
|
||||
|
||||
mean_score = sum(scores_repeat) / len(scores_repeat)
|
||||
mean_latency = sum(latencies) / len(latencies)
|
||||
total_latency = sum(latencies)
|
||||
scores_repeat = [f"{s:.3f}" for s in scores_repeat]
|
||||
print("=" * 20)
|
||||
print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
|
||||
print(f"Scores: {scores_repeat}")
|
||||
print(f"Mean latency: {mean_latency:.3f} s")
|
||||
print("=" * 20)
|
||||
metrics = result.metrics | {"scores": scores_repeat}
|
||||
metrics = metrics | {"mean_score": mean_score}
|
||||
metrics["latency"] = mean_latency
|
||||
|
||||
if total_completion_tokens > 0 and total_latency > 0:
|
||||
metrics["output_throughput"] = total_completion_tokens / total_latency
|
||||
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
|
||||
|
||||
# Report metrics to unified collection framework
|
||||
dump_metric(
|
||||
f"{args.eval_name}_mean_score",
|
||||
mean_score,
|
||||
labels={
|
||||
"model": sampler.model,
|
||||
"eval": args.eval_name,
|
||||
"repeat": args.repeat,
|
||||
},
|
||||
)
|
||||
|
||||
executor.shutdown()
|
||||
|
||||
# Dump reports
|
||||
file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
|
||||
report_filename = f"/tmp/{file_stem}.html"
|
||||
print(f"Writing report to {report_filename}")
|
||||
with open(report_filename, "w") as fh:
|
||||
fh.write(make_report(result))
|
||||
print(metrics)
|
||||
result_filename = f"/tmp/{file_stem}.json"
|
||||
with open(result_filename, "w") as f:
|
||||
f.write(json.dumps(metrics, indent=2))
|
||||
print(f"Writing results to {result_filename}")
|
||||
|
||||
if getattr(args, "return_latency", False):
|
||||
return metrics, latency
|
||||
return metrics
|
||||
@@ -1,125 +0,0 @@
|
||||
# Adapted from https://github.com/openai/simple-evals/
|
||||
|
||||
"""
|
||||
Measuring Massive Multitask Language Understanding
|
||||
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
|
||||
https://arxiv.org/abs/2009.03300
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
import pandas
|
||||
|
||||
from sglang.test import simple_eval_common as common
|
||||
from sglang.test.simple_eval_common import (
|
||||
ANSWER_PATTERN_MULTICHOICE,
|
||||
HTML_JINJA,
|
||||
Eval,
|
||||
EvalResult,
|
||||
SamplerBase,
|
||||
SingleEvalResult,
|
||||
format_multichoice_question,
|
||||
)
|
||||
from sglang.test.simple_eval_mmlu import subject2category
|
||||
|
||||
|
||||
def format_multichoice_question_example(row):
|
||||
return QUERY_TEMPLATE_MULTICHOICE.format(**row)
|
||||
|
||||
|
||||
QUERY_TEMPLATE_MULTICHOICE = """
|
||||
Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
|
||||
|
||||
{Question}
|
||||
|
||||
A) {A}
|
||||
B) {B}
|
||||
C) {C}
|
||||
D) {D}
|
||||
""".strip()
|
||||
|
||||
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """
|
||||
Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD.
|
||||
|
||||
"""
|
||||
|
||||
TEMPLATE_MULTICHOICE_EXAMPLE = """
|
||||
Example question:
|
||||
{Question}
|
||||
|
||||
A {A}
|
||||
B {B}
|
||||
C {C}
|
||||
D {D}
|
||||
|
||||
The last line of your response should be
|
||||
Answer: {Answer}
|
||||
""".strip()
|
||||
|
||||
|
||||
class MMLUEval(Eval):
|
||||
def __init__(
|
||||
self,
|
||||
filename: str,
|
||||
num_examples: Optional[int],
|
||||
num_threads: int,
|
||||
num_shots: int,
|
||||
):
|
||||
if "://" in filename:
|
||||
df = pandas.read_csv(filename, storage_options={"timeout": 30})
|
||||
else:
|
||||
df = pandas.read_csv(filename)
|
||||
examples = [row.to_dict() for _, row in df.iterrows()]
|
||||
if num_shots:
|
||||
example_questions = "".join(
|
||||
format_multichoice_question_example(row) + "\n\n"
|
||||
for row in examples[:num_shots]
|
||||
)
|
||||
self.template = (
|
||||
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN
|
||||
+ example_questions
|
||||
+ QUERY_TEMPLATE_MULTICHOICE
|
||||
)
|
||||
examples = examples[num_shots:]
|
||||
if num_examples:
|
||||
examples = random.Random(0).sample(examples, num_examples)
|
||||
self.examples = examples
|
||||
self.num_threads = num_threads
|
||||
self.num_shots = num_shots
|
||||
|
||||
def __call__(self, sampler: SamplerBase) -> EvalResult:
|
||||
def fn(row: dict):
|
||||
if self.num_shots:
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=self.template.format(**row), role="user"
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=format_multichoice_question(row), role="user"
|
||||
)
|
||||
]
|
||||
response_text = sampler(prompt_messages)
|
||||
response_text = response_text or ""
|
||||
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
|
||||
extracted_answer = match.group(1) if match else None
|
||||
score = 1.0 if extracted_answer == row["Answer"] else 0.0
|
||||
html = common.jinja_env.from_string(HTML_JINJA).render(
|
||||
prompt_messages=prompt_messages,
|
||||
next_message=dict(content=response_text, role="assistant"),
|
||||
score=score,
|
||||
correct_answer=row["Answer"],
|
||||
extracted_answer=extracted_answer,
|
||||
)
|
||||
convo = prompt_messages + [dict(content=response_text, role="assistant")]
|
||||
category = subject2category.get(row["Subject"], "other")
|
||||
return SingleEvalResult(
|
||||
html=html, score=score, metrics={category: score}, convo=convo
|
||||
)
|
||||
|
||||
results = common.map_with_progress(fn, self.examples, self.num_threads)
|
||||
return common.aggregate_results(results)
|
||||
@@ -15,7 +15,7 @@ class TestMMLU:
|
||||
"server": getattr(
|
||||
self, "server_cmd", subprocess.list2cmdline(map(str, self.other_args))
|
||||
),
|
||||
"client": "simple_eval_mmlu",
|
||||
"client": "sgl-eval",
|
||||
"accuracy_threshold": getattr(self, "accuracy_mmlu", "N/A"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
"""
|
||||
.. deprecated::
|
||||
This module is deprecated. Use ``sglang.test.run_eval`` with
|
||||
``eval_name="gsm8k"`` instead, which routes through the unified
|
||||
Chat API evaluation framework with dump_metric support.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
import sglang as sgl
|
||||
from sglang.srt.utils import get_or_create_event_loop
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
async def concurrent_generate(engine, prompts, sampling_param):
|
||||
tasks = []
|
||||
for prompt in prompts:
|
||||
tasks.append(asyncio.create_task(engine.async_generate(prompt, sampling_param)))
|
||||
|
||||
outputs = await asyncio.gather(*tasks)
|
||||
return outputs
|
||||
|
||||
|
||||
def run_eval(args):
|
||||
warnings.warn(
|
||||
"sglang.test.few_shot_gsm8k_engine is deprecated. "
|
||||
"Use sglang.test.run_eval with eval_name='gsm8k' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
# Select backend
|
||||
engine = sgl.Engine(model_path=args.model_path, log_level="error")
|
||||
|
||||
if args.local_data_path is None:
|
||||
# Read data
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
filename = download_and_cache_file(url)
|
||||
else:
|
||||
filename = args.local_data_path
|
||||
|
||||
lines = list(read_jsonl(filename))
|
||||
|
||||
# Construct prompts
|
||||
num_questions = args.num_questions
|
||||
num_shots = args.num_shots
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
# construct the prompts
|
||||
prompts = []
|
||||
for i, arg in enumerate(arguments):
|
||||
q = arg["question"]
|
||||
prompt = few_shot_examples + q
|
||||
prompts.append(prompt)
|
||||
|
||||
sampling_param = {
|
||||
"stop": ["Question", "Assistant:", "<|separator|>"],
|
||||
"max_new_tokens": 512,
|
||||
"temperature": 0,
|
||||
}
|
||||
|
||||
# Run requests
|
||||
tic = time.perf_counter()
|
||||
|
||||
loop = get_or_create_event_loop()
|
||||
|
||||
outputs = loop.run_until_complete(
|
||||
concurrent_generate(engine, prompts, sampling_param)
|
||||
)
|
||||
|
||||
# End requests
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
# Shutdown the engine
|
||||
engine.shutdown()
|
||||
|
||||
# Parse output
|
||||
preds = []
|
||||
|
||||
for output in outputs:
|
||||
preds.append(get_answer_value(output["text"]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
# Compute speed
|
||||
num_output_tokens = sum(
|
||||
output["meta_info"]["completion_tokens"] for output in outputs
|
||||
)
|
||||
output_throughput = num_output_tokens / latency
|
||||
|
||||
# Print results
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Latency: {latency:.3f} s")
|
||||
print(f"Output throughput: {output_throughput:.3f} token/s")
|
||||
|
||||
return {
|
||||
"accuracy": acc,
|
||||
"latency": latency,
|
||||
"output_throughput": output_throughput,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model-path", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct"
|
||||
)
|
||||
parser.add_argument("--local-data-path", type=Optional[str], default=None)
|
||||
parser.add_argument("--num-shots", type=int, default=5)
|
||||
parser.add_argument("--num-questions", type=int, default=200)
|
||||
args = parser.parse_args()
|
||||
metrics = run_eval(args)
|
||||
@@ -267,7 +267,6 @@ def run_eval(args):
|
||||
if args.eval_name == "mmlu":
|
||||
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
|
||||
# caller's threshold has to be measured against it, not inherited.
|
||||
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
|
||||
return _run_sgl_eval("mmlu", args)
|
||||
elif args.eval_name == "mgsm_en":
|
||||
from sglang.test.simple_eval_mgsm import MGSMEval
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
# Adapted from https://github.com/openai/simple-evals/
|
||||
|
||||
"""
|
||||
Measuring Massive Multitask Language Understanding
|
||||
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
|
||||
https://arxiv.org/abs/2009.03300
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
import pandas
|
||||
|
||||
from sglang.test import simple_eval_common as common
|
||||
from sglang.test.simple_eval_common import (
|
||||
ANSWER_PATTERN_MULTICHOICE,
|
||||
HTML_JINJA,
|
||||
Eval,
|
||||
EvalResult,
|
||||
SamplerBase,
|
||||
SingleEvalResult,
|
||||
format_multichoice_question,
|
||||
)
|
||||
|
||||
subject2category = {
|
||||
"abstract_algebra": "stem",
|
||||
"anatomy": "other",
|
||||
"astronomy": "stem",
|
||||
"business_ethics": "other",
|
||||
"clinical_knowledge": "other",
|
||||
"college_biology": "stem",
|
||||
"college_chemistry": "stem",
|
||||
"college_computer_science": "stem",
|
||||
"college_mathematics": "stem",
|
||||
"college_medicine": "other",
|
||||
"college_physics": "stem",
|
||||
"computer_security": "stem",
|
||||
"conceptual_physics": "stem",
|
||||
"econometrics": "social_sciences",
|
||||
"electrical_engineering": "stem",
|
||||
"elementary_mathematics": "stem",
|
||||
"formal_logic": "humanities",
|
||||
"global_facts": "other",
|
||||
"high_school_biology": "stem",
|
||||
"high_school_chemistry": "stem",
|
||||
"high_school_computer_science": "stem",
|
||||
"high_school_european_history": "humanities",
|
||||
"high_school_geography": "social_sciences",
|
||||
"high_school_government_and_politics": "social_sciences",
|
||||
"high_school_macroeconomics": "social_sciences",
|
||||
"high_school_mathematics": "stem",
|
||||
"high_school_microeconomics": "social_sciences",
|
||||
"high_school_physics": "stem",
|
||||
"high_school_psychology": "social_sciences",
|
||||
"high_school_statistics": "stem",
|
||||
"high_school_us_history": "humanities",
|
||||
"high_school_world_history": "humanities",
|
||||
"human_aging": "other",
|
||||
"human_sexuality": "social_sciences",
|
||||
"international_law": "humanities",
|
||||
"jurisprudence": "humanities",
|
||||
"logical_fallacies": "humanities",
|
||||
"machine_learning": "stem",
|
||||
"management": "other",
|
||||
"marketing": "other",
|
||||
"medical_genetics": "other",
|
||||
"miscellaneous": "other",
|
||||
"moral_disputes": "humanities",
|
||||
"moral_scenarios": "humanities",
|
||||
"nutrition": "other",
|
||||
"philosophy": "humanities",
|
||||
"prehistory": "humanities",
|
||||
"professional_accounting": "other",
|
||||
"professional_law": "humanities",
|
||||
"professional_medicine": "other",
|
||||
"professional_psychology": "social_sciences",
|
||||
"public_relations": "social_sciences",
|
||||
"security_studies": "social_sciences",
|
||||
"sociology": "social_sciences",
|
||||
"us_foreign_policy": "social_sciences",
|
||||
"virology": "other",
|
||||
"world_religions": "humanities",
|
||||
}
|
||||
|
||||
|
||||
class MMLUEval(Eval):
|
||||
def __init__(self, filename: str, num_examples: Optional[int], num_threads: int):
|
||||
if "://" in filename:
|
||||
df = pandas.read_csv(filename, storage_options={"timeout": 30})
|
||||
else:
|
||||
df = pandas.read_csv(filename)
|
||||
examples = [row.to_dict() for _, row in df.iterrows()]
|
||||
if num_examples:
|
||||
examples = random.Random(0).sample(examples, num_examples)
|
||||
self.examples = examples
|
||||
self.num_threads = num_threads
|
||||
|
||||
def __call__(self, sampler: SamplerBase) -> EvalResult:
|
||||
def fn(row: dict):
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=format_multichoice_question(row), role="user"
|
||||
)
|
||||
]
|
||||
response_text = sampler(prompt_messages)
|
||||
response_text = response_text or ""
|
||||
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
|
||||
extracted_answer = match.group(1) if match else None
|
||||
score = 1.0 if extracted_answer == row["Answer"] else 0.0
|
||||
html = common.jinja_env.from_string(HTML_JINJA).render(
|
||||
prompt_messages=prompt_messages,
|
||||
next_message=dict(content=response_text, role="assistant"),
|
||||
score=score,
|
||||
correct_answer=row["Answer"],
|
||||
extracted_answer=extracted_answer,
|
||||
)
|
||||
convo = prompt_messages + [dict(content=response_text, role="assistant")]
|
||||
category = subject2category.get(row["Subject"], "other")
|
||||
return SingleEvalResult(
|
||||
html=html, score=score, metrics={category: score}, convo=convo
|
||||
)
|
||||
|
||||
results = common.map_with_progress(fn, self.examples, self.num_threads)
|
||||
return common.aggregate_results(results)
|
||||
@@ -2409,38 +2409,6 @@ class CustomTestCase(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
def dump_bench_raw_result(
|
||||
path: str,
|
||||
states,
|
||||
preds,
|
||||
labels,
|
||||
):
|
||||
if not path:
|
||||
return
|
||||
|
||||
rows = []
|
||||
for i in range(len(states)):
|
||||
state = states[i]
|
||||
output = state["answer"]
|
||||
prompt = _ensure_remove_suffix(state.text(), output)
|
||||
rows.append(
|
||||
dict(
|
||||
prompt_id=i,
|
||||
prompt=prompt,
|
||||
output=output,
|
||||
correct=bool(preds[i] == labels[i]),
|
||||
)
|
||||
)
|
||||
|
||||
print(f"BenchRawResultDumper save results to {path}")
|
||||
Path(path).write_text("\n".join(json.dumps(row) for row in rows))
|
||||
|
||||
|
||||
def _ensure_remove_suffix(text: str, suffix: str):
|
||||
assert text.endswith(suffix)
|
||||
return text.removesuffix(suffix)
|
||||
|
||||
|
||||
class ModelLaunchSettings:
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user