[Test] Remove dead eval modules and point GSM8K/MMLU docs to sgl-eval (#41215)
This commit is contained in:
@@ -136,8 +136,8 @@ than restating.
|
||||
= `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. Flag output-only values.
|
||||
- **Consistent accuracy harness across entries**: every value under one `accuracyLabels`
|
||||
column must be produced by the SAME harness — flag a page that, say, measures one
|
||||
platform's GSM8K with `few_shot_gsm8k --num-questions 200` and another's with
|
||||
`run_eval --eval-name gsm8k --num-examples 1319` and shows both as one "GSM8K %"
|
||||
platform's GSM8K with a historical raw completion harness and another's with
|
||||
`sgl-eval run gsm8k --num-examples 1319` and shows both as one "GSM8K %"
|
||||
(the scores aren't comparable). Either standardize on one harness (matching
|
||||
`benchmarkCommands.accuracy`) or require an explicit per-entry note. Common when folding
|
||||
a second contributor's measurements (e.g. an AMD/ROCm PR) into the page.
|
||||
|
||||
@@ -46,13 +46,11 @@ Typical startup time: 30-90 seconds depending on model size and whether CUDA gra
|
||||
### Step 3: Validate accuracy (sanity check)
|
||||
|
||||
```bash
|
||||
python3 -m sglang.test.run_eval --host 127.0.0.1 --port <port> --eval-name gsm8k --num-examples 20
|
||||
sgl-eval run gsm8k --base-url http://127.0.0.1:<port>/v1 --num-examples 20
|
||||
```
|
||||
|
||||
- Expected accuracy: **> 0.8** for capable models (Qwen3-8B, Llama-3.1-8B-Instruct, etc.)
|
||||
- This is a quick sanity check, not a rigorous benchmark.
|
||||
- `sglang.test.few_shot_gsm8k` is deprecated; use the unified `run_eval` entrypoint.
|
||||
- If you intentionally need the old completion-style GSM8K path, add `--api completion`.
|
||||
- If accuracy is unexpectedly low, something is wrong — do not proceed to profiling.
|
||||
|
||||
### Step 4: Generate the profile
|
||||
|
||||
@@ -166,8 +166,6 @@ cython_debug/
|
||||
*.swp
|
||||
|
||||
# SGL
|
||||
benchmark/mmlu/data
|
||||
benchmark/mmlu/data.tar
|
||||
benchmark/llava_bench/images
|
||||
benchmark/llava_bench/mme_pack
|
||||
*.jsonl
|
||||
|
||||
@@ -271,7 +271,9 @@ Then we can benchmark the accuracy and latency by accessing the first node's exp
|
||||
|
||||
```bash
|
||||
# bench accuracy
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host 10.0.0.1 --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://10.0.0.1:30000/v1 \
|
||||
--num-examples 1319 --num-threads 64
|
||||
|
||||
# bench latency
|
||||
python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128
|
||||
@@ -322,13 +324,15 @@ Then on the **master node**, supposing the ShareGPT data is located at `/path/to
|
||||
|
||||
```bash
|
||||
# bench accuracy
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 --num-threads 64
|
||||
|
||||
# bench serving
|
||||
python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0
|
||||
```
|
||||
|
||||
> **Note: using `--parallel 200` can accelerate accuracy benchmarking**.
|
||||
> **Note: using `--num-threads 200` can accelerate accuracy benchmarking**.
|
||||
|
||||
### Example: Serving with 32 L40S with int8 Quantization
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
## Run benchmark
|
||||
|
||||
### Using GSM8K Platinum
|
||||
|
||||
GSM8K Platinum is a revised version of the GSM8K test set with corrected labels and removed ambiguous questions. It can be more stable than the original GSM8K dataset. It's a drop-in replacement that can be used by adding the `--platinum` flag:
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-shots 8 --num-questions 1209 --parallel 1209 --platinum
|
||||
```
|
||||
|
||||
For more information, see: https://huggingface.co/datasets/madrylab/gsm8k-platinum
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --num-questions 200
|
||||
```
|
||||
@@ -1,207 +0,0 @@
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from datasets import load_dataset
|
||||
|
||||
from sglang.lang.api import set_default_backend
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
dump_bench_raw_result,
|
||||
select_sglang_backend,
|
||||
)
|
||||
from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
def main(args):
|
||||
# Select backend
|
||||
set_default_backend(select_sglang_backend(args))
|
||||
|
||||
# Load tokenizer if enable_thinking is set
|
||||
tokenizer = None
|
||||
if args.enable_thinking:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
assert args.tokenizer_path is not None, (
|
||||
"--tokenizer-path is required when --enable-thinking is set"
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
args.tokenizer_path, trust_remote_code=True
|
||||
)
|
||||
|
||||
# Read data
|
||||
if args.platinum:
|
||||
print("Loading GSM8K Platinum dataset from HuggingFace...")
|
||||
dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test")
|
||||
lines = [
|
||||
{"question": item["question"], "answer": item["answer"]} for item in dataset
|
||||
]
|
||||
else:
|
||||
data_path = args.data_path
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
if not os.path.isfile(data_path):
|
||||
data_path = download_and_cache_file(url)
|
||||
lines = list(read_jsonl(data_path))
|
||||
|
||||
# Construct prompts
|
||||
num_questions = args.num_questions
|
||||
num_shots = args.num_shots
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
raw_question = few_shot_examples + get_one_example(lines, i, False)
|
||||
if tokenizer is not None:
|
||||
messages = [{"role": "user", "content": raw_question}]
|
||||
raw_question = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
**args.chat_template_kwargs,
|
||||
)
|
||||
questions.append(raw_question)
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
@sgl.function
|
||||
def few_shot_gsm8k(s, question):
|
||||
s += question
|
||||
s += sgl.gen(
|
||||
"answer",
|
||||
max_tokens=args.max_new_tokens,
|
||||
stop=["Question", "Assistant:", "<|separator|>"],
|
||||
)
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Run requests
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_gsm8k.run_batch(
|
||||
arguments,
|
||||
temperature=args.temperature,
|
||||
top_p=args.top_p,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
preds = []
|
||||
for i in range(len(states)):
|
||||
preds.append(get_answer_value(states[i]["answer"]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
# Compute speed
|
||||
num_output_tokens = sum(
|
||||
s.get_meta_info("answer")["completion_tokens"] for s in states
|
||||
)
|
||||
output_throughput = num_output_tokens / latency
|
||||
|
||||
# Print results
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Latency: {latency:.3f} s")
|
||||
print(f"Output throughput: {output_throughput:.3f} token/s")
|
||||
|
||||
# Dump results
|
||||
dump_state_text(f"tmp_output_{args.backend}.txt", states)
|
||||
dump_bench_raw_result(
|
||||
path=args.raw_result_file,
|
||||
states=states,
|
||||
preds=preds,
|
||||
labels=labels,
|
||||
)
|
||||
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "gsm8k-platinum" if args.platinum else "gsm8k",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(acc, 3),
|
||||
"num_requests": args.num_questions,
|
||||
"other": {
|
||||
"num_questions": args.num_questions,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--num-shots", type=int, default=5)
|
||||
parser.add_argument("--data-path", type=str, default="test.jsonl")
|
||||
parser.add_argument("--num-questions", type=int, default=200)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=512)
|
||||
parser.add_argument("--temperature", type=float, default=0.0)
|
||||
parser.add_argument("--top-p", type=float, default=1.0)
|
||||
parser.add_argument(
|
||||
"--enable-thinking",
|
||||
action="store_true",
|
||||
help="Enable thinking mode by wrapping prompts with chat template",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-template-kwargs",
|
||||
type=json.loads,
|
||||
default='{"enable_thinking": true}',
|
||||
help="JSON dict passed through to tokenizer.apply_chat_template. "
|
||||
"The thinking-toggle kwarg name is model-specific, e.g. "
|
||||
"'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to tokenizer (required when --enable-thinking is set)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--platinum",
|
||||
action="store_true",
|
||||
help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)",
|
||||
)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
main(args)
|
||||
@@ -1,20 +0,0 @@
|
||||
## Download data
|
||||
```
|
||||
bash download_data.sh
|
||||
```
|
||||
|
||||
## Run benchmark
|
||||
|
||||
### Benchmark sglang
|
||||
```
|
||||
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
|
||||
```
|
||||
|
||||
```
|
||||
python3 bench_sglang.py --nsub 10
|
||||
```
|
||||
|
||||
```
|
||||
# OpenAI models
|
||||
python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8
|
||||
```
|
||||
@@ -1,151 +0,0 @@
|
||||
"""
|
||||
Usage:
|
||||
python3 bench_hf.py --model-path meta-llama/Llama-2-7b-hf --data-dir data --ntrain 5
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
choices = ["A", "B", "C", "D"]
|
||||
|
||||
|
||||
def format_subject(subject):
|
||||
l = subject.split("_")
|
||||
s = ""
|
||||
for entry in l:
|
||||
s += " " + entry
|
||||
return s
|
||||
|
||||
|
||||
def format_example(df, idx, include_answer=True):
|
||||
prompt = df.iloc[idx, 0]
|
||||
k = df.shape[1] - 2
|
||||
for j in range(k):
|
||||
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
|
||||
prompt += "\nAnswer:"
|
||||
if include_answer:
|
||||
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
|
||||
return prompt
|
||||
|
||||
|
||||
def gen_prompt(train_df, subject, k=-1):
|
||||
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
|
||||
format_subject(subject)
|
||||
)
|
||||
if k == -1:
|
||||
k = train_df.shape[0]
|
||||
for i in range(k):
|
||||
prompt += format_example(train_df, i)
|
||||
return prompt
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def main(args):
|
||||
print(f"Loading model: {args.model_path}")
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.model_path,
|
||||
torch_dtype=torch.bfloat16,
|
||||
trust_remote_code=True,
|
||||
device_map="auto",
|
||||
).eval()
|
||||
|
||||
subjects = sorted(
|
||||
[
|
||||
f.split("_test.csv")[0]
|
||||
for f in os.listdir(os.path.join(args.data_dir, "test"))
|
||||
if "_test.csv" in f
|
||||
]
|
||||
)
|
||||
|
||||
all_cors = []
|
||||
num_requests = 0
|
||||
total_latency = 0
|
||||
|
||||
for subject in tqdm(subjects[: args.nsub]):
|
||||
dev_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
|
||||
)[: args.ntrain]
|
||||
test_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
|
||||
)
|
||||
|
||||
k = args.ntrain
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
while len(tokenizer.encode(few_shot_examples)) > 1536:
|
||||
k -= 1
|
||||
if k < 0:
|
||||
break
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
|
||||
preds = []
|
||||
labels = []
|
||||
tic = time.perf_counter()
|
||||
|
||||
for i in range(test_df.shape[0]):
|
||||
prompt_end = format_example(test_df, i, include_answer=False)
|
||||
prompt = few_shot_examples + prompt_end
|
||||
|
||||
input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device)
|
||||
output_ids = model.generate(
|
||||
input_ids,
|
||||
max_new_tokens=1,
|
||||
do_sample=False,
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
)
|
||||
|
||||
output_str = tokenizer.decode(
|
||||
output_ids[0][input_ids.shape[-1] :], skip_special_tokens=True
|
||||
)
|
||||
preds.append(output_str.strip()[0] if len(output_str.strip()) > 0 else "")
|
||||
labels.append(test_df.iloc[i, test_df.shape[1] - 1])
|
||||
|
||||
latency = time.perf_counter() - tic
|
||||
total_latency += latency
|
||||
|
||||
cors = [pred == label for pred, label in zip(preds, labels)]
|
||||
all_cors.append(cors)
|
||||
num_requests += len(test_df)
|
||||
|
||||
print(
|
||||
f"Subject: {subject}, Accuracy: {np.mean(cors):.3f}, Latency: {latency:.3f}s"
|
||||
)
|
||||
|
||||
weighted_acc = np.mean(np.concatenate(all_cors))
|
||||
print(f"Total Latency: {total_latency:.3f}s")
|
||||
print(f"Average Accuracy: {weighted_acc:.3f}")
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "a") as fout:
|
||||
value = {
|
||||
"task": "mmlu",
|
||||
"backend": "hf",
|
||||
"model": args.model_path,
|
||||
"latency": round(total_latency, 3),
|
||||
"accuracy": round(weighted_acc, 3),
|
||||
"num_requests": num_requests,
|
||||
"other": {
|
||||
"nsub": args.nsub,
|
||||
"ntrain": args.ntrain,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-path", type=str, required=True)
|
||||
parser.add_argument("--ntrain", type=int, default=5)
|
||||
parser.add_argument("--data-dir", type=str, default="data")
|
||||
parser.add_argument("--nsub", type=int, default=60)
|
||||
parser.add_argument("--output", type=str, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
@@ -1,210 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import tiktoken
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
add_common_sglang_args_and_parse,
|
||||
dump_bench_raw_result,
|
||||
select_sglang_backend,
|
||||
)
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
choices = ["A", "B", "C", "D"]
|
||||
|
||||
tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
||||
|
||||
|
||||
def format_subject(subject):
|
||||
l = subject.split("_")
|
||||
s = ""
|
||||
for entry in l:
|
||||
s += " " + entry
|
||||
return s
|
||||
|
||||
|
||||
def format_example(df, idx, include_answer=True):
|
||||
prompt = df.iloc[idx, 0]
|
||||
k = df.shape[1] - 2
|
||||
for j in range(k):
|
||||
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
|
||||
prompt += "\nAnswer:"
|
||||
if include_answer:
|
||||
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
|
||||
return prompt
|
||||
|
||||
|
||||
def gen_prompt(train_df, subject, k=-1):
|
||||
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
|
||||
format_subject(subject)
|
||||
)
|
||||
if k == -1:
|
||||
k = train_df.shape[0]
|
||||
for i in range(k):
|
||||
prompt += format_example(train_df, i)
|
||||
return prompt
|
||||
|
||||
|
||||
def download_data(data_dir):
|
||||
"""Download and extract MMLU data if it doesn't exist."""
|
||||
if os.path.isdir(os.path.join(data_dir, "test")):
|
||||
return
|
||||
print(f"Data not found at {data_dir}. Downloading...")
|
||||
os.makedirs(data_dir, exist_ok=True)
|
||||
tar_path = os.path.join(data_dir, "data.tar")
|
||||
subprocess.check_call(
|
||||
["wget", "-O", tar_path, "https://people.eecs.berkeley.edu/~hendrycks/data.tar"]
|
||||
)
|
||||
with tarfile.open(tar_path) as tar:
|
||||
tar.extractall(path=data_dir, filter="data")
|
||||
# The tarball extracts into a "data/" subdirectory; move contents up if needed
|
||||
nested = os.path.join(data_dir, "data")
|
||||
if os.path.isdir(nested):
|
||||
for item in os.listdir(nested):
|
||||
os.rename(os.path.join(nested, item), os.path.join(data_dir, item))
|
||||
os.rmdir(nested)
|
||||
os.remove(tar_path)
|
||||
print("Download complete.")
|
||||
|
||||
|
||||
def main(args):
|
||||
subjects = sorted(
|
||||
[
|
||||
f.split("_test.csv")[0]
|
||||
for f in os.listdir(os.path.join(args.data_dir, "test"))
|
||||
if "_test.csv" in f
|
||||
]
|
||||
)
|
||||
|
||||
# Build prompts
|
||||
arguments = []
|
||||
labels = []
|
||||
num_questions = []
|
||||
|
||||
for subject in subjects[: args.nsub]:
|
||||
dev_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
|
||||
)[: args.ntrain]
|
||||
test_df = pd.read_csv(
|
||||
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
|
||||
)
|
||||
num_questions.append(test_df.shape[0])
|
||||
|
||||
k = args.ntrain
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
while len(tokenizer.encode(few_shot_examples)) > 1536:
|
||||
k -= 1
|
||||
few_shot_examples = gen_prompt(dev_df, subject, k)
|
||||
|
||||
for i in range(test_df.shape[0]):
|
||||
prompt_end = format_example(test_df, i, include_answer=False)
|
||||
|
||||
arguments.append(
|
||||
{
|
||||
"examples": few_shot_examples,
|
||||
"question": prompt_end,
|
||||
}
|
||||
)
|
||||
|
||||
label = test_df.iloc[i, test_df.shape[1] - 1]
|
||||
labels.append(label)
|
||||
|
||||
#####################################
|
||||
######### SGL Program Begin #########
|
||||
#####################################
|
||||
|
||||
import sglang as sgl
|
||||
|
||||
if args.backend.startswith("gpt-"):
|
||||
|
||||
@sgl.function
|
||||
def few_shot_mmlu(s, examples, question):
|
||||
s += sgl.user(examples + question)
|
||||
s += sgl.assistant(sgl.gen("answer"))
|
||||
|
||||
else:
|
||||
|
||||
@sgl.function
|
||||
def few_shot_mmlu(s, examples, question):
|
||||
s += examples + question + sgl.gen("answer")
|
||||
|
||||
#####################################
|
||||
########## SGL Program End ##########
|
||||
#####################################
|
||||
|
||||
# Select backend
|
||||
backend = select_sglang_backend(args)
|
||||
|
||||
# Run
|
||||
tic = time.perf_counter()
|
||||
states = few_shot_mmlu.run_batch(
|
||||
arguments,
|
||||
temperature=0,
|
||||
max_new_tokens=1,
|
||||
backend=backend,
|
||||
num_threads=args.parallel,
|
||||
progress_bar=True,
|
||||
)
|
||||
preds = [
|
||||
s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states
|
||||
]
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
# Compute accuracy
|
||||
cors = [pred == label for pred, label in zip(preds, labels)]
|
||||
|
||||
pt = 0
|
||||
for subject, num_qs in zip(subjects[: args.nsub], num_questions):
|
||||
print(
|
||||
f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt : pt + num_qs]):.3f}"
|
||||
)
|
||||
pt += num_qs
|
||||
assert pt == len(cors)
|
||||
weighted_acc = np.mean(cors)
|
||||
|
||||
dump_bench_raw_result(
|
||||
path=args.raw_result_file,
|
||||
states=states,
|
||||
preds=preds,
|
||||
labels=labels,
|
||||
)
|
||||
|
||||
# Print results
|
||||
print("Total latency: {:.3f}".format(latency))
|
||||
print("Average accuracy: {:.3f}".format(weighted_acc))
|
||||
|
||||
# Write results
|
||||
with open(args.result_file, "a") as fout:
|
||||
value = {
|
||||
"task": "mmlu",
|
||||
"backend": args.backend,
|
||||
"num_gpus": 1,
|
||||
"latency": round(latency, 3),
|
||||
"accuracy": round(weighted_acc, 3),
|
||||
"num_requests": len(arguments),
|
||||
"other": {
|
||||
"nsub": args.nsub,
|
||||
"parallel": args.parallel,
|
||||
},
|
||||
}
|
||||
fout.write(json.dumps(value) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ntrain", "-k", type=int, default=5)
|
||||
parser.add_argument(
|
||||
"--data_dir", "-d", type=str, default=os.path.join(SCRIPT_DIR, "data")
|
||||
)
|
||||
parser.add_argument("--save_dir", "-s", type=str, default="results")
|
||||
parser.add_argument("--nsub", type=int, default=60)
|
||||
args = add_common_sglang_args_and_parse(parser)
|
||||
download_data(args.data_dir)
|
||||
main(args)
|
||||
@@ -1,2 +0,0 @@
|
||||
wget https://people.eecs.berkeley.edu/~hendrycks/data.tar
|
||||
tar xf data.tar
|
||||
@@ -388,9 +388,14 @@ Thus for all positive \(a,b,c\) with \(abc=1\),
|
||||
**Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -977,12 +977,14 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py \
|
||||
--num-shots 8 \
|
||||
--num-questions 1316 \
|
||||
--parallel 1316
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 1316
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -597,9 +597,13 @@ Max ITL (ms): 43465.60
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:8000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Test Results**:
|
||||
- DeepSeek-V3
|
||||
```text Output
|
||||
@@ -614,11 +618,13 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
cd sglang
|
||||
bash benchmark/mmlu/download_data.sh
|
||||
python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 8000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:8000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- **Test Results**:
|
||||
- DeepSeek-V3
|
||||
```text Output
|
||||
|
||||
@@ -973,12 +973,14 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py \
|
||||
--num-shots 8 \
|
||||
--num-questions 1316 \
|
||||
--parallel 1316
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 1316
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -709,9 +709,13 @@ Max ITL (ms): 975.03
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Test Results**:
|
||||
- DeepSeek-V3.2-Exp
|
||||
```
|
||||
@@ -721,12 +725,17 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000
|
||||
Output throughput: 965.919 token/s
|
||||
```
|
||||
|
||||
- **Full GSM8K (1319 questions)** — for a stricter accuracy check, run the full set 8-shot:
|
||||
- **Full GSM8K (1319 questions)** — evaluate the complete test set with sgl-eval:
|
||||
|
||||
```shell Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 1319
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- 8-shot:
|
||||
```
|
||||
Accuracy: 0.956
|
||||
@@ -747,11 +756,13 @@ python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --par
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
cd sglang
|
||||
bash benchmark/mmlu/download_data.sh
|
||||
python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- **Test Results**:
|
||||
- DeepSeek-V3.2-Exp
|
||||
```
|
||||
|
||||
@@ -680,12 +680,14 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command:
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py \
|
||||
--num-shots 8 \
|
||||
--num-questions 1316 \
|
||||
--parallel 1316
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 1316
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Results:
|
||||
- ERNIE-4.5-21B-A3B-PT
|
||||
```
|
||||
|
||||
@@ -510,7 +510,9 @@ Document model accuracy on standard benchmarks:
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python -m sglang.test.few_shot_gsm8k \
|
||||
--num-questions 200 \
|
||||
--port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
@@ -900,11 +900,13 @@ Document model accuracy on standard benchmarks:
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python -m sglang.test.few_shot_gsm8k \
|
||||
--num-questions 200 \
|
||||
--port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
Accuracy: 0.975
|
||||
|
||||
@@ -388,8 +388,13 @@ print(response)
|
||||
|
||||
#### Command
|
||||
```shell Command
|
||||
python3 ./benchmark/gsm8k/bench_sglang.py
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
#### Result Output
|
||||
```text Output
|
||||
Accuracy: 0.925
|
||||
|
||||
@@ -920,11 +920,13 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command
|
||||
|
||||
```bash Command
|
||||
python -m sglang.test.few_shot_gsm8k \
|
||||
--num-questions 200 \
|
||||
--port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- Result
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -941,12 +941,13 @@ Document model accuracy on standard benchmarks:
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python -m sglang.test.few_shot_gsm8k \
|
||||
--num-shots 5 \
|
||||
--num-questions 1319 \
|
||||
--port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- Test Result (NVIDIA B200, TP=2 NVFP4)
|
||||
```text Output
|
||||
Accuracy: 0.946
|
||||
|
||||
@@ -644,9 +644,14 @@ The accuracy benchmark results below are shared with GLM-5, as GLM-5.1 was not i
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
Accuracy: 0.955
|
||||
@@ -659,9 +664,13 @@ Output throughput: 642.044 token/s
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/mmlu/bench_sglang.py --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
subject: abstract_algebra, #q:100, acc: 0.860
|
||||
@@ -732,9 +741,14 @@ Average accuracy: 0.877
|
||||
- MI325/MI35x Test (GLM-5.1 BF16, `tp=8`, TileLang DSA backends)
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
```text Output
|
||||
Accuracy: 0.970
|
||||
Invalid: 0.000
|
||||
|
||||
@@ -575,9 +575,14 @@ The accuracy benchmark results below are shared with GLM-5.1, as GLM-5.1 was not
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
Accuracy: 0.955
|
||||
@@ -590,9 +595,13 @@ Output throughput: 642.044 token/s
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/mmlu/bench_sglang.py --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
subject: abstract_algebra, #q:100, acc: 0.860
|
||||
@@ -663,9 +672,14 @@ Average accuracy: 0.877
|
||||
- MI325/MI35x Test (GLM-5 BF16, `tp=8`, TileLang DSA backends)
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
```text Output
|
||||
Accuracy: 0.970
|
||||
Invalid: 0.000
|
||||
|
||||
@@ -815,10 +815,13 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command
|
||||
|
||||
```bash Command
|
||||
python -m sglang.test.few_shot_gsm8k \
|
||||
--num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- Test Result
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -1443,7 +1443,7 @@ Median ITL (ms): 15.11
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
> **Note**: These GSM8K numbers use the raw few-shot completion harness (`sglang.test.few_shot_gsm8k`). `gemma-4-12B-it` is reasoning-oriented and is under-elicited by raw few-shot prompting; with the chat template it scores **0.950** on the same 1319 GSM8K test questions (`sglang.test.run_eval --eval-name gsm8k`).
|
||||
> **Note**: These GSM8K numbers use the raw few-shot completion harness (now removed). `gemma-4-12B-it` is reasoning-oriented and is under-elicited by raw few-shot prompting; with the chat template it scores **0.950** on the same 1319 GSM8K test questions (also measured before the sgl-eval migration).
|
||||
|
||||
#### gemma-4-12B-it with sgl-eval
|
||||
|
||||
@@ -1467,6 +1467,8 @@ sgl-eval run gsm8k --base-url http://localhost:30000/v1
|
||||
sgl-eval run mmlu --base-url http://localhost:30000/v1 --num-examples 2000
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
#### MMMU
|
||||
|
||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||
|
||||
@@ -685,11 +685,13 @@ Max ITL (ms): 176.78
|
||||
#### 5.2.1 GSM8K Benchmark
|
||||
|
||||
```bash Command
|
||||
python -m sglang.test.few_shot_gsm8k \
|
||||
--num-questions 200 \
|
||||
--port 8000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:8000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
**Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -205,9 +205,14 @@ Output:
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
Accuracy: 0.960
|
||||
|
||||
@@ -213,9 +213,14 @@ For more API examples, see the [SGLang Basic Usage Guide](../../../docs/basic_us
|
||||
Reference run on a single GB300 node with `--tp 4`:
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
```text Output
|
||||
Accuracy: 0.9621 (1269 / 1319)
|
||||
```
|
||||
|
||||
@@ -250,9 +250,18 @@ sglang serve \
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --temperature 1.2 --top-p 0.8 --max-new-tokens 32768 --num-questions 200 --tokenizer-path inclusionAI/Ring-2.5-1T --enable-thinking
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64 \
|
||||
--temperature 1.2 \
|
||||
--top-p 0.8 \
|
||||
--max-tokens 32768 \
|
||||
--chat-template-kwarg enable_thinking=true
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
Accuracy: 0.955
|
||||
|
||||
@@ -337,10 +337,8 @@ Mean ITL (ms): 47.77
|
||||
- Benchmark Command:
|
||||
|
||||
```bash Command
|
||||
python3 -m sglang.test.run_eval \
|
||||
--eval-name gsm8k \
|
||||
--host 127.0.0.1 \
|
||||
--port ${PORT} \
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:${PORT}/v1 \
|
||||
--model auto \
|
||||
--num-examples 200 \
|
||||
--num-threads 64 \
|
||||
@@ -348,6 +346,8 @@ python3 -m sglang.test.run_eval \
|
||||
--reasoning-effort high
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- Test Results (B200 x8):
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -635,9 +635,13 @@ Max ITL (ms): 1970.03
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Results**:
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -488,8 +488,12 @@ Max ITL (ms): 268.17
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
- Llama-4-Scout-17B-16E-Instruct
|
||||
```text Output
|
||||
Accuracy: 0.945
|
||||
@@ -505,9 +509,9 @@ Latency: 9.739 s
|
||||
Output throughput: 2405.505 token/s
|
||||
```
|
||||
|
||||
#### 5.3.2 MMLU Pro with lm-eval
|
||||
#### 5.3.2 MMLU-Pro with sgl-eval
|
||||
|
||||
Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) on 8×H100 (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)):
|
||||
The table below records historical lm-eval measurements on 8×H100 alongside [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)). These scores were not produced by the sgl-eval commands below; rerun the evaluation for results using sgl-eval's prompts and grading.
|
||||
|
||||
<table style={{width: "100%", borderCollapse: "collapse"}}>
|
||||
<thead>
|
||||
@@ -531,6 +535,12 @@ Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
Install the evaluation package before running either command:
|
||||
|
||||
```bash
|
||||
pip install sgl-eval==0.1.0
|
||||
```
|
||||
|
||||
**Scout:**
|
||||
|
||||
```bash Command
|
||||
@@ -542,13 +552,12 @@ python -m sglang.launch_server \
|
||||
--mem-fraction-static 0.8 \
|
||||
--context-length 65536
|
||||
|
||||
# Run lm_eval
|
||||
lm_eval --model local-chat-completions \
|
||||
--model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \
|
||||
--tasks mmlu_pro \
|
||||
--batch_size 128 \
|
||||
--apply_chat_template \
|
||||
--num_fewshot 0
|
||||
# Run MMLU-Pro
|
||||
sgl-eval run mmlu_pro \
|
||||
--base-url http://localhost:30000/v1 \
|
||||
--model meta-llama/Llama-4-Scout-17B-16E-Instruct \
|
||||
--max-tokens 2048 \
|
||||
--num-threads 128
|
||||
```
|
||||
|
||||
**Maverick:**
|
||||
@@ -562,11 +571,10 @@ python -m sglang.launch_server \
|
||||
--mem-fraction-static 0.8 \
|
||||
--context-length 65536
|
||||
|
||||
# Run lm_eval
|
||||
lm_eval --model local-chat-completions \
|
||||
--model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \
|
||||
--tasks mmlu_pro \
|
||||
--batch_size 128 \
|
||||
--apply_chat_template \
|
||||
--num_fewshot 0
|
||||
# Run MMLU-Pro
|
||||
sgl-eval run mmlu_pro \
|
||||
--base-url http://localhost:30000/v1 \
|
||||
--model meta-llama/Llama-4-Maverick-17B-128E-Instruct \
|
||||
--max-tokens 2048 \
|
||||
--num-threads 128
|
||||
```
|
||||
|
||||
@@ -960,8 +960,13 @@ Max ITL (ms): 376.34
|
||||
#### 5.2.1 GSM8K Benchmark
|
||||
- Benchmark Command:
|
||||
```shell Command
|
||||
python benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
- Test Results:
|
||||
```text Output
|
||||
Accuracy: 0.950
|
||||
@@ -972,10 +977,13 @@ Output throughput: 1130.161 token/s
|
||||
#### 5.2.2 MMLU Benchmark
|
||||
- Benchmark Command:
|
||||
```shell Command
|
||||
cd benchmark/mmlu
|
||||
bash download_data.sh
|
||||
python3 bench_sglang.py --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Results:
|
||||
```text Output
|
||||
subject: abstract_algebra, #q:100, acc: 0.620
|
||||
|
||||
@@ -601,8 +601,12 @@ sglang serve \
|
||||
|
||||
- **Benchmark Command**:
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
- **Result**:
|
||||
- MiniMax-M2
|
||||
```text Output
|
||||
|
||||
@@ -501,13 +501,14 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py \
|
||||
--num-shots 8 \
|
||||
--num-questions 1316 \
|
||||
--parallel 1316 \
|
||||
--port 8888
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:8888/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 1316
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -272,12 +272,14 @@ Document model accuracy on standard benchmarks:
|
||||
- Benchmark Command
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py \
|
||||
--num-shots 8 \
|
||||
--num-questions 1316 \
|
||||
--parallel 1316
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 1316
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -336,9 +336,14 @@ Validation runs on 4× H200 with `--tp 4`, served via the `/v1/chat/completions`
|
||||
#### GSM8K
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -333,9 +333,14 @@ shapes to create a distinct symbol which signifies copyright protection.
|
||||
#### GSM8K
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
**Results:**
|
||||
|
||||
```text Output
|
||||
@@ -345,7 +350,9 @@ TODO
|
||||
#### MMLU
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/mmlu/bench_sglang.py --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
**Results:**
|
||||
|
||||
@@ -597,15 +597,17 @@ The following results are from the official Kimi-K2.7-Code model card. They were
|
||||
|
||||
#### 5.2.1 Accuracy Benchmark
|
||||
|
||||
GSM8K accuracy of the MXFP4 checkpoint, measured on the deployed server across several client parallelism levels (2,000 questions each):
|
||||
Evaluate GSM8K on the deployed MXFP4 checkpoint across client parallelism levels (all 1319 test questions via sgl-eval):
|
||||
|
||||
```shell Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py \
|
||||
--num-questions 2000 \
|
||||
--parallel <PARALLEL> \
|
||||
--port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads '<PARALLEL>'
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
|
||||
@@ -507,9 +507,13 @@ python3 -m sglang.launch_server \
|
||||
- Benchmark Command
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:8000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Result**:
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -284,9 +284,13 @@ SGLANG_ROCM_FUSED_DECODE_MLA=0 python3 -m sglang.launch_server \
|
||||
- Benchmark Command
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Result**:
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -572,9 +572,14 @@ sglang serve \
|
||||
|
||||
**Run Benchmark**
|
||||
```shell Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
```text Output
|
||||
Accuracy: 0.830
|
||||
@@ -587,9 +592,13 @@ Output throughput: 1611.623 token/s
|
||||
|
||||
**Run Benchmark**
|
||||
```shell Command
|
||||
python3 benchmark/mmlu/bench_sglang.py --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
```text Output
|
||||
subject: abstract_algebra, #q:100, acc: 0.510
|
||||
|
||||
@@ -235,7 +235,7 @@ python3 -m sglang.bench_serving \
|
||||
--max-concurrency 256
|
||||
```
|
||||
|
||||
- **Test Results:**
|
||||
- **Serving benchmark results:**
|
||||
|
||||
```text Output
|
||||
============ Serving Benchmark Result ============
|
||||
@@ -304,7 +304,7 @@ python3 -m sglang.bench_serving \
|
||||
--max-concurrency 256
|
||||
```
|
||||
|
||||
- **Test Results:**
|
||||
- **Serving benchmark results:**
|
||||
|
||||
```text Output
|
||||
============ Serving Benchmark Result ============
|
||||
@@ -361,14 +361,16 @@ python3 -m sglang.launch_server \
|
||||
--reasoning-parser nemotron_3
|
||||
```
|
||||
|
||||
**Run Benchmark with lm-eval**
|
||||
**Run Benchmark with sgl-eval**
|
||||
```bash Command
|
||||
pip install lm-eval[api]==0.4.9.2
|
||||
pip install sgl-eval
|
||||
|
||||
lm_eval --model local-completions --tasks gsm8k --model_args "model=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16,base_url=http://127.0.0.1:30000/v1/completions,num_concurrent=4,max_retries=3,tokenized_requests=False,max_lengths=16384" --gen_kwargs '{"chat_template_kwargs":{"thinking":true}}' --batch_size 256
|
||||
sgl-eval run gsm8k --base-url http://127.0.0.1:30000/v1 --model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --num-threads 4 --max-tokens 16384 --thinking
|
||||
```
|
||||
|
||||
**Test Results:**
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
**Historical lm-eval results (not measurements of the sgl-eval command above):**
|
||||
```text Output
|
||||
|Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr|
|
||||
|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:|
|
||||
|
||||
@@ -489,9 +489,14 @@ python3 -m sglang.launch_server \
|
||||
|
||||
**Run Benchmark**
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 5000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
```text Output
|
||||
Accuracy: 0.950
|
||||
@@ -504,9 +509,13 @@ Output throughput: 996.815 token/s
|
||||
|
||||
**Run Benchmark**
|
||||
```bash Command
|
||||
python3 benchmark/mmlu/bench_sglang.py --port 5000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
```text Output
|
||||
subject: abstract_algebra, #q:100, acc: 0.730
|
||||
|
||||
@@ -528,9 +528,14 @@ python3 -m sglang.launch_server \
|
||||
|
||||
**Run Benchmark**
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
**Test Results:**
|
||||
```text Output
|
||||
Accuracy: 0.970
|
||||
@@ -543,7 +548,9 @@ Output throughput: 745.333 token/s
|
||||
|
||||
**Run Benchmark**
|
||||
```bash Command
|
||||
python3 benchmark/mmlu/bench_sglang.py
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
**Test Results:**
|
||||
|
||||
@@ -640,9 +640,13 @@ Max ITL (ms): 621.42
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:8000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Results**:
|
||||
|
||||
- GPT-OSS-120b
|
||||
|
||||
@@ -818,9 +818,14 @@ Max ITL (ms): 2783.23
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- **Test Results:**
|
||||
|
||||
```text Output
|
||||
@@ -835,11 +840,13 @@ Output throughput: 929.132 token/s
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
cd benchmark/mmlu
|
||||
bash download_data.sh
|
||||
python3 bench_sglang.py --port 30000
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- **Test Results:**
|
||||
|
||||
```text Output
|
||||
|
||||
@@ -752,9 +752,13 @@ Max ITL (ms): 548.82
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
##### AMD (MI300X/MI325X/MI355X)
|
||||
|
||||
- **Results**:
|
||||
|
||||
@@ -726,9 +726,13 @@ Max ITL (ms): 486.38
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:8000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Results**:
|
||||
|
||||
- Qwen3-Next-80B-A3B-Instruct
|
||||
@@ -753,11 +757,13 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
cd sglang
|
||||
bash benchmark/mmlu/download_data.sh
|
||||
python3 benchmark/mmlu/bench_sglang.py --nsub 10
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- **Results**:
|
||||
|
||||
- Qwen3-Next-80B-A3B-Instruct
|
||||
|
||||
@@ -816,9 +816,14 @@ See [Decision models](/docs/supported-models/decision_models) for the request re
|
||||
|
||||
- Benchmark Command
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --port 30000
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
|
||||
|
||||
- Test Result
|
||||
```text Output
|
||||
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:31<00:00, 6.43it/s]
|
||||
@@ -828,21 +833,17 @@ Latency: 31.784 s
|
||||
Output throughput: 998.166 token/s
|
||||
```
|
||||
|
||||
#### 5.1.2 GSM8K with lm-eval (5-shot)
|
||||
|
||||
Evaluate using the industry-standard `lm-eval` harness for reproducible accuracy reporting:
|
||||
#### 5.1.2 Full GSM8K with sgl-eval
|
||||
|
||||
```bash Command
|
||||
pip install lm-eval[api]
|
||||
|
||||
lm_eval --model local-completions \
|
||||
--model_args '{"base_url": "http://localhost:30000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \
|
||||
--tasks gsm8k \
|
||||
--batch_size auto \
|
||||
--num_fewshot 5 \
|
||||
--trust_remote_code
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://localhost:30000/v1 \
|
||||
--model Qwen/Qwen3.5-397B-A17B \
|
||||
--num-examples 1319 --num-threads 256 --max-tokens 2048 --thinking
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
#### 5.1.3 MMMU Benchmark
|
||||
|
||||
- Benchmark Command
|
||||
|
||||
@@ -871,9 +871,13 @@ Max ITL (ms): 352.79
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Results**:
|
||||
|
||||
- Qwen/Qwen3-235B-A22B-Instruct-2507
|
||||
|
||||
@@ -514,9 +514,13 @@ Max ITL (ms): 1359.81
|
||||
- **Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- **Results**:
|
||||
|
||||
- Step-3.5-Flash
|
||||
|
||||
@@ -443,13 +443,18 @@ Tool[0] get_weather({"city": "Beijing", "unit": "fahrenheit"})
|
||||
|
||||
#### 5.1.1 GSM8K
|
||||
|
||||
- Benchmark Method: 5-shot CoT on 200 questions, evaluated via SGLang native backend
|
||||
- Benchmark Method: GSM8K on 200 questions using sgl-eval
|
||||
- Benchmark Command:
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --parallel 64
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200 \
|
||||
--num-threads 64
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
- Test Results:
|
||||
|
||||
```text Output
|
||||
@@ -458,11 +463,13 @@ TODO — replace with real GSM8K accuracy after benchmark run on Hy3-preview (BF
|
||||
|
||||
#### 5.1.2 MMLU
|
||||
|
||||
- Benchmark Method: 5-shot, all 57 subjects
|
||||
- Benchmark Method: zero-shot Chat API evaluation via sgl-eval, all 57 subjects
|
||||
- Benchmark Command:
|
||||
|
||||
```bash Command
|
||||
python3 benchmark/mmlu/bench_sglang.py --nsub 60 --parallel 64
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-threads 64 --max-tokens 2048
|
||||
```
|
||||
|
||||
- Test Results:
|
||||
|
||||
@@ -365,16 +365,17 @@ Standard 5-shot, `temperature=0`, `max_tokens=4096`, model defaults to thinking-
|
||||
**Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.run_eval \
|
||||
--base-url http://127.0.0.1:30000 \
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--model XiaomiMiMo/MiMo-V2.5 \
|
||||
--eval-name gsm8k \
|
||||
--num-examples 200 \
|
||||
--num-threads 8 \
|
||||
--max-tokens 4096 \
|
||||
--temperature 0.0
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
> `run_eval.py` automatically appends `/v1` to `--base-url`; pass the bare `host:port` URL (without trailing `/v1`), otherwise requests resolve to `/v1/v1/chat/completions` and 404.
|
||||
|
||||
- **Test Results:**
|
||||
@@ -748,14 +749,15 @@ Pro's 3-layer MTP behaves very differently on natural text vs uniform-random tok
|
||||
**Benchmark Command:**
|
||||
|
||||
```shell Command
|
||||
python3 -m sglang.test.run_eval \
|
||||
--base-url http://127.0.0.1:30000 \
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--model XiaomiMiMo/MiMo-V2.5-Pro \
|
||||
--eval-name gsm8k \
|
||||
--num-examples 200 \
|
||||
--num-threads 4
|
||||
```
|
||||
|
||||
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
|
||||
|
||||
The `accept_rate` and `accept_length` rows below are not part of `run_eval`'s own output — they were aggregated from the server-side `Decode batch ... accept rate: X accept len: Y` log lines emitted during the GSM8K run (307 batches total).
|
||||
|
||||
| Workload | accept_rate | accept_length (max = 4) |
|
||||
|
||||
@@ -90,9 +90,8 @@ If your code changes model output, run an accuracy evaluation appropriate for th
|
||||
sglang serve --model-path Qwen/Qwen3-8B
|
||||
|
||||
# Evaluate
|
||||
python3 -m sglang.test.run_eval \
|
||||
--eval-name gsm8k \
|
||||
--port 30000 \
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
|
||||
@@ -114,6 +114,9 @@ nsys profile -o deepseek_v3 python3 -m sglang.benchmark.one_batch --batch-size 1
|
||||
## Evaluation
|
||||
|
||||
```bash Command
|
||||
# e.g. gsm8k 8 shot
|
||||
python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8
|
||||
# GSM8K via sgl-eval: all 1319 test questions
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1319 \
|
||||
--num-threads 2000
|
||||
```
|
||||
|
||||
@@ -18,9 +18,8 @@ SGLang provides built-in scripts to evaluate common benchmarks.
|
||||
**MMLU**
|
||||
|
||||
```bash Command
|
||||
python -m sglang.test.run_eval \
|
||||
--eval-name mmlu \
|
||||
--port 30000 \
|
||||
sgl-eval run mmlu \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 1000 \
|
||||
--max-tokens 8192
|
||||
```
|
||||
@@ -28,12 +27,9 @@ python -m sglang.test.run_eval \
|
||||
**GSM8K**
|
||||
|
||||
```bash Command
|
||||
python -m sglang.test.run_eval \
|
||||
--eval-name gsm8k \
|
||||
--host 127.0.0.1 \
|
||||
--port 30000 \
|
||||
--num-examples 200 \
|
||||
--num-shots 5
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://127.0.0.1:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
**HellaSwag**
|
||||
|
||||
@@ -103,7 +103,7 @@ For more details, please refer to [docs/README.md](https://github.com/sgl-projec
|
||||
|
||||
## Test the accuracy
|
||||
|
||||
If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K.
|
||||
If your code changes the model output, run the accuracy tests. A quick sanity check is GSM8K through sgl-eval.
|
||||
|
||||
```bash
|
||||
# Launch a server
|
||||
@@ -112,7 +112,9 @@ python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct
|
||||
# Evaluate. --base-url must point at the server launched above.
|
||||
# The default SGLang server port is 30000; change it if you launched
|
||||
# the server with a different --port.
|
||||
python3 -m sglang.test.run_eval --base-url http://localhost:30000 --eval-name gsm8k --num-examples 200
|
||||
sgl-eval run gsm8k \
|
||||
--base-url http://localhost:30000/v1 \
|
||||
--num-examples 200
|
||||
```
|
||||
|
||||
Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test.
|
||||
|
||||
@@ -105,40 +105,25 @@ For more details of SGLang server, refer to the [Ascend NPU Quick Start](/docs/h
|
||||
|
||||
#### Execute Accuracy Evaluation
|
||||
|
||||
EvalScope connects to the SGLang server via OpenAI-compatible API. The following example uses the GSM8K dataset:
|
||||
Use sgl-eval for GSM8K and MMLU. It owns the datasets, chat prompts and graders. EvalScope remains available for the other benchmarks below:
|
||||
|
||||
```shell Command
|
||||
evalscope eval \
|
||||
pip install sgl-eval
|
||||
sgl-eval run gsm8k \
|
||||
--model /home/weights/Qwen2.5-7B-Instruct \
|
||||
--api-url http://localhost:30000/v1 \
|
||||
--api-key EMPTY \
|
||||
--eval-type openai_api \
|
||||
--datasets gsm8k \
|
||||
--limit 10
|
||||
--base-url http://localhost:30000/v1 \
|
||||
--num-examples 10 --max-tokens 2048
|
||||
```
|
||||
|
||||
Upon completion, results similar to the following will be displayed:
|
||||
|
||||
```text
|
||||
+---------------------+-----------+----------+----------+-------+---------+---------+
|
||||
| Model | Dataset | Metric | Subset | Num | Score | Cat.0 |
|
||||
+=====================+===========+==========+==========+=======+=========+=========+
|
||||
| Qwen2.5-7B-Instruct | gsm8k | mean_acc | main | 5 | 1.0 | default |
|
||||
+---------------------+-----------+----------+----------+-------+---------+---------+
|
||||
```
|
||||
|
||||
> **Note**: Output format may vary slightly across different EvalScope versions. The above example is from EvalScope 1.6.x. Ensure the `--model` parameter matches the model name returned by the SGLang server's `/v1/models` endpoint. When starting the server with an HF path (e.g., `Qwen/Qwen2.5-7B-Instruct`), use that path directly. For local paths, pass the full path or the model name returned by `/v1/models`.
|
||||
The run directory contains `metrics.json` and per-example predictions. Use the model name reported by `/v1/models`.
|
||||
|
||||
#### Common Datasets for Online Evaluation
|
||||
|
||||
```shell Command
|
||||
# MMLU
|
||||
evalscope eval \
|
||||
sgl-eval run mmlu \
|
||||
--model /home/weights/Qwen2.5-7B-Instruct \
|
||||
--api-url http://localhost:30000/v1 \
|
||||
--api-key EMPTY \
|
||||
--eval-type openai_api \
|
||||
--datasets mmlu
|
||||
--base-url http://localhost:30000/v1 --max-tokens 2048
|
||||
|
||||
# CEval (Chinese evaluation)
|
||||
evalscope eval \
|
||||
@@ -241,7 +226,7 @@ When using EvalScope commands without specifying a dataset or model path, it wil
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/adapters.py", line 676, in send
|
||||
raise SSLError(e, request=request)
|
||||
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/gsm8k (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)')))
|
||||
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/ceval (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)')))
|
||||
[ERROR] 2026-05-13-02:20:01 (PID:876, Device:-1, RankID:-1) ERR99999 UNKNOWN application exception
|
||||
```
|
||||
**Temporary workaround (test only):**
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
// Per-platform re-measurement under sgl-eval is in progress; rows still pending
|
||||
// show `gsm8k_pct: null` (no GSM8K row rendered) with the legacy-harness number
|
||||
// kept in a comment. Legacy harnesses were NOT comparable across platforms
|
||||
// (NVIDIA: few_shot_gsm8k --num-questions 200; AMD: run_eval gsm8k 1319 examples) —
|
||||
// (NVIDIA: removed completion harness, 200 questions; AMD: run_eval gsm8k 1319 examples) —
|
||||
// which is exactly why we re-measure on one harness.
|
||||
export const benchmarks = [
|
||||
{
|
||||
|
||||
@@ -9,7 +9,7 @@ _DESCRIPTION = """Compare and find differences to benchmark outputs.
|
||||
|
||||
Supported inputs:
|
||||
* The samples jsonl from `lm_eval --log_samples --output_path FOLDER_NAME`
|
||||
* The output from `gsm8k/bench_sglang.py --raw-result-file FILE_NAME` (or mmlu)
|
||||
* Legacy benchmark JSONL with prompt_id, prompt, output, and correct fields
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from sglang.test.run_eval import run_eval_once
|
||||
from sglang.test.simple_eval_common import (
|
||||
make_report,
|
||||
set_ulimit,
|
||||
)
|
||||
|
||||
|
||||
def run_eval(args):
|
||||
# Lazy import to avoid circular dependency with test_utils
|
||||
from sglang.test.test_utils import dump_metric
|
||||
|
||||
set_ulimit()
|
||||
|
||||
if "OPENAI_API_KEY" not in os.environ:
|
||||
os.environ["OPENAI_API_KEY"] = "EMPTY"
|
||||
|
||||
base_url = (
|
||||
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
|
||||
)
|
||||
|
||||
if args.eval_name == "mmlu":
|
||||
from sglang.test.ascend.simple_eval_mmlu import MMLUEval
|
||||
|
||||
filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
|
||||
eval_obj = MMLUEval(
|
||||
filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0)
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid eval name: {args.eval_name}")
|
||||
|
||||
if getattr(args, "repeat", 1) == 1:
|
||||
result, latency, sampler = run_eval_once(args, base_url, eval_obj)
|
||||
metrics = result.metrics | {"score": result.score}
|
||||
metrics["latency"] = latency
|
||||
print(f"Total latency: {latency:.3f} s")
|
||||
print(f"Score: {metrics['score']:.3f}")
|
||||
|
||||
# Compute output throughput from accumulated completion tokens
|
||||
total_completion_tokens = sum(sampler._completion_tokens)
|
||||
if total_completion_tokens > 0 and latency > 0:
|
||||
metrics["output_throughput"] = total_completion_tokens / latency
|
||||
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
|
||||
|
||||
# Report metrics to unified collection framework
|
||||
dump_metric(
|
||||
f"{args.eval_name}_score",
|
||||
metrics["score"],
|
||||
labels={"model": sampler.model, "eval": args.eval_name},
|
||||
)
|
||||
dump_metric(
|
||||
f"{args.eval_name}_latency",
|
||||
latency,
|
||||
labels={"model": sampler.model, "eval": args.eval_name},
|
||||
)
|
||||
else:
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
executor = ThreadPoolExecutor(max_workers=args.repeat)
|
||||
|
||||
futures = [
|
||||
executor.submit(run_eval_once, args, base_url, eval_obj)
|
||||
for _ in range(args.repeat)
|
||||
]
|
||||
|
||||
scores_repeat = []
|
||||
latencies = []
|
||||
total_completion_tokens = 0
|
||||
|
||||
for f in futures:
|
||||
result, latency, sampler = f.result()
|
||||
scores_repeat.append(result.score)
|
||||
latencies.append(latency)
|
||||
total_completion_tokens += sum(sampler._completion_tokens)
|
||||
|
||||
mean_score = sum(scores_repeat) / len(scores_repeat)
|
||||
mean_latency = sum(latencies) / len(latencies)
|
||||
total_latency = sum(latencies)
|
||||
scores_repeat = [f"{s:.3f}" for s in scores_repeat]
|
||||
print("=" * 20)
|
||||
print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
|
||||
print(f"Scores: {scores_repeat}")
|
||||
print(f"Mean latency: {mean_latency:.3f} s")
|
||||
print("=" * 20)
|
||||
metrics = result.metrics | {"scores": scores_repeat}
|
||||
metrics = metrics | {"mean_score": mean_score}
|
||||
metrics["latency"] = mean_latency
|
||||
|
||||
if total_completion_tokens > 0 and total_latency > 0:
|
||||
metrics["output_throughput"] = total_completion_tokens / total_latency
|
||||
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
|
||||
|
||||
# Report metrics to unified collection framework
|
||||
dump_metric(
|
||||
f"{args.eval_name}_mean_score",
|
||||
mean_score,
|
||||
labels={
|
||||
"model": sampler.model,
|
||||
"eval": args.eval_name,
|
||||
"repeat": args.repeat,
|
||||
},
|
||||
)
|
||||
|
||||
executor.shutdown()
|
||||
|
||||
# Dump reports
|
||||
file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
|
||||
report_filename = f"/tmp/{file_stem}.html"
|
||||
print(f"Writing report to {report_filename}")
|
||||
with open(report_filename, "w") as fh:
|
||||
fh.write(make_report(result))
|
||||
print(metrics)
|
||||
result_filename = f"/tmp/{file_stem}.json"
|
||||
with open(result_filename, "w") as f:
|
||||
f.write(json.dumps(metrics, indent=2))
|
||||
print(f"Writing results to {result_filename}")
|
||||
|
||||
if getattr(args, "return_latency", False):
|
||||
return metrics, latency
|
||||
return metrics
|
||||
@@ -1,125 +0,0 @@
|
||||
# Adapted from https://github.com/openai/simple-evals/
|
||||
|
||||
"""
|
||||
Measuring Massive Multitask Language Understanding
|
||||
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
|
||||
https://arxiv.org/abs/2009.03300
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
import pandas
|
||||
|
||||
from sglang.test import simple_eval_common as common
|
||||
from sglang.test.simple_eval_common import (
|
||||
ANSWER_PATTERN_MULTICHOICE,
|
||||
HTML_JINJA,
|
||||
Eval,
|
||||
EvalResult,
|
||||
SamplerBase,
|
||||
SingleEvalResult,
|
||||
format_multichoice_question,
|
||||
)
|
||||
from sglang.test.simple_eval_mmlu import subject2category
|
||||
|
||||
|
||||
def format_multichoice_question_example(row):
|
||||
return QUERY_TEMPLATE_MULTICHOICE.format(**row)
|
||||
|
||||
|
||||
QUERY_TEMPLATE_MULTICHOICE = """
|
||||
Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
|
||||
|
||||
{Question}
|
||||
|
||||
A) {A}
|
||||
B) {B}
|
||||
C) {C}
|
||||
D) {D}
|
||||
""".strip()
|
||||
|
||||
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """
|
||||
Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD.
|
||||
|
||||
"""
|
||||
|
||||
TEMPLATE_MULTICHOICE_EXAMPLE = """
|
||||
Example question:
|
||||
{Question}
|
||||
|
||||
A {A}
|
||||
B {B}
|
||||
C {C}
|
||||
D {D}
|
||||
|
||||
The last line of your response should be
|
||||
Answer: {Answer}
|
||||
""".strip()
|
||||
|
||||
|
||||
class MMLUEval(Eval):
|
||||
def __init__(
|
||||
self,
|
||||
filename: str,
|
||||
num_examples: Optional[int],
|
||||
num_threads: int,
|
||||
num_shots: int,
|
||||
):
|
||||
if "://" in filename:
|
||||
df = pandas.read_csv(filename, storage_options={"timeout": 30})
|
||||
else:
|
||||
df = pandas.read_csv(filename)
|
||||
examples = [row.to_dict() for _, row in df.iterrows()]
|
||||
if num_shots:
|
||||
example_questions = "".join(
|
||||
format_multichoice_question_example(row) + "\n\n"
|
||||
for row in examples[:num_shots]
|
||||
)
|
||||
self.template = (
|
||||
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN
|
||||
+ example_questions
|
||||
+ QUERY_TEMPLATE_MULTICHOICE
|
||||
)
|
||||
examples = examples[num_shots:]
|
||||
if num_examples:
|
||||
examples = random.Random(0).sample(examples, num_examples)
|
||||
self.examples = examples
|
||||
self.num_threads = num_threads
|
||||
self.num_shots = num_shots
|
||||
|
||||
def __call__(self, sampler: SamplerBase) -> EvalResult:
|
||||
def fn(row: dict):
|
||||
if self.num_shots:
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=self.template.format(**row), role="user"
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=format_multichoice_question(row), role="user"
|
||||
)
|
||||
]
|
||||
response_text = sampler(prompt_messages)
|
||||
response_text = response_text or ""
|
||||
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
|
||||
extracted_answer = match.group(1) if match else None
|
||||
score = 1.0 if extracted_answer == row["Answer"] else 0.0
|
||||
html = common.jinja_env.from_string(HTML_JINJA).render(
|
||||
prompt_messages=prompt_messages,
|
||||
next_message=dict(content=response_text, role="assistant"),
|
||||
score=score,
|
||||
correct_answer=row["Answer"],
|
||||
extracted_answer=extracted_answer,
|
||||
)
|
||||
convo = prompt_messages + [dict(content=response_text, role="assistant")]
|
||||
category = subject2category.get(row["Subject"], "other")
|
||||
return SingleEvalResult(
|
||||
html=html, score=score, metrics={category: score}, convo=convo
|
||||
)
|
||||
|
||||
results = common.map_with_progress(fn, self.examples, self.num_threads)
|
||||
return common.aggregate_results(results)
|
||||
@@ -15,7 +15,7 @@ class TestMMLU:
|
||||
"server": getattr(
|
||||
self, "server_cmd", subprocess.list2cmdline(map(str, self.other_args))
|
||||
),
|
||||
"client": "simple_eval_mmlu",
|
||||
"client": "sgl-eval",
|
||||
"accuracy_threshold": getattr(self, "accuracy_mmlu", "N/A"),
|
||||
}
|
||||
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
"""
|
||||
.. deprecated::
|
||||
This module is deprecated. Use ``sglang.test.run_eval`` with
|
||||
``eval_name="gsm8k"`` instead, which routes through the unified
|
||||
Chat API evaluation framework with dump_metric support.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
import warnings
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
|
||||
import sglang as sgl
|
||||
from sglang.srt.utils import get_or_create_event_loop
|
||||
from sglang.utils import download_and_cache_file, read_jsonl
|
||||
|
||||
INVALID = -9999999
|
||||
|
||||
|
||||
def get_one_example(lines, i, include_answer):
|
||||
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
|
||||
if include_answer:
|
||||
ret += " " + lines[i]["answer"]
|
||||
return ret
|
||||
|
||||
|
||||
def get_few_shot_examples(lines, k):
|
||||
ret = ""
|
||||
for i in range(k):
|
||||
ret += get_one_example(lines, i, True) + "\n\n"
|
||||
return ret
|
||||
|
||||
|
||||
def get_answer_value(answer_str):
|
||||
answer_str = answer_str.replace(",", "")
|
||||
numbers = re.findall(r"\d+", answer_str)
|
||||
if len(numbers) < 1:
|
||||
return INVALID
|
||||
try:
|
||||
return ast.literal_eval(numbers[-1])
|
||||
except SyntaxError:
|
||||
return INVALID
|
||||
|
||||
|
||||
async def concurrent_generate(engine, prompts, sampling_param):
|
||||
tasks = []
|
||||
for prompt in prompts:
|
||||
tasks.append(asyncio.create_task(engine.async_generate(prompt, sampling_param)))
|
||||
|
||||
outputs = await asyncio.gather(*tasks)
|
||||
return outputs
|
||||
|
||||
|
||||
def run_eval(args):
|
||||
warnings.warn(
|
||||
"sglang.test.few_shot_gsm8k_engine is deprecated. "
|
||||
"Use sglang.test.run_eval with eval_name='gsm8k' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
# Select backend
|
||||
engine = sgl.Engine(model_path=args.model_path, log_level="error")
|
||||
|
||||
if args.local_data_path is None:
|
||||
# Read data
|
||||
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
|
||||
filename = download_and_cache_file(url)
|
||||
else:
|
||||
filename = args.local_data_path
|
||||
|
||||
lines = list(read_jsonl(filename))
|
||||
|
||||
# Construct prompts
|
||||
num_questions = args.num_questions
|
||||
num_shots = args.num_shots
|
||||
few_shot_examples = get_few_shot_examples(lines, num_shots)
|
||||
|
||||
questions = []
|
||||
labels = []
|
||||
for i in range(len(lines[:num_questions])):
|
||||
questions.append(get_one_example(lines, i, False))
|
||||
labels.append(get_answer_value(lines[i]["answer"]))
|
||||
assert all(l != INVALID for l in labels)
|
||||
arguments = [{"question": q} for q in questions]
|
||||
|
||||
# construct the prompts
|
||||
prompts = []
|
||||
for i, arg in enumerate(arguments):
|
||||
q = arg["question"]
|
||||
prompt = few_shot_examples + q
|
||||
prompts.append(prompt)
|
||||
|
||||
sampling_param = {
|
||||
"stop": ["Question", "Assistant:", "<|separator|>"],
|
||||
"max_new_tokens": 512,
|
||||
"temperature": 0,
|
||||
}
|
||||
|
||||
# Run requests
|
||||
tic = time.perf_counter()
|
||||
|
||||
loop = get_or_create_event_loop()
|
||||
|
||||
outputs = loop.run_until_complete(
|
||||
concurrent_generate(engine, prompts, sampling_param)
|
||||
)
|
||||
|
||||
# End requests
|
||||
latency = time.perf_counter() - tic
|
||||
|
||||
# Shutdown the engine
|
||||
engine.shutdown()
|
||||
|
||||
# Parse output
|
||||
preds = []
|
||||
|
||||
for output in outputs:
|
||||
preds.append(get_answer_value(output["text"]))
|
||||
|
||||
# Compute accuracy
|
||||
acc = np.mean(np.array(preds) == np.array(labels))
|
||||
invalid = np.mean(np.array(preds) == INVALID)
|
||||
|
||||
# Compute speed
|
||||
num_output_tokens = sum(
|
||||
output["meta_info"]["completion_tokens"] for output in outputs
|
||||
)
|
||||
output_throughput = num_output_tokens / latency
|
||||
|
||||
# Print results
|
||||
print(f"Accuracy: {acc:.3f}")
|
||||
print(f"Invalid: {invalid:.3f}")
|
||||
print(f"Latency: {latency:.3f} s")
|
||||
print(f"Output throughput: {output_throughput:.3f} token/s")
|
||||
|
||||
return {
|
||||
"accuracy": acc,
|
||||
"latency": latency,
|
||||
"output_throughput": output_throughput,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model-path", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct"
|
||||
)
|
||||
parser.add_argument("--local-data-path", type=Optional[str], default=None)
|
||||
parser.add_argument("--num-shots", type=int, default=5)
|
||||
parser.add_argument("--num-questions", type=int, default=200)
|
||||
args = parser.parse_args()
|
||||
metrics = run_eval(args)
|
||||
@@ -267,7 +267,6 @@ def run_eval(args):
|
||||
if args.eval_name == "mmlu":
|
||||
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
|
||||
# caller's threshold has to be measured against it, not inherited.
|
||||
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
|
||||
return _run_sgl_eval("mmlu", args)
|
||||
elif args.eval_name == "mgsm_en":
|
||||
from sglang.test.simple_eval_mgsm import MGSMEval
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
# Adapted from https://github.com/openai/simple-evals/
|
||||
|
||||
"""
|
||||
Measuring Massive Multitask Language Understanding
|
||||
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
|
||||
https://arxiv.org/abs/2009.03300
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
import pandas
|
||||
|
||||
from sglang.test import simple_eval_common as common
|
||||
from sglang.test.simple_eval_common import (
|
||||
ANSWER_PATTERN_MULTICHOICE,
|
||||
HTML_JINJA,
|
||||
Eval,
|
||||
EvalResult,
|
||||
SamplerBase,
|
||||
SingleEvalResult,
|
||||
format_multichoice_question,
|
||||
)
|
||||
|
||||
subject2category = {
|
||||
"abstract_algebra": "stem",
|
||||
"anatomy": "other",
|
||||
"astronomy": "stem",
|
||||
"business_ethics": "other",
|
||||
"clinical_knowledge": "other",
|
||||
"college_biology": "stem",
|
||||
"college_chemistry": "stem",
|
||||
"college_computer_science": "stem",
|
||||
"college_mathematics": "stem",
|
||||
"college_medicine": "other",
|
||||
"college_physics": "stem",
|
||||
"computer_security": "stem",
|
||||
"conceptual_physics": "stem",
|
||||
"econometrics": "social_sciences",
|
||||
"electrical_engineering": "stem",
|
||||
"elementary_mathematics": "stem",
|
||||
"formal_logic": "humanities",
|
||||
"global_facts": "other",
|
||||
"high_school_biology": "stem",
|
||||
"high_school_chemistry": "stem",
|
||||
"high_school_computer_science": "stem",
|
||||
"high_school_european_history": "humanities",
|
||||
"high_school_geography": "social_sciences",
|
||||
"high_school_government_and_politics": "social_sciences",
|
||||
"high_school_macroeconomics": "social_sciences",
|
||||
"high_school_mathematics": "stem",
|
||||
"high_school_microeconomics": "social_sciences",
|
||||
"high_school_physics": "stem",
|
||||
"high_school_psychology": "social_sciences",
|
||||
"high_school_statistics": "stem",
|
||||
"high_school_us_history": "humanities",
|
||||
"high_school_world_history": "humanities",
|
||||
"human_aging": "other",
|
||||
"human_sexuality": "social_sciences",
|
||||
"international_law": "humanities",
|
||||
"jurisprudence": "humanities",
|
||||
"logical_fallacies": "humanities",
|
||||
"machine_learning": "stem",
|
||||
"management": "other",
|
||||
"marketing": "other",
|
||||
"medical_genetics": "other",
|
||||
"miscellaneous": "other",
|
||||
"moral_disputes": "humanities",
|
||||
"moral_scenarios": "humanities",
|
||||
"nutrition": "other",
|
||||
"philosophy": "humanities",
|
||||
"prehistory": "humanities",
|
||||
"professional_accounting": "other",
|
||||
"professional_law": "humanities",
|
||||
"professional_medicine": "other",
|
||||
"professional_psychology": "social_sciences",
|
||||
"public_relations": "social_sciences",
|
||||
"security_studies": "social_sciences",
|
||||
"sociology": "social_sciences",
|
||||
"us_foreign_policy": "social_sciences",
|
||||
"virology": "other",
|
||||
"world_religions": "humanities",
|
||||
}
|
||||
|
||||
|
||||
class MMLUEval(Eval):
|
||||
def __init__(self, filename: str, num_examples: Optional[int], num_threads: int):
|
||||
if "://" in filename:
|
||||
df = pandas.read_csv(filename, storage_options={"timeout": 30})
|
||||
else:
|
||||
df = pandas.read_csv(filename)
|
||||
examples = [row.to_dict() for _, row in df.iterrows()]
|
||||
if num_examples:
|
||||
examples = random.Random(0).sample(examples, num_examples)
|
||||
self.examples = examples
|
||||
self.num_threads = num_threads
|
||||
|
||||
def __call__(self, sampler: SamplerBase) -> EvalResult:
|
||||
def fn(row: dict):
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=format_multichoice_question(row), role="user"
|
||||
)
|
||||
]
|
||||
response_text = sampler(prompt_messages)
|
||||
response_text = response_text or ""
|
||||
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
|
||||
extracted_answer = match.group(1) if match else None
|
||||
score = 1.0 if extracted_answer == row["Answer"] else 0.0
|
||||
html = common.jinja_env.from_string(HTML_JINJA).render(
|
||||
prompt_messages=prompt_messages,
|
||||
next_message=dict(content=response_text, role="assistant"),
|
||||
score=score,
|
||||
correct_answer=row["Answer"],
|
||||
extracted_answer=extracted_answer,
|
||||
)
|
||||
convo = prompt_messages + [dict(content=response_text, role="assistant")]
|
||||
category = subject2category.get(row["Subject"], "other")
|
||||
return SingleEvalResult(
|
||||
html=html, score=score, metrics={category: score}, convo=convo
|
||||
)
|
||||
|
||||
results = common.map_with_progress(fn, self.examples, self.num_threads)
|
||||
return common.aggregate_results(results)
|
||||
@@ -2409,38 +2409,6 @@ class CustomTestCase(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
def dump_bench_raw_result(
|
||||
path: str,
|
||||
states,
|
||||
preds,
|
||||
labels,
|
||||
):
|
||||
if not path:
|
||||
return
|
||||
|
||||
rows = []
|
||||
for i in range(len(states)):
|
||||
state = states[i]
|
||||
output = state["answer"]
|
||||
prompt = _ensure_remove_suffix(state.text(), output)
|
||||
rows.append(
|
||||
dict(
|
||||
prompt_id=i,
|
||||
prompt=prompt,
|
||||
output=output,
|
||||
correct=bool(preds[i] == labels[i]),
|
||||
)
|
||||
)
|
||||
|
||||
print(f"BenchRawResultDumper save results to {path}")
|
||||
Path(path).write_text("\n".join(json.dumps(row) for row in rows))
|
||||
|
||||
|
||||
def _ensure_remove_suffix(text: str, suffix: str):
|
||||
assert text.endswith(suffix)
|
||||
return text.removesuffix(suffix)
|
||||
|
||||
|
||||
class ModelLaunchSettings:
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -63,7 +63,6 @@ from .process_utils import (
|
||||
wait_for_health,
|
||||
wait_for_workers_ready,
|
||||
)
|
||||
from .run_eval import run_eval
|
||||
|
||||
__all__ = [
|
||||
# Enums and Identity
|
||||
@@ -139,6 +138,4 @@ __all__ = [
|
||||
"FUNCTION_CALLING_MODELS",
|
||||
# Third-party models
|
||||
"THIRD_PARTY_MODELS",
|
||||
# Evaluation
|
||||
"run_eval",
|
||||
]
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
"""MMLU evaluation runner for E2E tests.
|
||||
|
||||
Simplified evaluation runner that uses local eval implementations
|
||||
with cleaner logging for CI/CD environments.
|
||||
|
||||
Usage:
|
||||
from infra.run_eval import run_eval
|
||||
from types import SimpleNamespace
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url="http://127.0.0.1:30000",
|
||||
model="meta-llama/Llama-3.1-8B-Instruct",
|
||||
eval_name="mmlu",
|
||||
num_examples=64,
|
||||
num_threads=32,
|
||||
temperature=0.1,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .simple_eval_common import Eval
|
||||
|
||||
from .simple_eval_common import ChatCompletionSampler, set_ulimit
|
||||
from .simple_eval_mmlu import MMLU_DATASET_URL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalConfig:
|
||||
"""Configuration for running an evaluation."""
|
||||
|
||||
base_url: str
|
||||
model: str | None = None
|
||||
eval_name: str = "mmlu"
|
||||
num_examples: int = 64
|
||||
num_threads: int = 32
|
||||
temperature: float = 0.0
|
||||
max_tokens: int = 2048
|
||||
host: str = "127.0.0.1"
|
||||
port: int = 30000
|
||||
|
||||
|
||||
def _get_eval(eval_name: str, num_examples: int, num_threads: int) -> "Eval":
|
||||
"""Get the evaluation object by name."""
|
||||
if eval_name == "mmlu":
|
||||
from .simple_eval_mmlu import MMLUEval
|
||||
|
||||
return MMLUEval(MMLU_DATASET_URL, num_examples, num_threads)
|
||||
else:
|
||||
raise ValueError(f"Unknown eval: {eval_name}. Supported: mmlu")
|
||||
|
||||
|
||||
def run_eval(args: Any) -> dict:
|
||||
"""Run an evaluation and return metrics.
|
||||
|
||||
Args:
|
||||
args: Configuration object with attributes:
|
||||
- base_url: Base URL of the server (e.g., "http://127.0.0.1:30000")
|
||||
- model: Model name/path (optional, will be auto-detected)
|
||||
- eval_name: Evaluation name ("mmlu")
|
||||
- num_examples: Number of examples to evaluate
|
||||
- num_threads: Number of parallel threads
|
||||
- temperature: Sampling temperature
|
||||
|
||||
Returns:
|
||||
Dict with metrics including 'score' key.
|
||||
"""
|
||||
set_ulimit()
|
||||
|
||||
if "OPENAI_API_KEY" not in os.environ:
|
||||
os.environ["OPENAI_API_KEY"] = "EMPTY"
|
||||
|
||||
# Build base URL
|
||||
base_url = getattr(args, "base_url", None)
|
||||
if base_url:
|
||||
base_url = base_url.rstrip("/") # Remove trailing slashes
|
||||
if not base_url.endswith("/v1"):
|
||||
base_url = f"{base_url}/v1"
|
||||
else:
|
||||
host = getattr(args, "host", "127.0.0.1")
|
||||
port = getattr(args, "port", 30000)
|
||||
base_url = f"http://{host}:{port}/v1"
|
||||
|
||||
eval_name = getattr(args, "eval_name", "mmlu")
|
||||
num_examples = getattr(args, "num_examples", 64)
|
||||
num_threads = getattr(args, "num_threads", 32)
|
||||
temperature = getattr(args, "temperature", 0.0)
|
||||
max_tokens = getattr(args, "max_tokens", 2048)
|
||||
model = getattr(args, "model", None)
|
||||
|
||||
logger.info(
|
||||
"Starting %s eval: %d examples, %d threads, temp=%.2f",
|
||||
eval_name,
|
||||
num_examples,
|
||||
num_threads,
|
||||
temperature,
|
||||
)
|
||||
|
||||
# Create sampler
|
||||
sampler = ChatCompletionSampler(
|
||||
model=model,
|
||||
max_tokens=max_tokens,
|
||||
base_url=base_url,
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
# Get eval object
|
||||
eval_obj = _get_eval(eval_name, num_examples, num_threads)
|
||||
|
||||
# Run evaluation
|
||||
start_time = time.perf_counter()
|
||||
result = eval_obj(sampler)
|
||||
latency = time.perf_counter() - start_time
|
||||
|
||||
# Build metrics
|
||||
metrics = result.metrics.copy() if result.metrics else {}
|
||||
metrics["score"] = result.score
|
||||
metrics["latency"] = latency
|
||||
|
||||
logger.info(
|
||||
"%s eval complete: score=%.3f, latency=%.1fs, model=%s",
|
||||
eval_name,
|
||||
result.score,
|
||||
latency,
|
||||
sampler.model,
|
||||
)
|
||||
|
||||
return metrics
|
||||
@@ -1,495 +0,0 @@
|
||||
# Adapted from https://github.com/openai/simple-evals/
|
||||
"""Common utilities for simple evaluations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from multiprocessing.pool import ThreadPool
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import jinja2
|
||||
import numpy as np
|
||||
import openai
|
||||
import requests
|
||||
from openai import OpenAI
|
||||
from tqdm import tqdm
|
||||
|
||||
from .constants import MAX_RETRY_ATTEMPTS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPENAI_SYSTEM_MESSAGE_API = "You are a helpful assistant."
|
||||
OPENAI_SYSTEM_MESSAGE_CHATGPT = (
|
||||
"You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture."
|
||||
+ "\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01"
|
||||
)
|
||||
|
||||
|
||||
Message = dict[str, Any] # keys role, content
|
||||
MessageList = list[Message]
|
||||
|
||||
|
||||
class SamplerBase:
|
||||
"""
|
||||
Base class for defining a sampling model, which can be evaluated,
|
||||
or used as part of the grading process.
|
||||
"""
|
||||
|
||||
def __call__(self, message_list: MessageList) -> str:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalResult:
|
||||
"""Result of running an evaluation (usually consisting of many samples)."""
|
||||
|
||||
score: float | None # top-line metric
|
||||
metrics: dict[str, float] | None # other metrics
|
||||
htmls: list[str] # strings of valid HTML
|
||||
convos: list[MessageList] # sampled conversations
|
||||
|
||||
|
||||
@dataclass
|
||||
class SingleEvalResult:
|
||||
"""Result of evaluating a single sample."""
|
||||
|
||||
score: float | None
|
||||
metrics: dict[str, float] = field(default_factory=dict)
|
||||
html: str | None = None
|
||||
convo: MessageList | None = None # sampled conversation
|
||||
|
||||
|
||||
class Eval:
|
||||
"""
|
||||
Base class for defining an evaluation.
|
||||
"""
|
||||
|
||||
def __call__(self, sampler: SamplerBase) -> EvalResult:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class LargerHttpxClient(httpx.Client):
|
||||
def __init__(self):
|
||||
timeout_config = httpx.Timeout(3600)
|
||||
limits = httpx.Limits(
|
||||
max_keepalive_connections=3600,
|
||||
max_connections=3600,
|
||||
)
|
||||
super().__init__(timeout=timeout_config, limits=limits)
|
||||
|
||||
|
||||
class ChatCompletionSampler(SamplerBase):
|
||||
"""Sample from OpenAI's chat completion API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str | None = None,
|
||||
model: str | None = None,
|
||||
system_message: str | None = None,
|
||||
temperature: float = 0.0,
|
||||
reasoning_effort: str | None = None,
|
||||
max_tokens: int = 2048,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
):
|
||||
self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
|
||||
|
||||
if model is None:
|
||||
model = self.client.models.list().data[0].id
|
||||
|
||||
self.model = model
|
||||
self.system_message = system_message
|
||||
self.temperature = temperature
|
||||
self.max_tokens = max_tokens
|
||||
self.reasoning_effort = reasoning_effort
|
||||
self.extra_body = extra_body
|
||||
self.image_format = "url"
|
||||
logger.debug(
|
||||
"ChatCompletionSampler: model=%s, temp=%.2f, max_tokens=%d",
|
||||
self.model,
|
||||
self.temperature,
|
||||
self.max_tokens,
|
||||
)
|
||||
|
||||
def _handle_image(
|
||||
self,
|
||||
image: str,
|
||||
encoding: str = "base64",
|
||||
format: str = "png",
|
||||
):
|
||||
new_image = {
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/{format};{encoding},{image}",
|
||||
},
|
||||
}
|
||||
return new_image
|
||||
|
||||
def _handle_text(self, text: str):
|
||||
return {"type": "text", "text": text}
|
||||
|
||||
def _pack_message(self, role: str, content: Any):
|
||||
return {"role": str(role), "content": content}
|
||||
|
||||
def __call__(self, message_list: MessageList) -> str:
|
||||
if self.system_message:
|
||||
message_list = [
|
||||
self._pack_message("system", self.system_message)
|
||||
] + message_list
|
||||
trial = 0
|
||||
while trial < MAX_RETRY_ATTEMPTS:
|
||||
try:
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=message_list,
|
||||
temperature=self.temperature,
|
||||
max_tokens=self.max_tokens,
|
||||
reasoning_effort=self.reasoning_effort,
|
||||
extra_body=self.extra_body,
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
except openai.BadRequestError as e:
|
||||
logger.warning("Bad request error: %s", e)
|
||||
return ""
|
||||
except Exception as e:
|
||||
exception_backoff = 2**trial # exponential back off
|
||||
# Log first few retries at debug, later ones at warning
|
||||
log_fn = logger.warning if trial >= 3 else logger.debug
|
||||
log_fn(
|
||||
"Request failed (retry %d/%d, backoff %ds): %s",
|
||||
trial + 1,
|
||||
MAX_RETRY_ATTEMPTS,
|
||||
exception_backoff,
|
||||
e,
|
||||
)
|
||||
time.sleep(exception_backoff)
|
||||
trial += 1
|
||||
logger.warning(
|
||||
"All retry attempts exhausted after %d retries, returning empty response",
|
||||
MAX_RETRY_ATTEMPTS,
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
QUERY_TEMPLATE_MULTICHOICE = """
|
||||
Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
|
||||
|
||||
{Question}
|
||||
|
||||
A) {A}
|
||||
B) {B}
|
||||
C) {C}
|
||||
D) {D}
|
||||
""".strip()
|
||||
|
||||
ANSWER_PATTERN_MULTICHOICE = r"(?i)Answer\s*:\s*([A-D])"
|
||||
ANSWER_PATTERN = r"(?i)Answer\s*:\s*([^\n]+)"
|
||||
|
||||
|
||||
EQUALITY_TEMPLATE = r"""
|
||||
Look at the following two expressions (answers to a math problem) and judge whether they are equivalent. Only perform trivial simplifications
|
||||
|
||||
Examples:
|
||||
|
||||
Expression 1: $2x+3$
|
||||
Expression 2: $3+2x$
|
||||
|
||||
Yes
|
||||
|
||||
Expression 1: 3/2
|
||||
Expression 2: 1.5
|
||||
|
||||
Yes
|
||||
|
||||
Expression 1: $x^2+2x+1$
|
||||
Expression 2: $y^2+2y+1$
|
||||
|
||||
No
|
||||
|
||||
Expression 1: $x^2+2x+1$
|
||||
Expression 2: $(x+1)^2$
|
||||
|
||||
Yes
|
||||
|
||||
Expression 1: 3245/5
|
||||
Expression 2: 649
|
||||
|
||||
No
|
||||
(these are actually equal, don't mark them equivalent if you need to do nontrivial simplifications)
|
||||
|
||||
Expression 1: 2/(-3)
|
||||
Expression 2: -2/3
|
||||
|
||||
Yes
|
||||
(trivial simplifications are allowed)
|
||||
|
||||
Expression 1: 72 degrees
|
||||
Expression 2: 72
|
||||
|
||||
Yes
|
||||
(give benefit of the doubt to units)
|
||||
|
||||
Expression 1: 64
|
||||
Expression 2: 64 square feet
|
||||
|
||||
Yes
|
||||
(give benefit of the doubt to units)
|
||||
|
||||
---
|
||||
|
||||
YOUR TASK
|
||||
|
||||
|
||||
Respond with only "Yes" or "No" (without quotes). Do not include a rationale.
|
||||
|
||||
Expression 1: %(expression1)s
|
||||
Expression 2: %(expression2)s
|
||||
""".strip()
|
||||
|
||||
|
||||
HTML_JINJA = """
|
||||
<h3>Prompt conversation</h3>
|
||||
{% for message in prompt_messages %}
|
||||
{{ message_to_html(message) | safe }}
|
||||
{% endfor %}
|
||||
<h3>Sampled message</h3>
|
||||
{{ message_to_html(next_message) | safe }}
|
||||
<h3>Results</h3>
|
||||
<p>Correct Answer: {{ correct_answer }}</p>
|
||||
<p>Extracted Answer: {{ extracted_answer }}</p>
|
||||
<p>Score: {{ score }}</p>
|
||||
"""
|
||||
|
||||
|
||||
def format_multichoice_question(row):
|
||||
return QUERY_TEMPLATE_MULTICHOICE.format(**row)
|
||||
|
||||
|
||||
def check_equality(sampler: SamplerBase, expr1: str, expr2: str):
|
||||
prompt = EQUALITY_TEMPLATE % {"expression1": expr1, "expression2": expr2}
|
||||
response = sampler([dict(content=prompt, role="user")])
|
||||
return (response or "").lower().strip() == "yes"
|
||||
|
||||
|
||||
def _compute_stat(values: list, stat: str):
|
||||
if stat == "mean":
|
||||
return np.mean(values)
|
||||
elif stat == "std":
|
||||
return np.std(values)
|
||||
elif stat == "min":
|
||||
return np.min(values)
|
||||
elif stat == "max":
|
||||
return np.max(values)
|
||||
else:
|
||||
raise ValueError(f"Unknown {stat =}")
|
||||
|
||||
|
||||
def aggregate_results(
|
||||
single_eval_results: list[SingleEvalResult],
|
||||
default_stats: tuple[str, ...] = ("mean", "std"),
|
||||
name2stats: dict[str, tuple[str, ...]] | None = None,
|
||||
) -> EvalResult:
|
||||
"""
|
||||
Aggregate results from multiple evaluations into a single EvalResult.
|
||||
"""
|
||||
name2stats = name2stats or {}
|
||||
name2values = defaultdict(list)
|
||||
htmls = []
|
||||
convos = []
|
||||
for single_eval_result in single_eval_results:
|
||||
# Skip None results
|
||||
if single_eval_result is None:
|
||||
continue
|
||||
for name, value in single_eval_result.metrics.items():
|
||||
name2values[name].append(value)
|
||||
if single_eval_result.score is not None:
|
||||
name2values["score"].append(single_eval_result.score)
|
||||
htmls.append(single_eval_result.html)
|
||||
convos.append(single_eval_result.convo)
|
||||
final_metrics = {}
|
||||
for name, values in name2values.items():
|
||||
stats = name2stats.get(name, default_stats)
|
||||
for stat in stats:
|
||||
key = name if stat == "mean" else f"{name}:{stat}"
|
||||
final_metrics[key] = _compute_stat(values, stat)
|
||||
return EvalResult(
|
||||
score=final_metrics.pop("score", None),
|
||||
metrics=final_metrics,
|
||||
htmls=htmls,
|
||||
convos=convos,
|
||||
)
|
||||
|
||||
|
||||
def map_with_progress(f: callable, xs: list[Any], num_threads: int) -> list[Any]:
|
||||
"""Apply f to each element of xs, using a ThreadPool, and show progress."""
|
||||
# Use quiet progress bar that doesn't pollute logs
|
||||
if os.getenv("debug"):
|
||||
return list(map(f, tqdm(xs, total=len(xs), leave=False)))
|
||||
else:
|
||||
with ThreadPool(min(num_threads, len(xs))) as pool:
|
||||
return list(tqdm(pool.imap(f, xs), total=len(xs), leave=False))
|
||||
|
||||
|
||||
jinja_env = jinja2.Environment(
|
||||
loader=jinja2.BaseLoader(),
|
||||
undefined=jinja2.StrictUndefined,
|
||||
autoescape=jinja2.select_autoescape(["html", "xml"]),
|
||||
)
|
||||
_message_template = """
|
||||
<div class="message {{ role }}">
|
||||
<div class="role">
|
||||
{{ role }}
|
||||
{% if variant %}<span class="variant">({{ variant }})</span>{% endif %}
|
||||
</div>
|
||||
<div class="content">
|
||||
<pre>{{ content }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
def message_to_html(message: Message) -> str:
|
||||
"""
|
||||
Generate HTML snippet (inside a <div>) for a message.
|
||||
"""
|
||||
return jinja_env.from_string(_message_template).render(
|
||||
role=message["role"],
|
||||
content=message["content"],
|
||||
variant=message.get("variant", None),
|
||||
)
|
||||
|
||||
|
||||
jinja_env.globals["message_to_html"] = message_to_html
|
||||
|
||||
|
||||
_report_template = """<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
.message {
|
||||
padding: 8px 16px;
|
||||
margin-bottom: 8px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
.message.user {
|
||||
background-color: #B2DFDB;
|
||||
color: #00695C;
|
||||
}
|
||||
.message.assistant {
|
||||
background-color: #B39DDB;
|
||||
color: #4527A0;
|
||||
}
|
||||
.message.system {
|
||||
background-color: #EEEEEE;
|
||||
color: #212121;
|
||||
}
|
||||
.role {
|
||||
font-weight: bold;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.variant {
|
||||
color: #795548;
|
||||
}
|
||||
table, th, td {
|
||||
border: 1px solid black;
|
||||
}
|
||||
pre {
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{% if metrics %}
|
||||
<h1>Metrics</h1>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Metric</th>
|
||||
<th>Value</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><b>Score</b></td>
|
||||
<td>{{ score | float | round(3) }}</td>
|
||||
</tr>
|
||||
{% for name, value in metrics.items() %}
|
||||
<tr>
|
||||
<td>{{ name }}</td>
|
||||
<td>{{ value }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
{% endif %}
|
||||
<h1>Examples</h1>
|
||||
{% for html in htmls %}
|
||||
{{ html | safe }}
|
||||
<hr>
|
||||
{% endfor %}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def make_report(eval_result: EvalResult) -> str:
|
||||
"""
|
||||
Create a standalone HTML report from an EvalResult.
|
||||
"""
|
||||
return jinja_env.from_string(_report_template).render(
|
||||
score=eval_result.score,
|
||||
metrics=eval_result.metrics,
|
||||
htmls=eval_result.htmls,
|
||||
)
|
||||
|
||||
|
||||
def make_report_from_example_htmls(htmls: list[str]):
|
||||
"""
|
||||
Create a standalone HTML report from a list of example htmls
|
||||
"""
|
||||
return jinja_env.from_string(_report_template).render(
|
||||
score=None, metrics={}, htmls=htmls
|
||||
)
|
||||
|
||||
|
||||
def download_dataset(path: str, url: str) -> None:
|
||||
"""Download a dataset from URL to path."""
|
||||
logger.info("Downloading dataset from %s", url)
|
||||
try:
|
||||
response = requests.get(url, stream=True, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
block_size = 8192
|
||||
|
||||
with (
|
||||
open(path, "wb") as f,
|
||||
tqdm(
|
||||
desc="Downloading",
|
||||
total=total_size,
|
||||
unit="iB",
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
leave=False,
|
||||
) as progress_bar,
|
||||
):
|
||||
for data in response.iter_content(block_size):
|
||||
size = f.write(data)
|
||||
progress_bar.update(size)
|
||||
|
||||
logger.debug("Dataset saved to %s", path)
|
||||
except requests.RequestException as e:
|
||||
raise RuntimeError(f"Failed to download dataset: {e}") from e
|
||||
|
||||
|
||||
def set_ulimit(target_soft_limit: int = 65535) -> None:
|
||||
"""Set the file descriptor limit for parallel requests."""
|
||||
resource_type = resource.RLIMIT_NOFILE
|
||||
current_soft, current_hard = resource.getrlimit(resource_type)
|
||||
|
||||
if current_soft < target_soft_limit:
|
||||
try:
|
||||
resource.setrlimit(resource_type, (target_soft_limit, current_hard))
|
||||
except ValueError as e:
|
||||
logger.debug("Could not set RLIMIT_NOFILE: %s", e)
|
||||
@@ -1,132 +0,0 @@
|
||||
# Adapted from https://github.com/openai/simple-evals/
|
||||
"""
|
||||
MMLU Evaluation - Measuring Massive Multitask Language Understanding
|
||||
Dan Hendrycks et al. https://arxiv.org/abs/2009.03300
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pandas
|
||||
|
||||
from . import simple_eval_common as common
|
||||
from .simple_eval_common import (
|
||||
ANSWER_PATTERN_MULTICHOICE,
|
||||
HTML_JINJA,
|
||||
Eval,
|
||||
EvalResult,
|
||||
SingleEvalResult,
|
||||
format_multichoice_question,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .simple_eval_common import SamplerBase
|
||||
|
||||
# MMLU dataset URL (hosted by OpenAI)
|
||||
MMLU_DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
|
||||
|
||||
SUBJECT_TO_CATEGORY = {
|
||||
"abstract_algebra": "stem",
|
||||
"anatomy": "other",
|
||||
"astronomy": "stem",
|
||||
"business_ethics": "other",
|
||||
"clinical_knowledge": "other",
|
||||
"college_biology": "stem",
|
||||
"college_chemistry": "stem",
|
||||
"college_computer_science": "stem",
|
||||
"college_mathematics": "stem",
|
||||
"college_medicine": "other",
|
||||
"college_physics": "stem",
|
||||
"computer_security": "stem",
|
||||
"conceptual_physics": "stem",
|
||||
"econometrics": "social_sciences",
|
||||
"electrical_engineering": "stem",
|
||||
"elementary_mathematics": "stem",
|
||||
"formal_logic": "humanities",
|
||||
"global_facts": "other",
|
||||
"high_school_biology": "stem",
|
||||
"high_school_chemistry": "stem",
|
||||
"high_school_computer_science": "stem",
|
||||
"high_school_european_history": "humanities",
|
||||
"high_school_geography": "social_sciences",
|
||||
"high_school_government_and_politics": "social_sciences",
|
||||
"high_school_macroeconomics": "social_sciences",
|
||||
"high_school_mathematics": "stem",
|
||||
"high_school_microeconomics": "social_sciences",
|
||||
"high_school_physics": "stem",
|
||||
"high_school_psychology": "social_sciences",
|
||||
"high_school_statistics": "stem",
|
||||
"high_school_us_history": "humanities",
|
||||
"high_school_world_history": "humanities",
|
||||
"human_aging": "other",
|
||||
"human_sexuality": "social_sciences",
|
||||
"international_law": "humanities",
|
||||
"jurisprudence": "humanities",
|
||||
"logical_fallacies": "humanities",
|
||||
"machine_learning": "stem",
|
||||
"management": "other",
|
||||
"marketing": "other",
|
||||
"medical_genetics": "other",
|
||||
"miscellaneous": "other",
|
||||
"moral_disputes": "humanities",
|
||||
"moral_scenarios": "humanities",
|
||||
"nutrition": "other",
|
||||
"philosophy": "humanities",
|
||||
"prehistory": "humanities",
|
||||
"professional_accounting": "other",
|
||||
"professional_law": "humanities",
|
||||
"professional_medicine": "other",
|
||||
"professional_psychology": "social_sciences",
|
||||
"public_relations": "social_sciences",
|
||||
"security_studies": "social_sciences",
|
||||
"sociology": "social_sciences",
|
||||
"us_foreign_policy": "social_sciences",
|
||||
"virology": "other",
|
||||
"world_religions": "humanities",
|
||||
}
|
||||
|
||||
|
||||
class MMLUEval(Eval):
|
||||
"""MMLU benchmark evaluation."""
|
||||
|
||||
def __init__(self, filename: str, num_examples: int | None, num_threads: int):
|
||||
if "://" in filename:
|
||||
df = pandas.read_csv(filename, storage_options={"timeout": 30})
|
||||
else:
|
||||
df = pandas.read_csv(filename)
|
||||
examples = [row.to_dict() for _, row in df.iterrows()]
|
||||
if num_examples:
|
||||
examples = random.Random(0).sample(examples, num_examples)
|
||||
self.examples = examples
|
||||
self.num_threads = num_threads
|
||||
|
||||
def __call__(self, sampler: "SamplerBase") -> EvalResult:
|
||||
def fn(row: dict) -> SingleEvalResult:
|
||||
prompt_messages = [
|
||||
sampler._pack_message(
|
||||
content=format_multichoice_question(row), role="user"
|
||||
)
|
||||
]
|
||||
response_text = sampler(prompt_messages)
|
||||
response_text = response_text or ""
|
||||
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
|
||||
extracted_answer = match.group(1) if match else None
|
||||
score = 1.0 if extracted_answer == row["Answer"] else 0.0
|
||||
html = common.jinja_env.from_string(HTML_JINJA).render(
|
||||
prompt_messages=prompt_messages,
|
||||
next_message=dict(content=response_text, role="assistant"),
|
||||
score=score,
|
||||
correct_answer=row["Answer"],
|
||||
extracted_answer=extracted_answer,
|
||||
)
|
||||
convo = prompt_messages + [dict(content=response_text, role="assistant")]
|
||||
category = SUBJECT_TO_CATEGORY.get(row["Subject"], "other")
|
||||
return SingleEvalResult(
|
||||
html=html, score=score, metrics={category: score}, convo=convo
|
||||
)
|
||||
|
||||
results = common.map_with_progress(fn, self.examples, self.num_threads)
|
||||
return common.aggregate_results(results)
|
||||
@@ -1,76 +0,0 @@
|
||||
"""MMLU evaluation tests for router functionality.
|
||||
|
||||
Tests the router's ability to handle MMLU benchmark evaluations across
|
||||
different backend configurations (gRPC and HTTP workers).
|
||||
|
||||
Usage:
|
||||
# Run with gRPC backend only
|
||||
pytest e2e_test/router/test_mmlu.py -v
|
||||
|
||||
# Run with specific backend
|
||||
pytest e2e_test/router/test_mmlu.py -v -k "grpc"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from infra import run_eval
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True)
|
||||
class TestMMLU:
|
||||
"""MMLU evaluation tests using local workers (gRPC and HTTP)."""
|
||||
|
||||
def test_mmlu_basic(self, setup_backend):
|
||||
"""Basic MMLU evaluation with score threshold.
|
||||
|
||||
Runs MMLU evaluation with 64 examples and validates that
|
||||
accuracy meets minimum threshold (>= 0.65).
|
||||
|
||||
Note: setup_backend fixture already waits for workers to be ready.
|
||||
"""
|
||||
backend, model, client, *_ = setup_backend
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=str(client.base_url),
|
||||
model=model,
|
||||
eval_name="mmlu",
|
||||
num_examples=64,
|
||||
num_threads=32,
|
||||
temperature=0.1,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
|
||||
assert metrics["score"] >= 0.65, (
|
||||
f"MMLU score {metrics['score']:.2f} below threshold 0.65"
|
||||
)
|
||||
logger.info("MMLU score: %.2f (threshold: 0.65)", metrics["score"])
|
||||
|
||||
def test_mmlu_extended(self, setup_backend):
|
||||
"""Extended MMLU evaluation with more examples.
|
||||
|
||||
Runs MMLU with 128 examples for more statistically
|
||||
significant results.
|
||||
"""
|
||||
backend, model, client, *_ = setup_backend
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=str(client.base_url),
|
||||
model=model,
|
||||
eval_name="mmlu",
|
||||
num_examples=128,
|
||||
num_threads=64,
|
||||
temperature=0.1,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
|
||||
assert metrics["score"] >= 0.65, (
|
||||
f"MMLU score {metrics['score']:.2f} below threshold 0.65"
|
||||
)
|
||||
logger.info("MMLU extended score: %.2f (threshold: 0.65)", metrics["score"])
|
||||
@@ -1,61 +0,0 @@
|
||||
"""MMLU evaluation tests for PD (Prefill-Decode) disaggregated routing.
|
||||
|
||||
PD disaggregation separates prefill and decode phases across different
|
||||
workers for improved throughput and resource utilization.
|
||||
|
||||
Requirements:
|
||||
- sgl_kernel package
|
||||
- GPUs: num_prefill + num_decode (default: 2 GPUs for 1+1)
|
||||
- Optional: InfiniBand for high-performance transfers
|
||||
|
||||
Configuration via markers:
|
||||
@pytest.mark.model("model-id") # Override default model
|
||||
@pytest.mark.workers(prefill=2, decode=2) # Custom worker counts
|
||||
@pytest.mark.gateway(policy="round_robin") # Gateway configuration
|
||||
|
||||
Usage:
|
||||
# Basic (1 prefill + 1 decode)
|
||||
pytest e2e_test/router/test_pd_mmlu.py -v
|
||||
|
||||
# Run specific test
|
||||
pytest e2e_test/router/test_pd_mmlu.py::TestPDMMLU::test_pd_mmlu_basic -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from infra import run_eval
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
@pytest.mark.parametrize("setup_backend", ["pd"], indirect=True)
|
||||
class TestPDMMLU:
|
||||
"""MMLU evaluation tests using PD disaggregated routing."""
|
||||
|
||||
def test_pd_mmlu_basic(self, setup_backend):
|
||||
"""Basic MMLU evaluation with PD disaggregation.
|
||||
|
||||
Runs MMLU with 1 prefill + 1 decode worker and validates
|
||||
accuracy meets threshold (>= 0.65).
|
||||
"""
|
||||
backend, model, client, *_ = setup_backend
|
||||
|
||||
args = SimpleNamespace(
|
||||
base_url=str(client.base_url),
|
||||
model=model,
|
||||
eval_name="mmlu",
|
||||
num_examples=64,
|
||||
num_threads=32,
|
||||
temperature=0.1,
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
|
||||
assert metrics["score"] >= 0.65, (
|
||||
f"PD MMLU score {metrics['score']:.2f} below threshold 0.65"
|
||||
)
|
||||
logger.info("PD MMLU score: %.2f (threshold: 0.65)", metrics["score"])
|
||||
@@ -137,6 +137,7 @@ This README mostly describes the NVIDIA GPU CI pipeline. Other hardware backends
|
||||
- Each test file should take < 500 seconds; split if longer.
|
||||
- Each GitHub Actions job should take < 30 minutes; split if longer.
|
||||
- If tests are too slow for per-commit, consider nightly suites.
|
||||
- New GSM8K or MMLU accuracy checks should use sgl-eval (`run_eval(..., api="sgl_eval")` or `sgl-eval run`) instead of adding another dataset or evaluator copy.
|
||||
|
||||
## Other Notes
|
||||
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
|
||||
tasks:
|
||||
- name: "gsm8k"
|
||||
metrics:
|
||||
- name: "exact_match,strict-match"
|
||||
value: 0.847
|
||||
- name: "exact_match,flexible-extract"
|
||||
value: 0.556
|
||||
limit: 1319
|
||||
num_concurrent: 128
|
||||
num_fewshot: 5
|
||||
apply_chat_template: false
|
||||
fewshot_as_multiturn: true
|
||||
@@ -1,13 +0,0 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
|
||||
tasks:
|
||||
- name: "gsm8k"
|
||||
metrics:
|
||||
- name: "exact_match,strict-match"
|
||||
value: 0.847
|
||||
- name: "exact_match,flexible-extract"
|
||||
value: 0.556
|
||||
limit: 1319
|
||||
num_concurrent: 128
|
||||
num_fewshot: 5
|
||||
apply_chat_template: false
|
||||
fewshot_as_multiturn: true
|
||||
@@ -1,13 +0,0 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
|
||||
tasks:
|
||||
- name: "gsm8k"
|
||||
metrics:
|
||||
- name: "exact_match,strict-match"
|
||||
value: 0.847
|
||||
- name: "exact_match,flexible-extract"
|
||||
value: 0.444
|
||||
limit: 1319
|
||||
num_concurrent: 128
|
||||
num_fewshot: 5
|
||||
apply_chat_template: false
|
||||
fewshot_as_multiturn: true
|
||||
@@ -1,69 +0,0 @@
|
||||
"""Archived test classes split out of test/registered/models_e2e/test_nvidia_nemotron_3_nano.py.
|
||||
|
||||
Originally registered with `register_cuda_ci(...)`. Moved here as part of
|
||||
the per-commit pruning effort to keep the code reachable manually.
|
||||
Run with `python3 test/manual/models/test_nvidia_nemotron_3_nano_archived.py`.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.utils import is_sm80_supported, is_sm90_supported
|
||||
from sglang.test.kits.lm_eval_kit import LMEvalMixin
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
|
||||
NEMOTRON_3_NANO_THINKING_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--tool-call-parser",
|
||||
"qwen3_coder",
|
||||
"--reasoning-parser",
|
||||
"deepseek-r1",
|
||||
]
|
||||
|
||||
|
||||
class TestNvidiaNemotron3Nano30BBF16(LMEvalMixin, DefaultServerBase):
|
||||
"""Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation."""
|
||||
|
||||
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
|
||||
model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml"
|
||||
other_args = [
|
||||
"--tp-size",
|
||||
"2",
|
||||
] + NEMOTRON_3_NANO_THINKING_ARGS
|
||||
|
||||
|
||||
class TestNvidiaNemotron3Nano30BBF16FlashInfer(LMEvalMixin, DefaultServerBase):
|
||||
"""Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation using flashinfer mamba backend."""
|
||||
|
||||
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
|
||||
model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml"
|
||||
other_args = [
|
||||
"--tp-size",
|
||||
"2",
|
||||
"--mamba-backend",
|
||||
"flashinfer",
|
||||
] + NEMOTRON_3_NANO_THINKING_ARGS
|
||||
|
||||
|
||||
@unittest.skipIf(
|
||||
not (is_sm80_supported() or is_sm90_supported()),
|
||||
"NVFP4 Marlin fallback test requires CUDA SM8X/SM9X",
|
||||
)
|
||||
class TestNvidiaNemotron3Nano30BNVFP4Marlin(LMEvalMixin, DefaultServerBase):
|
||||
"""Test Nemotron-3-Nano-30B NVFP4 model with the Marlin path."""
|
||||
|
||||
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
|
||||
model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.yaml"
|
||||
other_args = [
|
||||
"--tp-size",
|
||||
"1",
|
||||
"--quantization",
|
||||
"modelopt_fp4",
|
||||
"--fp4-gemm-backend",
|
||||
"marlin",
|
||||
"--moe-runner-backend",
|
||||
"marlin",
|
||||
] + NEMOTRON_3_NANO_THINKING_ARGS
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -17,7 +17,6 @@ import subprocess
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import requests
|
||||
@@ -43,10 +42,6 @@ SERVER_LAUNCH_TIMEOUT = 4800
|
||||
GSM8K_NUM_QUESTIONS = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
|
||||
ACCURACY_THRESHOLD = 0.94
|
||||
|
||||
# bench_sglang.py lives at the repo root (this file is 5 levels below), not under
|
||||
# test/. Resolve it absolutely so it works regardless of the CI working directory.
|
||||
REPO_ROOT = Path(__file__).resolve().parents[5]
|
||||
GSM8K_BENCH_SCRIPT = REPO_ROOT / "benchmark" / "gsm8k" / "bench_sglang.py"
|
||||
GSM8K_DATA_URL = (
|
||||
"https://raw.githubusercontent.com/openai/grade-school-math/"
|
||||
"master/grade_school_math/data/test.jsonl"
|
||||
@@ -145,7 +140,8 @@ class TestQwen35Fp8ArFusionMI35x(CustomTestCase):
|
||||
port = int(base_url.rsplit(":", 1)[-1])
|
||||
command = [
|
||||
"python3",
|
||||
str(GSM8K_BENCH_SCRIPT),
|
||||
"-m",
|
||||
"sglang.test.few_shot_gsm8k",
|
||||
"--num-questions",
|
||||
str(GSM8K_NUM_QUESTIONS),
|
||||
"--parallel",
|
||||
|
||||
Reference in New Issue
Block a user