[Test] Remove dead eval modules and point GSM8K/MMLU docs to sgl-eval (#41215)

This commit is contained in:
Liangsheng Yin
2026-09-25 12:50:31 -07:00
committed by GitHub
parent ac62ddf164
commit f337f01b80
80 changed files with 394 additions and 2388 deletions
+2 -2
View File
@@ -136,8 +136,8 @@ than restating.
= `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. Flag output-only values.
- **Consistent accuracy harness across entries**: every value under one `accuracyLabels`
column must be produced by the SAME harness — flag a page that, say, measures one
platform's GSM8K with `few_shot_gsm8k --num-questions 200` and another's with
`run_eval --eval-name gsm8k --num-examples 1319` and shows both as one "GSM8K %"
platform's GSM8K with a historical raw completion harness and another's with
`sgl-eval run gsm8k --num-examples 1319` and shows both as one "GSM8K %"
(the scores aren't comparable). Either standardize on one harness (matching
`benchmarkCommands.accuracy`) or require an explicit per-entry note. Common when folding
a second contributor's measurements (e.g. an AMD/ROCm PR) into the page.
+1 -3
View File
@@ -46,13 +46,11 @@ Typical startup time: 30-90 seconds depending on model size and whether CUDA gra
### Step 3: Validate accuracy (sanity check)
```bash
python3 -m sglang.test.run_eval --host 127.0.0.1 --port <port> --eval-name gsm8k --num-examples 20
sgl-eval run gsm8k --base-url http://127.0.0.1:<port>/v1 --num-examples 20
```
- Expected accuracy: **> 0.8** for capable models (Qwen3-8B, Llama-3.1-8B-Instruct, etc.)
- This is a quick sanity check, not a rigorous benchmark.
- `sglang.test.few_shot_gsm8k` is deprecated; use the unified `run_eval` entrypoint.
- If you intentionally need the old completion-style GSM8K path, add `--api completion`.
- If accuracy is unexpectedly low, something is wrong — do not proceed to profiling.
### Step 4: Generate the profile
-2
View File
@@ -166,8 +166,6 @@ cython_debug/
*.swp
# SGL
benchmark/mmlu/data
benchmark/mmlu/data.tar
benchmark/llava_bench/images
benchmark/llava_bench/mme_pack
*.jsonl
+7 -3
View File
@@ -271,7 +271,9 @@ Then we can benchmark the accuracy and latency by accessing the first node's exp
```bash
# bench accuracy
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host 10.0.0.1 --port 30000
sgl-eval run gsm8k \
--base-url http://10.0.0.1:30000/v1 \
--num-examples 1319 --num-threads 64
# bench latency
python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128
@@ -322,13 +324,15 @@ Then on the **master node**, supposing the ShareGPT data is located at `/path/to
```bash
# bench accuracy
python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 --num-threads 64
# bench serving
python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0
```
> **Note: using `--parallel 200` can accelerate accuracy benchmarking**.
> **Note: using `--num-threads 200` can accelerate accuracy benchmarking**.
### Example: Serving with 32 L40S with int8 Quantization
-20
View File
@@ -1,20 +0,0 @@
## Run benchmark
### Using GSM8K Platinum
GSM8K Platinum is a revised version of the GSM8K test set with corrected labels and removed ambiguous questions. It can be more stable than the original GSM8K dataset. It's a drop-in replacement that can be used by adding the `--platinum` flag:
```
python3 bench_sglang.py --num-shots 8 --num-questions 1209 --parallel 1209 --platinum
```
For more information, see: https://huggingface.co/datasets/madrylab/gsm8k-platinum
### Benchmark sglang
```
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
```
```
python3 bench_sglang.py --num-questions 200
```
-207
View File
@@ -1,207 +0,0 @@
import argparse
import ast
import json
import os
import re
import time
import numpy as np
from datasets import load_dataset
from sglang.lang.api import set_default_backend
from sglang.test.test_utils import (
add_common_sglang_args_and_parse,
dump_bench_raw_result,
select_sglang_backend,
)
from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def main(args):
# Select backend
set_default_backend(select_sglang_backend(args))
# Load tokenizer if enable_thinking is set
tokenizer = None
if args.enable_thinking:
from transformers import AutoTokenizer
assert args.tokenizer_path is not None, (
"--tokenizer-path is required when --enable-thinking is set"
)
tokenizer = AutoTokenizer.from_pretrained(
args.tokenizer_path, trust_remote_code=True
)
# Read data
if args.platinum:
print("Loading GSM8K Platinum dataset from HuggingFace...")
dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test")
lines = [
{"question": item["question"], "answer": item["answer"]} for item in dataset
]
else:
data_path = args.data_path
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
if not os.path.isfile(data_path):
data_path = download_and_cache_file(url)
lines = list(read_jsonl(data_path))
# Construct prompts
num_questions = args.num_questions
num_shots = args.num_shots
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
raw_question = few_shot_examples + get_one_example(lines, i, False)
if tokenizer is not None:
messages = [{"role": "user", "content": raw_question}]
raw_question = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
**args.chat_template_kwargs,
)
questions.append(raw_question)
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
#####################################
######### SGL Program Begin #########
#####################################
import sglang as sgl
@sgl.function
def few_shot_gsm8k(s, question):
s += question
s += sgl.gen(
"answer",
max_tokens=args.max_new_tokens,
stop=["Question", "Assistant:", "<|separator|>"],
)
#####################################
########## SGL Program End ##########
#####################################
# Run requests
tic = time.perf_counter()
states = few_shot_gsm8k.run_batch(
arguments,
temperature=args.temperature,
top_p=args.top_p,
num_threads=args.parallel,
progress_bar=True,
)
latency = time.perf_counter() - tic
preds = []
for i in range(len(states)):
preds.append(get_answer_value(states[i]["answer"]))
# Compute accuracy
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
# Compute speed
num_output_tokens = sum(
s.get_meta_info("answer")["completion_tokens"] for s in states
)
output_throughput = num_output_tokens / latency
# Print results
print(f"Accuracy: {acc:.3f}")
print(f"Invalid: {invalid:.3f}")
print(f"Latency: {latency:.3f} s")
print(f"Output throughput: {output_throughput:.3f} token/s")
# Dump results
dump_state_text(f"tmp_output_{args.backend}.txt", states)
dump_bench_raw_result(
path=args.raw_result_file,
states=states,
preds=preds,
labels=labels,
)
with open(args.result_file, "a") as fout:
value = {
"task": "gsm8k-platinum" if args.platinum else "gsm8k",
"backend": args.backend,
"num_gpus": 1,
"latency": round(latency, 3),
"accuracy": round(acc, 3),
"num_requests": args.num_questions,
"other": {
"num_questions": args.num_questions,
"parallel": args.parallel,
},
}
fout.write(json.dumps(value) + "\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--num-shots", type=int, default=5)
parser.add_argument("--data-path", type=str, default="test.jsonl")
parser.add_argument("--num-questions", type=int, default=200)
parser.add_argument("--max-new-tokens", type=int, default=512)
parser.add_argument("--temperature", type=float, default=0.0)
parser.add_argument("--top-p", type=float, default=1.0)
parser.add_argument(
"--enable-thinking",
action="store_true",
help="Enable thinking mode by wrapping prompts with chat template",
)
parser.add_argument(
"--chat-template-kwargs",
type=json.loads,
default='{"enable_thinking": true}',
help="JSON dict passed through to tokenizer.apply_chat_template. "
"The thinking-toggle kwarg name is model-specific, e.g. "
"'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).",
)
parser.add_argument(
"--tokenizer-path",
type=str,
default=None,
help="Path to tokenizer (required when --enable-thinking is set)",
)
parser.add_argument(
"--platinum",
action="store_true",
help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)",
)
args = add_common_sglang_args_and_parse(parser)
main(args)
-20
View File
@@ -1,20 +0,0 @@
## Download data
```
bash download_data.sh
```
## Run benchmark
### Benchmark sglang
```
python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000
```
```
python3 bench_sglang.py --nsub 10
```
```
# OpenAI models
python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8
```
-151
View File
@@ -1,151 +0,0 @@
"""
Usage:
python3 bench_hf.py --model-path meta-llama/Llama-2-7b-hf --data-dir data --ntrain 5
"""
import argparse
import json
import os
import time
import numpy as np
import pandas as pd
import torch
from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer
choices = ["A", "B", "C", "D"]
def format_subject(subject):
l = subject.split("_")
s = ""
for entry in l:
s += " " + entry
return s
def format_example(df, idx, include_answer=True):
prompt = df.iloc[idx, 0]
k = df.shape[1] - 2
for j in range(k):
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
prompt += "\nAnswer:"
if include_answer:
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
return prompt
def gen_prompt(train_df, subject, k=-1):
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
format_subject(subject)
)
if k == -1:
k = train_df.shape[0]
for i in range(k):
prompt += format_example(train_df, i)
return prompt
@torch.no_grad()
def main(args):
print(f"Loading model: {args.model_path}")
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
args.model_path,
torch_dtype=torch.bfloat16,
trust_remote_code=True,
device_map="auto",
).eval()
subjects = sorted(
[
f.split("_test.csv")[0]
for f in os.listdir(os.path.join(args.data_dir, "test"))
if "_test.csv" in f
]
)
all_cors = []
num_requests = 0
total_latency = 0
for subject in tqdm(subjects[: args.nsub]):
dev_df = pd.read_csv(
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
)[: args.ntrain]
test_df = pd.read_csv(
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
)
k = args.ntrain
few_shot_examples = gen_prompt(dev_df, subject, k)
while len(tokenizer.encode(few_shot_examples)) > 1536:
k -= 1
if k < 0:
break
few_shot_examples = gen_prompt(dev_df, subject, k)
preds = []
labels = []
tic = time.perf_counter()
for i in range(test_df.shape[0]):
prompt_end = format_example(test_df, i, include_answer=False)
prompt = few_shot_examples + prompt_end
input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device)
output_ids = model.generate(
input_ids,
max_new_tokens=1,
do_sample=False,
pad_token_id=tokenizer.eos_token_id,
)
output_str = tokenizer.decode(
output_ids[0][input_ids.shape[-1] :], skip_special_tokens=True
)
preds.append(output_str.strip()[0] if len(output_str.strip()) > 0 else "")
labels.append(test_df.iloc[i, test_df.shape[1] - 1])
latency = time.perf_counter() - tic
total_latency += latency
cors = [pred == label for pred, label in zip(preds, labels)]
all_cors.append(cors)
num_requests += len(test_df)
print(
f"Subject: {subject}, Accuracy: {np.mean(cors):.3f}, Latency: {latency:.3f}s"
)
weighted_acc = np.mean(np.concatenate(all_cors))
print(f"Total Latency: {total_latency:.3f}s")
print(f"Average Accuracy: {weighted_acc:.3f}")
if args.output:
with open(args.output, "a") as fout:
value = {
"task": "mmlu",
"backend": "hf",
"model": args.model_path,
"latency": round(total_latency, 3),
"accuracy": round(weighted_acc, 3),
"num_requests": num_requests,
"other": {
"nsub": args.nsub,
"ntrain": args.ntrain,
},
}
fout.write(json.dumps(value) + "\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", type=str, required=True)
parser.add_argument("--ntrain", type=int, default=5)
parser.add_argument("--data-dir", type=str, default="data")
parser.add_argument("--nsub", type=int, default=60)
parser.add_argument("--output", type=str, help="Output file path")
args = parser.parse_args()
main(args)
-210
View File
@@ -1,210 +0,0 @@
import argparse
import json
import os
import subprocess
import tarfile
import time
import numpy as np
import pandas as pd
import tiktoken
from sglang.test.test_utils import (
add_common_sglang_args_and_parse,
dump_bench_raw_result,
select_sglang_backend,
)
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
choices = ["A", "B", "C", "D"]
tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo")
def format_subject(subject):
l = subject.split("_")
s = ""
for entry in l:
s += " " + entry
return s
def format_example(df, idx, include_answer=True):
prompt = df.iloc[idx, 0]
k = df.shape[1] - 2
for j in range(k):
prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1])
prompt += "\nAnswer:"
if include_answer:
prompt += " {}\n\n".format(df.iloc[idx, k + 1])
return prompt
def gen_prompt(train_df, subject, k=-1):
prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format(
format_subject(subject)
)
if k == -1:
k = train_df.shape[0]
for i in range(k):
prompt += format_example(train_df, i)
return prompt
def download_data(data_dir):
"""Download and extract MMLU data if it doesn't exist."""
if os.path.isdir(os.path.join(data_dir, "test")):
return
print(f"Data not found at {data_dir}. Downloading...")
os.makedirs(data_dir, exist_ok=True)
tar_path = os.path.join(data_dir, "data.tar")
subprocess.check_call(
["wget", "-O", tar_path, "https://people.eecs.berkeley.edu/~hendrycks/data.tar"]
)
with tarfile.open(tar_path) as tar:
tar.extractall(path=data_dir, filter="data")
# The tarball extracts into a "data/" subdirectory; move contents up if needed
nested = os.path.join(data_dir, "data")
if os.path.isdir(nested):
for item in os.listdir(nested):
os.rename(os.path.join(nested, item), os.path.join(data_dir, item))
os.rmdir(nested)
os.remove(tar_path)
print("Download complete.")
def main(args):
subjects = sorted(
[
f.split("_test.csv")[0]
for f in os.listdir(os.path.join(args.data_dir, "test"))
if "_test.csv" in f
]
)
# Build prompts
arguments = []
labels = []
num_questions = []
for subject in subjects[: args.nsub]:
dev_df = pd.read_csv(
os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None
)[: args.ntrain]
test_df = pd.read_csv(
os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None
)
num_questions.append(test_df.shape[0])
k = args.ntrain
few_shot_examples = gen_prompt(dev_df, subject, k)
while len(tokenizer.encode(few_shot_examples)) > 1536:
k -= 1
few_shot_examples = gen_prompt(dev_df, subject, k)
for i in range(test_df.shape[0]):
prompt_end = format_example(test_df, i, include_answer=False)
arguments.append(
{
"examples": few_shot_examples,
"question": prompt_end,
}
)
label = test_df.iloc[i, test_df.shape[1] - 1]
labels.append(label)
#####################################
######### SGL Program Begin #########
#####################################
import sglang as sgl
if args.backend.startswith("gpt-"):
@sgl.function
def few_shot_mmlu(s, examples, question):
s += sgl.user(examples + question)
s += sgl.assistant(sgl.gen("answer"))
else:
@sgl.function
def few_shot_mmlu(s, examples, question):
s += examples + question + sgl.gen("answer")
#####################################
########## SGL Program End ##########
#####################################
# Select backend
backend = select_sglang_backend(args)
# Run
tic = time.perf_counter()
states = few_shot_mmlu.run_batch(
arguments,
temperature=0,
max_new_tokens=1,
backend=backend,
num_threads=args.parallel,
progress_bar=True,
)
preds = [
s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states
]
latency = time.perf_counter() - tic
# Compute accuracy
cors = [pred == label for pred, label in zip(preds, labels)]
pt = 0
for subject, num_qs in zip(subjects[: args.nsub], num_questions):
print(
f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt : pt + num_qs]):.3f}"
)
pt += num_qs
assert pt == len(cors)
weighted_acc = np.mean(cors)
dump_bench_raw_result(
path=args.raw_result_file,
states=states,
preds=preds,
labels=labels,
)
# Print results
print("Total latency: {:.3f}".format(latency))
print("Average accuracy: {:.3f}".format(weighted_acc))
# Write results
with open(args.result_file, "a") as fout:
value = {
"task": "mmlu",
"backend": args.backend,
"num_gpus": 1,
"latency": round(latency, 3),
"accuracy": round(weighted_acc, 3),
"num_requests": len(arguments),
"other": {
"nsub": args.nsub,
"parallel": args.parallel,
},
}
fout.write(json.dumps(value) + "\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--ntrain", "-k", type=int, default=5)
parser.add_argument(
"--data_dir", "-d", type=str, default=os.path.join(SCRIPT_DIR, "data")
)
parser.add_argument("--save_dir", "-s", type=str, default="results")
parser.add_argument("--nsub", type=int, default=60)
args = add_common_sglang_args_and_parse(parser)
download_data(args.data_dir)
main(args)
-2
View File
@@ -1,2 +0,0 @@
wget https://people.eecs.berkeley.edu/~hendrycks/data.tar
tar xf data.tar
@@ -388,9 +388,14 @@ Thus for all positive \(a,b,c\) with \(abc=1\),
**Benchmark Command:**
```shell Command
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
@@ -977,12 +977,14 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py \
--num-shots 8 \
--num-questions 1316 \
--parallel 1316
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads 1316
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
@@ -597,9 +597,13 @@ Max ITL (ms): 43465.60
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:8000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Test Results**:
- DeepSeek-V3
```text Output
@@ -614,11 +618,13 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
- **Benchmark Command:**
```shell Command
cd sglang
bash benchmark/mmlu/download_data.sh
python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 8000
sgl-eval run mmlu \
--base-url http://127.0.0.1:8000/v1 \
--num-threads 64 --max-tokens 2048
```
The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above.
- **Test Results**:
- DeepSeek-V3
```text Output
@@ -973,12 +973,14 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py \
--num-shots 8 \
--num-questions 1316 \
--parallel 1316
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads 1316
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
@@ -709,9 +709,13 @@ Max ITL (ms): 975.03
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Test Results**:
- DeepSeek-V3.2-Exp
```
@@ -721,12 +725,17 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000
Output throughput: 965.919 token/s
```
- **Full GSM8K (1319 questions)** — for a stricter accuracy check, run the full set 8-shot:
- **Full GSM8K (1319 questions)** — evaluate the complete test set with sgl-eval:
```shell Command
python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads 1319
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- 8-shot:
```
Accuracy: 0.956
@@ -747,11 +756,13 @@ python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --par
- **Benchmark Command:**
```shell Command
cd sglang
bash benchmark/mmlu/download_data.sh
python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above.
- **Test Results**:
- DeepSeek-V3.2-Exp
```
@@ -680,12 +680,14 @@ Document model accuracy on standard benchmarks:
- Benchmark Command:
```bash Command
python3 benchmark/gsm8k/bench_sglang.py \
--num-shots 8 \
--num-questions 1316 \
--parallel 1316
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads 1316
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Results:
- ERNIE-4.5-21B-A3B-PT
```
+5 -3
View File
@@ -510,7 +510,9 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python -m sglang.test.few_shot_gsm8k \
--num-questions 200 \
--port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
+5 -3
View File
@@ -900,11 +900,13 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python -m sglang.test.few_shot_gsm8k \
--num-questions 200 \
--port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Test Result
```text Output
Accuracy: 0.975
@@ -388,8 +388,13 @@ print(response)
#### Command
```shell Command
python3 ./benchmark/gsm8k/bench_sglang.py
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
#### Result Output
```text Output
Accuracy: 0.925
@@ -920,11 +920,13 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python -m sglang.test.few_shot_gsm8k \
--num-questions 200 \
--port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Result
```text Output
+5 -4
View File
@@ -941,12 +941,13 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python -m sglang.test.few_shot_gsm8k \
--num-shots 5 \
--num-questions 1319 \
--port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Test Result (NVIDIA B200, TP=2 NVFP4)
```text Output
Accuracy: 0.946
+17 -3
View File
@@ -644,9 +644,14 @@ The accuracy benchmark results below are shared with GLM-5, as GLM-5.1 was not i
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
Accuracy: 0.955
@@ -659,9 +664,13 @@ Output throughput: 642.044 token/s
- Benchmark Command
```bash Command
python3 benchmark/mmlu/bench_sglang.py --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
subject: abstract_algebra, #q:100, acc: 0.860
@@ -732,9 +741,14 @@ Average accuracy: 0.877
- MI325/MI35x Test (GLM-5.1 BF16, `tp=8`, TileLang DSA backends)
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
```text Output
Accuracy: 0.970
Invalid: 0.000
+17 -3
View File
@@ -575,9 +575,14 @@ The accuracy benchmark results below are shared with GLM-5.1, as GLM-5.1 was not
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
Accuracy: 0.955
@@ -590,9 +595,13 @@ Output throughput: 642.044 token/s
- Benchmark Command
```bash Command
python3 benchmark/mmlu/bench_sglang.py --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
subject: abstract_algebra, #q:100, acc: 0.860
@@ -663,9 +672,14 @@ Average accuracy: 0.877
- MI325/MI35x Test (GLM-5 BF16, `tp=8`, TileLang DSA backends)
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
```text Output
Accuracy: 0.970
Invalid: 0.000
@@ -815,10 +815,13 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python -m sglang.test.few_shot_gsm8k \
--num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Test Result
```text Output
@@ -1443,7 +1443,7 @@ Median ITL (ms): 15.11
</tbody>
</table>
> **Note**: These GSM8K numbers use the raw few-shot completion harness (`sglang.test.few_shot_gsm8k`). `gemma-4-12B-it` is reasoning-oriented and is under-elicited by raw few-shot prompting; with the chat template it scores **0.950** on the same 1319 GSM8K test questions (`sglang.test.run_eval --eval-name gsm8k`).
> **Note**: These GSM8K numbers use the raw few-shot completion harness (now removed). `gemma-4-12B-it` is reasoning-oriented and is under-elicited by raw few-shot prompting; with the chat template it scores **0.950** on the same 1319 GSM8K test questions (also measured before the sgl-eval migration).
#### gemma-4-12B-it with sgl-eval
@@ -1467,6 +1467,8 @@ sgl-eval run gsm8k --base-url http://localhost:30000/v1
sgl-eval run mmlu --base-url http://localhost:30000/v1 --num-examples 2000
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
#### MMMU
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
@@ -685,11 +685,13 @@ Max ITL (ms): 176.78
#### 5.2.1 GSM8K Benchmark
```bash Command
python -m sglang.test.few_shot_gsm8k \
--num-questions 200 \
--port 8000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:8000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
**Results:**
```text Output
@@ -205,9 +205,14 @@ Output:
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
Accuracy: 0.960
@@ -213,9 +213,14 @@ For more API examples, see the [SGLang Basic Usage Guide](../../../docs/basic_us
Reference run on a single GB300 node with `--tp 4`:
```bash Command
python3 benchmark/gsm8k/bench_sglang.py
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
```text Output
Accuracy: 0.9621 (1269 / 1319)
```
@@ -250,9 +250,18 @@ sglang serve \
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --temperature 1.2 --top-p 0.8 --max-new-tokens 32768 --num-questions 200 --tokenizer-path inclusionAI/Ring-2.5-1T --enable-thinking
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64 \
--temperature 1.2 \
--top-p 0.8 \
--max-tokens 32768 \
--chat-template-kwarg enable_thinking=true
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
Accuracy: 0.955
@@ -337,10 +337,8 @@ Mean ITL (ms): 47.77
- Benchmark Command:
```bash Command
python3 -m sglang.test.run_eval \
--eval-name gsm8k \
--host 127.0.0.1 \
--port ${PORT} \
sgl-eval run gsm8k \
--base-url http://127.0.0.1:${PORT}/v1 \
--model auto \
--num-examples 200 \
--num-threads 64 \
@@ -348,6 +346,8 @@ python3 -m sglang.test.run_eval \
--reasoning-effort high
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Test Results (B200 x8):
```text Output
@@ -635,9 +635,13 @@ Max ITL (ms): 1970.03
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Results**:
```text Output
+25 -17
View File
@@ -488,8 +488,12 @@ Max ITL (ms): 268.17
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Llama-4-Scout-17B-16E-Instruct
```text Output
Accuracy: 0.945
@@ -505,9 +509,9 @@ Latency: 9.739 s
Output throughput: 2405.505 token/s
```
#### 5.3.2 MMLU Pro with lm-eval
#### 5.3.2 MMLU-Pro with sgl-eval
Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) on 8×H100 (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)):
The table below records historical lm-eval measurements on 8×H100 alongside [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)). These scores were not produced by the sgl-eval commands below; rerun the evaluation for results using sgl-eval's prompts and grading.
<table style={{width: "100%", borderCollapse: "collapse"}}>
<thead>
@@ -531,6 +535,12 @@ Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta
</tbody>
</table>
Install the evaluation package before running either command:
```bash
pip install sgl-eval==0.1.0
```
**Scout:**
```bash Command
@@ -542,13 +552,12 @@ python -m sglang.launch_server \
--mem-fraction-static 0.8 \
--context-length 65536
# Run lm_eval
lm_eval --model local-chat-completions \
--model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \
--tasks mmlu_pro \
--batch_size 128 \
--apply_chat_template \
--num_fewshot 0
# Run MMLU-Pro
sgl-eval run mmlu_pro \
--base-url http://localhost:30000/v1 \
--model meta-llama/Llama-4-Scout-17B-16E-Instruct \
--max-tokens 2048 \
--num-threads 128
```
**Maverick:**
@@ -562,11 +571,10 @@ python -m sglang.launch_server \
--mem-fraction-static 0.8 \
--context-length 65536
# Run lm_eval
lm_eval --model local-chat-completions \
--model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \
--tasks mmlu_pro \
--batch_size 128 \
--apply_chat_template \
--num_fewshot 0
# Run MMLU-Pro
sgl-eval run mmlu_pro \
--base-url http://localhost:30000/v1 \
--model meta-llama/Llama-4-Maverick-17B-128E-Instruct \
--max-tokens 2048 \
--num-threads 128
```
@@ -960,8 +960,13 @@ Max ITL (ms): 376.34
#### 5.2.1 GSM8K Benchmark
- Benchmark Command:
```shell Command
python benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Results:
```text Output
Accuracy: 0.950
@@ -972,10 +977,13 @@ Output throughput: 1130.161 token/s
#### 5.2.2 MMLU Benchmark
- Benchmark Command:
```shell Command
cd benchmark/mmlu
bash download_data.sh
python3 bench_sglang.py --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Results:
```text Output
subject: abstract_algebra, #q:100, acc: 0.620
@@ -601,8 +601,12 @@ sglang serve \
- **Benchmark Command**:
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Result**:
- MiniMax-M2
```text Output
@@ -501,13 +501,14 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py \
--num-shots 8 \
--num-questions 1316 \
--parallel 1316 \
--port 8888
sgl-eval run gsm8k \
--base-url http://127.0.0.1:8888/v1 \
--num-examples 1319 \
--num-threads 1316
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
@@ -272,12 +272,14 @@ Document model accuracy on standard benchmarks:
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py \
--num-shots 8 \
--num-questions 1316 \
--parallel 1316
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads 1316
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
@@ -336,9 +336,14 @@ Validation runs on 4× H200 with `--tp 4`, served via the `/v1/chat/completions`
#### GSM8K
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Results:**
```text Output
@@ -333,9 +333,14 @@ shapes to create a distinct symbol which signifies copyright protection.
#### GSM8K
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
**Results:**
```text Output
@@ -345,7 +350,9 @@ TODO
#### MMLU
```bash Command
python3 benchmark/mmlu/bench_sglang.py --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
**Results:**
@@ -597,15 +597,17 @@ The following results are from the official Kimi-K2.7-Code model card. They were
#### 5.2.1 Accuracy Benchmark
GSM8K accuracy of the MXFP4 checkpoint, measured on the deployed server across several client parallelism levels (2,000 questions each):
Evaluate GSM8K on the deployed MXFP4 checkpoint across client parallelism levels (all 1319 test questions via sgl-eval):
```shell Command
python3 benchmark/gsm8k/bench_sglang.py \
--num-questions 2000 \
--parallel <PARALLEL> \
--port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads '<PARALLEL>'
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
<table>
<thead>
<tr>
@@ -507,9 +507,13 @@ python3 -m sglang.launch_server \
- Benchmark Command
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:8000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Result**:
```text Output
@@ -284,9 +284,13 @@ SGLANG_ROCM_FUSED_DECODE_MLA=0 python3 -m sglang.launch_server \
- Benchmark Command
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Result**:
```text Output
@@ -572,9 +572,14 @@ sglang serve \
**Run Benchmark**
```shell Command
python3 benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
Accuracy: 0.830
@@ -587,9 +592,13 @@ Output throughput: 1611.623 token/s
**Run Benchmark**
```shell Command
python3 benchmark/mmlu/bench_sglang.py --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
subject: abstract_algebra, #q:100, acc: 0.510
@@ -235,7 +235,7 @@ python3 -m sglang.bench_serving \
--max-concurrency 256
```
- **Test Results:**
- **Serving benchmark results:**
```text Output
============ Serving Benchmark Result ============
@@ -304,7 +304,7 @@ python3 -m sglang.bench_serving \
--max-concurrency 256
```
- **Test Results:**
- **Serving benchmark results:**
```text Output
============ Serving Benchmark Result ============
@@ -361,14 +361,16 @@ python3 -m sglang.launch_server \
--reasoning-parser nemotron_3
```
**Run Benchmark with lm-eval**
**Run Benchmark with sgl-eval**
```bash Command
pip install lm-eval[api]==0.4.9.2
pip install sgl-eval
lm_eval --model local-completions --tasks gsm8k --model_args "model=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16,base_url=http://127.0.0.1:30000/v1/completions,num_concurrent=4,max_retries=3,tokenized_requests=False,max_lengths=16384" --gen_kwargs '{"chat_template_kwargs":{"thinking":true}}' --batch_size 256
sgl-eval run gsm8k --base-url http://127.0.0.1:30000/v1 --model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --num-threads 4 --max-tokens 16384 --thinking
```
**Test Results:**
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
**Historical lm-eval results (not measurements of the sgl-eval command above):**
```text Output
|Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr|
|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:|
@@ -489,9 +489,14 @@ python3 -m sglang.launch_server \
**Run Benchmark**
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --port 5000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
Accuracy: 0.950
@@ -504,9 +509,13 @@ Output throughput: 996.815 token/s
**Run Benchmark**
```bash Command
python3 benchmark/mmlu/bench_sglang.py --port 5000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
subject: abstract_algebra, #q:100, acc: 0.730
@@ -528,9 +528,14 @@ python3 -m sglang.launch_server \
**Run Benchmark**
```bash Command
python3 benchmark/gsm8k/bench_sglang.py
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
**Test Results:**
```text Output
Accuracy: 0.970
@@ -543,7 +548,9 @@ Output throughput: 745.333 token/s
**Run Benchmark**
```bash Command
python3 benchmark/mmlu/bench_sglang.py
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
**Test Results:**
@@ -640,9 +640,13 @@ Max ITL (ms): 621.42
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:8000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Results**:
- GPT-OSS-120b
@@ -818,9 +818,14 @@ Max ITL (ms): 2783.23
- **Benchmark Command:**
```shell Command
python benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- **Test Results:**
```text Output
@@ -835,11 +840,13 @@ Output throughput: 929.132 token/s
- **Benchmark Command:**
```shell Command
cd benchmark/mmlu
bash download_data.sh
python3 bench_sglang.py --port 30000
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- **Test Results:**
```text Output
@@ -752,9 +752,13 @@ Max ITL (ms): 548.82
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
##### AMD (MI300X/MI325X/MI355X)
- **Results**:
@@ -726,9 +726,13 @@ Max ITL (ms): 486.38
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:8000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Results**:
- Qwen3-Next-80B-A3B-Instruct
@@ -753,11 +757,13 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000
- **Benchmark Command:**
```shell Command
cd sglang
bash benchmark/mmlu/download_data.sh
python3 benchmark/mmlu/bench_sglang.py --nsub 10
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above.
- **Results**:
- Qwen3-Next-80B-A3B-Instruct
+13 -12
View File
@@ -816,9 +816,14 @@ See [Decision models](/docs/supported-models/decision_models) for the request re
- Benchmark Command
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --port 30000
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
Historical results below use the removed harness and are not measurements of the sgl-eval command above.
- Test Result
```text Output
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:31<00:00, 6.43it/s]
@@ -828,21 +833,17 @@ Latency: 31.784 s
Output throughput: 998.166 token/s
```
#### 5.1.2 GSM8K with lm-eval (5-shot)
Evaluate using the industry-standard `lm-eval` harness for reproducible accuracy reporting:
#### 5.1.2 Full GSM8K with sgl-eval
```bash Command
pip install lm-eval[api]
lm_eval --model local-completions \
--model_args '{"base_url": "http://localhost:30000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \
--tasks gsm8k \
--batch_size auto \
--num_fewshot 5 \
--trust_remote_code
sgl-eval run gsm8k \
--base-url http://localhost:30000/v1 \
--model Qwen/Qwen3.5-397B-A17B \
--num-examples 1319 --num-threads 256 --max-tokens 2048 --thinking
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
#### 5.1.3 MMMU Benchmark
- Benchmark Command
+5 -1
View File
@@ -871,9 +871,13 @@ Max ITL (ms): 352.79
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Results**:
- Qwen/Qwen3-235B-A22B-Instruct-2507
@@ -514,9 +514,13 @@ Max ITL (ms): 1359.81
- **Benchmark Command:**
```shell Command
python3 -m sglang.test.few_shot_gsm8k --num-questions 200
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- **Results**:
- Step-3.5-Flash
@@ -443,13 +443,18 @@ Tool[0] get_weather({"city": "Beijing", "unit": "fahrenheit"})
#### 5.1.1 GSM8K
- Benchmark Method: 5-shot CoT on 200 questions, evaluated via SGLang native backend
- Benchmark Method: GSM8K on 200 questions using sgl-eval
- Benchmark Command:
```bash Command
python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --parallel 64
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200 \
--num-threads 64
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
- Test Results:
```text Output
@@ -458,11 +463,13 @@ TODO — replace with real GSM8K accuracy after benchmark run on Hy3-preview (BF
#### 5.1.2 MMLU
- Benchmark Method: 5-shot, all 57 subjects
- Benchmark Method: zero-shot Chat API evaluation via sgl-eval, all 57 subjects
- Benchmark Command:
```bash Command
python3 benchmark/mmlu/bench_sglang.py --nsub 60 --parallel 64
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-threads 64 --max-tokens 2048
```
- Test Results:
@@ -365,16 +365,17 @@ Standard 5-shot, `temperature=0`, `max_tokens=4096`, model defaults to thinking-
**Benchmark Command:**
```shell Command
python3 -m sglang.test.run_eval \
--base-url http://127.0.0.1:30000 \
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--model XiaomiMiMo/MiMo-V2.5 \
--eval-name gsm8k \
--num-examples 200 \
--num-threads 8 \
--max-tokens 4096 \
--temperature 0.0
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
> `run_eval.py` automatically appends `/v1` to `--base-url`; pass the bare `host:port` URL (without trailing `/v1`), otherwise requests resolve to `/v1/v1/chat/completions` and 404.
- **Test Results:**
@@ -748,14 +749,15 @@ Pro's 3-layer MTP behaves very differently on natural text vs uniform-random tok
**Benchmark Command:**
```shell Command
python3 -m sglang.test.run_eval \
--base-url http://127.0.0.1:30000 \
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--model XiaomiMiMo/MiMo-V2.5-Pro \
--eval-name gsm8k \
--num-examples 200 \
--num-threads 4
```
The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result.
The `accept_rate` and `accept_length` rows below are not part of `run_eval`'s own output — they were aggregated from the server-side `Decode batch ... accept rate: X accept len: Y` log lines emitted during the GSM8K run (307 batches total).
| Workload | accept_rate | accept_length (max = 4) |
@@ -90,9 +90,8 @@ If your code changes model output, run an accuracy evaluation appropriate for th
sglang serve --model-path Qwen/Qwen3-8B
# Evaluate
python3 -m sglang.test.run_eval \
--eval-name gsm8k \
--port 30000 \
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
@@ -114,6 +114,9 @@ nsys profile -o deepseek_v3 python3 -m sglang.benchmark.one_batch --batch-size 1
## Evaluation
```bash Command
# e.g. gsm8k 8 shot
python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8
# GSM8K via sgl-eval: all 1319 test questions
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1319 \
--num-threads 2000
```
@@ -18,9 +18,8 @@ SGLang provides built-in scripts to evaluate common benchmarks.
**MMLU**
```bash Command
python -m sglang.test.run_eval \
--eval-name mmlu \
--port 30000 \
sgl-eval run mmlu \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 1000 \
--max-tokens 8192
```
@@ -28,12 +27,9 @@ python -m sglang.test.run_eval \
**GSM8K**
```bash Command
python -m sglang.test.run_eval \
--eval-name gsm8k \
--host 127.0.0.1 \
--port 30000 \
--num-examples 200 \
--num-shots 5
sgl-eval run gsm8k \
--base-url http://127.0.0.1:30000/v1 \
--num-examples 200
```
**HellaSwag**
@@ -103,7 +103,7 @@ For more details, please refer to [docs/README.md](https://github.com/sgl-projec
## Test the accuracy
If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K.
If your code changes the model output, run the accuracy tests. A quick sanity check is GSM8K through sgl-eval.
```bash
# Launch a server
@@ -112,7 +112,9 @@ python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct
# Evaluate. --base-url must point at the server launched above.
# The default SGLang server port is 30000; change it if you launched
# the server with a different --port.
python3 -m sglang.test.run_eval --base-url http://localhost:30000 --eval-name gsm8k --num-examples 200
sgl-eval run gsm8k \
--base-url http://localhost:30000/v1 \
--num-examples 200
```
Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test.
@@ -105,40 +105,25 @@ For more details of SGLang server, refer to the [Ascend NPU Quick Start](/docs/h
#### Execute Accuracy Evaluation
EvalScope connects to the SGLang server via OpenAI-compatible API. The following example uses the GSM8K dataset:
Use sgl-eval for GSM8K and MMLU. It owns the datasets, chat prompts and graders. EvalScope remains available for the other benchmarks below:
```shell Command
evalscope eval \
pip install sgl-eval
sgl-eval run gsm8k \
--model /home/weights/Qwen2.5-7B-Instruct \
--api-url http://localhost:30000/v1 \
--api-key EMPTY \
--eval-type openai_api \
--datasets gsm8k \
--limit 10
--base-url http://localhost:30000/v1 \
--num-examples 10 --max-tokens 2048
```
Upon completion, results similar to the following will be displayed:
```text
+---------------------+-----------+----------+----------+-------+---------+---------+
| Model | Dataset | Metric | Subset | Num | Score | Cat.0 |
+=====================+===========+==========+==========+=======+=========+=========+
| Qwen2.5-7B-Instruct | gsm8k | mean_acc | main | 5 | 1.0 | default |
+---------------------+-----------+----------+----------+-------+---------+---------+
```
> **Note**: Output format may vary slightly across different EvalScope versions. The above example is from EvalScope 1.6.x. Ensure the `--model` parameter matches the model name returned by the SGLang server's `/v1/models` endpoint. When starting the server with an HF path (e.g., `Qwen/Qwen2.5-7B-Instruct`), use that path directly. For local paths, pass the full path or the model name returned by `/v1/models`.
The run directory contains `metrics.json` and per-example predictions. Use the model name reported by `/v1/models`.
#### Common Datasets for Online Evaluation
```shell Command
# MMLU
evalscope eval \
sgl-eval run mmlu \
--model /home/weights/Qwen2.5-7B-Instruct \
--api-url http://localhost:30000/v1 \
--api-key EMPTY \
--eval-type openai_api \
--datasets mmlu
--base-url http://localhost:30000/v1 --max-tokens 2048
# CEval (Chinese evaluation)
evalscope eval \
@@ -241,7 +226,7 @@ When using EvalScope commands without specifying a dataset or model path, it wil
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/adapters.py", line 676, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/gsm8k (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)')))
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/ceval (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)')))
[ERROR] 2026-05-13-02:20:01 (PID:876, Device:-1, RankID:-1) ERR99999 UNKNOWN application exception
```
**Temporary workaround (test only):**
@@ -26,7 +26,7 @@
// Per-platform re-measurement under sgl-eval is in progress; rows still pending
// show `gsm8k_pct: null` (no GSM8K row rendered) with the legacy-harness number
// kept in a comment. Legacy harnesses were NOT comparable across platforms
// (NVIDIA: few_shot_gsm8k --num-questions 200; AMD: run_eval gsm8k 1319 examples) —
// (NVIDIA: removed completion harness, 200 questions; AMD: run_eval gsm8k 1319 examples) —
// which is exactly why we re-measure on one harness.
export const benchmarks = [
{
@@ -9,7 +9,7 @@ _DESCRIPTION = """Compare and find differences to benchmark outputs.
Supported inputs:
* The samples jsonl from `lm_eval --log_samples --output_path FOLDER_NAME`
* The output from `gsm8k/bench_sglang.py --raw-result-file FILE_NAME` (or mmlu)
* Legacy benchmark JSONL with prompt_id, prompt, output, and correct fields
"""
-122
View File
@@ -1,122 +0,0 @@
import json
import os
from sglang.test.run_eval import run_eval_once
from sglang.test.simple_eval_common import (
make_report,
set_ulimit,
)
def run_eval(args):
# Lazy import to avoid circular dependency with test_utils
from sglang.test.test_utils import dump_metric
set_ulimit()
if "OPENAI_API_KEY" not in os.environ:
os.environ["OPENAI_API_KEY"] = "EMPTY"
base_url = (
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
if args.eval_name == "mmlu":
from sglang.test.ascend.simple_eval_mmlu import MMLUEval
filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
eval_obj = MMLUEval(
filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0)
)
else:
raise ValueError(f"Invalid eval name: {args.eval_name}")
if getattr(args, "repeat", 1) == 1:
result, latency, sampler = run_eval_once(args, base_url, eval_obj)
metrics = result.metrics | {"score": result.score}
metrics["latency"] = latency
print(f"Total latency: {latency:.3f} s")
print(f"Score: {metrics['score']:.3f}")
# Compute output throughput from accumulated completion tokens
total_completion_tokens = sum(sampler._completion_tokens)
if total_completion_tokens > 0 and latency > 0:
metrics["output_throughput"] = total_completion_tokens / latency
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
# Report metrics to unified collection framework
dump_metric(
f"{args.eval_name}_score",
metrics["score"],
labels={"model": sampler.model, "eval": args.eval_name},
)
dump_metric(
f"{args.eval_name}_latency",
latency,
labels={"model": sampler.model, "eval": args.eval_name},
)
else:
from concurrent.futures import ThreadPoolExecutor
executor = ThreadPoolExecutor(max_workers=args.repeat)
futures = [
executor.submit(run_eval_once, args, base_url, eval_obj)
for _ in range(args.repeat)
]
scores_repeat = []
latencies = []
total_completion_tokens = 0
for f in futures:
result, latency, sampler = f.result()
scores_repeat.append(result.score)
latencies.append(latency)
total_completion_tokens += sum(sampler._completion_tokens)
mean_score = sum(scores_repeat) / len(scores_repeat)
mean_latency = sum(latencies) / len(latencies)
total_latency = sum(latencies)
scores_repeat = [f"{s:.3f}" for s in scores_repeat]
print("=" * 20)
print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}")
print(f"Scores: {scores_repeat}")
print(f"Mean latency: {mean_latency:.3f} s")
print("=" * 20)
metrics = result.metrics | {"scores": scores_repeat}
metrics = metrics | {"mean_score": mean_score}
metrics["latency"] = mean_latency
if total_completion_tokens > 0 and total_latency > 0:
metrics["output_throughput"] = total_completion_tokens / total_latency
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
# Report metrics to unified collection framework
dump_metric(
f"{args.eval_name}_mean_score",
mean_score,
labels={
"model": sampler.model,
"eval": args.eval_name,
"repeat": args.repeat,
},
)
executor.shutdown()
# Dump reports
file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}"
report_filename = f"/tmp/{file_stem}.html"
print(f"Writing report to {report_filename}")
with open(report_filename, "w") as fh:
fh.write(make_report(result))
print(metrics)
result_filename = f"/tmp/{file_stem}.json"
with open(result_filename, "w") as f:
f.write(json.dumps(metrics, indent=2))
print(f"Writing results to {result_filename}")
if getattr(args, "return_latency", False):
return metrics, latency
return metrics
@@ -1,125 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
Measuring Massive Multitask Language Understanding
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
https://arxiv.org/abs/2009.03300
"""
import random
import re
from typing import Optional
import pandas
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
format_multichoice_question,
)
from sglang.test.simple_eval_mmlu import subject2category
def format_multichoice_question_example(row):
return QUERY_TEMPLATE_MULTICHOICE.format(**row)
QUERY_TEMPLATE_MULTICHOICE = """
Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
{Question}
A) {A}
B) {B}
C) {C}
D) {D}
""".strip()
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """
Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD.
"""
TEMPLATE_MULTICHOICE_EXAMPLE = """
Example question:
{Question}
A {A}
B {B}
C {C}
D {D}
The last line of your response should be
Answer: {Answer}
""".strip()
class MMLUEval(Eval):
def __init__(
self,
filename: str,
num_examples: Optional[int],
num_threads: int,
num_shots: int,
):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
if num_shots:
example_questions = "".join(
format_multichoice_question_example(row) + "\n\n"
for row in examples[:num_shots]
)
self.template = (
TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN
+ example_questions
+ QUERY_TEMPLATE_MULTICHOICE
)
examples = examples[num_shots:]
if num_examples:
examples = random.Random(0).sample(examples, num_examples)
self.examples = examples
self.num_threads = num_threads
self.num_shots = num_shots
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
if self.num_shots:
prompt_messages = [
sampler._pack_message(
content=self.template.format(**row), role="user"
)
]
else:
prompt_messages = [
sampler._pack_message(
content=format_multichoice_question(row), role="user"
)
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
extracted_answer = match.group(1) if match else None
score = 1.0 if extracted_answer == row["Answer"] else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["Answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
category = subject2category.get(row["Subject"], "other")
return SingleEvalResult(
html=html, score=score, metrics={category: score}, convo=convo
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
+1 -1
View File
@@ -15,7 +15,7 @@ class TestMMLU:
"server": getattr(
self, "server_cmd", subprocess.list2cmdline(map(str, self.other_args))
),
"client": "simple_eval_mmlu",
"client": "sgl-eval",
"accuracy_threshold": getattr(self, "accuracy_mmlu", "N/A"),
}
-157
View File
@@ -1,157 +0,0 @@
"""
.. deprecated::
This module is deprecated. Use ``sglang.test.run_eval`` with
``eval_name="gsm8k"`` instead, which routes through the unified
Chat API evaluation framework with dump_metric support.
"""
import argparse
import ast
import asyncio
import re
import time
import warnings
from typing import Optional
import numpy as np
import sglang as sgl
from sglang.srt.utils import get_or_create_event_loop
from sglang.utils import download_and_cache_file, read_jsonl
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
async def concurrent_generate(engine, prompts, sampling_param):
tasks = []
for prompt in prompts:
tasks.append(asyncio.create_task(engine.async_generate(prompt, sampling_param)))
outputs = await asyncio.gather(*tasks)
return outputs
def run_eval(args):
warnings.warn(
"sglang.test.few_shot_gsm8k_engine is deprecated. "
"Use sglang.test.run_eval with eval_name='gsm8k' instead.",
DeprecationWarning,
stacklevel=2,
)
# Select backend
engine = sgl.Engine(model_path=args.model_path, log_level="error")
if args.local_data_path is None:
# Read data
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
filename = download_and_cache_file(url)
else:
filename = args.local_data_path
lines = list(read_jsonl(filename))
# Construct prompts
num_questions = args.num_questions
num_shots = args.num_shots
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
# construct the prompts
prompts = []
for i, arg in enumerate(arguments):
q = arg["question"]
prompt = few_shot_examples + q
prompts.append(prompt)
sampling_param = {
"stop": ["Question", "Assistant:", "<|separator|>"],
"max_new_tokens": 512,
"temperature": 0,
}
# Run requests
tic = time.perf_counter()
loop = get_or_create_event_loop()
outputs = loop.run_until_complete(
concurrent_generate(engine, prompts, sampling_param)
)
# End requests
latency = time.perf_counter() - tic
# Shutdown the engine
engine.shutdown()
# Parse output
preds = []
for output in outputs:
preds.append(get_answer_value(output["text"]))
# Compute accuracy
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
# Compute speed
num_output_tokens = sum(
output["meta_info"]["completion_tokens"] for output in outputs
)
output_throughput = num_output_tokens / latency
# Print results
print(f"Accuracy: {acc:.3f}")
print(f"Invalid: {invalid:.3f}")
print(f"Latency: {latency:.3f} s")
print(f"Output throughput: {output_throughput:.3f} token/s")
return {
"accuracy": acc,
"latency": latency,
"output_throughput": output_throughput,
}
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--model-path", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct"
)
parser.add_argument("--local-data-path", type=Optional[str], default=None)
parser.add_argument("--num-shots", type=int, default=5)
parser.add_argument("--num-questions", type=int, default=200)
args = parser.parse_args()
metrics = run_eval(args)
-1
View File
@@ -267,7 +267,6 @@ def run_eval(args):
if args.eval_name == "mmlu":
# Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a
# caller's threshold has to be measured against it, not inherited.
# `simple_eval_mmlu` stays: the ascend eval imports its subject2category.
return _run_sgl_eval("mmlu", args)
elif args.eval_name == "mgsm_en":
from sglang.test.simple_eval_mgsm import MGSMEval
-125
View File
@@ -1,125 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
Measuring Massive Multitask Language Understanding
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt
https://arxiv.org/abs/2009.03300
"""
import random
import re
from typing import Optional
import pandas
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
format_multichoice_question,
)
subject2category = {
"abstract_algebra": "stem",
"anatomy": "other",
"astronomy": "stem",
"business_ethics": "other",
"clinical_knowledge": "other",
"college_biology": "stem",
"college_chemistry": "stem",
"college_computer_science": "stem",
"college_mathematics": "stem",
"college_medicine": "other",
"college_physics": "stem",
"computer_security": "stem",
"conceptual_physics": "stem",
"econometrics": "social_sciences",
"electrical_engineering": "stem",
"elementary_mathematics": "stem",
"formal_logic": "humanities",
"global_facts": "other",
"high_school_biology": "stem",
"high_school_chemistry": "stem",
"high_school_computer_science": "stem",
"high_school_european_history": "humanities",
"high_school_geography": "social_sciences",
"high_school_government_and_politics": "social_sciences",
"high_school_macroeconomics": "social_sciences",
"high_school_mathematics": "stem",
"high_school_microeconomics": "social_sciences",
"high_school_physics": "stem",
"high_school_psychology": "social_sciences",
"high_school_statistics": "stem",
"high_school_us_history": "humanities",
"high_school_world_history": "humanities",
"human_aging": "other",
"human_sexuality": "social_sciences",
"international_law": "humanities",
"jurisprudence": "humanities",
"logical_fallacies": "humanities",
"machine_learning": "stem",
"management": "other",
"marketing": "other",
"medical_genetics": "other",
"miscellaneous": "other",
"moral_disputes": "humanities",
"moral_scenarios": "humanities",
"nutrition": "other",
"philosophy": "humanities",
"prehistory": "humanities",
"professional_accounting": "other",
"professional_law": "humanities",
"professional_medicine": "other",
"professional_psychology": "social_sciences",
"public_relations": "social_sciences",
"security_studies": "social_sciences",
"sociology": "social_sciences",
"us_foreign_policy": "social_sciences",
"virology": "other",
"world_religions": "humanities",
}
class MMLUEval(Eval):
def __init__(self, filename: str, num_examples: Optional[int], num_threads: int):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
if num_examples:
examples = random.Random(0).sample(examples, num_examples)
self.examples = examples
self.num_threads = num_threads
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(row: dict):
prompt_messages = [
sampler._pack_message(
content=format_multichoice_question(row), role="user"
)
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
extracted_answer = match.group(1) if match else None
score = 1.0 if extracted_answer == row["Answer"] else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["Answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
category = subject2category.get(row["Subject"], "other")
return SingleEvalResult(
html=html, score=score, metrics={category: score}, convo=convo
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
-32
View File
@@ -2409,38 +2409,6 @@ class CustomTestCase(unittest.TestCase):
)
def dump_bench_raw_result(
path: str,
states,
preds,
labels,
):
if not path:
return
rows = []
for i in range(len(states)):
state = states[i]
output = state["answer"]
prompt = _ensure_remove_suffix(state.text(), output)
rows.append(
dict(
prompt_id=i,
prompt=prompt,
output=output,
correct=bool(preds[i] == labels[i]),
)
)
print(f"BenchRawResultDumper save results to {path}")
Path(path).write_text("\n".join(json.dumps(row) for row in rows))
def _ensure_remove_suffix(text: str, suffix: str):
assert text.endswith(suffix)
return text.removesuffix(suffix)
class ModelLaunchSettings:
def __init__(
self,
@@ -63,7 +63,6 @@ from .process_utils import (
wait_for_health,
wait_for_workers_ready,
)
from .run_eval import run_eval
__all__ = [
# Enums and Identity
@@ -139,6 +138,4 @@ __all__ = [
"FUNCTION_CALLING_MODELS",
# Third-party models
"THIRD_PARTY_MODELS",
# Evaluation
"run_eval",
]
@@ -1,138 +0,0 @@
"""MMLU evaluation runner for E2E tests.
Simplified evaluation runner that uses local eval implementations
with cleaner logging for CI/CD environments.
Usage:
from infra.run_eval import run_eval
from types import SimpleNamespace
args = SimpleNamespace(
base_url="http://127.0.0.1:30000",
model="meta-llama/Llama-3.1-8B-Instruct",
eval_name="mmlu",
num_examples=64,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
"""
from __future__ import annotations
import logging
import os
import time
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .simple_eval_common import Eval
from .simple_eval_common import ChatCompletionSampler, set_ulimit
from .simple_eval_mmlu import MMLU_DATASET_URL
logger = logging.getLogger(__name__)
@dataclass
class EvalConfig:
"""Configuration for running an evaluation."""
base_url: str
model: str | None = None
eval_name: str = "mmlu"
num_examples: int = 64
num_threads: int = 32
temperature: float = 0.0
max_tokens: int = 2048
host: str = "127.0.0.1"
port: int = 30000
def _get_eval(eval_name: str, num_examples: int, num_threads: int) -> "Eval":
"""Get the evaluation object by name."""
if eval_name == "mmlu":
from .simple_eval_mmlu import MMLUEval
return MMLUEval(MMLU_DATASET_URL, num_examples, num_threads)
else:
raise ValueError(f"Unknown eval: {eval_name}. Supported: mmlu")
def run_eval(args: Any) -> dict:
"""Run an evaluation and return metrics.
Args:
args: Configuration object with attributes:
- base_url: Base URL of the server (e.g., "http://127.0.0.1:30000")
- model: Model name/path (optional, will be auto-detected)
- eval_name: Evaluation name ("mmlu")
- num_examples: Number of examples to evaluate
- num_threads: Number of parallel threads
- temperature: Sampling temperature
Returns:
Dict with metrics including 'score' key.
"""
set_ulimit()
if "OPENAI_API_KEY" not in os.environ:
os.environ["OPENAI_API_KEY"] = "EMPTY"
# Build base URL
base_url = getattr(args, "base_url", None)
if base_url:
base_url = base_url.rstrip("/") # Remove trailing slashes
if not base_url.endswith("/v1"):
base_url = f"{base_url}/v1"
else:
host = getattr(args, "host", "127.0.0.1")
port = getattr(args, "port", 30000)
base_url = f"http://{host}:{port}/v1"
eval_name = getattr(args, "eval_name", "mmlu")
num_examples = getattr(args, "num_examples", 64)
num_threads = getattr(args, "num_threads", 32)
temperature = getattr(args, "temperature", 0.0)
max_tokens = getattr(args, "max_tokens", 2048)
model = getattr(args, "model", None)
logger.info(
"Starting %s eval: %d examples, %d threads, temp=%.2f",
eval_name,
num_examples,
num_threads,
temperature,
)
# Create sampler
sampler = ChatCompletionSampler(
model=model,
max_tokens=max_tokens,
base_url=base_url,
temperature=temperature,
)
# Get eval object
eval_obj = _get_eval(eval_name, num_examples, num_threads)
# Run evaluation
start_time = time.perf_counter()
result = eval_obj(sampler)
latency = time.perf_counter() - start_time
# Build metrics
metrics = result.metrics.copy() if result.metrics else {}
metrics["score"] = result.score
metrics["latency"] = latency
logger.info(
"%s eval complete: score=%.3f, latency=%.1fs, model=%s",
eval_name,
result.score,
latency,
sampler.model,
)
return metrics
@@ -1,495 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""Common utilities for simple evaluations."""
from __future__ import annotations
import logging
import os
import resource
import time
from collections import defaultdict
from dataclasses import dataclass, field
from multiprocessing.pool import ThreadPool
from typing import Any
import httpx
import jinja2
import numpy as np
import openai
import requests
from openai import OpenAI
from tqdm import tqdm
from .constants import MAX_RETRY_ATTEMPTS
logger = logging.getLogger(__name__)
OPENAI_SYSTEM_MESSAGE_API = "You are a helpful assistant."
OPENAI_SYSTEM_MESSAGE_CHATGPT = (
"You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture."
+ "\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01"
)
Message = dict[str, Any] # keys role, content
MessageList = list[Message]
class SamplerBase:
"""
Base class for defining a sampling model, which can be evaluated,
or used as part of the grading process.
"""
def __call__(self, message_list: MessageList) -> str:
raise NotImplementedError()
@dataclass
class EvalResult:
"""Result of running an evaluation (usually consisting of many samples)."""
score: float | None # top-line metric
metrics: dict[str, float] | None # other metrics
htmls: list[str] # strings of valid HTML
convos: list[MessageList] # sampled conversations
@dataclass
class SingleEvalResult:
"""Result of evaluating a single sample."""
score: float | None
metrics: dict[str, float] = field(default_factory=dict)
html: str | None = None
convo: MessageList | None = None # sampled conversation
class Eval:
"""
Base class for defining an evaluation.
"""
def __call__(self, sampler: SamplerBase) -> EvalResult:
raise NotImplementedError()
class LargerHttpxClient(httpx.Client):
def __init__(self):
timeout_config = httpx.Timeout(3600)
limits = httpx.Limits(
max_keepalive_connections=3600,
max_connections=3600,
)
super().__init__(timeout=timeout_config, limits=limits)
class ChatCompletionSampler(SamplerBase):
"""Sample from OpenAI's chat completion API."""
def __init__(
self,
base_url: str | None = None,
model: str | None = None,
system_message: str | None = None,
temperature: float = 0.0,
reasoning_effort: str | None = None,
max_tokens: int = 2048,
extra_body: dict[str, Any] | None = None,
):
self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
if model is None:
model = self.client.models.list().data[0].id
self.model = model
self.system_message = system_message
self.temperature = temperature
self.max_tokens = max_tokens
self.reasoning_effort = reasoning_effort
self.extra_body = extra_body
self.image_format = "url"
logger.debug(
"ChatCompletionSampler: model=%s, temp=%.2f, max_tokens=%d",
self.model,
self.temperature,
self.max_tokens,
)
def _handle_image(
self,
image: str,
encoding: str = "base64",
format: str = "png",
):
new_image = {
"type": "image_url",
"image_url": {
"url": f"data:image/{format};{encoding},{image}",
},
}
return new_image
def _handle_text(self, text: str):
return {"type": "text", "text": text}
def _pack_message(self, role: str, content: Any):
return {"role": str(role), "content": content}
def __call__(self, message_list: MessageList) -> str:
if self.system_message:
message_list = [
self._pack_message("system", self.system_message)
] + message_list
trial = 0
while trial < MAX_RETRY_ATTEMPTS:
try:
response = self.client.chat.completions.create(
model=self.model,
messages=message_list,
temperature=self.temperature,
max_tokens=self.max_tokens,
reasoning_effort=self.reasoning_effort,
extra_body=self.extra_body,
)
return response.choices[0].message.content or ""
except openai.BadRequestError as e:
logger.warning("Bad request error: %s", e)
return ""
except Exception as e:
exception_backoff = 2**trial # exponential back off
# Log first few retries at debug, later ones at warning
log_fn = logger.warning if trial >= 3 else logger.debug
log_fn(
"Request failed (retry %d/%d, backoff %ds): %s",
trial + 1,
MAX_RETRY_ATTEMPTS,
exception_backoff,
e,
)
time.sleep(exception_backoff)
trial += 1
logger.warning(
"All retry attempts exhausted after %d retries, returning empty response",
MAX_RETRY_ATTEMPTS,
)
return ""
QUERY_TEMPLATE_MULTICHOICE = """
Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
{Question}
A) {A}
B) {B}
C) {C}
D) {D}
""".strip()
ANSWER_PATTERN_MULTICHOICE = r"(?i)Answer\s*:\s*([A-D])"
ANSWER_PATTERN = r"(?i)Answer\s*:\s*([^\n]+)"
EQUALITY_TEMPLATE = r"""
Look at the following two expressions (answers to a math problem) and judge whether they are equivalent. Only perform trivial simplifications
Examples:
Expression 1: $2x+3$
Expression 2: $3+2x$
Yes
Expression 1: 3/2
Expression 2: 1.5
Yes
Expression 1: $x^2+2x+1$
Expression 2: $y^2+2y+1$
No
Expression 1: $x^2+2x+1$
Expression 2: $(x+1)^2$
Yes
Expression 1: 3245/5
Expression 2: 649
No
(these are actually equal, don't mark them equivalent if you need to do nontrivial simplifications)
Expression 1: 2/(-3)
Expression 2: -2/3
Yes
(trivial simplifications are allowed)
Expression 1: 72 degrees
Expression 2: 72
Yes
(give benefit of the doubt to units)
Expression 1: 64
Expression 2: 64 square feet
Yes
(give benefit of the doubt to units)
---
YOUR TASK
Respond with only "Yes" or "No" (without quotes). Do not include a rationale.
Expression 1: %(expression1)s
Expression 2: %(expression2)s
""".strip()
HTML_JINJA = """
<h3>Prompt conversation</h3>
{% for message in prompt_messages %}
{{ message_to_html(message) | safe }}
{% endfor %}
<h3>Sampled message</h3>
{{ message_to_html(next_message) | safe }}
<h3>Results</h3>
<p>Correct Answer: {{ correct_answer }}</p>
<p>Extracted Answer: {{ extracted_answer }}</p>
<p>Score: {{ score }}</p>
"""
def format_multichoice_question(row):
return QUERY_TEMPLATE_MULTICHOICE.format(**row)
def check_equality(sampler: SamplerBase, expr1: str, expr2: str):
prompt = EQUALITY_TEMPLATE % {"expression1": expr1, "expression2": expr2}
response = sampler([dict(content=prompt, role="user")])
return (response or "").lower().strip() == "yes"
def _compute_stat(values: list, stat: str):
if stat == "mean":
return np.mean(values)
elif stat == "std":
return np.std(values)
elif stat == "min":
return np.min(values)
elif stat == "max":
return np.max(values)
else:
raise ValueError(f"Unknown {stat =}")
def aggregate_results(
single_eval_results: list[SingleEvalResult],
default_stats: tuple[str, ...] = ("mean", "std"),
name2stats: dict[str, tuple[str, ...]] | None = None,
) -> EvalResult:
"""
Aggregate results from multiple evaluations into a single EvalResult.
"""
name2stats = name2stats or {}
name2values = defaultdict(list)
htmls = []
convos = []
for single_eval_result in single_eval_results:
# Skip None results
if single_eval_result is None:
continue
for name, value in single_eval_result.metrics.items():
name2values[name].append(value)
if single_eval_result.score is not None:
name2values["score"].append(single_eval_result.score)
htmls.append(single_eval_result.html)
convos.append(single_eval_result.convo)
final_metrics = {}
for name, values in name2values.items():
stats = name2stats.get(name, default_stats)
for stat in stats:
key = name if stat == "mean" else f"{name}:{stat}"
final_metrics[key] = _compute_stat(values, stat)
return EvalResult(
score=final_metrics.pop("score", None),
metrics=final_metrics,
htmls=htmls,
convos=convos,
)
def map_with_progress(f: callable, xs: list[Any], num_threads: int) -> list[Any]:
"""Apply f to each element of xs, using a ThreadPool, and show progress."""
# Use quiet progress bar that doesn't pollute logs
if os.getenv("debug"):
return list(map(f, tqdm(xs, total=len(xs), leave=False)))
else:
with ThreadPool(min(num_threads, len(xs))) as pool:
return list(tqdm(pool.imap(f, xs), total=len(xs), leave=False))
jinja_env = jinja2.Environment(
loader=jinja2.BaseLoader(),
undefined=jinja2.StrictUndefined,
autoescape=jinja2.select_autoescape(["html", "xml"]),
)
_message_template = """
<div class="message {{ role }}">
<div class="role">
{{ role }}
{% if variant %}<span class="variant">({{ variant }})</span>{% endif %}
</div>
<div class="content">
<pre>{{ content }}</pre>
</div>
</div>
"""
def message_to_html(message: Message) -> str:
"""
Generate HTML snippet (inside a <div>) for a message.
"""
return jinja_env.from_string(_message_template).render(
role=message["role"],
content=message["content"],
variant=message.get("variant", None),
)
jinja_env.globals["message_to_html"] = message_to_html
_report_template = """<!DOCTYPE html>
<html>
<head>
<style>
.message {
padding: 8px 16px;
margin-bottom: 8px;
border-radius: 4px;
}
.message.user {
background-color: #B2DFDB;
color: #00695C;
}
.message.assistant {
background-color: #B39DDB;
color: #4527A0;
}
.message.system {
background-color: #EEEEEE;
color: #212121;
}
.role {
font-weight: bold;
margin-bottom: 4px;
}
.variant {
color: #795548;
}
table, th, td {
border: 1px solid black;
}
pre {
white-space: pre-wrap;
}
</style>
</head>
<body>
{% if metrics %}
<h1>Metrics</h1>
<table>
<tr>
<th>Metric</th>
<th>Value</th>
</tr>
<tr>
<td><b>Score</b></td>
<td>{{ score | float | round(3) }}</td>
</tr>
{% for name, value in metrics.items() %}
<tr>
<td>{{ name }}</td>
<td>{{ value }}</td>
</tr>
{% endfor %}
</table>
{% endif %}
<h1>Examples</h1>
{% for html in htmls %}
{{ html | safe }}
<hr>
{% endfor %}
</body>
</html>
"""
def make_report(eval_result: EvalResult) -> str:
"""
Create a standalone HTML report from an EvalResult.
"""
return jinja_env.from_string(_report_template).render(
score=eval_result.score,
metrics=eval_result.metrics,
htmls=eval_result.htmls,
)
def make_report_from_example_htmls(htmls: list[str]):
"""
Create a standalone HTML report from a list of example htmls
"""
return jinja_env.from_string(_report_template).render(
score=None, metrics={}, htmls=htmls
)
def download_dataset(path: str, url: str) -> None:
"""Download a dataset from URL to path."""
logger.info("Downloading dataset from %s", url)
try:
response = requests.get(url, stream=True, timeout=30)
response.raise_for_status()
total_size = int(response.headers.get("content-length", 0))
block_size = 8192
with (
open(path, "wb") as f,
tqdm(
desc="Downloading",
total=total_size,
unit="iB",
unit_scale=True,
unit_divisor=1024,
leave=False,
) as progress_bar,
):
for data in response.iter_content(block_size):
size = f.write(data)
progress_bar.update(size)
logger.debug("Dataset saved to %s", path)
except requests.RequestException as e:
raise RuntimeError(f"Failed to download dataset: {e}") from e
def set_ulimit(target_soft_limit: int = 65535) -> None:
"""Set the file descriptor limit for parallel requests."""
resource_type = resource.RLIMIT_NOFILE
current_soft, current_hard = resource.getrlimit(resource_type)
if current_soft < target_soft_limit:
try:
resource.setrlimit(resource_type, (target_soft_limit, current_hard))
except ValueError as e:
logger.debug("Could not set RLIMIT_NOFILE: %s", e)
@@ -1,132 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
"""
MMLU Evaluation - Measuring Massive Multitask Language Understanding
Dan Hendrycks et al. https://arxiv.org/abs/2009.03300
"""
from __future__ import annotations
import random
import re
from typing import TYPE_CHECKING
import pandas
from . import simple_eval_common as common
from .simple_eval_common import (
ANSWER_PATTERN_MULTICHOICE,
HTML_JINJA,
Eval,
EvalResult,
SingleEvalResult,
format_multichoice_question,
)
if TYPE_CHECKING:
from .simple_eval_common import SamplerBase
# MMLU dataset URL (hosted by OpenAI)
MMLU_DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv"
SUBJECT_TO_CATEGORY = {
"abstract_algebra": "stem",
"anatomy": "other",
"astronomy": "stem",
"business_ethics": "other",
"clinical_knowledge": "other",
"college_biology": "stem",
"college_chemistry": "stem",
"college_computer_science": "stem",
"college_mathematics": "stem",
"college_medicine": "other",
"college_physics": "stem",
"computer_security": "stem",
"conceptual_physics": "stem",
"econometrics": "social_sciences",
"electrical_engineering": "stem",
"elementary_mathematics": "stem",
"formal_logic": "humanities",
"global_facts": "other",
"high_school_biology": "stem",
"high_school_chemistry": "stem",
"high_school_computer_science": "stem",
"high_school_european_history": "humanities",
"high_school_geography": "social_sciences",
"high_school_government_and_politics": "social_sciences",
"high_school_macroeconomics": "social_sciences",
"high_school_mathematics": "stem",
"high_school_microeconomics": "social_sciences",
"high_school_physics": "stem",
"high_school_psychology": "social_sciences",
"high_school_statistics": "stem",
"high_school_us_history": "humanities",
"high_school_world_history": "humanities",
"human_aging": "other",
"human_sexuality": "social_sciences",
"international_law": "humanities",
"jurisprudence": "humanities",
"logical_fallacies": "humanities",
"machine_learning": "stem",
"management": "other",
"marketing": "other",
"medical_genetics": "other",
"miscellaneous": "other",
"moral_disputes": "humanities",
"moral_scenarios": "humanities",
"nutrition": "other",
"philosophy": "humanities",
"prehistory": "humanities",
"professional_accounting": "other",
"professional_law": "humanities",
"professional_medicine": "other",
"professional_psychology": "social_sciences",
"public_relations": "social_sciences",
"security_studies": "social_sciences",
"sociology": "social_sciences",
"us_foreign_policy": "social_sciences",
"virology": "other",
"world_religions": "humanities",
}
class MMLUEval(Eval):
"""MMLU benchmark evaluation."""
def __init__(self, filename: str, num_examples: int | None, num_threads: int):
if "://" in filename:
df = pandas.read_csv(filename, storage_options={"timeout": 30})
else:
df = pandas.read_csv(filename)
examples = [row.to_dict() for _, row in df.iterrows()]
if num_examples:
examples = random.Random(0).sample(examples, num_examples)
self.examples = examples
self.num_threads = num_threads
def __call__(self, sampler: "SamplerBase") -> EvalResult:
def fn(row: dict) -> SingleEvalResult:
prompt_messages = [
sampler._pack_message(
content=format_multichoice_question(row), role="user"
)
]
response_text = sampler(prompt_messages)
response_text = response_text or ""
match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text)
extracted_answer = match.group(1) if match else None
score = 1.0 if extracted_answer == row["Answer"] else 0.0
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=row["Answer"],
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
category = SUBJECT_TO_CATEGORY.get(row["Subject"], "other")
return SingleEvalResult(
html=html, score=score, metrics={category: score}, convo=convo
)
results = common.map_with_progress(fn, self.examples, self.num_threads)
return common.aggregate_results(results)
@@ -1,76 +0,0 @@
"""MMLU evaluation tests for router functionality.
Tests the router's ability to handle MMLU benchmark evaluations across
different backend configurations (gRPC and HTTP workers).
Usage:
# Run with gRPC backend only
pytest e2e_test/router/test_mmlu.py -v
# Run with specific backend
pytest e2e_test/router/test_mmlu.py -v -k "grpc"
"""
from __future__ import annotations
import logging
from types import SimpleNamespace
import pytest
from infra import run_eval
logger = logging.getLogger(__name__)
@pytest.mark.e2e
@pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True)
class TestMMLU:
"""MMLU evaluation tests using local workers (gRPC and HTTP)."""
def test_mmlu_basic(self, setup_backend):
"""Basic MMLU evaluation with score threshold.
Runs MMLU evaluation with 64 examples and validates that
accuracy meets minimum threshold (>= 0.65).
Note: setup_backend fixture already waits for workers to be ready.
"""
backend, model, client, *_ = setup_backend
args = SimpleNamespace(
base_url=str(client.base_url),
model=model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
assert metrics["score"] >= 0.65, (
f"MMLU score {metrics['score']:.2f} below threshold 0.65"
)
logger.info("MMLU score: %.2f (threshold: 0.65)", metrics["score"])
def test_mmlu_extended(self, setup_backend):
"""Extended MMLU evaluation with more examples.
Runs MMLU with 128 examples for more statistically
significant results.
"""
backend, model, client, *_ = setup_backend
args = SimpleNamespace(
base_url=str(client.base_url),
model=model,
eval_name="mmlu",
num_examples=128,
num_threads=64,
temperature=0.1,
)
metrics = run_eval(args)
assert metrics["score"] >= 0.65, (
f"MMLU score {metrics['score']:.2f} below threshold 0.65"
)
logger.info("MMLU extended score: %.2f (threshold: 0.65)", metrics["score"])
@@ -1,61 +0,0 @@
"""MMLU evaluation tests for PD (Prefill-Decode) disaggregated routing.
PD disaggregation separates prefill and decode phases across different
workers for improved throughput and resource utilization.
Requirements:
- sgl_kernel package
- GPUs: num_prefill + num_decode (default: 2 GPUs for 1+1)
- Optional: InfiniBand for high-performance transfers
Configuration via markers:
@pytest.mark.model("model-id") # Override default model
@pytest.mark.workers(prefill=2, decode=2) # Custom worker counts
@pytest.mark.gateway(policy="round_robin") # Gateway configuration
Usage:
# Basic (1 prefill + 1 decode)
pytest e2e_test/router/test_pd_mmlu.py -v
# Run specific test
pytest e2e_test/router/test_pd_mmlu.py::TestPDMMLU::test_pd_mmlu_basic -v
"""
from __future__ import annotations
import logging
from types import SimpleNamespace
import pytest
from infra import run_eval
logger = logging.getLogger(__name__)
@pytest.mark.e2e
@pytest.mark.parametrize("setup_backend", ["pd"], indirect=True)
class TestPDMMLU:
"""MMLU evaluation tests using PD disaggregated routing."""
def test_pd_mmlu_basic(self, setup_backend):
"""Basic MMLU evaluation with PD disaggregation.
Runs MMLU with 1 prefill + 1 decode worker and validates
accuracy meets threshold (>= 0.65).
"""
backend, model, client, *_ = setup_backend
args = SimpleNamespace(
base_url=str(client.base_url),
model=model,
eval_name="mmlu",
num_examples=64,
num_threads=32,
temperature=0.1,
)
metrics = run_eval(args)
assert metrics["score"] >= 0.65, (
f"PD MMLU score {metrics['score']:.2f} below threshold 0.65"
)
logger.info("PD MMLU score: %.2f (threshold: 0.65)", metrics["score"])
+1
View File
@@ -137,6 +137,7 @@ This README mostly describes the NVIDIA GPU CI pipeline. Other hardware backends
- Each test file should take < 500 seconds; split if longer.
- Each GitHub Actions job should take < 30 minutes; split if longer.
- If tests are too slow for per-commit, consider nightly suites.
- New GSM8K or MMLU accuracy checks should use sgl-eval (`run_eval(..., api="sgl_eval")` or `sgl-eval run`) instead of adding another dataset or evaluator copy.
## Other Notes
@@ -1,13 +0,0 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
tasks:
- name: "gsm8k"
metrics:
- name: "exact_match,strict-match"
value: 0.847
- name: "exact_match,flexible-extract"
value: 0.556
limit: 1319
num_concurrent: 128
num_fewshot: 5
apply_chat_template: false
fewshot_as_multiturn: true
@@ -1,13 +0,0 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
tasks:
- name: "gsm8k"
metrics:
- name: "exact_match,strict-match"
value: 0.847
- name: "exact_match,flexible-extract"
value: 0.556
limit: 1319
num_concurrent: 128
num_fewshot: 5
apply_chat_template: false
fewshot_as_multiturn: true
@@ -1,13 +0,0 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
tasks:
- name: "gsm8k"
metrics:
- name: "exact_match,strict-match"
value: 0.847
- name: "exact_match,flexible-extract"
value: 0.444
limit: 1319
num_concurrent: 128
num_fewshot: 5
apply_chat_template: false
fewshot_as_multiturn: true
@@ -1,69 +0,0 @@
"""Archived test classes split out of test/registered/models_e2e/test_nvidia_nemotron_3_nano.py.
Originally registered with `register_cuda_ci(...)`. Moved here as part of
the per-commit pruning effort to keep the code reachable manually.
Run with `python3 test/manual/models/test_nvidia_nemotron_3_nano_archived.py`.
"""
import unittest
from sglang.srt.utils import is_sm80_supported, is_sm90_supported
from sglang.test.kits.lm_eval_kit import LMEvalMixin
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
NEMOTRON_3_NANO_THINKING_ARGS = [
"--trust-remote-code",
"--tool-call-parser",
"qwen3_coder",
"--reasoning-parser",
"deepseek-r1",
]
class TestNvidiaNemotron3Nano30BBF16(LMEvalMixin, DefaultServerBase):
"""Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation."""
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml"
other_args = [
"--tp-size",
"2",
] + NEMOTRON_3_NANO_THINKING_ARGS
class TestNvidiaNemotron3Nano30BBF16FlashInfer(LMEvalMixin, DefaultServerBase):
"""Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation using flashinfer mamba backend."""
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml"
other_args = [
"--tp-size",
"2",
"--mamba-backend",
"flashinfer",
] + NEMOTRON_3_NANO_THINKING_ARGS
@unittest.skipIf(
not (is_sm80_supported() or is_sm90_supported()),
"NVFP4 Marlin fallback test requires CUDA SM8X/SM9X",
)
class TestNvidiaNemotron3Nano30BNVFP4Marlin(LMEvalMixin, DefaultServerBase):
"""Test Nemotron-3-Nano-30B NVFP4 model with the Marlin path."""
model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.yaml"
other_args = [
"--tp-size",
"1",
"--quantization",
"modelopt_fp4",
"--fp4-gemm-backend",
"marlin",
"--moe-runner-backend",
"marlin",
] + NEMOTRON_3_NANO_THINKING_ARGS
if __name__ == "__main__":
unittest.main()
@@ -17,7 +17,6 @@ import subprocess
import unittest
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List
import requests
@@ -43,10 +42,6 @@ SERVER_LAUNCH_TIMEOUT = 4800
GSM8K_NUM_QUESTIONS = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
ACCURACY_THRESHOLD = 0.94
# bench_sglang.py lives at the repo root (this file is 5 levels below), not under
# test/. Resolve it absolutely so it works regardless of the CI working directory.
REPO_ROOT = Path(__file__).resolve().parents[5]
GSM8K_BENCH_SCRIPT = REPO_ROOT / "benchmark" / "gsm8k" / "bench_sglang.py"
GSM8K_DATA_URL = (
"https://raw.githubusercontent.com/openai/grade-school-math/"
"master/grade_school_math/data/test.jsonl"
@@ -145,7 +140,8 @@ class TestQwen35Fp8ArFusionMI35x(CustomTestCase):
port = int(base_url.rsplit(":", 1)[-1])
command = [
"python3",
str(GSM8K_BENCH_SCRIPT),
"-m",
"sglang.test.few_shot_gsm8k",
"--num-questions",
str(GSM8K_NUM_QUESTIONS),
"--parallel",