diff --git a/.claude/skills/cookbook-review-pr/SKILL.md b/.claude/skills/cookbook-review-pr/SKILL.md index 27d29f0877..536ec8ea64 100644 --- a/.claude/skills/cookbook-review-pr/SKILL.md +++ b/.claude/skills/cookbook-review-pr/SKILL.md @@ -136,8 +136,8 @@ than restating. = `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. Flag output-only values. - **Consistent accuracy harness across entries**: every value under one `accuracyLabels` column must be produced by the SAME harness — flag a page that, say, measures one - platform's GSM8K with `few_shot_gsm8k --num-questions 200` and another's with - `run_eval --eval-name gsm8k --num-examples 1319` and shows both as one "GSM8K %" + platform's GSM8K with a historical raw completion harness and another's with + `sgl-eval run gsm8k --num-examples 1319` and shows both as one "GSM8K %" (the scores aren't comparable). Either standardize on one harness (matching `benchmarkCommands.accuracy`) or require an explicit per-entry note. Common when folding a second contributor's measurements (e.g. an AMD/ROCm PR) into the page. diff --git a/.claude/skills/generate-profile/SKILL.md b/.claude/skills/generate-profile/SKILL.md index dae475cfaf..dc744cf4fe 100644 --- a/.claude/skills/generate-profile/SKILL.md +++ b/.claude/skills/generate-profile/SKILL.md @@ -46,13 +46,11 @@ Typical startup time: 30-90 seconds depending on model size and whether CUDA gra ### Step 3: Validate accuracy (sanity check) ```bash -python3 -m sglang.test.run_eval --host 127.0.0.1 --port --eval-name gsm8k --num-examples 20 +sgl-eval run gsm8k --base-url http://127.0.0.1:/v1 --num-examples 20 ``` - Expected accuracy: **> 0.8** for capable models (Qwen3-8B, Llama-3.1-8B-Instruct, etc.) - This is a quick sanity check, not a rigorous benchmark. -- `sglang.test.few_shot_gsm8k` is deprecated; use the unified `run_eval` entrypoint. -- If you intentionally need the old completion-style GSM8K path, add `--api completion`. - If accuracy is unexpectedly low, something is wrong — do not proceed to profiling. ### Step 4: Generate the profile diff --git a/.gitignore b/.gitignore index 6009240589..09b2f034c8 100644 --- a/.gitignore +++ b/.gitignore @@ -166,8 +166,6 @@ cython_debug/ *.swp # SGL -benchmark/mmlu/data -benchmark/mmlu/data.tar benchmark/llava_bench/images benchmark/llava_bench/mme_pack *.jsonl diff --git a/benchmark/deepseek_v3/README.md b/benchmark/deepseek_v3/README.md index 2699596281..14c67c8cb0 100644 --- a/benchmark/deepseek_v3/README.md +++ b/benchmark/deepseek_v3/README.md @@ -271,7 +271,9 @@ Then we can benchmark the accuracy and latency by accessing the first node's exp ```bash # bench accuracy -python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host 10.0.0.1 --port 30000 +sgl-eval run gsm8k \ + --base-url http://10.0.0.1:30000/v1 \ + --num-examples 1319 --num-threads 64 # bench latency python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128 @@ -322,13 +324,15 @@ Then on the **master node**, supposing the ShareGPT data is located at `/path/to ```bash # bench accuracy -python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 --num-threads 64 # bench serving python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0 ``` -> **Note: using `--parallel 200` can accelerate accuracy benchmarking**. +> **Note: using `--num-threads 200` can accelerate accuracy benchmarking**. ### Example: Serving with 32 L40S with int8 Quantization diff --git a/benchmark/gsm8k/README.md b/benchmark/gsm8k/README.md deleted file mode 100644 index 5ff077d93f..0000000000 --- a/benchmark/gsm8k/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## Run benchmark - -### Using GSM8K Platinum - -GSM8K Platinum is a revised version of the GSM8K test set with corrected labels and removed ambiguous questions. It can be more stable than the original GSM8K dataset. It's a drop-in replacement that can be used by adding the `--platinum` flag: - -``` -python3 bench_sglang.py --num-shots 8 --num-questions 1209 --parallel 1209 --platinum -``` - -For more information, see: https://huggingface.co/datasets/madrylab/gsm8k-platinum - -### Benchmark sglang -``` -python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 -``` - -``` -python3 bench_sglang.py --num-questions 200 -``` diff --git a/benchmark/gsm8k/bench_sglang.py b/benchmark/gsm8k/bench_sglang.py deleted file mode 100644 index b3989673a4..0000000000 --- a/benchmark/gsm8k/bench_sglang.py +++ /dev/null @@ -1,207 +0,0 @@ -import argparse -import ast -import json -import os -import re -import time - -import numpy as np -from datasets import load_dataset - -from sglang.lang.api import set_default_backend -from sglang.test.test_utils import ( - add_common_sglang_args_and_parse, - dump_bench_raw_result, - select_sglang_backend, -) -from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl - -INVALID = -9999999 - - -def get_one_example(lines, i, include_answer): - ret = "Question: " + lines[i]["question"] + "\nAnswer:" - if include_answer: - ret += " " + lines[i]["answer"] - return ret - - -def get_few_shot_examples(lines, k): - ret = "" - for i in range(k): - ret += get_one_example(lines, i, True) + "\n\n" - return ret - - -def get_answer_value(answer_str): - answer_str = answer_str.replace(",", "") - numbers = re.findall(r"\d+", answer_str) - if len(numbers) < 1: - return INVALID - try: - return ast.literal_eval(numbers[-1]) - except SyntaxError: - return INVALID - - -def main(args): - # Select backend - set_default_backend(select_sglang_backend(args)) - - # Load tokenizer if enable_thinking is set - tokenizer = None - if args.enable_thinking: - from transformers import AutoTokenizer - - assert args.tokenizer_path is not None, ( - "--tokenizer-path is required when --enable-thinking is set" - ) - tokenizer = AutoTokenizer.from_pretrained( - args.tokenizer_path, trust_remote_code=True - ) - - # Read data - if args.platinum: - print("Loading GSM8K Platinum dataset from HuggingFace...") - dataset = load_dataset("madrylab/gsm8k-platinum", "main", split="test") - lines = [ - {"question": item["question"], "answer": item["answer"]} for item in dataset - ] - else: - data_path = args.data_path - url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" - if not os.path.isfile(data_path): - data_path = download_and_cache_file(url) - lines = list(read_jsonl(data_path)) - - # Construct prompts - num_questions = args.num_questions - num_shots = args.num_shots - few_shot_examples = get_few_shot_examples(lines, num_shots) - - questions = [] - labels = [] - for i in range(len(lines[:num_questions])): - raw_question = few_shot_examples + get_one_example(lines, i, False) - if tokenizer is not None: - messages = [{"role": "user", "content": raw_question}] - raw_question = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - **args.chat_template_kwargs, - ) - questions.append(raw_question) - labels.append(get_answer_value(lines[i]["answer"])) - assert all(l != INVALID for l in labels) - arguments = [{"question": q} for q in questions] - - ##################################### - ######### SGL Program Begin ######### - ##################################### - - import sglang as sgl - - @sgl.function - def few_shot_gsm8k(s, question): - s += question - s += sgl.gen( - "answer", - max_tokens=args.max_new_tokens, - stop=["Question", "Assistant:", "<|separator|>"], - ) - - ##################################### - ########## SGL Program End ########## - ##################################### - - # Run requests - tic = time.perf_counter() - states = few_shot_gsm8k.run_batch( - arguments, - temperature=args.temperature, - top_p=args.top_p, - num_threads=args.parallel, - progress_bar=True, - ) - latency = time.perf_counter() - tic - - preds = [] - for i in range(len(states)): - preds.append(get_answer_value(states[i]["answer"])) - - # Compute accuracy - acc = np.mean(np.array(preds) == np.array(labels)) - invalid = np.mean(np.array(preds) == INVALID) - - # Compute speed - num_output_tokens = sum( - s.get_meta_info("answer")["completion_tokens"] for s in states - ) - output_throughput = num_output_tokens / latency - - # Print results - print(f"Accuracy: {acc:.3f}") - print(f"Invalid: {invalid:.3f}") - print(f"Latency: {latency:.3f} s") - print(f"Output throughput: {output_throughput:.3f} token/s") - - # Dump results - dump_state_text(f"tmp_output_{args.backend}.txt", states) - dump_bench_raw_result( - path=args.raw_result_file, - states=states, - preds=preds, - labels=labels, - ) - - with open(args.result_file, "a") as fout: - value = { - "task": "gsm8k-platinum" if args.platinum else "gsm8k", - "backend": args.backend, - "num_gpus": 1, - "latency": round(latency, 3), - "accuracy": round(acc, 3), - "num_requests": args.num_questions, - "other": { - "num_questions": args.num_questions, - "parallel": args.parallel, - }, - } - fout.write(json.dumps(value) + "\n") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--num-shots", type=int, default=5) - parser.add_argument("--data-path", type=str, default="test.jsonl") - parser.add_argument("--num-questions", type=int, default=200) - parser.add_argument("--max-new-tokens", type=int, default=512) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--top-p", type=float, default=1.0) - parser.add_argument( - "--enable-thinking", - action="store_true", - help="Enable thinking mode by wrapping prompts with chat template", - ) - parser.add_argument( - "--chat-template-kwargs", - type=json.loads, - default='{"enable_thinking": true}', - help="JSON dict passed through to tokenizer.apply_chat_template. " - "The thinking-toggle kwarg name is model-specific, e.g. " - "'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).", - ) - parser.add_argument( - "--tokenizer-path", - type=str, - default=None, - help="Path to tokenizer (required when --enable-thinking is set)", - ) - parser.add_argument( - "--platinum", - action="store_true", - help="Use GSM8K Platinum dataset (drop-in replacement with corrected labels)", - ) - args = add_common_sglang_args_and_parse(parser) - main(args) diff --git a/benchmark/mmlu/README.md b/benchmark/mmlu/README.md deleted file mode 100644 index 54176add23..0000000000 --- a/benchmark/mmlu/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## Download data -``` -bash download_data.sh -``` - -## Run benchmark - -### Benchmark sglang -``` -python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 -``` - -``` -python3 bench_sglang.py --nsub 10 -``` - -``` -# OpenAI models -python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8 -``` diff --git a/benchmark/mmlu/bench_hf.py b/benchmark/mmlu/bench_hf.py deleted file mode 100644 index c76a18db68..0000000000 --- a/benchmark/mmlu/bench_hf.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -Usage: -python3 bench_hf.py --model-path meta-llama/Llama-2-7b-hf --data-dir data --ntrain 5 -""" - -import argparse -import json -import os -import time - -import numpy as np -import pandas as pd -import torch -from tqdm import tqdm -from transformers import AutoModelForCausalLM, AutoTokenizer - -choices = ["A", "B", "C", "D"] - - -def format_subject(subject): - l = subject.split("_") - s = "" - for entry in l: - s += " " + entry - return s - - -def format_example(df, idx, include_answer=True): - prompt = df.iloc[idx, 0] - k = df.shape[1] - 2 - for j in range(k): - prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) - prompt += "\nAnswer:" - if include_answer: - prompt += " {}\n\n".format(df.iloc[idx, k + 1]) - return prompt - - -def gen_prompt(train_df, subject, k=-1): - prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( - format_subject(subject) - ) - if k == -1: - k = train_df.shape[0] - for i in range(k): - prompt += format_example(train_df, i) - return prompt - - -@torch.no_grad() -def main(args): - print(f"Loading model: {args.model_path}") - tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) - model = AutoModelForCausalLM.from_pretrained( - args.model_path, - torch_dtype=torch.bfloat16, - trust_remote_code=True, - device_map="auto", - ).eval() - - subjects = sorted( - [ - f.split("_test.csv")[0] - for f in os.listdir(os.path.join(args.data_dir, "test")) - if "_test.csv" in f - ] - ) - - all_cors = [] - num_requests = 0 - total_latency = 0 - - for subject in tqdm(subjects[: args.nsub]): - dev_df = pd.read_csv( - os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None - )[: args.ntrain] - test_df = pd.read_csv( - os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None - ) - - k = args.ntrain - few_shot_examples = gen_prompt(dev_df, subject, k) - while len(tokenizer.encode(few_shot_examples)) > 1536: - k -= 1 - if k < 0: - break - few_shot_examples = gen_prompt(dev_df, subject, k) - - preds = [] - labels = [] - tic = time.perf_counter() - - for i in range(test_df.shape[0]): - prompt_end = format_example(test_df, i, include_answer=False) - prompt = few_shot_examples + prompt_end - - input_ids = tokenizer.encode(prompt, return_tensors="pt").to(model.device) - output_ids = model.generate( - input_ids, - max_new_tokens=1, - do_sample=False, - pad_token_id=tokenizer.eos_token_id, - ) - - output_str = tokenizer.decode( - output_ids[0][input_ids.shape[-1] :], skip_special_tokens=True - ) - preds.append(output_str.strip()[0] if len(output_str.strip()) > 0 else "") - labels.append(test_df.iloc[i, test_df.shape[1] - 1]) - - latency = time.perf_counter() - tic - total_latency += latency - - cors = [pred == label for pred, label in zip(preds, labels)] - all_cors.append(cors) - num_requests += len(test_df) - - print( - f"Subject: {subject}, Accuracy: {np.mean(cors):.3f}, Latency: {latency:.3f}s" - ) - - weighted_acc = np.mean(np.concatenate(all_cors)) - print(f"Total Latency: {total_latency:.3f}s") - print(f"Average Accuracy: {weighted_acc:.3f}") - - if args.output: - with open(args.output, "a") as fout: - value = { - "task": "mmlu", - "backend": "hf", - "model": args.model_path, - "latency": round(total_latency, 3), - "accuracy": round(weighted_acc, 3), - "num_requests": num_requests, - "other": { - "nsub": args.nsub, - "ntrain": args.ntrain, - }, - } - fout.write(json.dumps(value) + "\n") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--model-path", type=str, required=True) - parser.add_argument("--ntrain", type=int, default=5) - parser.add_argument("--data-dir", type=str, default="data") - parser.add_argument("--nsub", type=int, default=60) - parser.add_argument("--output", type=str, help="Output file path") - args = parser.parse_args() - main(args) diff --git a/benchmark/mmlu/bench_sglang.py b/benchmark/mmlu/bench_sglang.py deleted file mode 100644 index b86aec46c5..0000000000 --- a/benchmark/mmlu/bench_sglang.py +++ /dev/null @@ -1,210 +0,0 @@ -import argparse -import json -import os -import subprocess -import tarfile -import time - -import numpy as np -import pandas as pd -import tiktoken - -from sglang.test.test_utils import ( - add_common_sglang_args_and_parse, - dump_bench_raw_result, - select_sglang_backend, -) - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -choices = ["A", "B", "C", "D"] - -tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") - - -def format_subject(subject): - l = subject.split("_") - s = "" - for entry in l: - s += " " + entry - return s - - -def format_example(df, idx, include_answer=True): - prompt = df.iloc[idx, 0] - k = df.shape[1] - 2 - for j in range(k): - prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) - prompt += "\nAnswer:" - if include_answer: - prompt += " {}\n\n".format(df.iloc[idx, k + 1]) - return prompt - - -def gen_prompt(train_df, subject, k=-1): - prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( - format_subject(subject) - ) - if k == -1: - k = train_df.shape[0] - for i in range(k): - prompt += format_example(train_df, i) - return prompt - - -def download_data(data_dir): - """Download and extract MMLU data if it doesn't exist.""" - if os.path.isdir(os.path.join(data_dir, "test")): - return - print(f"Data not found at {data_dir}. Downloading...") - os.makedirs(data_dir, exist_ok=True) - tar_path = os.path.join(data_dir, "data.tar") - subprocess.check_call( - ["wget", "-O", tar_path, "https://people.eecs.berkeley.edu/~hendrycks/data.tar"] - ) - with tarfile.open(tar_path) as tar: - tar.extractall(path=data_dir, filter="data") - # The tarball extracts into a "data/" subdirectory; move contents up if needed - nested = os.path.join(data_dir, "data") - if os.path.isdir(nested): - for item in os.listdir(nested): - os.rename(os.path.join(nested, item), os.path.join(data_dir, item)) - os.rmdir(nested) - os.remove(tar_path) - print("Download complete.") - - -def main(args): - subjects = sorted( - [ - f.split("_test.csv")[0] - for f in os.listdir(os.path.join(args.data_dir, "test")) - if "_test.csv" in f - ] - ) - - # Build prompts - arguments = [] - labels = [] - num_questions = [] - - for subject in subjects[: args.nsub]: - dev_df = pd.read_csv( - os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None - )[: args.ntrain] - test_df = pd.read_csv( - os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None - ) - num_questions.append(test_df.shape[0]) - - k = args.ntrain - few_shot_examples = gen_prompt(dev_df, subject, k) - while len(tokenizer.encode(few_shot_examples)) > 1536: - k -= 1 - few_shot_examples = gen_prompt(dev_df, subject, k) - - for i in range(test_df.shape[0]): - prompt_end = format_example(test_df, i, include_answer=False) - - arguments.append( - { - "examples": few_shot_examples, - "question": prompt_end, - } - ) - - label = test_df.iloc[i, test_df.shape[1] - 1] - labels.append(label) - - ##################################### - ######### SGL Program Begin ######### - ##################################### - - import sglang as sgl - - if args.backend.startswith("gpt-"): - - @sgl.function - def few_shot_mmlu(s, examples, question): - s += sgl.user(examples + question) - s += sgl.assistant(sgl.gen("answer")) - - else: - - @sgl.function - def few_shot_mmlu(s, examples, question): - s += examples + question + sgl.gen("answer") - - ##################################### - ########## SGL Program End ########## - ##################################### - - # Select backend - backend = select_sglang_backend(args) - - # Run - tic = time.perf_counter() - states = few_shot_mmlu.run_batch( - arguments, - temperature=0, - max_new_tokens=1, - backend=backend, - num_threads=args.parallel, - progress_bar=True, - ) - preds = [ - s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states - ] - latency = time.perf_counter() - tic - - # Compute accuracy - cors = [pred == label for pred, label in zip(preds, labels)] - - pt = 0 - for subject, num_qs in zip(subjects[: args.nsub], num_questions): - print( - f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt : pt + num_qs]):.3f}" - ) - pt += num_qs - assert pt == len(cors) - weighted_acc = np.mean(cors) - - dump_bench_raw_result( - path=args.raw_result_file, - states=states, - preds=preds, - labels=labels, - ) - - # Print results - print("Total latency: {:.3f}".format(latency)) - print("Average accuracy: {:.3f}".format(weighted_acc)) - - # Write results - with open(args.result_file, "a") as fout: - value = { - "task": "mmlu", - "backend": args.backend, - "num_gpus": 1, - "latency": round(latency, 3), - "accuracy": round(weighted_acc, 3), - "num_requests": len(arguments), - "other": { - "nsub": args.nsub, - "parallel": args.parallel, - }, - } - fout.write(json.dumps(value) + "\n") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--ntrain", "-k", type=int, default=5) - parser.add_argument( - "--data_dir", "-d", type=str, default=os.path.join(SCRIPT_DIR, "data") - ) - parser.add_argument("--save_dir", "-s", type=str, default="results") - parser.add_argument("--nsub", type=int, default=60) - args = add_common_sglang_args_and_parse(parser) - download_data(args.data_dir) - main(args) diff --git a/benchmark/mmlu/download_data.sh b/benchmark/mmlu/download_data.sh deleted file mode 100644 index fb224165e3..0000000000 --- a/benchmark/mmlu/download_data.sh +++ /dev/null @@ -1,2 +0,0 @@ -wget https://people.eecs.berkeley.edu/~hendrycks/data.tar -tar xf data.tar diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx index 539f0d3e6f..a89618b870 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-Math-V2.mdx @@ -388,9 +388,14 @@ Thus for all positive \(a,b,c\) with \(abc=1\), **Benchmark Command:** ```shell Command -python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx index 19c4cac293..436c7add6b 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx @@ -977,12 +977,14 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py \ - --num-shots 8 \ - --num-questions 1316 \ - --parallel 1316 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads 1316 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index ec65a58668..a69839f1e1 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -597,9 +597,13 @@ Max ITL (ms): 43465.60 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:8000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Test Results**: - DeepSeek-V3 ```text Output @@ -614,11 +618,13 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000 - **Benchmark Command:** ```shell Command -cd sglang -bash benchmark/mmlu/download_data.sh -python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 8000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:8000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above. + - **Test Results**: - DeepSeek-V3 ```text Output diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx index 883ac1077d..0dcad01acd 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_1.mdx @@ -973,12 +973,14 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py \ - --num-shots 8 \ - --num-questions 1316 \ - --parallel 1316 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads 1316 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index f0c4a7df41..e082532e95 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -709,9 +709,13 @@ Max ITL (ms): 975.03 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Test Results**: - DeepSeek-V3.2-Exp ``` @@ -721,12 +725,17 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 30000 Output throughput: 965.919 token/s ``` -- **Full GSM8K (1319 questions)** — for a stricter accuracy check, run the full set 8-shot: +- **Full GSM8K (1319 questions)** — evaluate the complete test set with sgl-eval: ```shell Command -python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --parallel 1319 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads 1319 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - 8-shot: ``` Accuracy: 0.956 @@ -747,11 +756,13 @@ python3 benchmark/gsm8k/bench_sglang.py --num-shots 8 --num-questions 1319 --par - **Benchmark Command:** ```shell Command -cd sglang -bash benchmark/mmlu/download_data.sh -python3 benchmark/mmlu/bench_sglang.py --nsub 10 --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above. + - **Test Results**: - DeepSeek-V3.2-Exp ``` diff --git a/docs/cookbook/autoregressive/Ernie/Ernie4.5.mdx b/docs/cookbook/autoregressive/Ernie/Ernie4.5.mdx index c2dc5fda56..9f4d35afbf 100644 --- a/docs/cookbook/autoregressive/Ernie/Ernie4.5.mdx +++ b/docs/cookbook/autoregressive/Ernie/Ernie4.5.mdx @@ -680,12 +680,14 @@ Document model accuracy on standard benchmarks: - Benchmark Command: ```bash Command -python3 benchmark/gsm8k/bench_sglang.py \ - --num-shots 8 \ - --num-questions 1316 \ - --parallel 1316 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads 1316 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Results: - ERNIE-4.5-21B-A3B-PT ``` diff --git a/docs/cookbook/autoregressive/GLM/GLM-4.5.mdx b/docs/cookbook/autoregressive/GLM/GLM-4.5.mdx index b9d5500332..9cd21ae13a 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-4.5.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-4.5.mdx @@ -510,7 +510,9 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python -m sglang.test.few_shot_gsm8k \ - --num-questions 200 \ - --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` + +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. diff --git a/docs/cookbook/autoregressive/GLM/GLM-4.6.mdx b/docs/cookbook/autoregressive/GLM/GLM-4.6.mdx index 97fa60417d..9bb8a61b9b 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-4.6.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-4.6.mdx @@ -900,11 +900,13 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python -m sglang.test.few_shot_gsm8k \ - --num-questions 200 \ - --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - Test Result ```text Output Accuracy: 0.975 diff --git a/docs/cookbook/autoregressive/GLM/GLM-4.6V.mdx b/docs/cookbook/autoregressive/GLM/GLM-4.6V.mdx index 5836859764..44dfcad4a0 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-4.6V.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-4.6V.mdx @@ -388,8 +388,13 @@ print(response) #### Command ```shell Command -python3 ./benchmark/gsm8k/bench_sglang.py +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` + +Historical results below use the removed harness and are not measurements of the sgl-eval command above. #### Result Output ```text Output Accuracy: 0.925 diff --git a/docs/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx index 1789c0831c..c4716c4df9 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx @@ -920,11 +920,13 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python -m sglang.test.few_shot_gsm8k \ - --num-questions 200 \ - --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - Result ```text Output diff --git a/docs/cookbook/autoregressive/GLM/GLM-4.7.mdx b/docs/cookbook/autoregressive/GLM/GLM-4.7.mdx index 846dc551a5..9c93c195c6 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-4.7.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-4.7.mdx @@ -941,12 +941,13 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python -m sglang.test.few_shot_gsm8k \ - --num-shots 5 \ - --num-questions 1319 \ - --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - Test Result (NVIDIA B200, TP=2 NVFP4) ```text Output Accuracy: 0.946 diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx index bfa362e97e..f8a3308efa 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -644,9 +644,14 @@ The accuracy benchmark results below are shared with GLM-5, as GLM-5.1 was not i - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output Accuracy: 0.955 @@ -659,9 +664,13 @@ Output throughput: 642.044 token/s - Benchmark Command ```bash Command -python3 benchmark/mmlu/bench_sglang.py --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output subject: abstract_algebra, #q:100, acc: 0.860 @@ -732,9 +741,14 @@ Average accuracy: 0.877 - MI325/MI35x Test (GLM-5.1 BF16, `tp=8`, TileLang DSA backends) ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + ```text Output Accuracy: 0.970 Invalid: 0.000 diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.mdx index dfd9b6aa9b..791ebb4189 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.mdx @@ -575,9 +575,14 @@ The accuracy benchmark results below are shared with GLM-5.1, as GLM-5.1 was not - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output Accuracy: 0.955 @@ -590,9 +595,13 @@ Output throughput: 642.044 token/s - Benchmark Command ```bash Command -python3 benchmark/mmlu/bench_sglang.py --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output subject: abstract_algebra, #q:100, acc: 0.860 @@ -663,9 +672,14 @@ Average accuracy: 0.877 - MI325/MI35x Test (GLM-5 BF16, `tp=8`, TileLang DSA backends) ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + ```text Output Accuracy: 0.970 Invalid: 0.000 diff --git a/docs/cookbook/autoregressive/GLM/GLM-Glyph.mdx b/docs/cookbook/autoregressive/GLM/GLM-Glyph.mdx index cc676c5713..65e77071cf 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-Glyph.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-Glyph.mdx @@ -815,10 +815,13 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python -m sglang.test.few_shot_gsm8k \ - --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - Test Result ```text Output diff --git a/docs/cookbook/autoregressive/Google/Gemma4.mdx b/docs/cookbook/autoregressive/Google/Gemma4.mdx index 01ddccfc20..b87670a56e 100644 --- a/docs/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs/cookbook/autoregressive/Google/Gemma4.mdx @@ -1443,7 +1443,7 @@ Median ITL (ms): 15.11 -> **Note**: These GSM8K numbers use the raw few-shot completion harness (`sglang.test.few_shot_gsm8k`). `gemma-4-12B-it` is reasoning-oriented and is under-elicited by raw few-shot prompting; with the chat template it scores **0.950** on the same 1319 GSM8K test questions (`sglang.test.run_eval --eval-name gsm8k`). +> **Note**: These GSM8K numbers use the raw few-shot completion harness (now removed). `gemma-4-12B-it` is reasoning-oriented and is under-elicited by raw few-shot prompting; with the chat template it scores **0.950** on the same 1319 GSM8K test questions (also measured before the sgl-eval migration). #### gemma-4-12B-it with sgl-eval @@ -1467,6 +1467,8 @@ sgl-eval run gsm8k --base-url http://localhost:30000/v1 sgl-eval run mmlu --base-url http://localhost:30000/v1 --num-examples 2000 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + #### MMMU diff --git a/docs/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx b/docs/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx index aab3190ef0..4b17e3586d 100644 --- a/docs/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/LLaDA-2.1.mdx @@ -685,11 +685,13 @@ Max ITL (ms): 176.78 #### 5.2.1 GSM8K Benchmark ```bash Command -python -m sglang.test.few_shot_gsm8k \ - --num-questions 200 \ - --port 8000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:8000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + **Results:** ```text Output diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx index 008c2dd0c3..3c2eaa8877 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-2.5-1T.mdx @@ -205,9 +205,14 @@ Output: - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output Accuracy: 0.960 diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-2.6.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-2.6.mdx index c6cd1ebe9f..6559db96e8 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-2.6.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-2.6.mdx @@ -213,9 +213,14 @@ For more API examples, see the [SGLang Basic Usage Guide](../../../docs/basic_us Reference run on a single GB300 node with `--tp 4`: ```bash Command -python3 benchmark/gsm8k/bench_sglang.py +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + ```text Output Accuracy: 0.9621 (1269 / 1319) ``` diff --git a/docs/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx b/docs/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx index 42fb93cd65..77bdf971b4 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ring-2.5-1T.mdx @@ -250,9 +250,18 @@ sglang serve \ - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --temperature 1.2 --top-p 0.8 --max-new-tokens 32768 --num-questions 200 --tokenizer-path inclusionAI/Ring-2.5-1T --enable-thinking +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 \ + --temperature 1.2 \ + --top-p 0.8 \ + --max-tokens 32768 \ + --chat-template-kwarg enable_thinking=true ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output Accuracy: 0.955 diff --git a/docs/cookbook/autoregressive/InclusionAI/Ring-2.6-1T.mdx b/docs/cookbook/autoregressive/InclusionAI/Ring-2.6-1T.mdx index 42a9eff2c3..554de3e303 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ring-2.6-1T.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ring-2.6-1T.mdx @@ -337,10 +337,8 @@ Mean ITL (ms): 47.77 - Benchmark Command: ```bash Command -python3 -m sglang.test.run_eval \ - --eval-name gsm8k \ - --host 127.0.0.1 \ - --port ${PORT} \ +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:${PORT}/v1 \ --model auto \ --num-examples 200 \ --num-threads 64 \ @@ -348,6 +346,8 @@ python3 -m sglang.test.run_eval \ --reasoning-effort high ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - Test Results (B200 x8): ```text Output diff --git a/docs/cookbook/autoregressive/Meta/Llama3.1.mdx b/docs/cookbook/autoregressive/Meta/Llama3.1.mdx index 2bccc224eb..0448b36394 100644 --- a/docs/cookbook/autoregressive/Meta/Llama3.1.mdx +++ b/docs/cookbook/autoregressive/Meta/Llama3.1.mdx @@ -635,9 +635,13 @@ Max ITL (ms): 1970.03 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Results**: ```text Output diff --git a/docs/cookbook/autoregressive/Meta/Llama4.mdx b/docs/cookbook/autoregressive/Meta/Llama4.mdx index 7c70e05f8c..7dea7a9577 100644 --- a/docs/cookbook/autoregressive/Meta/Llama4.mdx +++ b/docs/cookbook/autoregressive/Meta/Llama4.mdx @@ -488,8 +488,12 @@ Max ITL (ms): 268.17 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` + +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. - Llama-4-Scout-17B-16E-Instruct ```text Output Accuracy: 0.945 @@ -505,9 +509,9 @@ Latency: 9.739 s Output throughput: 2405.505 token/s ``` -#### 5.3.2 MMLU Pro with lm-eval +#### 5.3.2 MMLU-Pro with sgl-eval -Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) on 8×H100 (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)): +The table below records historical lm-eval measurements on 8×H100 alongside [Meta's official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/) (reproduction details: [PR #5092](https://github.com/sgl-project/sglang/pull/5092)). These scores were not produced by the sgl-eval commands below; rerun the evaluation for results using sgl-eval's prompts and grading.
@@ -531,6 +535,12 @@ Accuracy on MMLU Pro matches [Meta's official benchmark numbers](https://ai.meta
+Install the evaluation package before running either command: + +```bash +pip install sgl-eval==0.1.0 +``` + **Scout:** ```bash Command @@ -542,13 +552,12 @@ python -m sglang.launch_server \ --mem-fraction-static 0.8 \ --context-length 65536 -# Run lm_eval -lm_eval --model local-chat-completions \ - --model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \ - --tasks mmlu_pro \ - --batch_size 128 \ - --apply_chat_template \ - --num_fewshot 0 +# Run MMLU-Pro +sgl-eval run mmlu_pro \ + --base-url http://localhost:30000/v1 \ + --model meta-llama/Llama-4-Scout-17B-16E-Instruct \ + --max-tokens 2048 \ + --num-threads 128 ``` **Maverick:** @@ -562,11 +571,10 @@ python -m sglang.launch_server \ --mem-fraction-static 0.8 \ --context-length 65536 -# Run lm_eval -lm_eval --model local-chat-completions \ - --model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 \ - --tasks mmlu_pro \ - --batch_size 128 \ - --apply_chat_template \ - --num_fewshot 0 +# Run MMLU-Pro +sgl-eval run mmlu_pro \ + --base-url http://localhost:30000/v1 \ + --model meta-llama/Llama-4-Maverick-17B-128E-Instruct \ + --max-tokens 2048 \ + --num-threads 128 ``` diff --git a/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx b/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx index d4c46623fa..77a7851f03 100644 --- a/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx +++ b/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.5.mdx @@ -960,8 +960,13 @@ Max ITL (ms): 376.34 #### 5.2.1 GSM8K Benchmark - Benchmark Command: ```shell Command -python benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` + +Historical results below use the removed harness and are not measurements of the sgl-eval command above. - Test Results: ```text Output Accuracy: 0.950 @@ -972,10 +977,13 @@ Output throughput: 1130.161 token/s #### 5.2.2 MMLU Benchmark - Benchmark Command: ```shell Command -cd benchmark/mmlu -bash download_data.sh -python3 bench_sglang.py --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` + +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Results: ```text Output subject: abstract_algebra, #q:100, acc: 0.620 diff --git a/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx b/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx index 0226d23148..6bf2f79b6d 100644 --- a/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx +++ b/docs/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx @@ -601,8 +601,12 @@ sglang serve \ - **Benchmark Command**: ```shell Command - python3 -m sglang.test.few_shot_gsm8k --num-questions 200 + sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` + +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. - **Result**: - MiniMax-M2 ```text Output diff --git a/docs/cookbook/autoregressive/Mistral/Devstral-2.mdx b/docs/cookbook/autoregressive/Mistral/Devstral-2.mdx index 845db315df..1b9f01eb17 100644 --- a/docs/cookbook/autoregressive/Mistral/Devstral-2.mdx +++ b/docs/cookbook/autoregressive/Mistral/Devstral-2.mdx @@ -501,13 +501,14 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py \ - --num-shots 8 \ - --num-questions 1316 \ - --parallel 1316 \ - --port 8888 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:8888/v1 \ + --num-examples 1319 \ + --num-threads 1316 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output diff --git a/docs/cookbook/autoregressive/Mistral/Ministral-3.mdx b/docs/cookbook/autoregressive/Mistral/Ministral-3.mdx index dd2dd8eb1a..85e8beb109 100644 --- a/docs/cookbook/autoregressive/Mistral/Ministral-3.mdx +++ b/docs/cookbook/autoregressive/Mistral/Ministral-3.mdx @@ -272,12 +272,14 @@ Document model accuracy on standard benchmarks: - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py \ - --num-shots 8 \ - --num-questions 1316 \ - --parallel 1316 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads 1316 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output diff --git a/docs/cookbook/autoregressive/Mistral/Mistral-Medium-3.5.mdx b/docs/cookbook/autoregressive/Mistral/Mistral-Medium-3.5.mdx index d62b597d98..0eeb3f4d40 100644 --- a/docs/cookbook/autoregressive/Mistral/Mistral-Medium-3.5.mdx +++ b/docs/cookbook/autoregressive/Mistral/Mistral-Medium-3.5.mdx @@ -336,9 +336,14 @@ Validation runs on 4× H200 with `--tp 4`, served via the `/v1/chat/completions` #### GSM8K ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Results:** ```text Output diff --git a/docs/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx b/docs/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx index 23ffd42580..12e533f6b6 100644 --- a/docs/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx +++ b/docs/cookbook/autoregressive/Mistral/Mistral-Small-4.mdx @@ -333,9 +333,14 @@ shapes to create a distinct symbol which signifies copyright protection. #### GSM8K ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + **Results:** ```text Output @@ -345,7 +350,9 @@ TODO #### MMLU ```bash Command -python3 benchmark/mmlu/bench_sglang.py --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` **Results:** diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx index af0ac209f0..6d2b93eac7 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx @@ -597,15 +597,17 @@ The following results are from the official Kimi-K2.7-Code model card. They were #### 5.2.1 Accuracy Benchmark -GSM8K accuracy of the MXFP4 checkpoint, measured on the deployed server across several client parallelism levels (2,000 questions each): +Evaluate GSM8K on the deployed MXFP4 checkpoint across client parallelism levels (all 1319 test questions via sgl-eval): ```shell Command -python3 benchmark/gsm8k/bench_sglang.py \ - --num-questions 2000 \ - --parallel \ - --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads '' ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx index abe1abea1f..af58a3e854 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx @@ -507,9 +507,13 @@ python3 -m sglang.launch_server \ - Benchmark Command ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:8000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Result**: ```text Output diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx index 4c9d1158de..442b877ee7 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-Linear.mdx @@ -284,9 +284,13 @@ SGLANG_ROCM_FUSED_DECODE_MLA=0 python3 -m sglang.launch_server \ - Benchmark Command ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Result**: ```text Output diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx index 49d2a25dba..4bcdb375f3 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano-Omni.mdx @@ -572,9 +572,14 @@ sglang serve \ **Run Benchmark** ```shell Command -python3 benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output Accuracy: 0.830 @@ -587,9 +592,13 @@ Output throughput: 1611.623 token/s **Run Benchmark** ```shell Command -python3 benchmark/mmlu/bench_sglang.py --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output subject: abstract_algebra, #q:100, acc: 0.510 diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx index 7ec09ef402..fc3947e6f5 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Nano.mdx @@ -235,7 +235,7 @@ python3 -m sglang.bench_serving \ --max-concurrency 256 ``` -- **Test Results:** +- **Serving benchmark results:** ```text Output ============ Serving Benchmark Result ============ @@ -304,7 +304,7 @@ python3 -m sglang.bench_serving \ --max-concurrency 256 ``` -- **Test Results:** +- **Serving benchmark results:** ```text Output ============ Serving Benchmark Result ============ @@ -361,14 +361,16 @@ python3 -m sglang.launch_server \ --reasoning-parser nemotron_3 ``` -**Run Benchmark with lm-eval** +**Run Benchmark with sgl-eval** ```bash Command -pip install lm-eval[api]==0.4.9.2 +pip install sgl-eval -lm_eval --model local-completions --tasks gsm8k --model_args "model=nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16,base_url=http://127.0.0.1:30000/v1/completions,num_concurrent=4,max_retries=3,tokenized_requests=False,max_lengths=16384" --gen_kwargs '{"chat_template_kwargs":{"thinking":true}}' --batch_size 256 +sgl-eval run gsm8k --base-url http://127.0.0.1:30000/v1 --model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 --num-threads 4 --max-tokens 16384 --thinking ``` -**Test Results:** +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + +**Historical lm-eval results (not measurements of the sgl-eval command above):** ```text Output |Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr| |-----|------:|----------------|-----:|-----------|---|-----:|---|-----:| diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx index 1a070ea7e1..1de82a3eef 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Super.mdx @@ -489,9 +489,14 @@ python3 -m sglang.launch_server \ **Run Benchmark** ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 5000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output Accuracy: 0.950 @@ -504,9 +509,13 @@ Output throughput: 996.815 token/s **Run Benchmark** ```bash Command -python3 benchmark/mmlu/bench_sglang.py --port 5000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output subject: abstract_algebra, #q:100, acc: 0.730 diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx index 66e6525e25..68af9b0ac8 100644 --- a/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx @@ -528,9 +528,14 @@ python3 -m sglang.launch_server \ **Run Benchmark** ```bash Command -python3 benchmark/gsm8k/bench_sglang.py +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + **Test Results:** ```text Output Accuracy: 0.970 @@ -543,7 +548,9 @@ Output throughput: 745.333 token/s **Run Benchmark** ```bash Command -python3 benchmark/mmlu/bench_sglang.py +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` **Test Results:** diff --git a/docs/cookbook/autoregressive/OpenAI/GPT-OSS.mdx b/docs/cookbook/autoregressive/OpenAI/GPT-OSS.mdx index 89d6e546bd..6e94653d66 100644 --- a/docs/cookbook/autoregressive/OpenAI/GPT-OSS.mdx +++ b/docs/cookbook/autoregressive/OpenAI/GPT-OSS.mdx @@ -640,9 +640,13 @@ Max ITL (ms): 621.42 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:8000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Results**: - GPT-OSS-120b diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx index a6d900d5bc..6df97fffde 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder-Next.mdx @@ -818,9 +818,14 @@ Max ITL (ms): 2783.23 - **Benchmark Command:** ```shell Command -python benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - **Test Results:** ```text Output @@ -835,11 +840,13 @@ Output throughput: 929.132 token/s - **Benchmark Command:** ```shell Command -cd benchmark/mmlu -bash download_data.sh -python3 bench_sglang.py --port 30000 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - **Test Results:** ```text Output diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx index 0b51ab4a7a..9a0d8974da 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3-Coder.mdx @@ -752,9 +752,13 @@ Max ITL (ms): 548.82 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + ##### AMD (MI300X/MI325X/MI355X) - **Results**: diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3-Next.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3-Next.mdx index 7577d7ceab..2dde9f8b8b 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3-Next.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3-Next.mdx @@ -726,9 +726,13 @@ Max ITL (ms): 486.38 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:8000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Results**: - Qwen3-Next-80B-A3B-Instruct @@ -753,11 +757,13 @@ python3 -m sglang.test.few_shot_gsm8k --num-questions 200 --port 8000 - **Benchmark Command:** ```shell Command -cd sglang -bash benchmark/mmlu/download_data.sh -python3 benchmark/mmlu/bench_sglang.py --nsub 10 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` +The command evaluates all 57 subjects. Historical results below use the removed 10-subject harness and are not measurements of the sgl-eval command above. + - **Results**: - Qwen3-Next-80B-A3B-Instruct diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx index 78681d399d..b3ab91357a 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx @@ -816,9 +816,14 @@ See [Decision models](/docs/supported-models/decision_models) for the request re - Benchmark Command ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 30000 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +Historical results below use the removed harness and are not measurements of the sgl-eval command above. + - Test Result ```text Output 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:31<00:00, 6.43it/s] @@ -828,21 +833,17 @@ Latency: 31.784 s Output throughput: 998.166 token/s ``` -#### 5.1.2 GSM8K with lm-eval (5-shot) - -Evaluate using the industry-standard `lm-eval` harness for reproducible accuracy reporting: +#### 5.1.2 Full GSM8K with sgl-eval ```bash Command -pip install lm-eval[api] - -lm_eval --model local-completions \ - --model_args '{"base_url": "http://localhost:30000/v1/completions", "model": "Qwen/Qwen3.5-397B-A17B", "num_concurrent": 256, "max_retries": 10, "max_gen_toks": 2048}' \ - --tasks gsm8k \ - --batch_size auto \ - --num_fewshot 5 \ - --trust_remote_code +sgl-eval run gsm8k \ + --base-url http://localhost:30000/v1 \ + --model Qwen/Qwen3.5-397B-A17B \ + --num-examples 1319 --num-threads 256 --max-tokens 2048 --thinking ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + #### 5.1.3 MMMU Benchmark - Benchmark Command diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.mdx index 4e5d64e5ed..973367bf3d 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.mdx @@ -871,9 +871,13 @@ Max ITL (ms): 352.79 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Results**: - Qwen/Qwen3-235B-A22B-Instruct-2507 diff --git a/docs/cookbook/autoregressive/StepFun/Step3.5.mdx b/docs/cookbook/autoregressive/StepFun/Step3.5.mdx index 1ff5a82b6b..c89dc6fddc 100644 --- a/docs/cookbook/autoregressive/StepFun/Step3.5.mdx +++ b/docs/cookbook/autoregressive/StepFun/Step3.5.mdx @@ -514,9 +514,13 @@ Max ITL (ms): 1359.81 - **Benchmark Command:** ```shell Command -python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - **Results**: - Step-3.5-Flash diff --git a/docs/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx b/docs/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx index f9b3dea579..11784644a0 100644 --- a/docs/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx +++ b/docs/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx @@ -443,13 +443,18 @@ Tool[0] get_weather({"city": "Beijing", "unit": "fahrenheit"}) #### 5.1.1 GSM8K -- Benchmark Method: 5-shot CoT on 200 questions, evaluated via SGLang native backend +- Benchmark Method: GSM8K on 200 questions using sgl-eval - Benchmark Command: ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --num-questions 200 --parallel 64 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 \ + --num-threads 64 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + - Test Results: ```text Output @@ -458,11 +463,13 @@ TODO — replace with real GSM8K accuracy after benchmark run on Hy3-preview (BF #### 5.1.2 MMLU -- Benchmark Method: 5-shot, all 57 subjects +- Benchmark Method: zero-shot Chat API evaluation via sgl-eval, all 57 subjects - Benchmark Command: ```bash Command -python3 benchmark/mmlu/bench_sglang.py --nsub 60 --parallel 64 +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ + --num-threads 64 --max-tokens 2048 ``` - Test Results: diff --git a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index 22f34259dd..53067a9a0f 100644 --- a/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -365,16 +365,17 @@ Standard 5-shot, `temperature=0`, `max_tokens=4096`, model defaults to thinking- **Benchmark Command:** ```shell Command -python3 -m sglang.test.run_eval \ - --base-url http://127.0.0.1:30000 \ +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ --model XiaomiMiMo/MiMo-V2.5 \ - --eval-name gsm8k \ --num-examples 200 \ --num-threads 8 \ --max-tokens 4096 \ --temperature 0.0 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + > `run_eval.py` automatically appends `/v1` to `--base-url`; pass the bare `host:port` URL (without trailing `/v1`), otherwise requests resolve to `/v1/v1/chat/completions` and 404. - **Test Results:** @@ -748,14 +749,15 @@ Pro's 3-layer MTP behaves very differently on natural text vs uniform-random tok **Benchmark Command:** ```shell Command -python3 -m sglang.test.run_eval \ - --base-url http://127.0.0.1:30000 \ +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ --model XiaomiMiMo/MiMo-V2.5-Pro \ - --eval-name gsm8k \ --num-examples 200 \ --num-threads 4 ``` +The command above uses sgl-eval. Historical scores from other harnesses are not directly comparable; rerun this command for a matching result. + The `accept_rate` and `accept_length` rows below are not part of `run_eval`'s own output — they were aggregated from the server-side `Decode batch ... accept rate: X accept len: Y` log lines emitted during the GSM8K run (307 batches total). | Workload | accept_rate | accept_length (max = 4) | diff --git a/docs/docs/developer_guide/contribution_guide.mdx b/docs/docs/developer_guide/contribution_guide.mdx index 08b212d315..a5d8ae1793 100644 --- a/docs/docs/developer_guide/contribution_guide.mdx +++ b/docs/docs/developer_guide/contribution_guide.mdx @@ -90,9 +90,8 @@ If your code changes model output, run an accuracy evaluation appropriate for th sglang serve --model-path Qwen/Qwen3-8B # Evaluate -python3 -m sglang.test.run_eval \ - --eval-name gsm8k \ - --port 30000 \ +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ --num-examples 200 ``` diff --git a/docs/docs/developer_guide/development_guide_using_docker.mdx b/docs/docs/developer_guide/development_guide_using_docker.mdx index 0e94bccd08..3c3a568b28 100644 --- a/docs/docs/developer_guide/development_guide_using_docker.mdx +++ b/docs/docs/developer_guide/development_guide_using_docker.mdx @@ -114,6 +114,9 @@ nsys profile -o deepseek_v3 python3 -m sglang.benchmark.one_batch --batch-size 1 ## Evaluation ```bash Command -# e.g. gsm8k 8 shot -python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8 +# GSM8K via sgl-eval: all 1319 test questions +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 1319 \ + --num-threads 2000 ``` diff --git a/docs/docs/developer_guide/evaluating_new_models.mdx b/docs/docs/developer_guide/evaluating_new_models.mdx index d06ba18e54..e20a2f54f9 100644 --- a/docs/docs/developer_guide/evaluating_new_models.mdx +++ b/docs/docs/developer_guide/evaluating_new_models.mdx @@ -18,9 +18,8 @@ SGLang provides built-in scripts to evaluate common benchmarks. **MMLU** ```bash Command -python -m sglang.test.run_eval \ - --eval-name mmlu \ - --port 30000 \ +sgl-eval run mmlu \ + --base-url http://127.0.0.1:30000/v1 \ --num-examples 1000 \ --max-tokens 8192 ``` @@ -28,12 +27,9 @@ python -m sglang.test.run_eval \ **GSM8K** ```bash Command -python -m sglang.test.run_eval \ - --eval-name gsm8k \ - --host 127.0.0.1 \ - --port 30000 \ - --num-examples 200 \ - --num-shots 5 +sgl-eval run gsm8k \ + --base-url http://127.0.0.1:30000/v1 \ + --num-examples 200 ``` **HellaSwag** diff --git a/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx b/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx index acaf5e683c..9e2c12ea22 100644 --- a/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx @@ -103,7 +103,7 @@ For more details, please refer to [docs/README.md](https://github.com/sgl-projec ## Test the accuracy -If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. +If your code changes the model output, run the accuracy tests. A quick sanity check is GSM8K through sgl-eval. ```bash # Launch a server @@ -112,7 +112,9 @@ python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct # Evaluate. --base-url must point at the server launched above. # The default SGLang server port is 30000; change it if you launched # the server with a different --port. -python3 -m sglang.test.run_eval --base-url http://localhost:30000 --eval-name gsm8k --num-examples 200 +sgl-eval run gsm8k \ + --base-url http://localhost:30000/v1 \ + --num-examples 200 ``` Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. diff --git a/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx b/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx index 5b441474ef..060d5bde1e 100644 --- a/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation.mdx @@ -105,40 +105,25 @@ For more details of SGLang server, refer to the [Ascend NPU Quick Start](/docs/h #### Execute Accuracy Evaluation -EvalScope connects to the SGLang server via OpenAI-compatible API. The following example uses the GSM8K dataset: +Use sgl-eval for GSM8K and MMLU. It owns the datasets, chat prompts and graders. EvalScope remains available for the other benchmarks below: ```shell Command -evalscope eval \ +pip install sgl-eval +sgl-eval run gsm8k \ --model /home/weights/Qwen2.5-7B-Instruct \ - --api-url http://localhost:30000/v1 \ - --api-key EMPTY \ - --eval-type openai_api \ - --datasets gsm8k \ - --limit 10 + --base-url http://localhost:30000/v1 \ + --num-examples 10 --max-tokens 2048 ``` -Upon completion, results similar to the following will be displayed: - -```text -+---------------------+-----------+----------+----------+-------+---------+---------+ -| Model | Dataset | Metric | Subset | Num | Score | Cat.0 | -+=====================+===========+==========+==========+=======+=========+=========+ -| Qwen2.5-7B-Instruct | gsm8k | mean_acc | main | 5 | 1.0 | default | -+---------------------+-----------+----------+----------+-------+---------+---------+ -``` - -> **Note**: Output format may vary slightly across different EvalScope versions. The above example is from EvalScope 1.6.x. Ensure the `--model` parameter matches the model name returned by the SGLang server's `/v1/models` endpoint. When starting the server with an HF path (e.g., `Qwen/Qwen2.5-7B-Instruct`), use that path directly. For local paths, pass the full path or the model name returned by `/v1/models`. +The run directory contains `metrics.json` and per-example predictions. Use the model name reported by `/v1/models`. #### Common Datasets for Online Evaluation ```shell Command # MMLU -evalscope eval \ +sgl-eval run mmlu \ --model /home/weights/Qwen2.5-7B-Instruct \ - --api-url http://localhost:30000/v1 \ - --api-key EMPTY \ - --eval-type openai_api \ - --datasets mmlu + --base-url http://localhost:30000/v1 --max-tokens 2048 # CEval (Chinese evaluation) evalscope eval \ @@ -241,7 +226,7 @@ When using EvalScope commands without specifying a dataset or model path, it wil ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/python3.11.14/lib/python3.11/site-packages/requests/adapters.py", line 676, in send raise SSLError(e, request=request) -requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/gsm8k (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)'))) +requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port=443): Max retries exceeded with url: /api/v1/datasets/AI-ModelScope/ceval (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1016)'))) [ERROR] 2026-05-13-02:20:01 (PID:876, Device:-1, RankID:-1) ERR99999 UNKNOWN application exception ``` **Temporary workaround (test only):** diff --git a/docs/src/snippets/configs/MiniMaxAI/minimax-m3-benchmarks.jsx b/docs/src/snippets/configs/MiniMaxAI/minimax-m3-benchmarks.jsx index 696ca3de19..c4a3b4555c 100644 --- a/docs/src/snippets/configs/MiniMaxAI/minimax-m3-benchmarks.jsx +++ b/docs/src/snippets/configs/MiniMaxAI/minimax-m3-benchmarks.jsx @@ -26,7 +26,7 @@ // Per-platform re-measurement under sgl-eval is in progress; rows still pending // show `gsm8k_pct: null` (no GSM8K row rendered) with the legacy-harness number // kept in a comment. Legacy harnesses were NOT comparable across platforms -// (NVIDIA: few_shot_gsm8k --num-questions 200; AMD: run_eval gsm8k 1319 examples) — +// (NVIDIA: removed completion harness, 200 questions; AMD: run_eval gsm8k 1319 examples) — // which is exactly why we re-measure on one harness. export const benchmarks = [ { diff --git a/python/sglang/srt/debug_utils/text_comparator.py b/python/sglang/srt/debug_utils/text_comparator.py index 3a6df19b9e..3bf64f7b1e 100644 --- a/python/sglang/srt/debug_utils/text_comparator.py +++ b/python/sglang/srt/debug_utils/text_comparator.py @@ -9,7 +9,7 @@ _DESCRIPTION = """Compare and find differences to benchmark outputs. Supported inputs: * The samples jsonl from `lm_eval --log_samples --output_path FOLDER_NAME` -* The output from `gsm8k/bench_sglang.py --raw-result-file FILE_NAME` (or mmlu) +* Legacy benchmark JSONL with prompt_id, prompt, output, and correct fields """ diff --git a/python/sglang/test/ascend/run_eval.py b/python/sglang/test/ascend/run_eval.py deleted file mode 100644 index c56aa7b5f8..0000000000 --- a/python/sglang/test/ascend/run_eval.py +++ /dev/null @@ -1,122 +0,0 @@ -import json -import os - -from sglang.test.run_eval import run_eval_once -from sglang.test.simple_eval_common import ( - make_report, - set_ulimit, -) - - -def run_eval(args): - # Lazy import to avoid circular dependency with test_utils - from sglang.test.test_utils import dump_metric - - set_ulimit() - - if "OPENAI_API_KEY" not in os.environ: - os.environ["OPENAI_API_KEY"] = "EMPTY" - - base_url = ( - f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1" - ) - - if args.eval_name == "mmlu": - from sglang.test.ascend.simple_eval_mmlu import MMLUEval - - filename = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv" - eval_obj = MMLUEval( - filename, args.num_examples, args.num_threads, getattr(args, "num_shots", 0) - ) - else: - raise ValueError(f"Invalid eval name: {args.eval_name}") - - if getattr(args, "repeat", 1) == 1: - result, latency, sampler = run_eval_once(args, base_url, eval_obj) - metrics = result.metrics | {"score": result.score} - metrics["latency"] = latency - print(f"Total latency: {latency:.3f} s") - print(f"Score: {metrics['score']:.3f}") - - # Compute output throughput from accumulated completion tokens - total_completion_tokens = sum(sampler._completion_tokens) - if total_completion_tokens > 0 and latency > 0: - metrics["output_throughput"] = total_completion_tokens / latency - print(f"Output throughput: {metrics['output_throughput']:.3f} token/s") - - # Report metrics to unified collection framework - dump_metric( - f"{args.eval_name}_score", - metrics["score"], - labels={"model": sampler.model, "eval": args.eval_name}, - ) - dump_metric( - f"{args.eval_name}_latency", - latency, - labels={"model": sampler.model, "eval": args.eval_name}, - ) - else: - from concurrent.futures import ThreadPoolExecutor - - executor = ThreadPoolExecutor(max_workers=args.repeat) - - futures = [ - executor.submit(run_eval_once, args, base_url, eval_obj) - for _ in range(args.repeat) - ] - - scores_repeat = [] - latencies = [] - total_completion_tokens = 0 - - for f in futures: - result, latency, sampler = f.result() - scores_repeat.append(result.score) - latencies.append(latency) - total_completion_tokens += sum(sampler._completion_tokens) - - mean_score = sum(scores_repeat) / len(scores_repeat) - mean_latency = sum(latencies) / len(latencies) - total_latency = sum(latencies) - scores_repeat = [f"{s:.3f}" for s in scores_repeat] - print("=" * 20) - print(f"Repeat: {args.repeat}, mean: {mean_score:.3f}") - print(f"Scores: {scores_repeat}") - print(f"Mean latency: {mean_latency:.3f} s") - print("=" * 20) - metrics = result.metrics | {"scores": scores_repeat} - metrics = metrics | {"mean_score": mean_score} - metrics["latency"] = mean_latency - - if total_completion_tokens > 0 and total_latency > 0: - metrics["output_throughput"] = total_completion_tokens / total_latency - print(f"Output throughput: {metrics['output_throughput']:.3f} token/s") - - # Report metrics to unified collection framework - dump_metric( - f"{args.eval_name}_mean_score", - mean_score, - labels={ - "model": sampler.model, - "eval": args.eval_name, - "repeat": args.repeat, - }, - ) - - executor.shutdown() - - # Dump reports - file_stem = f"{args.eval_name}_{sampler.model.replace('/', '_')}" - report_filename = f"/tmp/{file_stem}.html" - print(f"Writing report to {report_filename}") - with open(report_filename, "w") as fh: - fh.write(make_report(result)) - print(metrics) - result_filename = f"/tmp/{file_stem}.json" - with open(result_filename, "w") as f: - f.write(json.dumps(metrics, indent=2)) - print(f"Writing results to {result_filename}") - - if getattr(args, "return_latency", False): - return metrics, latency - return metrics diff --git a/python/sglang/test/ascend/simple_eval_mmlu.py b/python/sglang/test/ascend/simple_eval_mmlu.py deleted file mode 100644 index 5888786942..0000000000 --- a/python/sglang/test/ascend/simple_eval_mmlu.py +++ /dev/null @@ -1,125 +0,0 @@ -# Adapted from https://github.com/openai/simple-evals/ - -""" -Measuring Massive Multitask Language Understanding -Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt -https://arxiv.org/abs/2009.03300 -""" - -import random -import re -from typing import Optional - -import pandas - -from sglang.test import simple_eval_common as common -from sglang.test.simple_eval_common import ( - ANSWER_PATTERN_MULTICHOICE, - HTML_JINJA, - Eval, - EvalResult, - SamplerBase, - SingleEvalResult, - format_multichoice_question, -) -from sglang.test.simple_eval_mmlu import subject2category - - -def format_multichoice_question_example(row): - return QUERY_TEMPLATE_MULTICHOICE.format(**row) - - -QUERY_TEMPLATE_MULTICHOICE = """ -Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. - -{Question} - -A) {A} -B) {B} -C) {C} -D) {D} -""".strip() - -TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN = """ -Answer the multiple-choice questions following the examples below. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. - -""" - -TEMPLATE_MULTICHOICE_EXAMPLE = """ -Example question: -{Question} - -A {A} -B {B} -C {C} -D {D} - -The last line of your response should be -Answer: {Answer} -""".strip() - - -class MMLUEval(Eval): - def __init__( - self, - filename: str, - num_examples: Optional[int], - num_threads: int, - num_shots: int, - ): - if "://" in filename: - df = pandas.read_csv(filename, storage_options={"timeout": 30}) - else: - df = pandas.read_csv(filename) - examples = [row.to_dict() for _, row in df.iterrows()] - if num_shots: - example_questions = "".join( - format_multichoice_question_example(row) + "\n\n" - for row in examples[:num_shots] - ) - self.template = ( - TEMPLATE_MULTICHOICE_EXAMPLE_BEGIN - + example_questions - + QUERY_TEMPLATE_MULTICHOICE - ) - examples = examples[num_shots:] - if num_examples: - examples = random.Random(0).sample(examples, num_examples) - self.examples = examples - self.num_threads = num_threads - self.num_shots = num_shots - - def __call__(self, sampler: SamplerBase) -> EvalResult: - def fn(row: dict): - if self.num_shots: - prompt_messages = [ - sampler._pack_message( - content=self.template.format(**row), role="user" - ) - ] - else: - prompt_messages = [ - sampler._pack_message( - content=format_multichoice_question(row), role="user" - ) - ] - response_text = sampler(prompt_messages) - response_text = response_text or "" - match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) - extracted_answer = match.group(1) if match else None - score = 1.0 if extracted_answer == row["Answer"] else 0.0 - html = common.jinja_env.from_string(HTML_JINJA).render( - prompt_messages=prompt_messages, - next_message=dict(content=response_text, role="assistant"), - score=score, - correct_answer=row["Answer"], - extracted_answer=extracted_answer, - ) - convo = prompt_messages + [dict(content=response_text, role="assistant")] - category = subject2category.get(row["Subject"], "other") - return SingleEvalResult( - html=html, score=score, metrics={category: score}, convo=convo - ) - - results = common.map_with_progress(fn, self.examples, self.num_threads) - return common.aggregate_results(results) diff --git a/python/sglang/test/ascend/test_mmlu.py b/python/sglang/test/ascend/test_mmlu.py index 84f8a25f53..8a46b5ba1e 100644 --- a/python/sglang/test/ascend/test_mmlu.py +++ b/python/sglang/test/ascend/test_mmlu.py @@ -15,7 +15,7 @@ class TestMMLU: "server": getattr( self, "server_cmd", subprocess.list2cmdline(map(str, self.other_args)) ), - "client": "simple_eval_mmlu", + "client": "sgl-eval", "accuracy_threshold": getattr(self, "accuracy_mmlu", "N/A"), } diff --git a/python/sglang/test/few_shot_gsm8k_engine.py b/python/sglang/test/few_shot_gsm8k_engine.py deleted file mode 100644 index d06e15d35f..0000000000 --- a/python/sglang/test/few_shot_gsm8k_engine.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -.. deprecated:: - This module is deprecated. Use ``sglang.test.run_eval`` with - ``eval_name="gsm8k"`` instead, which routes through the unified - Chat API evaluation framework with dump_metric support. -""" - -import argparse -import ast -import asyncio -import re -import time -import warnings -from typing import Optional - -import numpy as np - -import sglang as sgl -from sglang.srt.utils import get_or_create_event_loop -from sglang.utils import download_and_cache_file, read_jsonl - -INVALID = -9999999 - - -def get_one_example(lines, i, include_answer): - ret = "Question: " + lines[i]["question"] + "\nAnswer:" - if include_answer: - ret += " " + lines[i]["answer"] - return ret - - -def get_few_shot_examples(lines, k): - ret = "" - for i in range(k): - ret += get_one_example(lines, i, True) + "\n\n" - return ret - - -def get_answer_value(answer_str): - answer_str = answer_str.replace(",", "") - numbers = re.findall(r"\d+", answer_str) - if len(numbers) < 1: - return INVALID - try: - return ast.literal_eval(numbers[-1]) - except SyntaxError: - return INVALID - - -async def concurrent_generate(engine, prompts, sampling_param): - tasks = [] - for prompt in prompts: - tasks.append(asyncio.create_task(engine.async_generate(prompt, sampling_param))) - - outputs = await asyncio.gather(*tasks) - return outputs - - -def run_eval(args): - warnings.warn( - "sglang.test.few_shot_gsm8k_engine is deprecated. " - "Use sglang.test.run_eval with eval_name='gsm8k' instead.", - DeprecationWarning, - stacklevel=2, - ) - # Select backend - engine = sgl.Engine(model_path=args.model_path, log_level="error") - - if args.local_data_path is None: - # Read data - url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" - filename = download_and_cache_file(url) - else: - filename = args.local_data_path - - lines = list(read_jsonl(filename)) - - # Construct prompts - num_questions = args.num_questions - num_shots = args.num_shots - few_shot_examples = get_few_shot_examples(lines, num_shots) - - questions = [] - labels = [] - for i in range(len(lines[:num_questions])): - questions.append(get_one_example(lines, i, False)) - labels.append(get_answer_value(lines[i]["answer"])) - assert all(l != INVALID for l in labels) - arguments = [{"question": q} for q in questions] - - # construct the prompts - prompts = [] - for i, arg in enumerate(arguments): - q = arg["question"] - prompt = few_shot_examples + q - prompts.append(prompt) - - sampling_param = { - "stop": ["Question", "Assistant:", "<|separator|>"], - "max_new_tokens": 512, - "temperature": 0, - } - - # Run requests - tic = time.perf_counter() - - loop = get_or_create_event_loop() - - outputs = loop.run_until_complete( - concurrent_generate(engine, prompts, sampling_param) - ) - - # End requests - latency = time.perf_counter() - tic - - # Shutdown the engine - engine.shutdown() - - # Parse output - preds = [] - - for output in outputs: - preds.append(get_answer_value(output["text"])) - - # Compute accuracy - acc = np.mean(np.array(preds) == np.array(labels)) - invalid = np.mean(np.array(preds) == INVALID) - - # Compute speed - num_output_tokens = sum( - output["meta_info"]["completion_tokens"] for output in outputs - ) - output_throughput = num_output_tokens / latency - - # Print results - print(f"Accuracy: {acc:.3f}") - print(f"Invalid: {invalid:.3f}") - print(f"Latency: {latency:.3f} s") - print(f"Output throughput: {output_throughput:.3f} token/s") - - return { - "accuracy": acc, - "latency": latency, - "output_throughput": output_throughput, - } - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--model-path", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct" - ) - parser.add_argument("--local-data-path", type=Optional[str], default=None) - parser.add_argument("--num-shots", type=int, default=5) - parser.add_argument("--num-questions", type=int, default=200) - args = parser.parse_args() - metrics = run_eval(args) diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index 7b6a4117dd..63bafe9fe1 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -267,7 +267,6 @@ def run_eval(args): if args.eval_name == "mmlu": # Scored by sgl-eval (NeMo-Skills' mcq prompt + eval_mcq grader), so a # caller's threshold has to be measured against it, not inherited. - # `simple_eval_mmlu` stays: the ascend eval imports its subject2category. return _run_sgl_eval("mmlu", args) elif args.eval_name == "mgsm_en": from sglang.test.simple_eval_mgsm import MGSMEval diff --git a/python/sglang/test/simple_eval_mmlu.py b/python/sglang/test/simple_eval_mmlu.py deleted file mode 100644 index 281da9e802..0000000000 --- a/python/sglang/test/simple_eval_mmlu.py +++ /dev/null @@ -1,125 +0,0 @@ -# Adapted from https://github.com/openai/simple-evals/ - -""" -Measuring Massive Multitask Language Understanding -Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt -https://arxiv.org/abs/2009.03300 -""" - -import random -import re -from typing import Optional - -import pandas - -from sglang.test import simple_eval_common as common -from sglang.test.simple_eval_common import ( - ANSWER_PATTERN_MULTICHOICE, - HTML_JINJA, - Eval, - EvalResult, - SamplerBase, - SingleEvalResult, - format_multichoice_question, -) - -subject2category = { - "abstract_algebra": "stem", - "anatomy": "other", - "astronomy": "stem", - "business_ethics": "other", - "clinical_knowledge": "other", - "college_biology": "stem", - "college_chemistry": "stem", - "college_computer_science": "stem", - "college_mathematics": "stem", - "college_medicine": "other", - "college_physics": "stem", - "computer_security": "stem", - "conceptual_physics": "stem", - "econometrics": "social_sciences", - "electrical_engineering": "stem", - "elementary_mathematics": "stem", - "formal_logic": "humanities", - "global_facts": "other", - "high_school_biology": "stem", - "high_school_chemistry": "stem", - "high_school_computer_science": "stem", - "high_school_european_history": "humanities", - "high_school_geography": "social_sciences", - "high_school_government_and_politics": "social_sciences", - "high_school_macroeconomics": "social_sciences", - "high_school_mathematics": "stem", - "high_school_microeconomics": "social_sciences", - "high_school_physics": "stem", - "high_school_psychology": "social_sciences", - "high_school_statistics": "stem", - "high_school_us_history": "humanities", - "high_school_world_history": "humanities", - "human_aging": "other", - "human_sexuality": "social_sciences", - "international_law": "humanities", - "jurisprudence": "humanities", - "logical_fallacies": "humanities", - "machine_learning": "stem", - "management": "other", - "marketing": "other", - "medical_genetics": "other", - "miscellaneous": "other", - "moral_disputes": "humanities", - "moral_scenarios": "humanities", - "nutrition": "other", - "philosophy": "humanities", - "prehistory": "humanities", - "professional_accounting": "other", - "professional_law": "humanities", - "professional_medicine": "other", - "professional_psychology": "social_sciences", - "public_relations": "social_sciences", - "security_studies": "social_sciences", - "sociology": "social_sciences", - "us_foreign_policy": "social_sciences", - "virology": "other", - "world_religions": "humanities", -} - - -class MMLUEval(Eval): - def __init__(self, filename: str, num_examples: Optional[int], num_threads: int): - if "://" in filename: - df = pandas.read_csv(filename, storage_options={"timeout": 30}) - else: - df = pandas.read_csv(filename) - examples = [row.to_dict() for _, row in df.iterrows()] - if num_examples: - examples = random.Random(0).sample(examples, num_examples) - self.examples = examples - self.num_threads = num_threads - - def __call__(self, sampler: SamplerBase) -> EvalResult: - def fn(row: dict): - prompt_messages = [ - sampler._pack_message( - content=format_multichoice_question(row), role="user" - ) - ] - response_text = sampler(prompt_messages) - response_text = response_text or "" - match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) - extracted_answer = match.group(1) if match else None - score = 1.0 if extracted_answer == row["Answer"] else 0.0 - html = common.jinja_env.from_string(HTML_JINJA).render( - prompt_messages=prompt_messages, - next_message=dict(content=response_text, role="assistant"), - score=score, - correct_answer=row["Answer"], - extracted_answer=extracted_answer, - ) - convo = prompt_messages + [dict(content=response_text, role="assistant")] - category = subject2category.get(row["Subject"], "other") - return SingleEvalResult( - html=html, score=score, metrics={category: score}, convo=convo - ) - - results = common.map_with_progress(fn, self.examples, self.num_threads) - return common.aggregate_results(results) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index fa59e41a65..521f330cb4 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -2409,38 +2409,6 @@ class CustomTestCase(unittest.TestCase): ) -def dump_bench_raw_result( - path: str, - states, - preds, - labels, -): - if not path: - return - - rows = [] - for i in range(len(states)): - state = states[i] - output = state["answer"] - prompt = _ensure_remove_suffix(state.text(), output) - rows.append( - dict( - prompt_id=i, - prompt=prompt, - output=output, - correct=bool(preds[i] == labels[i]), - ) - ) - - print(f"BenchRawResultDumper save results to {path}") - Path(path).write_text("\n".join(json.dumps(row) for row in rows)) - - -def _ensure_remove_suffix(text: str, suffix: str): - assert text.endswith(suffix) - return text.removesuffix(suffix) - - class ModelLaunchSettings: def __init__( self, diff --git a/sgl-model-gateway/e2e_test/infra/__init__.py b/sgl-model-gateway/e2e_test/infra/__init__.py index a138b3ce21..5f3e6fc836 100644 --- a/sgl-model-gateway/e2e_test/infra/__init__.py +++ b/sgl-model-gateway/e2e_test/infra/__init__.py @@ -63,7 +63,6 @@ from .process_utils import ( wait_for_health, wait_for_workers_ready, ) -from .run_eval import run_eval __all__ = [ # Enums and Identity @@ -139,6 +138,4 @@ __all__ = [ "FUNCTION_CALLING_MODELS", # Third-party models "THIRD_PARTY_MODELS", - # Evaluation - "run_eval", ] diff --git a/sgl-model-gateway/e2e_test/infra/run_eval.py b/sgl-model-gateway/e2e_test/infra/run_eval.py deleted file mode 100644 index fc87852db6..0000000000 --- a/sgl-model-gateway/e2e_test/infra/run_eval.py +++ /dev/null @@ -1,138 +0,0 @@ -"""MMLU evaluation runner for E2E tests. - -Simplified evaluation runner that uses local eval implementations -with cleaner logging for CI/CD environments. - -Usage: - from infra.run_eval import run_eval - from types import SimpleNamespace - - args = SimpleNamespace( - base_url="http://127.0.0.1:30000", - model="meta-llama/Llama-3.1-8B-Instruct", - eval_name="mmlu", - num_examples=64, - num_threads=32, - temperature=0.1, - ) - metrics = run_eval(args) -""" - -from __future__ import annotations - -import logging -import os -import time -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from .simple_eval_common import Eval - -from .simple_eval_common import ChatCompletionSampler, set_ulimit -from .simple_eval_mmlu import MMLU_DATASET_URL - -logger = logging.getLogger(__name__) - - -@dataclass -class EvalConfig: - """Configuration for running an evaluation.""" - - base_url: str - model: str | None = None - eval_name: str = "mmlu" - num_examples: int = 64 - num_threads: int = 32 - temperature: float = 0.0 - max_tokens: int = 2048 - host: str = "127.0.0.1" - port: int = 30000 - - -def _get_eval(eval_name: str, num_examples: int, num_threads: int) -> "Eval": - """Get the evaluation object by name.""" - if eval_name == "mmlu": - from .simple_eval_mmlu import MMLUEval - - return MMLUEval(MMLU_DATASET_URL, num_examples, num_threads) - else: - raise ValueError(f"Unknown eval: {eval_name}. Supported: mmlu") - - -def run_eval(args: Any) -> dict: - """Run an evaluation and return metrics. - - Args: - args: Configuration object with attributes: - - base_url: Base URL of the server (e.g., "http://127.0.0.1:30000") - - model: Model name/path (optional, will be auto-detected) - - eval_name: Evaluation name ("mmlu") - - num_examples: Number of examples to evaluate - - num_threads: Number of parallel threads - - temperature: Sampling temperature - - Returns: - Dict with metrics including 'score' key. - """ - set_ulimit() - - if "OPENAI_API_KEY" not in os.environ: - os.environ["OPENAI_API_KEY"] = "EMPTY" - - # Build base URL - base_url = getattr(args, "base_url", None) - if base_url: - base_url = base_url.rstrip("/") # Remove trailing slashes - if not base_url.endswith("/v1"): - base_url = f"{base_url}/v1" - else: - host = getattr(args, "host", "127.0.0.1") - port = getattr(args, "port", 30000) - base_url = f"http://{host}:{port}/v1" - - eval_name = getattr(args, "eval_name", "mmlu") - num_examples = getattr(args, "num_examples", 64) - num_threads = getattr(args, "num_threads", 32) - temperature = getattr(args, "temperature", 0.0) - max_tokens = getattr(args, "max_tokens", 2048) - model = getattr(args, "model", None) - - logger.info( - "Starting %s eval: %d examples, %d threads, temp=%.2f", - eval_name, - num_examples, - num_threads, - temperature, - ) - - # Create sampler - sampler = ChatCompletionSampler( - model=model, - max_tokens=max_tokens, - base_url=base_url, - temperature=temperature, - ) - - # Get eval object - eval_obj = _get_eval(eval_name, num_examples, num_threads) - - # Run evaluation - start_time = time.perf_counter() - result = eval_obj(sampler) - latency = time.perf_counter() - start_time - - # Build metrics - metrics = result.metrics.copy() if result.metrics else {} - metrics["score"] = result.score - metrics["latency"] = latency - - logger.info( - "%s eval complete: score=%.3f, latency=%.1fs, model=%s", - eval_name, - result.score, - latency, - sampler.model, - ) - - return metrics diff --git a/sgl-model-gateway/e2e_test/infra/simple_eval_common.py b/sgl-model-gateway/e2e_test/infra/simple_eval_common.py deleted file mode 100644 index 93767f3ef9..0000000000 --- a/sgl-model-gateway/e2e_test/infra/simple_eval_common.py +++ /dev/null @@ -1,495 +0,0 @@ -# Adapted from https://github.com/openai/simple-evals/ -"""Common utilities for simple evaluations.""" - -from __future__ import annotations - -import logging -import os -import resource -import time -from collections import defaultdict -from dataclasses import dataclass, field -from multiprocessing.pool import ThreadPool -from typing import Any - -import httpx -import jinja2 -import numpy as np -import openai -import requests -from openai import OpenAI -from tqdm import tqdm - -from .constants import MAX_RETRY_ATTEMPTS - -logger = logging.getLogger(__name__) - -OPENAI_SYSTEM_MESSAGE_API = "You are a helpful assistant." -OPENAI_SYSTEM_MESSAGE_CHATGPT = ( - "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture." - + "\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01" -) - - -Message = dict[str, Any] # keys role, content -MessageList = list[Message] - - -class SamplerBase: - """ - Base class for defining a sampling model, which can be evaluated, - or used as part of the grading process. - """ - - def __call__(self, message_list: MessageList) -> str: - raise NotImplementedError() - - -@dataclass -class EvalResult: - """Result of running an evaluation (usually consisting of many samples).""" - - score: float | None # top-line metric - metrics: dict[str, float] | None # other metrics - htmls: list[str] # strings of valid HTML - convos: list[MessageList] # sampled conversations - - -@dataclass -class SingleEvalResult: - """Result of evaluating a single sample.""" - - score: float | None - metrics: dict[str, float] = field(default_factory=dict) - html: str | None = None - convo: MessageList | None = None # sampled conversation - - -class Eval: - """ - Base class for defining an evaluation. - """ - - def __call__(self, sampler: SamplerBase) -> EvalResult: - raise NotImplementedError() - - -class LargerHttpxClient(httpx.Client): - def __init__(self): - timeout_config = httpx.Timeout(3600) - limits = httpx.Limits( - max_keepalive_connections=3600, - max_connections=3600, - ) - super().__init__(timeout=timeout_config, limits=limits) - - -class ChatCompletionSampler(SamplerBase): - """Sample from OpenAI's chat completion API.""" - - def __init__( - self, - base_url: str | None = None, - model: str | None = None, - system_message: str | None = None, - temperature: float = 0.0, - reasoning_effort: str | None = None, - max_tokens: int = 2048, - extra_body: dict[str, Any] | None = None, - ): - self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient()) - - if model is None: - model = self.client.models.list().data[0].id - - self.model = model - self.system_message = system_message - self.temperature = temperature - self.max_tokens = max_tokens - self.reasoning_effort = reasoning_effort - self.extra_body = extra_body - self.image_format = "url" - logger.debug( - "ChatCompletionSampler: model=%s, temp=%.2f, max_tokens=%d", - self.model, - self.temperature, - self.max_tokens, - ) - - def _handle_image( - self, - image: str, - encoding: str = "base64", - format: str = "png", - ): - new_image = { - "type": "image_url", - "image_url": { - "url": f"data:image/{format};{encoding},{image}", - }, - } - return new_image - - def _handle_text(self, text: str): - return {"type": "text", "text": text} - - def _pack_message(self, role: str, content: Any): - return {"role": str(role), "content": content} - - def __call__(self, message_list: MessageList) -> str: - if self.system_message: - message_list = [ - self._pack_message("system", self.system_message) - ] + message_list - trial = 0 - while trial < MAX_RETRY_ATTEMPTS: - try: - response = self.client.chat.completions.create( - model=self.model, - messages=message_list, - temperature=self.temperature, - max_tokens=self.max_tokens, - reasoning_effort=self.reasoning_effort, - extra_body=self.extra_body, - ) - return response.choices[0].message.content or "" - except openai.BadRequestError as e: - logger.warning("Bad request error: %s", e) - return "" - except Exception as e: - exception_backoff = 2**trial # exponential back off - # Log first few retries at debug, later ones at warning - log_fn = logger.warning if trial >= 3 else logger.debug - log_fn( - "Request failed (retry %d/%d, backoff %ds): %s", - trial + 1, - MAX_RETRY_ATTEMPTS, - exception_backoff, - e, - ) - time.sleep(exception_backoff) - trial += 1 - logger.warning( - "All retry attempts exhausted after %d retries, returning empty response", - MAX_RETRY_ATTEMPTS, - ) - return "" - - -QUERY_TEMPLATE_MULTICHOICE = """ -Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. - -{Question} - -A) {A} -B) {B} -C) {C} -D) {D} -""".strip() - -ANSWER_PATTERN_MULTICHOICE = r"(?i)Answer\s*:\s*([A-D])" -ANSWER_PATTERN = r"(?i)Answer\s*:\s*([^\n]+)" - - -EQUALITY_TEMPLATE = r""" -Look at the following two expressions (answers to a math problem) and judge whether they are equivalent. Only perform trivial simplifications - -Examples: - - Expression 1: $2x+3$ - Expression 2: $3+2x$ - -Yes - - Expression 1: 3/2 - Expression 2: 1.5 - -Yes - - Expression 1: $x^2+2x+1$ - Expression 2: $y^2+2y+1$ - -No - - Expression 1: $x^2+2x+1$ - Expression 2: $(x+1)^2$ - -Yes - - Expression 1: 3245/5 - Expression 2: 649 - -No -(these are actually equal, don't mark them equivalent if you need to do nontrivial simplifications) - - Expression 1: 2/(-3) - Expression 2: -2/3 - -Yes -(trivial simplifications are allowed) - - Expression 1: 72 degrees - Expression 2: 72 - -Yes -(give benefit of the doubt to units) - - Expression 1: 64 - Expression 2: 64 square feet - -Yes -(give benefit of the doubt to units) - ---- - -YOUR TASK - - -Respond with only "Yes" or "No" (without quotes). Do not include a rationale. - - Expression 1: %(expression1)s - Expression 2: %(expression2)s -""".strip() - - -HTML_JINJA = """ -

Prompt conversation

-{% for message in prompt_messages %} -{{ message_to_html(message) | safe }} -{% endfor %} -

Sampled message

-{{ message_to_html(next_message) | safe }} -

Results

-

Correct Answer: {{ correct_answer }}

-

Extracted Answer: {{ extracted_answer }}

-

Score: {{ score }}

-""" - - -def format_multichoice_question(row): - return QUERY_TEMPLATE_MULTICHOICE.format(**row) - - -def check_equality(sampler: SamplerBase, expr1: str, expr2: str): - prompt = EQUALITY_TEMPLATE % {"expression1": expr1, "expression2": expr2} - response = sampler([dict(content=prompt, role="user")]) - return (response or "").lower().strip() == "yes" - - -def _compute_stat(values: list, stat: str): - if stat == "mean": - return np.mean(values) - elif stat == "std": - return np.std(values) - elif stat == "min": - return np.min(values) - elif stat == "max": - return np.max(values) - else: - raise ValueError(f"Unknown {stat =}") - - -def aggregate_results( - single_eval_results: list[SingleEvalResult], - default_stats: tuple[str, ...] = ("mean", "std"), - name2stats: dict[str, tuple[str, ...]] | None = None, -) -> EvalResult: - """ - Aggregate results from multiple evaluations into a single EvalResult. - """ - name2stats = name2stats or {} - name2values = defaultdict(list) - htmls = [] - convos = [] - for single_eval_result in single_eval_results: - # Skip None results - if single_eval_result is None: - continue - for name, value in single_eval_result.metrics.items(): - name2values[name].append(value) - if single_eval_result.score is not None: - name2values["score"].append(single_eval_result.score) - htmls.append(single_eval_result.html) - convos.append(single_eval_result.convo) - final_metrics = {} - for name, values in name2values.items(): - stats = name2stats.get(name, default_stats) - for stat in stats: - key = name if stat == "mean" else f"{name}:{stat}" - final_metrics[key] = _compute_stat(values, stat) - return EvalResult( - score=final_metrics.pop("score", None), - metrics=final_metrics, - htmls=htmls, - convos=convos, - ) - - -def map_with_progress(f: callable, xs: list[Any], num_threads: int) -> list[Any]: - """Apply f to each element of xs, using a ThreadPool, and show progress.""" - # Use quiet progress bar that doesn't pollute logs - if os.getenv("debug"): - return list(map(f, tqdm(xs, total=len(xs), leave=False))) - else: - with ThreadPool(min(num_threads, len(xs))) as pool: - return list(tqdm(pool.imap(f, xs), total=len(xs), leave=False)) - - -jinja_env = jinja2.Environment( - loader=jinja2.BaseLoader(), - undefined=jinja2.StrictUndefined, - autoescape=jinja2.select_autoescape(["html", "xml"]), -) -_message_template = """ -
-
- {{ role }} - {% if variant %}({{ variant }}){% endif %} -
-
-
{{ content }}
-
-
-""" - - -def message_to_html(message: Message) -> str: - """ - Generate HTML snippet (inside a
) for a message. - """ - return jinja_env.from_string(_message_template).render( - role=message["role"], - content=message["content"], - variant=message.get("variant", None), - ) - - -jinja_env.globals["message_to_html"] = message_to_html - - -_report_template = """ - - - - - - {% if metrics %} -

Metrics

-
- - - - - - - - - {% for name, value in metrics.items() %} - - - - - {% endfor %} -
MetricValue
Score{{ score | float | round(3) }}
{{ name }}{{ value }}
- {% endif %} -

Examples

- {% for html in htmls %} - {{ html | safe }} -
- {% endfor %} - - -""" - - -def make_report(eval_result: EvalResult) -> str: - """ - Create a standalone HTML report from an EvalResult. - """ - return jinja_env.from_string(_report_template).render( - score=eval_result.score, - metrics=eval_result.metrics, - htmls=eval_result.htmls, - ) - - -def make_report_from_example_htmls(htmls: list[str]): - """ - Create a standalone HTML report from a list of example htmls - """ - return jinja_env.from_string(_report_template).render( - score=None, metrics={}, htmls=htmls - ) - - -def download_dataset(path: str, url: str) -> None: - """Download a dataset from URL to path.""" - logger.info("Downloading dataset from %s", url) - try: - response = requests.get(url, stream=True, timeout=30) - response.raise_for_status() - - total_size = int(response.headers.get("content-length", 0)) - block_size = 8192 - - with ( - open(path, "wb") as f, - tqdm( - desc="Downloading", - total=total_size, - unit="iB", - unit_scale=True, - unit_divisor=1024, - leave=False, - ) as progress_bar, - ): - for data in response.iter_content(block_size): - size = f.write(data) - progress_bar.update(size) - - logger.debug("Dataset saved to %s", path) - except requests.RequestException as e: - raise RuntimeError(f"Failed to download dataset: {e}") from e - - -def set_ulimit(target_soft_limit: int = 65535) -> None: - """Set the file descriptor limit for parallel requests.""" - resource_type = resource.RLIMIT_NOFILE - current_soft, current_hard = resource.getrlimit(resource_type) - - if current_soft < target_soft_limit: - try: - resource.setrlimit(resource_type, (target_soft_limit, current_hard)) - except ValueError as e: - logger.debug("Could not set RLIMIT_NOFILE: %s", e) diff --git a/sgl-model-gateway/e2e_test/infra/simple_eval_mmlu.py b/sgl-model-gateway/e2e_test/infra/simple_eval_mmlu.py deleted file mode 100644 index a83ed1d2ea..0000000000 --- a/sgl-model-gateway/e2e_test/infra/simple_eval_mmlu.py +++ /dev/null @@ -1,132 +0,0 @@ -# Adapted from https://github.com/openai/simple-evals/ -""" -MMLU Evaluation - Measuring Massive Multitask Language Understanding -Dan Hendrycks et al. https://arxiv.org/abs/2009.03300 -""" - -from __future__ import annotations - -import random -import re -from typing import TYPE_CHECKING - -import pandas - -from . import simple_eval_common as common -from .simple_eval_common import ( - ANSWER_PATTERN_MULTICHOICE, - HTML_JINJA, - Eval, - EvalResult, - SingleEvalResult, - format_multichoice_question, -) - -if TYPE_CHECKING: - from .simple_eval_common import SamplerBase - -# MMLU dataset URL (hosted by OpenAI) -MMLU_DATASET_URL = "https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv" - -SUBJECT_TO_CATEGORY = { - "abstract_algebra": "stem", - "anatomy": "other", - "astronomy": "stem", - "business_ethics": "other", - "clinical_knowledge": "other", - "college_biology": "stem", - "college_chemistry": "stem", - "college_computer_science": "stem", - "college_mathematics": "stem", - "college_medicine": "other", - "college_physics": "stem", - "computer_security": "stem", - "conceptual_physics": "stem", - "econometrics": "social_sciences", - "electrical_engineering": "stem", - "elementary_mathematics": "stem", - "formal_logic": "humanities", - "global_facts": "other", - "high_school_biology": "stem", - "high_school_chemistry": "stem", - "high_school_computer_science": "stem", - "high_school_european_history": "humanities", - "high_school_geography": "social_sciences", - "high_school_government_and_politics": "social_sciences", - "high_school_macroeconomics": "social_sciences", - "high_school_mathematics": "stem", - "high_school_microeconomics": "social_sciences", - "high_school_physics": "stem", - "high_school_psychology": "social_sciences", - "high_school_statistics": "stem", - "high_school_us_history": "humanities", - "high_school_world_history": "humanities", - "human_aging": "other", - "human_sexuality": "social_sciences", - "international_law": "humanities", - "jurisprudence": "humanities", - "logical_fallacies": "humanities", - "machine_learning": "stem", - "management": "other", - "marketing": "other", - "medical_genetics": "other", - "miscellaneous": "other", - "moral_disputes": "humanities", - "moral_scenarios": "humanities", - "nutrition": "other", - "philosophy": "humanities", - "prehistory": "humanities", - "professional_accounting": "other", - "professional_law": "humanities", - "professional_medicine": "other", - "professional_psychology": "social_sciences", - "public_relations": "social_sciences", - "security_studies": "social_sciences", - "sociology": "social_sciences", - "us_foreign_policy": "social_sciences", - "virology": "other", - "world_religions": "humanities", -} - - -class MMLUEval(Eval): - """MMLU benchmark evaluation.""" - - def __init__(self, filename: str, num_examples: int | None, num_threads: int): - if "://" in filename: - df = pandas.read_csv(filename, storage_options={"timeout": 30}) - else: - df = pandas.read_csv(filename) - examples = [row.to_dict() for _, row in df.iterrows()] - if num_examples: - examples = random.Random(0).sample(examples, num_examples) - self.examples = examples - self.num_threads = num_threads - - def __call__(self, sampler: "SamplerBase") -> EvalResult: - def fn(row: dict) -> SingleEvalResult: - prompt_messages = [ - sampler._pack_message( - content=format_multichoice_question(row), role="user" - ) - ] - response_text = sampler(prompt_messages) - response_text = response_text or "" - match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) - extracted_answer = match.group(1) if match else None - score = 1.0 if extracted_answer == row["Answer"] else 0.0 - html = common.jinja_env.from_string(HTML_JINJA).render( - prompt_messages=prompt_messages, - next_message=dict(content=response_text, role="assistant"), - score=score, - correct_answer=row["Answer"], - extracted_answer=extracted_answer, - ) - convo = prompt_messages + [dict(content=response_text, role="assistant")] - category = SUBJECT_TO_CATEGORY.get(row["Subject"], "other") - return SingleEvalResult( - html=html, score=score, metrics={category: score}, convo=convo - ) - - results = common.map_with_progress(fn, self.examples, self.num_threads) - return common.aggregate_results(results) diff --git a/sgl-model-gateway/e2e_test/router/test_mmlu.py b/sgl-model-gateway/e2e_test/router/test_mmlu.py deleted file mode 100644 index dbbe9cf8d1..0000000000 --- a/sgl-model-gateway/e2e_test/router/test_mmlu.py +++ /dev/null @@ -1,76 +0,0 @@ -"""MMLU evaluation tests for router functionality. - -Tests the router's ability to handle MMLU benchmark evaluations across -different backend configurations (gRPC and HTTP workers). - -Usage: - # Run with gRPC backend only - pytest e2e_test/router/test_mmlu.py -v - - # Run with specific backend - pytest e2e_test/router/test_mmlu.py -v -k "grpc" -""" - -from __future__ import annotations - -import logging -from types import SimpleNamespace - -import pytest -from infra import run_eval - -logger = logging.getLogger(__name__) - - -@pytest.mark.e2e -@pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True) -class TestMMLU: - """MMLU evaluation tests using local workers (gRPC and HTTP).""" - - def test_mmlu_basic(self, setup_backend): - """Basic MMLU evaluation with score threshold. - - Runs MMLU evaluation with 64 examples and validates that - accuracy meets minimum threshold (>= 0.65). - - Note: setup_backend fixture already waits for workers to be ready. - """ - backend, model, client, *_ = setup_backend - - args = SimpleNamespace( - base_url=str(client.base_url), - model=model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - temperature=0.1, - ) - metrics = run_eval(args) - - assert metrics["score"] >= 0.65, ( - f"MMLU score {metrics['score']:.2f} below threshold 0.65" - ) - logger.info("MMLU score: %.2f (threshold: 0.65)", metrics["score"]) - - def test_mmlu_extended(self, setup_backend): - """Extended MMLU evaluation with more examples. - - Runs MMLU with 128 examples for more statistically - significant results. - """ - backend, model, client, *_ = setup_backend - - args = SimpleNamespace( - base_url=str(client.base_url), - model=model, - eval_name="mmlu", - num_examples=128, - num_threads=64, - temperature=0.1, - ) - metrics = run_eval(args) - - assert metrics["score"] >= 0.65, ( - f"MMLU score {metrics['score']:.2f} below threshold 0.65" - ) - logger.info("MMLU extended score: %.2f (threshold: 0.65)", metrics["score"]) diff --git a/sgl-model-gateway/e2e_test/router/test_pd_mmlu.py b/sgl-model-gateway/e2e_test/router/test_pd_mmlu.py deleted file mode 100644 index 0c2e0f368d..0000000000 --- a/sgl-model-gateway/e2e_test/router/test_pd_mmlu.py +++ /dev/null @@ -1,61 +0,0 @@ -"""MMLU evaluation tests for PD (Prefill-Decode) disaggregated routing. - -PD disaggregation separates prefill and decode phases across different -workers for improved throughput and resource utilization. - -Requirements: - - sgl_kernel package - - GPUs: num_prefill + num_decode (default: 2 GPUs for 1+1) - - Optional: InfiniBand for high-performance transfers - -Configuration via markers: - @pytest.mark.model("model-id") # Override default model - @pytest.mark.workers(prefill=2, decode=2) # Custom worker counts - @pytest.mark.gateway(policy="round_robin") # Gateway configuration - -Usage: - # Basic (1 prefill + 1 decode) - pytest e2e_test/router/test_pd_mmlu.py -v - - # Run specific test - pytest e2e_test/router/test_pd_mmlu.py::TestPDMMLU::test_pd_mmlu_basic -v -""" - -from __future__ import annotations - -import logging -from types import SimpleNamespace - -import pytest -from infra import run_eval - -logger = logging.getLogger(__name__) - - -@pytest.mark.e2e -@pytest.mark.parametrize("setup_backend", ["pd"], indirect=True) -class TestPDMMLU: - """MMLU evaluation tests using PD disaggregated routing.""" - - def test_pd_mmlu_basic(self, setup_backend): - """Basic MMLU evaluation with PD disaggregation. - - Runs MMLU with 1 prefill + 1 decode worker and validates - accuracy meets threshold (>= 0.65). - """ - backend, model, client, *_ = setup_backend - - args = SimpleNamespace( - base_url=str(client.base_url), - model=model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - temperature=0.1, - ) - metrics = run_eval(args) - - assert metrics["score"] >= 0.65, ( - f"PD MMLU score {metrics['score']:.2f} below threshold 0.65" - ) - logger.info("PD MMLU score: %.2f (threshold: 0.65)", metrics["score"]) diff --git a/test/README.md b/test/README.md index a991583285..1d76282438 100644 --- a/test/README.md +++ b/test/README.md @@ -137,6 +137,7 @@ This README mostly describes the NVIDIA GPU CI pipeline. Other hardware backends - Each test file should take < 500 seconds; split if longer. - Each GitHub Actions job should take < 30 minutes; split if longer. - If tests are too slow for per-commit, consider nightly suites. +- New GSM8K or MMLU accuracy checks should use sgl-eval (`run_eval(..., api="sgl_eval")` or `sgl-eval run`) instead of adding another dataset or evaluator copy. ## Other Notes diff --git a/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml b/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml deleted file mode 100644 index b97c4f8a5a..0000000000 --- a/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.847 - - name: "exact_match,flexible-extract" - value: 0.556 -limit: 1319 -num_concurrent: 128 -num_fewshot: 5 -apply_chat_template: false -fewshot_as_multiturn: true diff --git a/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml b/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml deleted file mode 100644 index af7180b3cd..0000000000 --- a/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.847 - - name: "exact_match,flexible-extract" - value: 0.556 -limit: 1319 -num_concurrent: 128 -num_fewshot: 5 -apply_chat_template: false -fewshot_as_multiturn: true diff --git a/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.yaml b/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.yaml deleted file mode 100644 index 0d8a8dfa2a..0000000000 --- a/test/lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.yaml +++ /dev/null @@ -1,13 +0,0 @@ -model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4" -tasks: -- name: "gsm8k" - metrics: - - name: "exact_match,strict-match" - value: 0.847 - - name: "exact_match,flexible-extract" - value: 0.444 -limit: 1319 -num_concurrent: 128 -num_fewshot: 5 -apply_chat_template: false -fewshot_as_multiturn: true diff --git a/test/manual/models/test_nvidia_nemotron_3_nano_archived.py b/test/manual/models/test_nvidia_nemotron_3_nano_archived.py deleted file mode 100644 index 39a06b27ed..0000000000 --- a/test/manual/models/test_nvidia_nemotron_3_nano_archived.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Archived test classes split out of test/registered/models_e2e/test_nvidia_nemotron_3_nano.py. - -Originally registered with `register_cuda_ci(...)`. Moved here as part of -the per-commit pruning effort to keep the code reachable manually. -Run with `python3 test/manual/models/test_nvidia_nemotron_3_nano_archived.py`. -""" - -import unittest - -from sglang.srt.utils import is_sm80_supported, is_sm90_supported -from sglang.test.kits.lm_eval_kit import LMEvalMixin -from sglang.test.server_fixtures.default_fixture import DefaultServerBase - -NEMOTRON_3_NANO_THINKING_ARGS = [ - "--trust-remote-code", - "--tool-call-parser", - "qwen3_coder", - "--reasoning-parser", - "deepseek-r1", -] - - -class TestNvidiaNemotron3Nano30BBF16(LMEvalMixin, DefaultServerBase): - """Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation.""" - - model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml" - other_args = [ - "--tp-size", - "2", - ] + NEMOTRON_3_NANO_THINKING_ARGS - - -class TestNvidiaNemotron3Nano30BBF16FlashInfer(LMEvalMixin, DefaultServerBase): - """Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation using flashinfer mamba backend.""" - - model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml" - other_args = [ - "--tp-size", - "2", - "--mamba-backend", - "flashinfer", - ] + NEMOTRON_3_NANO_THINKING_ARGS - - -@unittest.skipIf( - not (is_sm80_supported() or is_sm90_supported()), - "NVFP4 Marlin fallback test requires CUDA SM8X/SM9X", -) -class TestNvidiaNemotron3Nano30BNVFP4Marlin(LMEvalMixin, DefaultServerBase): - """Test Nemotron-3-Nano-30B NVFP4 model with the Marlin path.""" - - model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4" - model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.yaml" - other_args = [ - "--tp-size", - "1", - "--quantization", - "modelopt_fp4", - "--fp4-gemm-backend", - "marlin", - "--moe-runner-backend", - "marlin", - ] + NEMOTRON_3_NANO_THINKING_ARGS - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_qwen35_fp8_ar_fusion_mi35x.py b/test/registered/amd/perf/mi35x/test_qwen35_fp8_ar_fusion_mi35x.py index a60373316b..5ff50e81a7 100644 --- a/test/registered/amd/perf/mi35x/test_qwen35_fp8_ar_fusion_mi35x.py +++ b/test/registered/amd/perf/mi35x/test_qwen35_fp8_ar_fusion_mi35x.py @@ -17,7 +17,6 @@ import subprocess import unittest from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field -from pathlib import Path from typing import Dict, List import requests @@ -43,10 +42,6 @@ SERVER_LAUNCH_TIMEOUT = 4800 GSM8K_NUM_QUESTIONS = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319")) ACCURACY_THRESHOLD = 0.94 -# bench_sglang.py lives at the repo root (this file is 5 levels below), not under -# test/. Resolve it absolutely so it works regardless of the CI working directory. -REPO_ROOT = Path(__file__).resolve().parents[5] -GSM8K_BENCH_SCRIPT = REPO_ROOT / "benchmark" / "gsm8k" / "bench_sglang.py" GSM8K_DATA_URL = ( "https://raw.githubusercontent.com/openai/grade-school-math/" "master/grade_school_math/data/test.jsonl" @@ -145,7 +140,8 @@ class TestQwen35Fp8ArFusionMI35x(CustomTestCase): port = int(base_url.rsplit(":", 1)[-1]) command = [ "python3", - str(GSM8K_BENCH_SCRIPT), + "-m", + "sglang.test.few_shot_gsm8k", "--num-questions", str(GSM8K_NUM_QUESTIONS), "--parallel",