minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix Mistral GSM8K chat eval (#27757)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-07-09 21:08:48 -07:00
committed by GitHub
parent a38cfc6768
commit b76dd0be69
10 changed files with 429 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/pyproject.toml
+6
View File
@@ -148,6 +148,12 @@ test = [
"jsonlines",
"lm-eval[api]>=0.4.9.2",
"matplotlib",
# Pin sgl-eval to a git SHA: upgrading changes zero-shot \boxed{} grading, so
# re-baseline MODEL_SCORE_THRESHOLDS in test_text_models_gsm8k_eval.py first.
# antlr4 4.9.3 is forced because latex2sympy2_extended raises ImportError on
# 4.7.x, and an older transitive pin can win during install.
"antlr4-python3-runtime==4.9.3",
"sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@b2a2703c42cae379bbcb8b7ff092df6601a61694",
"pandas",
"parameterized",
"peft>=0.18.0",
python/sglang/test/run_eval.py
+110 -1
View File
@@ -7,7 +7,10 @@ import argparse
import json
import os
import statistics
import subprocess
import time
import uuid
from pathlib import Path
from sglang.test.simple_eval_common import (
ChatCompletionSampler,
@@ -94,6 +97,105 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
return result, latency, sampler
def _run_sgl_eval(eval_name, args) -> dict:
# Returns a metrics dict (score, latency, output_throughput) so the
# existing write_results_to_json + threshold gate keep working.
from sglang.test.test_utils import dump_metric
base_url = (
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
out_parent = Path(
getattr(args, "sgl_eval_out_dir", None)
or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex)
).expanduser()
out_parent.mkdir(parents=True, exist_ok=True)
cmd = [
"sgl-eval",
"run",
eval_name,
"--base-url",
base_url,
"--num-threads",
str(getattr(args, "num_threads", 64)),
"--temperature",
str(getattr(args, "temperature", 0.0)),
"--out-dir",
str(out_parent),
]
if getattr(args, "model", None):
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
else:
cmd += ["--max-tokens", "2048"]
# Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
# without --thinking their message.content is empty and sgl-eval scores 0.
if getattr(args, "sgl_eval_thinking", None) is None:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif args.sgl_eval_thinking:
cmd += ["--thinking"]
try:
completed = subprocess.run(
cmd,
text=True,
capture_output=True,
check=False,
timeout=getattr(args, "sgl_eval_timeout", None),
)
except subprocess.TimeoutExpired as e:
raise TimeoutError(
f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n"
f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}"
) from e
if completed.returncode != 0:
raise RuntimeError(
f"sgl-eval failed with exit code {completed.returncode}: "
f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
)
metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json"))
if len(metrics_files) != 1:
raise FileNotFoundError(
f"Expected exactly one metrics.json under {out_parent}, "
f"found {len(metrics_files)}"
)
payload = json.loads(metrics_files[0].read_text())
aggregate = payload.get("aggregate")
if not isinstance(aggregate, dict) or "score" not in aggregate:
raise KeyError(f"{metrics_files[0]} missing aggregate.score")
metrics = dict(aggregate)
metrics["latency"] = payload.get("latency_seconds", 0.0)
metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0)
metrics["sgl_eval_metrics_path"] = str(metrics_files[0])
model = payload.get("model") or getattr(args, "model", None)
dump_metric(
f"{eval_name}_score",
metrics["score"],
labels={"model": model, "eval": eval_name},
)
dump_metric(
f"{eval_name}_latency",
metrics["latency"],
labels={"model": model, "eval": eval_name},
)
print(f"Score: {metrics['score']:.3f}")
print(f"Total latency: {metrics['latency']:.3f} s")
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
print(f"sgl-eval metrics: {metrics_files[0]}")
return metrics
def print_accept_length_summary(samplers: list) -> None:
accept_lengths = [
m["spec_accept_length"]
@@ -196,7 +298,14 @@ def run_eval(args):
eval_obj = AIME25Eval(args.num_examples, args.num_threads)
elif args.eval_name == "gsm8k":
from sglang.test.simple_eval_gsm8k import GSM8KEval
if getattr(args, "api", None) == "sgl_eval":
# Only the nightly correctness eval opts into sgl-eval (zero-shot
# chat, \boxed{}, math_verify). Every other gsm8k caller — spec
# decoding perf/accuracy, disaggregation, quant, model e2e — uses
# the 5-shot completion last-number scorer and relies on
# max_tokens/throughput behavior sgl-eval cannot provide.
return _run_sgl_eval("gsm8k", args)
from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval
eval_obj = GSM8KEval(
num_examples=args.num_examples,
python/sglang/test/simple_eval_common.py
+4 -1
View File
@@ -95,6 +95,7 @@ class ChatCompletionSampler(SamplerBase):
reasoning_effort: Optional[str] = None,
max_tokens: int = 2048,
extra_body: Optional[Dict[str, Any]] = None,
stop: Optional[List[str]] = None,
record_meta_info: bool = False,
):
self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
@@ -109,12 +110,13 @@ class ChatCompletionSampler(SamplerBase):
self.max_tokens = max_tokens
self.reasoning_effort = reasoning_effort
self.extra_body = extra_body
self.stop = stop
self.image_format = "url"
self._completion_tokens: list[int] = []
self.record_meta_info = record_meta_info
self._meta_infos: List[Dict[str, Any]] = []
print(
f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=} {self.record_meta_info=}"
f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=} {self.stop=} {self.record_meta_info=}"
)
def _handle_image(
@@ -157,6 +159,7 @@ class ChatCompletionSampler(SamplerBase):
max_tokens=self.max_tokens,
reasoning_effort=self.reasoning_effort,
extra_body=extra_body,
stop=self.stop,
)
if self.record_meta_info:
meta_info = getattr(response.choices[0], "meta_info", None)
python/sglang/test/simple_eval_gsm8k.py
-108
View File
@@ -1,108 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
import ast
import re
from typing import Optional
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
)
from sglang.utils import download_and_cache_file, read_jsonl
GSM8K_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = f"Question: {lines[i]['question']}\nAnswer:"
if include_answer:
ret += f" {lines[i]['answer']}"
return ret
def get_few_shot_examples(lines, k):
return "".join(get_one_example(lines, i, True) + "\n\n" for i in range(k))
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"-?\d+\.?\d*", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except (SyntaxError, ValueError):
return INVALID
class GSM8KEval(Eval):
def __init__(
self,
num_examples: Optional[int] = None,
num_threads: int = 64,
num_shots: int = 5,
data_path: Optional[str] = None,
):
self._num_threads = num_threads
self._num_shots = num_shots
if data_path:
filename = data_path
else:
filename = download_and_cache_file(GSM8K_URL)
all_lines = list(read_jsonl(filename))
pool_size = self._setup_prefix_pool(all_lines, num_shots)
# The evaluation data should not include the few-shot examples to prevent data leakage.
self._lines = all_lines[pool_size:]
if num_examples is not None:
# Slice caps silently when num_examples exceeds the available lines,
# matching upstream: callers like test_basic_sanity_eagle3 pass a
# num_examples larger than the dataset on purpose.
self._lines = self._lines[:num_examples]
def _setup_prefix_pool(self, all_lines: list, num_shots: int) -> int:
self._few_shot_prompt = get_few_shot_examples(all_lines, num_shots)
return num_shots
def _build_prefix(self, idx: int) -> str:
return self._few_shot_prompt
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(idx: int) -> SingleEvalResult:
question = get_one_example(self._lines, idx, include_answer=False)
correct_answer = get_answer_value(self._lines[idx]["answer"])
prompt_content = self._build_prefix(idx) + question
prompt_messages = [
sampler._pack_message(content=prompt_content, role="user")
]
try:
response_text = sampler(prompt_messages)
except Exception:
response_text = ""
extracted_answer = get_answer_value(response_text)
score = float(extracted_answer == correct_answer)
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=correct_answer,
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(html=html, score=score, convo=convo)
results = common.map_with_progress(
fn, list(range(len(self._lines))), num_threads=self._num_threads
)
return common.aggregate_results(results, default_stats=("mean", "std"))
python/sglang/test/simple_eval_mixed_prefix_gsm8k.py
+108 -1
View File
@@ -1,7 +1,114 @@
# Adapted from https://github.com/openai/simple-evals/
# Hand-rolled GSM8K scorer kept only for the mixed-prefix KV-correctness test,
# which needs randomized completion-style prefixes sgl-eval does not produce.
import ast
import random
import re
from typing import Optional
from sglang.test.simple_eval_gsm8k import GSM8KEval, get_one_example
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
)
from sglang.utils import download_and_cache_file, read_jsonl
GSM8K_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = f"Question: {lines[i]['question']}\nAnswer:"
if include_answer:
ret += f" {lines[i]['answer']}"
return ret
def get_few_shot_examples(lines, k):
return "".join(get_one_example(lines, i, True) + "\n\n" for i in range(k))
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"-?\d+\.?\d*", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except (SyntaxError, ValueError):
return INVALID
class GSM8KEval(Eval):
def __init__(
self,
num_examples: Optional[int] = None,
num_threads: int = 64,
num_shots: int = 5,
data_path: Optional[str] = None,
):
self._num_threads = num_threads
self._num_shots = num_shots
if data_path:
filename = data_path
else:
filename = download_and_cache_file(GSM8K_URL)
all_lines = list(read_jsonl(filename))
pool_size = self._setup_prefix_pool(all_lines, num_shots)
# The evaluation data should not include the few-shot examples to prevent data leakage.
self._lines = all_lines[pool_size:]
if num_examples is not None:
# Slice caps silently when num_examples exceeds the available lines,
# matching upstream: callers like test_basic_sanity_eagle3 pass a
# num_examples larger than the dataset on purpose.
self._lines = self._lines[:num_examples]
def _setup_prefix_pool(self, all_lines: list, num_shots: int) -> int:
self._few_shot_prompt = get_few_shot_examples(all_lines, num_shots)
return num_shots
def _build_prefix(self, idx: int) -> str:
return self._few_shot_prompt
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(idx: int) -> SingleEvalResult:
question = get_one_example(self._lines, idx, include_answer=False)
correct_answer = get_answer_value(self._lines[idx]["answer"])
prompt_content = self._build_prefix(idx) + question
prompt_messages = [
sampler._pack_message(content=prompt_content, role="user")
]
try:
response_text = sampler(prompt_messages)
except Exception:
response_text = ""
extracted_answer = get_answer_value(response_text)
score = float(extracted_answer == correct_answer)
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=correct_answer,
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(html=html, score=score, convo=convo)
results = common.map_with_progress(
fn, list(range(len(self._lines))), num_threads=self._num_threads
)
return common.aggregate_results(results, default_stats=("mean", "std"))
class MixedPrefixGSM8KEval(GSM8KEval):
python/sglang/test/test_utils.py
+3 -1
View File
@@ -150,7 +150,9 @@ DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8"
DEFAULT_ENABLE_ROUTED_EXPERTS_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B"
# Nightly tests
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = "meta-llama/Llama-3.1-8B-Instruct,mistralai/Mistral-7B-Instruct-v0.3,deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct,google/gemma-2-27b-it"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = (
"meta-llama/Llama-3.1-8B-Instruct,Qwen/Qwen3-8B,Qwen/Qwen3-4B"
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8"