minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix Mistral GSM8K chat eval (#27757)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-07-09 21:08:48 -07:00
committed by GitHub
parent a38cfc6768
commit b76dd0be69
10 changed files with 429 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/pyproject.toml
+6
View File
@@ -148,6 +148,12 @@ test = [
"jsonlines",
"lm-eval[api]>=0.4.9.2",
"matplotlib",
# Pin sgl-eval to a git SHA: upgrading changes zero-shot \boxed{} grading, so
# re-baseline MODEL_SCORE_THRESHOLDS in test_text_models_gsm8k_eval.py first.
# antlr4 4.9.3 is forced because latex2sympy2_extended raises ImportError on
# 4.7.x, and an older transitive pin can win during install.
"antlr4-python3-runtime==4.9.3",
"sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@b2a2703c42cae379bbcb8b7ff092df6601a61694",
"pandas",
"parameterized",
"peft>=0.18.0",
python/sglang/test/run_eval.py
+110 -1
View File
@@ -7,7 +7,10 @@ import argparse
import json
import os
import statistics
import subprocess
import time
import uuid
from pathlib import Path
from sglang.test.simple_eval_common import (
ChatCompletionSampler,
@@ -94,6 +97,105 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
return result, latency, sampler
def _run_sgl_eval(eval_name, args) -> dict:
# Returns a metrics dict (score, latency, output_throughput) so the
# existing write_results_to_json + threshold gate keep working.
from sglang.test.test_utils import dump_metric
base_url = (
f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1"
)
out_parent = Path(
getattr(args, "sgl_eval_out_dir", None)
or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex)
).expanduser()
out_parent.mkdir(parents=True, exist_ok=True)
cmd = [
"sgl-eval",
"run",
eval_name,
"--base-url",
base_url,
"--num-threads",
str(getattr(args, "num_threads", 64)),
"--temperature",
str(getattr(args, "temperature", 0.0)),
"--out-dir",
str(out_parent),
]
if getattr(args, "model", None):
cmd += ["--model", args.model]
if getattr(args, "num_examples", None) is not None:
cmd += ["--num-examples", str(args.num_examples)]
# Bound generation length so long-reasoning models don't stall the eval.
if getattr(args, "max_tokens", None) is not None:
cmd += ["--max-tokens", str(args.max_tokens)]
else:
cmd += ["--max-tokens", "2048"]
# Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel;
# without --thinking their message.content is empty and sgl-eval scores 0.
if getattr(args, "sgl_eval_thinking", None) is None:
model_l = (getattr(args, "model", None) or "").lower()
if "qwen3.5" in model_l or "qwen3-thinking" in model_l:
cmd += ["--thinking"]
elif args.sgl_eval_thinking:
cmd += ["--thinking"]
try:
completed = subprocess.run(
cmd,
text=True,
capture_output=True,
check=False,
timeout=getattr(args, "sgl_eval_timeout", None),
)
except subprocess.TimeoutExpired as e:
raise TimeoutError(
f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n"
f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}"
) from e
if completed.returncode != 0:
raise RuntimeError(
f"sgl-eval failed with exit code {completed.returncode}: "
f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}"
)
metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json"))
if len(metrics_files) != 1:
raise FileNotFoundError(
f"Expected exactly one metrics.json under {out_parent}, "
f"found {len(metrics_files)}"
)
payload = json.loads(metrics_files[0].read_text())
aggregate = payload.get("aggregate")
if not isinstance(aggregate, dict) or "score" not in aggregate:
raise KeyError(f"{metrics_files[0]} missing aggregate.score")
metrics = dict(aggregate)
metrics["latency"] = payload.get("latency_seconds", 0.0)
metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0)
metrics["sgl_eval_metrics_path"] = str(metrics_files[0])
model = payload.get("model") or getattr(args, "model", None)
dump_metric(
f"{eval_name}_score",
metrics["score"],
labels={"model": model, "eval": eval_name},
)
dump_metric(
f"{eval_name}_latency",
metrics["latency"],
labels={"model": model, "eval": eval_name},
)
print(f"Score: {metrics['score']:.3f}")
print(f"Total latency: {metrics['latency']:.3f} s")
print(f"Output throughput: {metrics['output_throughput']:.3f} token/s")
print(f"sgl-eval metrics: {metrics_files[0]}")
return metrics
def print_accept_length_summary(samplers: list) -> None:
accept_lengths = [
m["spec_accept_length"]
@@ -196,7 +298,14 @@ def run_eval(args):
eval_obj = AIME25Eval(args.num_examples, args.num_threads)
elif args.eval_name == "gsm8k":
from sglang.test.simple_eval_gsm8k import GSM8KEval
if getattr(args, "api", None) == "sgl_eval":
# Only the nightly correctness eval opts into sgl-eval (zero-shot
# chat, \boxed{}, math_verify). Every other gsm8k caller — spec
# decoding perf/accuracy, disaggregation, quant, model e2e — uses
# the 5-shot completion last-number scorer and relies on
# max_tokens/throughput behavior sgl-eval cannot provide.
return _run_sgl_eval("gsm8k", args)
from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval
eval_obj = GSM8KEval(
num_examples=args.num_examples,
python/sglang/test/simple_eval_common.py
+4 -1
View File
@@ -95,6 +95,7 @@ class ChatCompletionSampler(SamplerBase):
reasoning_effort: Optional[str] = None,
max_tokens: int = 2048,
extra_body: Optional[Dict[str, Any]] = None,
stop: Optional[List[str]] = None,
record_meta_info: bool = False,
):
self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
@@ -109,12 +110,13 @@ class ChatCompletionSampler(SamplerBase):
self.max_tokens = max_tokens
self.reasoning_effort = reasoning_effort
self.extra_body = extra_body
self.stop = stop
self.image_format = "url"
self._completion_tokens: list[int] = []
self.record_meta_info = record_meta_info
self._meta_infos: List[Dict[str, Any]] = []
print(
f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=} {self.record_meta_info=}"
f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=} {self.stop=} {self.record_meta_info=}"
)
def _handle_image(
@@ -157,6 +159,7 @@ class ChatCompletionSampler(SamplerBase):
max_tokens=self.max_tokens,
reasoning_effort=self.reasoning_effort,
extra_body=extra_body,
stop=self.stop,
)
if self.record_meta_info:
meta_info = getattr(response.choices[0], "meta_info", None)
python/sglang/test/simple_eval_gsm8k.py
-108
View File
@@ -1,108 +0,0 @@
# Adapted from https://github.com/openai/simple-evals/
import ast
import re
from typing import Optional
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
)
from sglang.utils import download_and_cache_file, read_jsonl
GSM8K_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = f"Question: {lines[i]['question']}\nAnswer:"
if include_answer:
ret += f" {lines[i]['answer']}"
return ret
def get_few_shot_examples(lines, k):
return "".join(get_one_example(lines, i, True) + "\n\n" for i in range(k))
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"-?\d+\.?\d*", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except (SyntaxError, ValueError):
return INVALID
class GSM8KEval(Eval):
def __init__(
self,
num_examples: Optional[int] = None,
num_threads: int = 64,
num_shots: int = 5,
data_path: Optional[str] = None,
):
self._num_threads = num_threads
self._num_shots = num_shots
if data_path:
filename = data_path
else:
filename = download_and_cache_file(GSM8K_URL)
all_lines = list(read_jsonl(filename))
pool_size = self._setup_prefix_pool(all_lines, num_shots)
# The evaluation data should not include the few-shot examples to prevent data leakage.
self._lines = all_lines[pool_size:]
if num_examples is not None:
# Slice caps silently when num_examples exceeds the available lines,
# matching upstream: callers like test_basic_sanity_eagle3 pass a
# num_examples larger than the dataset on purpose.
self._lines = self._lines[:num_examples]
def _setup_prefix_pool(self, all_lines: list, num_shots: int) -> int:
self._few_shot_prompt = get_few_shot_examples(all_lines, num_shots)
return num_shots
def _build_prefix(self, idx: int) -> str:
return self._few_shot_prompt
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(idx: int) -> SingleEvalResult:
question = get_one_example(self._lines, idx, include_answer=False)
correct_answer = get_answer_value(self._lines[idx]["answer"])
prompt_content = self._build_prefix(idx) + question
prompt_messages = [
sampler._pack_message(content=prompt_content, role="user")
]
try:
response_text = sampler(prompt_messages)
except Exception:
response_text = ""
extracted_answer = get_answer_value(response_text)
score = float(extracted_answer == correct_answer)
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=correct_answer,
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(html=html, score=score, convo=convo)
results = common.map_with_progress(
fn, list(range(len(self._lines))), num_threads=self._num_threads
)
return common.aggregate_results(results, default_stats=("mean", "std"))
python/sglang/test/simple_eval_mixed_prefix_gsm8k.py
+108 -1
View File
@@ -1,7 +1,114 @@
# Adapted from https://github.com/openai/simple-evals/
# Hand-rolled GSM8K scorer kept only for the mixed-prefix KV-correctness test,
# which needs randomized completion-style prefixes sgl-eval does not produce.
import ast
import random
import re
from typing import Optional
from sglang.test.simple_eval_gsm8k import GSM8KEval, get_one_example
from sglang.test import simple_eval_common as common
from sglang.test.simple_eval_common import (
HTML_JINJA,
Eval,
EvalResult,
SamplerBase,
SingleEvalResult,
)
from sglang.utils import download_and_cache_file, read_jsonl
GSM8K_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
INVALID = -9999999
def get_one_example(lines, i, include_answer):
ret = f"Question: {lines[i]['question']}\nAnswer:"
if include_answer:
ret += f" {lines[i]['answer']}"
return ret
def get_few_shot_examples(lines, k):
return "".join(get_one_example(lines, i, True) + "\n\n" for i in range(k))
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"-?\d+\.?\d*", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except (SyntaxError, ValueError):
return INVALID
class GSM8KEval(Eval):
def __init__(
self,
num_examples: Optional[int] = None,
num_threads: int = 64,
num_shots: int = 5,
data_path: Optional[str] = None,
):
self._num_threads = num_threads
self._num_shots = num_shots
if data_path:
filename = data_path
else:
filename = download_and_cache_file(GSM8K_URL)
all_lines = list(read_jsonl(filename))
pool_size = self._setup_prefix_pool(all_lines, num_shots)
# The evaluation data should not include the few-shot examples to prevent data leakage.
self._lines = all_lines[pool_size:]
if num_examples is not None:
# Slice caps silently when num_examples exceeds the available lines,
# matching upstream: callers like test_basic_sanity_eagle3 pass a
# num_examples larger than the dataset on purpose.
self._lines = self._lines[:num_examples]
def _setup_prefix_pool(self, all_lines: list, num_shots: int) -> int:
self._few_shot_prompt = get_few_shot_examples(all_lines, num_shots)
return num_shots
def _build_prefix(self, idx: int) -> str:
return self._few_shot_prompt
def __call__(self, sampler: SamplerBase) -> EvalResult:
def fn(idx: int) -> SingleEvalResult:
question = get_one_example(self._lines, idx, include_answer=False)
correct_answer = get_answer_value(self._lines[idx]["answer"])
prompt_content = self._build_prefix(idx) + question
prompt_messages = [
sampler._pack_message(content=prompt_content, role="user")
]
try:
response_text = sampler(prompt_messages)
except Exception:
response_text = ""
extracted_answer = get_answer_value(response_text)
score = float(extracted_answer == correct_answer)
html = common.jinja_env.from_string(HTML_JINJA).render(
prompt_messages=prompt_messages,
next_message=dict(content=response_text, role="assistant"),
score=score,
correct_answer=correct_answer,
extracted_answer=extracted_answer,
)
convo = prompt_messages + [dict(content=response_text, role="assistant")]
return SingleEvalResult(html=html, score=score, convo=convo)
results = common.map_with_progress(
fn, list(range(len(self._lines))), num_threads=self._num_threads
)
return common.aggregate_results(results, default_stats=("mean", "std"))
class MixedPrefixGSM8KEval(GSM8KEval):
python/sglang/test/test_utils.py
+3 -1
View File
@@ -150,7 +150,9 @@ DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8"
DEFAULT_ENABLE_ROUTED_EXPERTS_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B"
# Nightly tests
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = "meta-llama/Llama-3.1-8B-Instruct,mistralai/Mistral-7B-Instruct-v0.3,deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct,google/gemma-2-27b-it"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = (
"meta-llama/Llama-3.1-8B-Instruct,Qwen/Qwen3-8B,Qwen/Qwen3-4B"
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8"
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8"
scripts/ci/cuda/ci_install_dependency.sh
+4
View File
@@ -514,6 +514,10 @@ install_extra_deps() {
if [ "$IS_BLACKWELL" != "1" ]; then
git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git
$PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX
# lmms-eval v0.5 pulls antlr4-python3-runtime==4.7.2, clobbering the
# 4.9.3 that sgl-eval's latex2sympy2_extended needs (4.7.2 ImportError
# at sgl-eval import). Pin it back so the nightly sgl-eval path works.
$PIP_CMD install "antlr4-python3-runtime==4.9.3" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX
fi
$PIP_CMD uninstall xformers || true
test/registered/eval/test_text_models_gsm8k_eval.py
+18 -17
View File
@@ -26,23 +26,23 @@ NIGHTLY_EVAL_SERVER_TIMEOUT = 1800
register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True)
MODEL_SCORE_THRESHOLDS = {
# Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores
"meta-llama/Llama-3.1-8B-Instruct": 0.80, # 84.5% - 5%
"mistralai/Mistral-7B-Instruct-v0.3": 0.47, # 52.1% - 5%
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81, # 86.4% - 5%
"google/gemma-2-27b-it": 0.81, # 85.5% measured - 5%
"meta-llama/Llama-3.1-70B-Instruct": 0.89, # 94.1% - 5%
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69, # 74.4% - 5%
"Qwen/Qwen2-57B-A14B-Instruct": 0.76, # 80.7% - 5% (official A14B score; 88.2% was the 72B)
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80, # 84.5% - 5%
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.47, # 52.1% - 5%
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81, # 86.4% - 5%
"zai-org/GLM-4.5-Air-FP8": 0.80, # ~85% - 5%
"neuralmagic/gemma-2-2b-it-FP8": 0.53, # 58.4% measured - 5%
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.1% - 5%
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69, # 74.4% - 5%
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.86, # 91.1% - 5%
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76, # 80.7% - 5% (official A14B score)
# sgl-eval (zero-shot chat, \boxed{}, math_verify grading). Thresholds are
# measured_score - 0.05, baselined on H100 2-GPU over the full 1319 split.
"meta-llama/Llama-3.1-8B-Instruct": 0.77, # 81.05% measured - 5%
"Qwen/Qwen3-8B": 0.76, # 81.43% measured - 5%
"Qwen/Qwen3-4B": 0.77, # 82.41% measured - 5%
"meta-llama/Llama-3.1-70B-Instruct": 0.90, # 94.77% measured - 5%
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.39, # 43.52% measured - 5%
"Qwen/Qwen2-57B-A14B-Instruct": 0.46, # 50.87% measured - 5%
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.77, # 82.34% measured - 5%
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.23, # 27.82% measured - 5%
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.80, # 84.91% measured - 5%
"zai-org/GLM-4.5-Air-FP8": 0.73, # 77.48% measured - 5%
"neuralmagic/gemma-2-2b-it-FP8": 0.02, # 6.52% measured - 5%
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.01% measured - 5%
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.35, # 40.33% measured - 5%
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.83, # 87.64% measured - 5%
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.40, # 44.66% measured - 5%
}
@@ -91,6 +91,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
base_url=self.base_url,
model=model_setup.model_path,
eval_name="gsm8k",
api="sgl_eval",
num_examples=None,
num_threads=1024,
)
test/registered/unit/bench/test_mixed_prefix_gsm8k.py
+4 -2
View File
@@ -5,8 +5,10 @@ import unittest
from typing import List, Tuple
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.simple_eval_gsm8k import get_one_example
from sglang.test.simple_eval_mixed_prefix_gsm8k import MixedPrefixGSM8KEval
from sglang.test.simple_eval_mixed_prefix_gsm8k import (
MixedPrefixGSM8KEval,
get_one_example,
)
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-c-test-cpu")
test/registered/unit/bench/test_simple_eval_gsm8k.py
+172
View File
@@ -0,0 +1,172 @@
import json
import subprocess
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.run_eval import _run_sgl_eval
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-b-test-cpu")
def _write_fake_metrics(out_parent: Path, eval_name: str, payload: dict) -> None:
run_dir = out_parent / f"sgl_eval_{eval_name}_20260101-000000"
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / "metrics.json").write_text(json.dumps(payload))
class TestRunSglEval(CustomTestCase):
"""sgl-eval is a black box, so these mock subprocess.run and assert the shim
builds the CLI and parses metrics.json's aggregate.score (not top-level)."""
def _args(self, out_dir: str, **overrides):
defaults = dict(
base_url="http://127.0.0.1:30000",
model="test-model",
num_examples=7,
num_threads=8,
temperature=0.0,
sgl_eval_out_dir=out_dir,
)
defaults.update(overrides)
return SimpleNamespace(**defaults)
def _fake_run_factory(self, out_dir: Path, eval_name: str, payload: dict):
def fake_run(cmd, **kwargs):
_write_fake_metrics(out_dir, eval_name, payload)
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
return fake_run
def test_parses_aggregate_score_and_maps_latency(self):
payload = {
"name": "gsm8k",
"model": "test-model",
"num_examples": 7,
"n_repeats": 1,
"latency_seconds": 12.5,
"output_throughput_tps": 34.0,
"aggregate": {"score": 0.75, "no_answer": 0.1},
}
with tempfile.TemporaryDirectory() as td:
out_dir = Path(td)
args = self._args(td)
with patch(
"sglang.test.run_eval.subprocess.run",
side_effect=self._fake_run_factory(out_dir, "gsm8k", payload),
):
metrics = _run_sgl_eval("gsm8k", args)
self.assertAlmostEqual(metrics["score"], 0.75)
self.assertAlmostEqual(metrics["latency"], 12.5)
self.assertAlmostEqual(metrics["output_throughput"], 34.0)
self.assertEqual(metrics["no_answer"], 0.1)
self.assertTrue(metrics["sgl_eval_metrics_path"].endswith("metrics.json"))
def test_builds_cli_with_required_flags(self):
captured = {}
def fake_run(cmd, **kwargs):
captured["cmd"] = cmd
out_dir = Path(captured["cmd"][captured["cmd"].index("--out-dir") + 1])
_write_fake_metrics(
out_dir,
"gsm8k",
{
"name": "gsm8k",
"model": "test-model",
"latency_seconds": 1.0,
"output_throughput_tps": 1.0,
"aggregate": {"score": 0.5},
},
)
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
with tempfile.TemporaryDirectory() as td:
args = self._args(td)
with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
_run_sgl_eval("gsm8k", args)
cmd = captured["cmd"]
self.assertEqual(cmd[0:3], ["sgl-eval", "run", "gsm8k"])
self.assertIn("--base-url", cmd)
self.assertIn("http://127.0.0.1:30000/v1", cmd)
self.assertIn("--model", cmd)
self.assertIn("test-model", cmd)
self.assertIn("--num-threads", cmd)
self.assertIn("8", cmd)
self.assertIn("--temperature", cmd)
self.assertIn("0.0", cmd)
self.assertIn("--num-examples", cmd)
self.assertIn("7", cmd)
def test_omits_num_examples_when_none(self):
captured = {}
def fake_run(cmd, **kwargs):
captured["cmd"] = cmd
out_dir = Path(cmd[cmd.index("--out-dir") + 1])
_write_fake_metrics(
out_dir,
"gsm8k",
{
"model": "test-model",
"latency_seconds": 1.0,
"output_throughput_tps": 1.0,
"aggregate": {"score": 0.5},
},
)
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
with tempfile.TemporaryDirectory() as td:
args = self._args(td, num_examples=None)
with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
_run_sgl_eval("gsm8k", args)
self.assertNotIn("--num-examples", captured["cmd"])
def test_raises_on_nonzero_exit(self):
def fake_run(cmd, **kwargs):
return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom")
with tempfile.TemporaryDirectory() as td:
args = self._args(td)
with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
with self.assertRaises(RuntimeError) as cm:
_run_sgl_eval("gsm8k", args)
self.assertIn("exit code 2", str(cm.exception))
def test_raises_when_metrics_json_missing(self):
def fake_run(cmd, **kwargs):
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
with tempfile.TemporaryDirectory() as td:
args = self._args(td)
with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run):
with self.assertRaises(FileNotFoundError):
_run_sgl_eval("gsm8k", args)
def test_raises_when_aggregate_score_missing(self):
payload = {
"name": "gsm8k",
"latency_seconds": 1.0,
"output_throughput_tps": 1.0,
"aggregate": {"no_answer": 0.5}, # no score key
}
with tempfile.TemporaryDirectory() as td:
out_dir = Path(td)
args = self._args(td)
with patch(
"sglang.test.run_eval.subprocess.run",
side_effect=self._fake_run_factory(out_dir, "gsm8k", payload),
):
with self.assertRaises(KeyError):
_run_sgl_eval("gsm8k", args)
if __name__ == "__main__":
unittest.main()