From b76dd0be699e13f28077eb6804cffc2654e084a9 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:08:48 +0800 Subject: [PATCH] Fix Mistral GSM8K chat eval (#27757) --- python/pyproject.toml | 6 + python/sglang/test/run_eval.py | 111 ++++++++++- python/sglang/test/simple_eval_common.py | 5 +- python/sglang/test/simple_eval_gsm8k.py | 108 ----------- .../test/simple_eval_mixed_prefix_gsm8k.py | 109 ++++++++++- python/sglang/test/test_utils.py | 4 +- scripts/ci/cuda/ci_install_dependency.sh | 4 + .../eval/test_text_models_gsm8k_eval.py | 35 ++-- .../unit/bench/test_mixed_prefix_gsm8k.py | 6 +- .../unit/bench/test_simple_eval_gsm8k.py | 172 ++++++++++++++++++ 10 files changed, 429 insertions(+), 131 deletions(-) delete mode 100644 python/sglang/test/simple_eval_gsm8k.py create mode 100644 test/registered/unit/bench/test_simple_eval_gsm8k.py diff --git a/python/pyproject.toml b/python/pyproject.toml index d09352abe..0b38dd679 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -148,6 +148,12 @@ test = [ "jsonlines", "lm-eval[api]>=0.4.9.2", "matplotlib", + # Pin sgl-eval to a git SHA: upgrading changes zero-shot \boxed{} grading, so + # re-baseline MODEL_SCORE_THRESHOLDS in test_text_models_gsm8k_eval.py first. + # antlr4 4.9.3 is forced because latex2sympy2_extended raises ImportError on + # 4.7.x, and an older transitive pin can win during install. + "antlr4-python3-runtime==4.9.3", + "sgl-eval @ git+https://github.com/sgl-project/sgl-eval.git@b2a2703c42cae379bbcb8b7ff092df6601a61694", "pandas", "parameterized", "peft>=0.18.0", diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index e5dcffeb8..3ea3db287 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -7,7 +7,10 @@ import argparse import json import os import statistics +import subprocess import time +import uuid +from pathlib import Path from sglang.test.simple_eval_common import ( ChatCompletionSampler, @@ -94,6 +97,105 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict: return result, latency, sampler +def _run_sgl_eval(eval_name, args) -> dict: + # Returns a metrics dict (score, latency, output_throughput) so the + # existing write_results_to_json + threshold gate keep working. + from sglang.test.test_utils import dump_metric + + base_url = ( + f"{args.base_url}/v1" if args.base_url else f"http://{args.host}:{args.port}/v1" + ) + out_parent = Path( + getattr(args, "sgl_eval_out_dir", None) + or (Path.home() / ".sgl_eval" / "sglang_run_eval" / uuid.uuid4().hex) + ).expanduser() + out_parent.mkdir(parents=True, exist_ok=True) + + cmd = [ + "sgl-eval", + "run", + eval_name, + "--base-url", + base_url, + "--num-threads", + str(getattr(args, "num_threads", 64)), + "--temperature", + str(getattr(args, "temperature", 0.0)), + "--out-dir", + str(out_parent), + ] + if getattr(args, "model", None): + cmd += ["--model", args.model] + if getattr(args, "num_examples", None) is not None: + cmd += ["--num-examples", str(args.num_examples)] + # Bound generation length so long-reasoning models don't stall the eval. + if getattr(args, "max_tokens", None) is not None: + cmd += ["--max-tokens", str(args.max_tokens)] + else: + cmd += ["--max-tokens", "2048"] + # Reasoning models (e.g. Qwen3.5) put their answer in the reasoning channel; + # without --thinking their message.content is empty and sgl-eval scores 0. + if getattr(args, "sgl_eval_thinking", None) is None: + model_l = (getattr(args, "model", None) or "").lower() + if "qwen3.5" in model_l or "qwen3-thinking" in model_l: + cmd += ["--thinking"] + elif args.sgl_eval_thinking: + cmd += ["--thinking"] + + try: + completed = subprocess.run( + cmd, + text=True, + capture_output=True, + check=False, + timeout=getattr(args, "sgl_eval_timeout", None), + ) + except subprocess.TimeoutExpired as e: + raise TimeoutError( + f"sgl-eval timed out after {e.timeout}s: {' '.join(cmd)}\n" + f"stdout:\n{e.stdout or ''}\nstderr:\n{e.stderr or ''}" + ) from e + + if completed.returncode != 0: + raise RuntimeError( + f"sgl-eval failed with exit code {completed.returncode}: " + f"{' '.join(cmd)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + + metrics_files = sorted(out_parent.glob(f"sgl_eval_{eval_name}_*/metrics.json")) + if len(metrics_files) != 1: + raise FileNotFoundError( + f"Expected exactly one metrics.json under {out_parent}, " + f"found {len(metrics_files)}" + ) + payload = json.loads(metrics_files[0].read_text()) + aggregate = payload.get("aggregate") + if not isinstance(aggregate, dict) or "score" not in aggregate: + raise KeyError(f"{metrics_files[0]} missing aggregate.score") + + metrics = dict(aggregate) + metrics["latency"] = payload.get("latency_seconds", 0.0) + metrics["output_throughput"] = payload.get("output_throughput_tps", 0.0) + metrics["sgl_eval_metrics_path"] = str(metrics_files[0]) + + model = payload.get("model") or getattr(args, "model", None) + dump_metric( + f"{eval_name}_score", + metrics["score"], + labels={"model": model, "eval": eval_name}, + ) + dump_metric( + f"{eval_name}_latency", + metrics["latency"], + labels={"model": model, "eval": eval_name}, + ) + print(f"Score: {metrics['score']:.3f}") + print(f"Total latency: {metrics['latency']:.3f} s") + print(f"Output throughput: {metrics['output_throughput']:.3f} token/s") + print(f"sgl-eval metrics: {metrics_files[0]}") + return metrics + + def print_accept_length_summary(samplers: list) -> None: accept_lengths = [ m["spec_accept_length"] @@ -196,7 +298,14 @@ def run_eval(args): eval_obj = AIME25Eval(args.num_examples, args.num_threads) elif args.eval_name == "gsm8k": - from sglang.test.simple_eval_gsm8k import GSM8KEval + if getattr(args, "api", None) == "sgl_eval": + # Only the nightly correctness eval opts into sgl-eval (zero-shot + # chat, \boxed{}, math_verify). Every other gsm8k caller — spec + # decoding perf/accuracy, disaggregation, quant, model e2e — uses + # the 5-shot completion last-number scorer and relies on + # max_tokens/throughput behavior sgl-eval cannot provide. + return _run_sgl_eval("gsm8k", args) + from sglang.test.simple_eval_mixed_prefix_gsm8k import GSM8KEval eval_obj = GSM8KEval( num_examples=args.num_examples, diff --git a/python/sglang/test/simple_eval_common.py b/python/sglang/test/simple_eval_common.py index e010e1194..235916554 100644 --- a/python/sglang/test/simple_eval_common.py +++ b/python/sglang/test/simple_eval_common.py @@ -95,6 +95,7 @@ class ChatCompletionSampler(SamplerBase): reasoning_effort: Optional[str] = None, max_tokens: int = 2048, extra_body: Optional[Dict[str, Any]] = None, + stop: Optional[List[str]] = None, record_meta_info: bool = False, ): self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient()) @@ -109,12 +110,13 @@ class ChatCompletionSampler(SamplerBase): self.max_tokens = max_tokens self.reasoning_effort = reasoning_effort self.extra_body = extra_body + self.stop = stop self.image_format = "url" self._completion_tokens: list[int] = [] self.record_meta_info = record_meta_info self._meta_infos: List[Dict[str, Any]] = [] print( - f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=} {self.record_meta_info=}" + f"ChatCompletionSampler initialized with {self.system_message=} {self.temperature=} {self.max_tokens=} {self.reasoning_effort=} {self.extra_body=} {self.stop=} {self.record_meta_info=}" ) def _handle_image( @@ -157,6 +159,7 @@ class ChatCompletionSampler(SamplerBase): max_tokens=self.max_tokens, reasoning_effort=self.reasoning_effort, extra_body=extra_body, + stop=self.stop, ) if self.record_meta_info: meta_info = getattr(response.choices[0], "meta_info", None) diff --git a/python/sglang/test/simple_eval_gsm8k.py b/python/sglang/test/simple_eval_gsm8k.py deleted file mode 100644 index 9408d7006..000000000 --- a/python/sglang/test/simple_eval_gsm8k.py +++ /dev/null @@ -1,108 +0,0 @@ -# Adapted from https://github.com/openai/simple-evals/ - -import ast -import re -from typing import Optional - -from sglang.test import simple_eval_common as common -from sglang.test.simple_eval_common import ( - HTML_JINJA, - Eval, - EvalResult, - SamplerBase, - SingleEvalResult, -) -from sglang.utils import download_and_cache_file, read_jsonl - -GSM8K_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" -INVALID = -9999999 - - -def get_one_example(lines, i, include_answer): - ret = f"Question: {lines[i]['question']}\nAnswer:" - if include_answer: - ret += f" {lines[i]['answer']}" - return ret - - -def get_few_shot_examples(lines, k): - return "".join(get_one_example(lines, i, True) + "\n\n" for i in range(k)) - - -def get_answer_value(answer_str): - answer_str = answer_str.replace(",", "") - numbers = re.findall(r"-?\d+\.?\d*", answer_str) - if len(numbers) < 1: - return INVALID - try: - return ast.literal_eval(numbers[-1]) - except (SyntaxError, ValueError): - return INVALID - - -class GSM8KEval(Eval): - def __init__( - self, - num_examples: Optional[int] = None, - num_threads: int = 64, - num_shots: int = 5, - data_path: Optional[str] = None, - ): - self._num_threads = num_threads - self._num_shots = num_shots - - if data_path: - filename = data_path - else: - filename = download_and_cache_file(GSM8K_URL) - - all_lines = list(read_jsonl(filename)) - pool_size = self._setup_prefix_pool(all_lines, num_shots) - # The evaluation data should not include the few-shot examples to prevent data leakage. - self._lines = all_lines[pool_size:] - if num_examples is not None: - # Slice caps silently when num_examples exceeds the available lines, - # matching upstream: callers like test_basic_sanity_eagle3 pass a - # num_examples larger than the dataset on purpose. - self._lines = self._lines[:num_examples] - - def _setup_prefix_pool(self, all_lines: list, num_shots: int) -> int: - self._few_shot_prompt = get_few_shot_examples(all_lines, num_shots) - return num_shots - - def _build_prefix(self, idx: int) -> str: - return self._few_shot_prompt - - def __call__(self, sampler: SamplerBase) -> EvalResult: - def fn(idx: int) -> SingleEvalResult: - question = get_one_example(self._lines, idx, include_answer=False) - correct_answer = get_answer_value(self._lines[idx]["answer"]) - - prompt_content = self._build_prefix(idx) + question - prompt_messages = [ - sampler._pack_message(content=prompt_content, role="user") - ] - - try: - response_text = sampler(prompt_messages) - except Exception: - response_text = "" - - extracted_answer = get_answer_value(response_text) - score = float(extracted_answer == correct_answer) - - html = common.jinja_env.from_string(HTML_JINJA).render( - prompt_messages=prompt_messages, - next_message=dict(content=response_text, role="assistant"), - score=score, - correct_answer=correct_answer, - extracted_answer=extracted_answer, - ) - convo = prompt_messages + [dict(content=response_text, role="assistant")] - - return SingleEvalResult(html=html, score=score, convo=convo) - - results = common.map_with_progress( - fn, list(range(len(self._lines))), num_threads=self._num_threads - ) - return common.aggregate_results(results, default_stats=("mean", "std")) diff --git a/python/sglang/test/simple_eval_mixed_prefix_gsm8k.py b/python/sglang/test/simple_eval_mixed_prefix_gsm8k.py index 1a62d2c61..34a0db672 100644 --- a/python/sglang/test/simple_eval_mixed_prefix_gsm8k.py +++ b/python/sglang/test/simple_eval_mixed_prefix_gsm8k.py @@ -1,7 +1,114 @@ +# Adapted from https://github.com/openai/simple-evals/ +# Hand-rolled GSM8K scorer kept only for the mixed-prefix KV-correctness test, +# which needs randomized completion-style prefixes sgl-eval does not produce. + +import ast import random +import re from typing import Optional -from sglang.test.simple_eval_gsm8k import GSM8KEval, get_one_example +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) +from sglang.utils import download_and_cache_file, read_jsonl + +GSM8K_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = f"Question: {lines[i]['question']}\nAnswer:" + if include_answer: + ret += f" {lines[i]['answer']}" + return ret + + +def get_few_shot_examples(lines, k): + return "".join(get_one_example(lines, i, True) + "\n\n" for i in range(k)) + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"-?\d+\.?\d*", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except (SyntaxError, ValueError): + return INVALID + + +class GSM8KEval(Eval): + def __init__( + self, + num_examples: Optional[int] = None, + num_threads: int = 64, + num_shots: int = 5, + data_path: Optional[str] = None, + ): + self._num_threads = num_threads + self._num_shots = num_shots + + if data_path: + filename = data_path + else: + filename = download_and_cache_file(GSM8K_URL) + + all_lines = list(read_jsonl(filename)) + pool_size = self._setup_prefix_pool(all_lines, num_shots) + # The evaluation data should not include the few-shot examples to prevent data leakage. + self._lines = all_lines[pool_size:] + if num_examples is not None: + # Slice caps silently when num_examples exceeds the available lines, + # matching upstream: callers like test_basic_sanity_eagle3 pass a + # num_examples larger than the dataset on purpose. + self._lines = self._lines[:num_examples] + + def _setup_prefix_pool(self, all_lines: list, num_shots: int) -> int: + self._few_shot_prompt = get_few_shot_examples(all_lines, num_shots) + return num_shots + + def _build_prefix(self, idx: int) -> str: + return self._few_shot_prompt + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(idx: int) -> SingleEvalResult: + question = get_one_example(self._lines, idx, include_answer=False) + correct_answer = get_answer_value(self._lines[idx]["answer"]) + + prompt_content = self._build_prefix(idx) + question + prompt_messages = [ + sampler._pack_message(content=prompt_content, role="user") + ] + + try: + response_text = sampler(prompt_messages) + except Exception: + response_text = "" + + extracted_answer = get_answer_value(response_text) + score = float(extracted_answer == correct_answer) + + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + + return SingleEvalResult(html=html, score=score, convo=convo) + + results = common.map_with_progress( + fn, list(range(len(self._lines))), num_threads=self._num_threads + ) + return common.aggregate_results(results, default_stats=("mean", "std")) class MixedPrefixGSM8KEval(GSM8KEval): diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index dc5262332..b8864b88d 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -150,7 +150,9 @@ DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8" DEFAULT_ENABLE_ROUTED_EXPERTS_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B" # Nightly tests -DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = "meta-llama/Llama-3.1-8B-Instruct,mistralai/Mistral-7B-Instruct-v0.3,deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct,google/gemma-2-27b-it" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = ( + "meta-llama/Llama-3.1-8B-Instruct,Qwen/Qwen3-8B,Qwen/Qwen3-4B" +) DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct" DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8" DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8" diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index 666335820..9bd9969e3 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -514,6 +514,10 @@ install_extra_deps() { if [ "$IS_BLACKWELL" != "1" ]; then git clone --branch v0.5 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX + # lmms-eval v0.5 pulls antlr4-python3-runtime==4.7.2, clobbering the + # 4.9.3 that sgl-eval's latex2sympy2_extended needs (4.7.2 ImportError + # at sgl-eval import). Pin it back so the nightly sgl-eval path works. + $PIP_CMD install "antlr4-python3-runtime==4.9.3" --force-reinstall --no-deps $PIP_INSTALL_SUFFIX fi $PIP_CMD uninstall xformers || true diff --git a/test/registered/eval/test_text_models_gsm8k_eval.py b/test/registered/eval/test_text_models_gsm8k_eval.py index 275430a01..cccba0337 100644 --- a/test/registered/eval/test_text_models_gsm8k_eval.py +++ b/test/registered/eval/test_text_models_gsm8k_eval.py @@ -26,23 +26,23 @@ NIGHTLY_EVAL_SERVER_TIMEOUT = 1800 register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True) MODEL_SCORE_THRESHOLDS = { - # Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores - "meta-llama/Llama-3.1-8B-Instruct": 0.80, # 84.5% - 5% - "mistralai/Mistral-7B-Instruct-v0.3": 0.47, # 52.1% - 5% - "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81, # 86.4% - 5% - "google/gemma-2-27b-it": 0.81, # 85.5% measured - 5% - "meta-llama/Llama-3.1-70B-Instruct": 0.89, # 94.1% - 5% - "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69, # 74.4% - 5% - "Qwen/Qwen2-57B-A14B-Instruct": 0.76, # 80.7% - 5% (official A14B score; 88.2% was the 72B) - "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80, # 84.5% - 5% - "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.47, # 52.1% - 5% - "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81, # 86.4% - 5% - "zai-org/GLM-4.5-Air-FP8": 0.80, # ~85% - 5% - "neuralmagic/gemma-2-2b-it-FP8": 0.53, # 58.4% measured - 5% - "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.1% - 5% - "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69, # 74.4% - 5% - "neuralmagic/Qwen2-72B-Instruct-FP8": 0.86, # 91.1% - 5% - "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76, # 80.7% - 5% (official A14B score) + # sgl-eval (zero-shot chat, \boxed{}, math_verify grading). Thresholds are + # measured_score - 0.05, baselined on H100 2-GPU over the full 1319 split. + "meta-llama/Llama-3.1-8B-Instruct": 0.77, # 81.05% measured - 5% + "Qwen/Qwen3-8B": 0.76, # 81.43% measured - 5% + "Qwen/Qwen3-4B": 0.77, # 82.41% measured - 5% + "meta-llama/Llama-3.1-70B-Instruct": 0.90, # 94.77% measured - 5% + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.39, # 43.52% measured - 5% + "Qwen/Qwen2-57B-A14B-Instruct": 0.46, # 50.87% measured - 5% + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.77, # 82.34% measured - 5% + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.23, # 27.82% measured - 5% + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.80, # 84.91% measured - 5% + "zai-org/GLM-4.5-Air-FP8": 0.73, # 77.48% measured - 5% + "neuralmagic/gemma-2-2b-it-FP8": 0.02, # 6.52% measured - 5% + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.01% measured - 5% + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.35, # 40.33% measured - 5% + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.83, # 87.64% measured - 5% + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.40, # 44.66% measured - 5% } @@ -91,6 +91,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): base_url=self.base_url, model=model_setup.model_path, eval_name="gsm8k", + api="sgl_eval", num_examples=None, num_threads=1024, ) diff --git a/test/registered/unit/bench/test_mixed_prefix_gsm8k.py b/test/registered/unit/bench/test_mixed_prefix_gsm8k.py index 4338f13ed..4d7626fd6 100644 --- a/test/registered/unit/bench/test_mixed_prefix_gsm8k.py +++ b/test/registered/unit/bench/test_mixed_prefix_gsm8k.py @@ -5,8 +5,10 @@ import unittest from typing import List, Tuple from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.simple_eval_gsm8k import get_one_example -from sglang.test.simple_eval_mixed_prefix_gsm8k import MixedPrefixGSM8KEval +from sglang.test.simple_eval_mixed_prefix_gsm8k import ( + MixedPrefixGSM8KEval, + get_one_example, +) from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-c-test-cpu") diff --git a/test/registered/unit/bench/test_simple_eval_gsm8k.py b/test/registered/unit/bench/test_simple_eval_gsm8k.py new file mode 100644 index 000000000..7d80e28c4 --- /dev/null +++ b/test/registered/unit/bench/test_simple_eval_gsm8k.py @@ -0,0 +1,172 @@ +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.run_eval import _run_sgl_eval +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-b-test-cpu") + + +def _write_fake_metrics(out_parent: Path, eval_name: str, payload: dict) -> None: + run_dir = out_parent / f"sgl_eval_{eval_name}_20260101-000000" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "metrics.json").write_text(json.dumps(payload)) + + +class TestRunSglEval(CustomTestCase): + """sgl-eval is a black box, so these mock subprocess.run and assert the shim + builds the CLI and parses metrics.json's aggregate.score (not top-level).""" + + def _args(self, out_dir: str, **overrides): + defaults = dict( + base_url="http://127.0.0.1:30000", + model="test-model", + num_examples=7, + num_threads=8, + temperature=0.0, + sgl_eval_out_dir=out_dir, + ) + defaults.update(overrides) + return SimpleNamespace(**defaults) + + def _fake_run_factory(self, out_dir: Path, eval_name: str, payload: dict): + def fake_run(cmd, **kwargs): + _write_fake_metrics(out_dir, eval_name, payload) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + return fake_run + + def test_parses_aggregate_score_and_maps_latency(self): + payload = { + "name": "gsm8k", + "model": "test-model", + "num_examples": 7, + "n_repeats": 1, + "latency_seconds": 12.5, + "output_throughput_tps": 34.0, + "aggregate": {"score": 0.75, "no_answer": 0.1}, + } + with tempfile.TemporaryDirectory() as td: + out_dir = Path(td) + args = self._args(td) + with patch( + "sglang.test.run_eval.subprocess.run", + side_effect=self._fake_run_factory(out_dir, "gsm8k", payload), + ): + metrics = _run_sgl_eval("gsm8k", args) + + self.assertAlmostEqual(metrics["score"], 0.75) + self.assertAlmostEqual(metrics["latency"], 12.5) + self.assertAlmostEqual(metrics["output_throughput"], 34.0) + self.assertEqual(metrics["no_answer"], 0.1) + self.assertTrue(metrics["sgl_eval_metrics_path"].endswith("metrics.json")) + + def test_builds_cli_with_required_flags(self): + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + out_dir = Path(captured["cmd"][captured["cmd"].index("--out-dir") + 1]) + _write_fake_metrics( + out_dir, + "gsm8k", + { + "name": "gsm8k", + "model": "test-model", + "latency_seconds": 1.0, + "output_throughput_tps": 1.0, + "aggregate": {"score": 0.5}, + }, + ) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + with tempfile.TemporaryDirectory() as td: + args = self._args(td) + with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): + _run_sgl_eval("gsm8k", args) + + cmd = captured["cmd"] + self.assertEqual(cmd[0:3], ["sgl-eval", "run", "gsm8k"]) + self.assertIn("--base-url", cmd) + self.assertIn("http://127.0.0.1:30000/v1", cmd) + self.assertIn("--model", cmd) + self.assertIn("test-model", cmd) + self.assertIn("--num-threads", cmd) + self.assertIn("8", cmd) + self.assertIn("--temperature", cmd) + self.assertIn("0.0", cmd) + self.assertIn("--num-examples", cmd) + self.assertIn("7", cmd) + + def test_omits_num_examples_when_none(self): + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + out_dir = Path(cmd[cmd.index("--out-dir") + 1]) + _write_fake_metrics( + out_dir, + "gsm8k", + { + "model": "test-model", + "latency_seconds": 1.0, + "output_throughput_tps": 1.0, + "aggregate": {"score": 0.5}, + }, + ) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + with tempfile.TemporaryDirectory() as td: + args = self._args(td, num_examples=None) + with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): + _run_sgl_eval("gsm8k", args) + + self.assertNotIn("--num-examples", captured["cmd"]) + + def test_raises_on_nonzero_exit(self): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 2, stdout="", stderr="boom") + + with tempfile.TemporaryDirectory() as td: + args = self._args(td) + with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): + with self.assertRaises(RuntimeError) as cm: + _run_sgl_eval("gsm8k", args) + self.assertIn("exit code 2", str(cm.exception)) + + def test_raises_when_metrics_json_missing(self): + def fake_run(cmd, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + with tempfile.TemporaryDirectory() as td: + args = self._args(td) + with patch("sglang.test.run_eval.subprocess.run", side_effect=fake_run): + with self.assertRaises(FileNotFoundError): + _run_sgl_eval("gsm8k", args) + + def test_raises_when_aggregate_score_missing(self): + payload = { + "name": "gsm8k", + "latency_seconds": 1.0, + "output_throughput_tps": 1.0, + "aggregate": {"no_answer": 0.5}, # no score key + } + with tempfile.TemporaryDirectory() as td: + out_dir = Path(td) + args = self._args(td) + with patch( + "sglang.test.run_eval.subprocess.run", + side_effect=self._fake_run_factory(out_dir, "gsm8k", payload), + ): + with self.assertRaises(KeyError): + _run_sgl_eval("gsm8k", args) + + +if __name__ == "__main__": + unittest.main()