[Test] Add in-process sgl-eval adapter and move validated GSM8K tests to it (#41216)

This commit is contained in:
Liangsheng Yin
2026-09-25 13:59:17 -07:00
committed by GitHub
parent f337f01b80
commit f88572a57c
58 changed files with 412 additions and 345 deletions
+5 -26
View File
@@ -12,6 +12,11 @@ import time
import uuid
from pathlib import Path
from sglang.test.sgl_eval_utils import (
THINKING_MODE_CHOICES,
get_thinking_kwargs,
parse_json_object,
)
from sglang.test.simple_eval_common import (
ChatCompletionSampler,
CompletionSampler,
@@ -22,30 +27,6 @@ from sglang.test.simple_eval_common import (
)
def get_thinking_kwargs(args):
thinking_mode = getattr(args, "thinking_mode", None)
if thinking_mode in THINKING_MODE_CHOICES:
if thinking_mode in ["deepseek-v3", "kimi-k2"]:
thinking_param = "thinking"
else:
# All models other than dpsk v3/kimi_k2
thinking_param = "enable_thinking"
return {thinking_param: True}
return {}
def parse_json_object(value: str) -> dict:
try:
parsed = json.loads(value)
except json.JSONDecodeError as e:
raise argparse.ArgumentTypeError("must be a valid JSON object string") from e
if not isinstance(parsed, dict):
raise argparse.ArgumentTypeError("must be a JSON object")
return parsed
def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
chat_template_kwargs = getattr(args, "chat_template_kwargs", None)
if isinstance(chat_template_kwargs, str):
@@ -426,8 +407,6 @@ def run_eval(args):
return metrics
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
+192
View File
@@ -0,0 +1,192 @@
"""SGLang CI transport and metric reporting for sgl-eval benchmarks."""
import argparse
import json
import multiprocessing
import os
import threading
import uuid
from concurrent.futures import ProcessPoolExecutor
from dataclasses import replace
from itertools import islice
from pathlib import Path
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
def get_thinking_kwargs(args):
thinking_mode = getattr(args, "thinking_mode", None)
if thinking_mode in THINKING_MODE_CHOICES:
if thinking_mode in ["deepseek-v3", "kimi-k2"]:
thinking_param = "thinking"
else:
# All models other than dpsk v3/kimi_k2
thinking_param = "enable_thinking"
return {thinking_param: True}
return {}
def parse_json_object(value: str) -> dict:
try:
parsed = json.loads(value)
except json.JSONDecodeError as e:
raise argparse.ArgumentTypeError("must be a valid JSON object string") from e
if not isinstance(parsed, dict):
raise argparse.ArgumentTypeError("must be a JSON object")
return parsed
def api_base_url(args):
base_url = getattr(args, "base_url", None)
if not base_url:
host = getattr(args, "host", "127.0.0.1").rstrip("/")
if "://" not in host:
host = f"http://{host}"
base_url = f"{host}:{getattr(args, 'port', None) or 30000}"
base_url = base_url.rstrip("/")
return base_url if base_url.endswith("/v1") else base_url + "/v1"
def _print_truncated_samples(result):
truncated = (
(example, repeat, sample)
for example in result.per_example
for repeat, sample in enumerate(example.samples)
if sample.finish_reason == "length"
)
for example, repeat, sample in islice(truncated, 3):
preview = {
"example_id": example.example.id,
"repeat": repeat,
"finish_reason": sample.finish_reason,
"completion_tokens": sample.completion_tokens,
"score": example.scores[repeat],
"text_chars": len(sample.text),
"head": sample.text[:500],
"tail": sample.text[-500:],
}
print(f"sgl-eval truncated sample: {json.dumps(preview)}", flush=True)
def run_sgl_eval(args):
# math_verify uses SIGALRM, which requires the process's main thread.
if threading.current_thread() is not threading.main_thread():
with ProcessPoolExecutor(
max_workers=1, mp_context=multiprocessing.get_context("spawn")
) as executor:
return executor.submit(run_sgl_eval, args).result()
from sgl_eval.predictions import PredictionsWriter
from sgl_eval.registry import get
from sgl_eval.sampler import ChatCompletionSampler
from sglang.test.test_utils import dump_metric
spec = get(args.eval_name)
overrides = {"max_tokens": 2048}
for key in (
"max_tokens",
"temperature",
"top_p",
"min_p",
"seed",
"reasoning_effort",
):
value = getattr(args, key, None)
if value is not None:
overrides[key] = value
chat_kwargs = getattr(args, "chat_template_kwargs", None) or {}
if isinstance(chat_kwargs, str):
chat_kwargs = parse_json_object(chat_kwargs)
chat_kwargs = {**get_thinking_kwargs(args), **chat_kwargs}
thinking = getattr(args, "sgl_eval_thinking", None)
if thinking is not None:
chat_kwargs.setdefault("thinking", thinking)
chat_kwargs.setdefault("enable_thinking", thinking)
if chat_kwargs:
overrides["chat_template_kwargs"] = {
**(spec.default_gen.chat_template_kwargs or {}),
**chat_kwargs,
}
if getattr(args, "top_k", None) is not None:
overrides["extra_body"] = {"top_k": args.top_k}
gen = replace(spec.default_gen, **overrides)
generation = {
key: getattr(gen, key)
for key in ("max_tokens", "temperature", "top_p", "min_p", "seed")
} | {"chat_template_kwargs": gen.chat_template_kwargs, **(gen.extra_body or {})}
print(f"sgl-eval {spec.name} generation: {generation}", flush=True)
sampler = ChatCompletionSampler(
base_url=api_base_url(args),
model=getattr(args, "model", None),
api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"),
)
out_parent = Path(
getattr(args, "sgl_eval_out_dir", None)
or Path.home() / ".sgl_eval" / "sglang_run_eval"
).expanduser()
out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}"
out_dir.mkdir(parents=True, exist_ok=True)
repeats = getattr(args, "repeat", 1)
writer = PredictionsWriter(out_dir, repeats, spec.pred_schema)
try:
result = spec.run(
sampler=sampler,
gen=gen,
n_repeats=repeats,
num_examples=getattr(args, "num_examples", None),
num_threads=getattr(args, "num_threads", None) or spec.default_num_threads,
predictions_writer=writer,
load_examples=None,
)
finally:
writer.close()
sampler.abort()
if result.partial:
raise RuntimeError(f"Incomplete {spec.name} evaluation; results: {out_dir}")
extracted = [
answer for example in result.per_example for answer in example.extracted
]
if not extracted:
raise RuntimeError(f"Empty {spec.name} evaluation; results: {out_dir}")
metrics = {
**result.aggregate,
"accuracy": result.aggregate["score"],
"invalid": sum(answer is None for answer in extracted) / len(extracted),
"latency": result.latency,
"output_throughput": result.output_throughput,
"sgl_eval_metrics_path": str(out_dir / "metrics.json"),
}
(out_dir / "metrics.json").write_text(
json.dumps(
{
"name": spec.name,
"model": sampler.model,
"generation": generation,
"aggregate": result.aggregate,
"num_examples": result.num_examples,
"n_repeats": result.n_repeats,
"latency_seconds": result.latency,
"output_throughput_tps": result.output_throughput,
},
indent=2,
)
)
_print_truncated_samples(result)
if result.aggregate.get("error_rate", 0) > 0:
raise RuntimeError(
f"{spec.name} evaluation had request errors "
f"(error_rate={result.aggregate['error_rate']}); results: {out_dir}"
)
for name, value in (("score", metrics["score"]), ("latency", metrics["latency"])):
dump_metric(
f"{spec.name}_{name}",
value,
labels={"model": sampler.model, "eval": spec.name},
)
print(f"sgl-eval {spec.name}: {metrics}")
return (
(metrics, result.latency) if getattr(args, "return_latency", False) else metrics
)