[Test] Add in-process sgl-eval adapter and move validated GSM8K tests to it (#41216)
This commit is contained in:
@@ -12,6 +12,11 @@ import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from sglang.test.sgl_eval_utils import (
|
||||
THINKING_MODE_CHOICES,
|
||||
get_thinking_kwargs,
|
||||
parse_json_object,
|
||||
)
|
||||
from sglang.test.simple_eval_common import (
|
||||
ChatCompletionSampler,
|
||||
CompletionSampler,
|
||||
@@ -22,30 +27,6 @@ from sglang.test.simple_eval_common import (
|
||||
)
|
||||
|
||||
|
||||
def get_thinking_kwargs(args):
|
||||
thinking_mode = getattr(args, "thinking_mode", None)
|
||||
if thinking_mode in THINKING_MODE_CHOICES:
|
||||
if thinking_mode in ["deepseek-v3", "kimi-k2"]:
|
||||
thinking_param = "thinking"
|
||||
else:
|
||||
# All models other than dpsk v3/kimi_k2
|
||||
thinking_param = "enable_thinking"
|
||||
return {thinking_param: True}
|
||||
return {}
|
||||
|
||||
|
||||
def parse_json_object(value: str) -> dict:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError("must be a valid JSON object string") from e
|
||||
|
||||
if not isinstance(parsed, dict):
|
||||
raise argparse.ArgumentTypeError("must be a JSON object")
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
|
||||
chat_template_kwargs = getattr(args, "chat_template_kwargs", None)
|
||||
if isinstance(chat_template_kwargs, str):
|
||||
@@ -426,8 +407,6 @@ def run_eval(args):
|
||||
return metrics
|
||||
|
||||
|
||||
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
"""SGLang CI transport and metric reporting for sgl-eval benchmarks."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import replace
|
||||
from itertools import islice
|
||||
from pathlib import Path
|
||||
|
||||
THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"]
|
||||
|
||||
|
||||
def get_thinking_kwargs(args):
|
||||
thinking_mode = getattr(args, "thinking_mode", None)
|
||||
if thinking_mode in THINKING_MODE_CHOICES:
|
||||
if thinking_mode in ["deepseek-v3", "kimi-k2"]:
|
||||
thinking_param = "thinking"
|
||||
else:
|
||||
# All models other than dpsk v3/kimi_k2
|
||||
thinking_param = "enable_thinking"
|
||||
return {thinking_param: True}
|
||||
return {}
|
||||
|
||||
|
||||
def parse_json_object(value: str) -> dict:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError("must be a valid JSON object string") from e
|
||||
|
||||
if not isinstance(parsed, dict):
|
||||
raise argparse.ArgumentTypeError("must be a JSON object")
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def api_base_url(args):
|
||||
base_url = getattr(args, "base_url", None)
|
||||
if not base_url:
|
||||
host = getattr(args, "host", "127.0.0.1").rstrip("/")
|
||||
if "://" not in host:
|
||||
host = f"http://{host}"
|
||||
base_url = f"{host}:{getattr(args, 'port', None) or 30000}"
|
||||
base_url = base_url.rstrip("/")
|
||||
return base_url if base_url.endswith("/v1") else base_url + "/v1"
|
||||
|
||||
|
||||
def _print_truncated_samples(result):
|
||||
truncated = (
|
||||
(example, repeat, sample)
|
||||
for example in result.per_example
|
||||
for repeat, sample in enumerate(example.samples)
|
||||
if sample.finish_reason == "length"
|
||||
)
|
||||
for example, repeat, sample in islice(truncated, 3):
|
||||
preview = {
|
||||
"example_id": example.example.id,
|
||||
"repeat": repeat,
|
||||
"finish_reason": sample.finish_reason,
|
||||
"completion_tokens": sample.completion_tokens,
|
||||
"score": example.scores[repeat],
|
||||
"text_chars": len(sample.text),
|
||||
"head": sample.text[:500],
|
||||
"tail": sample.text[-500:],
|
||||
}
|
||||
print(f"sgl-eval truncated sample: {json.dumps(preview)}", flush=True)
|
||||
|
||||
|
||||
def run_sgl_eval(args):
|
||||
# math_verify uses SIGALRM, which requires the process's main thread.
|
||||
if threading.current_thread() is not threading.main_thread():
|
||||
with ProcessPoolExecutor(
|
||||
max_workers=1, mp_context=multiprocessing.get_context("spawn")
|
||||
) as executor:
|
||||
return executor.submit(run_sgl_eval, args).result()
|
||||
|
||||
from sgl_eval.predictions import PredictionsWriter
|
||||
from sgl_eval.registry import get
|
||||
from sgl_eval.sampler import ChatCompletionSampler
|
||||
|
||||
from sglang.test.test_utils import dump_metric
|
||||
|
||||
spec = get(args.eval_name)
|
||||
overrides = {"max_tokens": 2048}
|
||||
for key in (
|
||||
"max_tokens",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"min_p",
|
||||
"seed",
|
||||
"reasoning_effort",
|
||||
):
|
||||
value = getattr(args, key, None)
|
||||
if value is not None:
|
||||
overrides[key] = value
|
||||
chat_kwargs = getattr(args, "chat_template_kwargs", None) or {}
|
||||
if isinstance(chat_kwargs, str):
|
||||
chat_kwargs = parse_json_object(chat_kwargs)
|
||||
chat_kwargs = {**get_thinking_kwargs(args), **chat_kwargs}
|
||||
thinking = getattr(args, "sgl_eval_thinking", None)
|
||||
if thinking is not None:
|
||||
chat_kwargs.setdefault("thinking", thinking)
|
||||
chat_kwargs.setdefault("enable_thinking", thinking)
|
||||
if chat_kwargs:
|
||||
overrides["chat_template_kwargs"] = {
|
||||
**(spec.default_gen.chat_template_kwargs or {}),
|
||||
**chat_kwargs,
|
||||
}
|
||||
if getattr(args, "top_k", None) is not None:
|
||||
overrides["extra_body"] = {"top_k": args.top_k}
|
||||
gen = replace(spec.default_gen, **overrides)
|
||||
generation = {
|
||||
key: getattr(gen, key)
|
||||
for key in ("max_tokens", "temperature", "top_p", "min_p", "seed")
|
||||
} | {"chat_template_kwargs": gen.chat_template_kwargs, **(gen.extra_body or {})}
|
||||
print(f"sgl-eval {spec.name} generation: {generation}", flush=True)
|
||||
sampler = ChatCompletionSampler(
|
||||
base_url=api_base_url(args),
|
||||
model=getattr(args, "model", None),
|
||||
api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"),
|
||||
)
|
||||
out_parent = Path(
|
||||
getattr(args, "sgl_eval_out_dir", None)
|
||||
or Path.home() / ".sgl_eval" / "sglang_run_eval"
|
||||
).expanduser()
|
||||
out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
repeats = getattr(args, "repeat", 1)
|
||||
writer = PredictionsWriter(out_dir, repeats, spec.pred_schema)
|
||||
try:
|
||||
result = spec.run(
|
||||
sampler=sampler,
|
||||
gen=gen,
|
||||
n_repeats=repeats,
|
||||
num_examples=getattr(args, "num_examples", None),
|
||||
num_threads=getattr(args, "num_threads", None) or spec.default_num_threads,
|
||||
predictions_writer=writer,
|
||||
load_examples=None,
|
||||
)
|
||||
finally:
|
||||
writer.close()
|
||||
sampler.abort()
|
||||
if result.partial:
|
||||
raise RuntimeError(f"Incomplete {spec.name} evaluation; results: {out_dir}")
|
||||
extracted = [
|
||||
answer for example in result.per_example for answer in example.extracted
|
||||
]
|
||||
if not extracted:
|
||||
raise RuntimeError(f"Empty {spec.name} evaluation; results: {out_dir}")
|
||||
metrics = {
|
||||
**result.aggregate,
|
||||
"accuracy": result.aggregate["score"],
|
||||
"invalid": sum(answer is None for answer in extracted) / len(extracted),
|
||||
"latency": result.latency,
|
||||
"output_throughput": result.output_throughput,
|
||||
"sgl_eval_metrics_path": str(out_dir / "metrics.json"),
|
||||
}
|
||||
(out_dir / "metrics.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"name": spec.name,
|
||||
"model": sampler.model,
|
||||
"generation": generation,
|
||||
"aggregate": result.aggregate,
|
||||
"num_examples": result.num_examples,
|
||||
"n_repeats": result.n_repeats,
|
||||
"latency_seconds": result.latency,
|
||||
"output_throughput_tps": result.output_throughput,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
_print_truncated_samples(result)
|
||||
if result.aggregate.get("error_rate", 0) > 0:
|
||||
raise RuntimeError(
|
||||
f"{spec.name} evaluation had request errors "
|
||||
f"(error_rate={result.aggregate['error_rate']}); results: {out_dir}"
|
||||
)
|
||||
for name, value in (("score", metrics["score"]), ("latency", metrics["latency"])):
|
||||
dump_metric(
|
||||
f"{spec.name}_{name}",
|
||||
value,
|
||||
labels={"model": sampler.model, "eval": spec.name},
|
||||
)
|
||||
print(f"sgl-eval {spec.name}: {metrics}")
|
||||
return (
|
||||
(metrics, result.latency) if getattr(args, "return_latency", False) else metrics
|
||||
)
|
||||
Reference in New Issue
Block a user