diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index 63bafe9fe1..5b74d93fdd 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -12,6 +12,11 @@ import time import uuid from pathlib import Path +from sglang.test.sgl_eval_utils import ( + THINKING_MODE_CHOICES, + get_thinking_kwargs, + parse_json_object, +) from sglang.test.simple_eval_common import ( ChatCompletionSampler, CompletionSampler, @@ -22,30 +27,6 @@ from sglang.test.simple_eval_common import ( ) -def get_thinking_kwargs(args): - thinking_mode = getattr(args, "thinking_mode", None) - if thinking_mode in THINKING_MODE_CHOICES: - if thinking_mode in ["deepseek-v3", "kimi-k2"]: - thinking_param = "thinking" - else: - # All models other than dpsk v3/kimi_k2 - thinking_param = "enable_thinking" - return {thinking_param: True} - return {} - - -def parse_json_object(value: str) -> dict: - try: - parsed = json.loads(value) - except json.JSONDecodeError as e: - raise argparse.ArgumentTypeError("must be a valid JSON object string") from e - - if not isinstance(parsed, dict): - raise argparse.ArgumentTypeError("must be a JSON object") - - return parsed - - def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict: chat_template_kwargs = getattr(args, "chat_template_kwargs", None) if isinstance(chat_template_kwargs, str): @@ -426,8 +407,6 @@ def run_eval(args): return metrics -THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"] - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( diff --git a/python/sglang/test/sgl_eval_utils.py b/python/sglang/test/sgl_eval_utils.py new file mode 100644 index 0000000000..01f7c651ce --- /dev/null +++ b/python/sglang/test/sgl_eval_utils.py @@ -0,0 +1,192 @@ +"""SGLang CI transport and metric reporting for sgl-eval benchmarks.""" + +import argparse +import json +import multiprocessing +import os +import threading +import uuid +from concurrent.futures import ProcessPoolExecutor +from dataclasses import replace +from itertools import islice +from pathlib import Path + +THINKING_MODE_CHOICES = ["deepseek-v3", "qwen-3", "glm-45", "kimi-k2"] + + +def get_thinking_kwargs(args): + thinking_mode = getattr(args, "thinking_mode", None) + if thinking_mode in THINKING_MODE_CHOICES: + if thinking_mode in ["deepseek-v3", "kimi-k2"]: + thinking_param = "thinking" + else: + # All models other than dpsk v3/kimi_k2 + thinking_param = "enable_thinking" + return {thinking_param: True} + return {} + + +def parse_json_object(value: str) -> dict: + try: + parsed = json.loads(value) + except json.JSONDecodeError as e: + raise argparse.ArgumentTypeError("must be a valid JSON object string") from e + + if not isinstance(parsed, dict): + raise argparse.ArgumentTypeError("must be a JSON object") + + return parsed + + +def api_base_url(args): + base_url = getattr(args, "base_url", None) + if not base_url: + host = getattr(args, "host", "127.0.0.1").rstrip("/") + if "://" not in host: + host = f"http://{host}" + base_url = f"{host}:{getattr(args, 'port', None) or 30000}" + base_url = base_url.rstrip("/") + return base_url if base_url.endswith("/v1") else base_url + "/v1" + + +def _print_truncated_samples(result): + truncated = ( + (example, repeat, sample) + for example in result.per_example + for repeat, sample in enumerate(example.samples) + if sample.finish_reason == "length" + ) + for example, repeat, sample in islice(truncated, 3): + preview = { + "example_id": example.example.id, + "repeat": repeat, + "finish_reason": sample.finish_reason, + "completion_tokens": sample.completion_tokens, + "score": example.scores[repeat], + "text_chars": len(sample.text), + "head": sample.text[:500], + "tail": sample.text[-500:], + } + print(f"sgl-eval truncated sample: {json.dumps(preview)}", flush=True) + + +def run_sgl_eval(args): + # math_verify uses SIGALRM, which requires the process's main thread. + if threading.current_thread() is not threading.main_thread(): + with ProcessPoolExecutor( + max_workers=1, mp_context=multiprocessing.get_context("spawn") + ) as executor: + return executor.submit(run_sgl_eval, args).result() + + from sgl_eval.predictions import PredictionsWriter + from sgl_eval.registry import get + from sgl_eval.sampler import ChatCompletionSampler + + from sglang.test.test_utils import dump_metric + + spec = get(args.eval_name) + overrides = {"max_tokens": 2048} + for key in ( + "max_tokens", + "temperature", + "top_p", + "min_p", + "seed", + "reasoning_effort", + ): + value = getattr(args, key, None) + if value is not None: + overrides[key] = value + chat_kwargs = getattr(args, "chat_template_kwargs", None) or {} + if isinstance(chat_kwargs, str): + chat_kwargs = parse_json_object(chat_kwargs) + chat_kwargs = {**get_thinking_kwargs(args), **chat_kwargs} + thinking = getattr(args, "sgl_eval_thinking", None) + if thinking is not None: + chat_kwargs.setdefault("thinking", thinking) + chat_kwargs.setdefault("enable_thinking", thinking) + if chat_kwargs: + overrides["chat_template_kwargs"] = { + **(spec.default_gen.chat_template_kwargs or {}), + **chat_kwargs, + } + if getattr(args, "top_k", None) is not None: + overrides["extra_body"] = {"top_k": args.top_k} + gen = replace(spec.default_gen, **overrides) + generation = { + key: getattr(gen, key) + for key in ("max_tokens", "temperature", "top_p", "min_p", "seed") + } | {"chat_template_kwargs": gen.chat_template_kwargs, **(gen.extra_body or {})} + print(f"sgl-eval {spec.name} generation: {generation}", flush=True) + sampler = ChatCompletionSampler( + base_url=api_base_url(args), + model=getattr(args, "model", None), + api_key=os.environ.get("OPENAI_API_KEY", "EMPTY"), + ) + out_parent = Path( + getattr(args, "sgl_eval_out_dir", None) + or Path.home() / ".sgl_eval" / "sglang_run_eval" + ).expanduser() + out_dir = out_parent / f"sgl_eval_{spec.name}_{uuid.uuid4().hex}" + out_dir.mkdir(parents=True, exist_ok=True) + repeats = getattr(args, "repeat", 1) + writer = PredictionsWriter(out_dir, repeats, spec.pred_schema) + try: + result = spec.run( + sampler=sampler, + gen=gen, + n_repeats=repeats, + num_examples=getattr(args, "num_examples", None), + num_threads=getattr(args, "num_threads", None) or spec.default_num_threads, + predictions_writer=writer, + load_examples=None, + ) + finally: + writer.close() + sampler.abort() + if result.partial: + raise RuntimeError(f"Incomplete {spec.name} evaluation; results: {out_dir}") + extracted = [ + answer for example in result.per_example for answer in example.extracted + ] + if not extracted: + raise RuntimeError(f"Empty {spec.name} evaluation; results: {out_dir}") + metrics = { + **result.aggregate, + "accuracy": result.aggregate["score"], + "invalid": sum(answer is None for answer in extracted) / len(extracted), + "latency": result.latency, + "output_throughput": result.output_throughput, + "sgl_eval_metrics_path": str(out_dir / "metrics.json"), + } + (out_dir / "metrics.json").write_text( + json.dumps( + { + "name": spec.name, + "model": sampler.model, + "generation": generation, + "aggregate": result.aggregate, + "num_examples": result.num_examples, + "n_repeats": result.n_repeats, + "latency_seconds": result.latency, + "output_throughput_tps": result.output_throughput, + }, + indent=2, + ) + ) + _print_truncated_samples(result) + if result.aggregate.get("error_rate", 0) > 0: + raise RuntimeError( + f"{spec.name} evaluation had request errors " + f"(error_rate={result.aggregate['error_rate']}); results: {out_dir}" + ) + for name, value in (("score", metrics["score"]), ("latency", metrics["latency"])): + dump_metric( + f"{spec.name}_{name}", + value, + labels={"model": sampler.model, "eval": spec.name}, + ) + print(f"sgl-eval {spec.name}: {metrics}") + return ( + (metrics, result.latency) if getattr(args, "return_latency", False) else metrics + ) diff --git a/test/README.md b/test/README.md index 1d76282438..b0a9e1aa0b 100644 --- a/test/README.md +++ b/test/README.md @@ -137,7 +137,7 @@ This README mostly describes the NVIDIA GPU CI pipeline. Other hardware backends - Each test file should take < 500 seconds; split if longer. - Each GitHub Actions job should take < 30 minutes; split if longer. - If tests are too slow for per-commit, consider nightly suites. -- New GSM8K or MMLU accuracy checks should use sgl-eval (`run_eval(..., api="sgl_eval")` or `sgl-eval run`) instead of adding another dataset or evaluator copy. +- New GSM8K or MMLU accuracy checks should use sgl-eval (`sglang.test.sgl_eval_utils.run_sgl_eval` or `sgl-eval run`) instead of adding another dataset or evaluator copy. ## Other Notes diff --git a/test/manual/4-gpu-models/test_qwen35_models_archived.py b/test/manual/4-gpu-models/test_qwen35_models_archived.py index a5a90bfea1..aaf942c5ad 100644 --- a/test/manual/4-gpu-models/test_qwen35_models_archived.py +++ b/test/manual/4-gpu-models/test_qwen35_models_archived.py @@ -17,7 +17,7 @@ from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin # This eval harness applies the chat_template, which is critical for qwen3.5 # to get good accuracy on gsm8k from sglang.test.run_combined_tests import run_combined_tests -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -140,7 +140,6 @@ class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): args = SimpleNamespace( model=self.model, eval_name="gsm8k", - num_shots=5, num_examples=200, max_tokens=16000, num_threads=128, @@ -152,7 +151,7 @@ class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"]) diff --git a/test/manual/8-gpu-models/test_deepseek_v3_basic.py b/test/manual/8-gpu-models/test_deepseek_v3_basic.py index 57c34c3a8c..e48199aadf 100644 --- a/test/manual/8-gpu-models/test_deepseek_v3_basic.py +++ b/test/manual/8-gpu-models/test_deepseek_v3_basic.py @@ -2,8 +2,8 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -47,13 +47,11 @@ class TestDeepseekV3Basic(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1400, num_threads=1400, - num_shots=8, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/manual/8-gpu-models/test_dsa_models_basic.py b/test/manual/8-gpu-models/test_dsa_models_basic.py index 14dfde0c92..672d365c49 100644 --- a/test/manual/8-gpu-models/test_dsa_models_basic.py +++ b/test/manual/8-gpu-models/test_dsa_models_basic.py @@ -2,8 +2,8 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -50,13 +50,11 @@ class TestDeepseekV32DP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1400, num_threads=1400, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): @@ -108,13 +106,11 @@ class TestDeepseekV32TP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1400, num_threads=1400, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): @@ -169,13 +165,11 @@ class TestGLM5DP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1400, num_threads=1400, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): @@ -227,13 +221,11 @@ class TestGLM5TP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1400, num_threads=1400, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/manual/attention/test_fa3.py b/test/manual/attention/test_fa3.py index 1737a84fdd..6bb866a333 100644 --- a/test/manual/attention/test_fa3.py +++ b/test/manual/attention/test_fa3.py @@ -5,7 +5,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import get_device_sm, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, DEFAULT_MODEL_NAME_FOR_TEST, @@ -97,14 +97,11 @@ class BaseFlashAttentionTest(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=100, num_threads=128, - num_shots=4, - gsm8k_data_path=GSM_DATASET_PATH, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") # Use the appropriate metric key based on the test class diff --git a/test/manual/attention/test_local_attn.py b/test/manual/attention/test_local_attn.py index 4f00e26914..352e0e450b 100644 --- a/test/manual/attention/test_local_attn.py +++ b/test/manual/attention/test_local_attn.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import get_device_sm, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -56,13 +56,11 @@ class TestFlashAttention3LocalAttn(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=100, num_threads=128, - num_shots=4, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") # Use the appropriate metric key based on the test class diff --git a/test/manual/distributed/test_dp_attention_archived.py b/test/manual/distributed/test_dp_attention_archived.py index 8b07ff2803..40f7ff7b10 100644 --- a/test/manual/distributed/test_dp_attention_archived.py +++ b/test/manual/distributed/test_dp_attention_archived.py @@ -14,7 +14,7 @@ from sglang.srt.utils import kill_process_tree from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, @@ -75,12 +75,11 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/manual/distributed/test_dp_attention_large.py b/test/manual/distributed/test_dp_attention_large.py index 997491a1c1..7186a71333 100644 --- a/test/manual/distributed/test_dp_attention_large.py +++ b/test/manual/distributed/test_dp_attention_large.py @@ -7,7 +7,7 @@ from sglang.srt.utils import kill_process_tree from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_IMAGE_URL, DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -60,7 +60,7 @@ class TestDPAttentionDP2TP4( num_threads=1024, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.8) @@ -112,12 +112,11 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py index 9954353281..5274621b24 100644 --- a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py @@ -23,7 +23,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -128,15 +128,14 @@ class TestDSV4FlashMTPBasic(DSV4FlashMTPServerBase): """Accuracy + spec path full forward.""" requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, + eval_name="gsm8k", + num_examples=200, + max_tokens=512, + num_threads=128, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_gsm8k_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["accuracy"], 0.95) @@ -144,15 +143,14 @@ class TestDSV4FlashMTPBasic(DSV4FlashMTPServerBase): """Degenerate spec step (still cuda-graph captured).""" requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=100, - max_new_tokens=1, - parallel=128, + eval_name="gsm8k", + num_examples=100, + max_tokens=1, + num_threads=128, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_gsm8k_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["output_throughput"], 50) def test_request_abort(self): diff --git a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py index fed6e21240..ac84120248 100644 --- a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py +++ b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py @@ -9,10 +9,10 @@ path, so any layout mismatch would trip the item_len assert in import unittest from types import SimpleNamespace -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, popen_launch_pd_server, @@ -131,15 +131,14 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase): def test_gsm8k(self): """End-to-end PD-disagg accuracy through the LB.""" args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=64, + eval_name="gsm8k", + num_examples=200, + max_tokens=512, + num_threads=64, host=self.base_host, port=int(self.lb_port), ) - metrics = run_gsm8k_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["accuracy"], 0.95) diff --git a/test/manual/ep/test_elastic_scale.py b/test/manual/ep/test_elastic_scale.py index 6262be1e04..092e639bf9 100644 --- a/test/manual/ep/test_elastic_scale.py +++ b/test/manual/ep/test_elastic_scale.py @@ -25,8 +25,8 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -389,12 +389,11 @@ class _ElasticScaleUpEndToEndBase(CustomTestCase): self.fail("Timed out waiting for scaling to complete (300s)") def _run_post_scale_gsm8k(self) -> None: - metrics = run_eval( + metrics = run_sgl_eval( SimpleNamespace( base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=256, num_threads=50, diff --git a/test/manual/ep/test_eplb_mori.py b/test/manual/ep/test_eplb_mori.py index 24b49713c2..5710e4d995 100644 --- a/test/manual/ep/test_eplb_mori.py +++ b/test/manual/ep/test_eplb_mori.py @@ -5,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.server_args import ZMQ_TCP_PORT_DELTA from sglang.srt.utils import kill_process_tree from sglang.srt.utils.network import is_port_available -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -131,15 +131,14 @@ class TestEPLBMoriStat(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1209, - max_new_tokens=512, - parallel=1209, + eval_name="gsm8k", + num_examples=1209, + max_tokens=512, + num_threads=1209, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["accuracy"], 0.9) @@ -177,15 +176,14 @@ class TestEPLBMoriStatApprox(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1209, - max_new_tokens=512, - parallel=1209, + eval_name="gsm8k", + num_examples=1209, + max_tokens=512, + num_threads=1209, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["accuracy"], 0.9) @@ -225,15 +223,14 @@ class TestEPLBMoriMultiChunk(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1209, - max_new_tokens=512, - parallel=1209, + eval_name="gsm8k", + num_examples=1209, + max_tokens=512, + num_threads=1209, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["accuracy"], 0.9) diff --git a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py index 2db5881f3d..0b990f3b35 100644 --- a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py +++ b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py @@ -8,6 +8,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -46,13 +47,11 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=64, - num_shots=8, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") self.assertGreater(metrics["score"], 0.93) diff --git a/test/manual/ep/test_mooncake_expert_backup.py b/test/manual/ep/test_mooncake_expert_backup.py index b846d6ad7e..5298ed31e4 100644 --- a/test/manual/ep/test_mooncake_expert_backup.py +++ b/test/manual/ep/test_mooncake_expert_backup.py @@ -5,8 +5,8 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -128,12 +128,11 @@ class TestBackup(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/manual/ep/test_nixl_ep.py b/test/manual/ep/test_nixl_ep.py index d395aa33e2..6341fe9bc3 100644 --- a/test/manual/ep/test_nixl_ep.py +++ b/test/manual/ep/test_nixl_ep.py @@ -7,8 +7,8 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -77,12 +77,11 @@ class _EPTestBase(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) return metrics diff --git a/test/manual/ep/test_pplx_small.py b/test/manual/ep/test_pplx_small.py index 382103a641..3e4c3c4143 100644 --- a/test/manual/ep/test_pplx_small.py +++ b/test/manual/ep/test_pplx_small.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -57,12 +57,11 @@ class TestPureDP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) @@ -77,12 +76,11 @@ class TestPureDP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=40, num_threads=1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.50) @@ -130,12 +128,11 @@ class TestHybridDPTP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/manual/mla/test_flashmla_archived.py b/test/manual/mla/test_flashmla_archived.py index eb1bdcf1a6..2b99c65b85 100644 --- a/test/manual/mla/test_flashmla_archived.py +++ b/test/manual/mla/test_flashmla_archived.py @@ -16,7 +16,7 @@ from types import SimpleNamespace import torch from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -58,12 +58,11 @@ class TestFlashMLAAttnBackend(unittest.TestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/manual/mla/test_mla_deepseek_v3.py b/test/manual/mla/test_mla_deepseek_v3.py index 4ddc1827cd..389d4ad13a 100644 --- a/test/manual/mla/test_mla_deepseek_v3.py +++ b/test/manual/mla/test_mla_deepseek_v3.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import is_cuda, is_hip, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -41,12 +41,11 @@ class TestMLADeepseekV3(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) @@ -78,12 +77,11 @@ class TestMLADeepseekV3DisableFusedFunc(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.62) @@ -129,12 +127,11 @@ class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) @@ -182,12 +179,11 @@ class TestDeepseekV3MTP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/manual/mla/test_mla_flashinfer_archived.py b/test/manual/mla/test_mla_flashinfer_archived.py index f120a20229..f18914ebf1 100644 --- a/test/manual/mla/test_mla_flashinfer_archived.py +++ b/test/manual/mla/test_mla_flashinfer_archived.py @@ -11,7 +11,7 @@ from types import SimpleNamespace import torch from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -53,12 +53,11 @@ class TestFlashinferMLA(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.615) diff --git a/test/manual/mla/test_mla_int8_deepseek_v3_archived.py b/test/manual/mla/test_mla_int8_deepseek_v3_archived.py index 7b7ce3d0e2..94cafd2e12 100644 --- a/test/manual/mla/test_mla_int8_deepseek_v3_archived.py +++ b/test/manual/mla/test_mla_int8_deepseek_v3_archived.py @@ -11,7 +11,7 @@ from types import SimpleNamespace import torch from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -54,12 +54,11 @@ class TestMLADeepseekV3ChannelInt8(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreaterEqual(metrics["score"], 0.61) @@ -98,12 +97,11 @@ class TestMLADeepseekV3BlockInt8(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.62) diff --git a/test/manual/models/test_falcon_h1_models.py b/test/manual/models/test_falcon_h1_models.py index 4630e1cdb6..71779ee3ac 100644 --- a/test/manual/models/test_falcon_h1_models.py +++ b/test/manual/models/test_falcon_h1_models.py @@ -1,7 +1,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -34,12 +34,11 @@ class TestFalconH1(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.74) @@ -68,12 +67,11 @@ class TestFalconH1TP4(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.74) @@ -102,12 +100,11 @@ class TestFalconH1NoGatedRMS(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.74) @@ -136,11 +133,10 @@ class TestFalconH1NoGatedTP4(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.74) diff --git a/test/manual/models/test_grok_models.py b/test/manual/models/test_grok_models.py index 9a3b0e516f..b70d85167e 100644 --- a/test/manual/models/test_grok_models.py +++ b/test/manual/models/test_grok_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -37,12 +37,11 @@ class TestGrok(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=64, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") # It is dummy weights so we only assert the output throughput instead of accuracy. diff --git a/test/manual/models/test_kimi_k2_models.py b/test/manual/models/test_kimi_k2_models.py index 812c1589e5..368fa1bd75 100644 --- a/test/manual/models/test_kimi_k2_models.py +++ b/test/manual/models/test_kimi_k2_models.py @@ -4,7 +4,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -51,12 +51,11 @@ class TestKimiK2Thinking(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/manual/models/test_llama4_models.py b/test/manual/models/test_llama4_models.py index 70c4210fe9..6a7974085b 100644 --- a/test/manual/models/test_llama4_models.py +++ b/test/manual/models/test_llama4_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -46,12 +46,11 @@ class TestLlama4(CustomTestCase): args = SimpleNamespace( base_url=self.base_url, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["score"], model.accuracy) except Exception as e: diff --git a/test/manual/models/test_mistral_large3_basic.py b/test/manual/models/test_mistral_large3_basic.py index 7004b5a2d3..1cc50c7dfb 100644 --- a/test/manual/models/test_mistral_large3_basic.py +++ b/test/manual/models/test_mistral_large3_basic.py @@ -3,8 +3,8 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -56,13 +56,11 @@ class TestMistralLarge3Basic(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1400, num_threads=1400, - num_shots=8, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/manual/models/test_mtp_models.py b/test/manual/models/test_mtp_models.py index c5f3fc5cd3..939da04a31 100644 --- a/test/manual/models/test_mtp_models.py +++ b/test/manual/models/test_mtp_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -44,12 +44,11 @@ class TestMiMoMTP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.7) diff --git a/test/manual/models/test_qwen_models.py b/test/manual/models/test_qwen_models.py index 92429b9425..49486aa1f1 100644 --- a/test/manual/models/test_qwen_models.py +++ b/test/manual/models/test_qwen_models.py @@ -4,7 +4,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -34,12 +34,11 @@ class TestQwen2(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.78) @@ -65,12 +64,11 @@ class TestQwen2FP8(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.78) diff --git a/test/manual/models/test_unsloth_models.py b/test/manual/models/test_unsloth_models.py index 9f71ff1634..dfe35454aa 100644 --- a/test/manual/models/test_unsloth_models.py +++ b/test/manual/models/test_unsloth_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -32,12 +32,11 @@ class TestUnslothPhi4(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.78) @@ -66,12 +65,11 @@ class TestUnslothPhi4Bnb4bit(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.75) @@ -100,12 +98,11 @@ class TestUnslothPhi4UnslothBnb4bit(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.75) @@ -131,12 +128,11 @@ class TestUnslothPhi4MiniInstruct(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.65) @@ -165,12 +161,11 @@ class TestUnslothPhi4MiniBnb4bit(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.6) @@ -199,12 +194,11 @@ class TestUnslothPhi4MiniUnslothBnb4bit(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.6) diff --git a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py index 4246c977ef..9b7b2ca245 100644 --- a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py +++ b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py @@ -1,10 +1,10 @@ import unittest from types import SimpleNamespace -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -73,12 +73,11 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"GSM8K accuracy with piecewise cuda graph: {metrics['score']:.3f}") self.assertGreater(metrics["score"], 0.62) diff --git a/test/manual/quant/test_deepseek_v32_fp4_4gpu.py b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py index df7e8e9675..773029805a 100644 --- a/test/manual/quant/test_deepseek_v32_fp4_4gpu.py +++ b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py @@ -2,8 +2,8 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -56,13 +56,11 @@ class TestDeepseekV32FP4DP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=500, num_threads=500, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): @@ -124,13 +122,11 @@ class TestDeepseekV32FP4TP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=500, num_threads=500, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py b/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py index 742882d4e3..78a8958d59 100644 --- a/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py +++ b/test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py @@ -8,8 +8,8 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -59,13 +59,11 @@ class TestDeepseekV3FP4(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1319, num_threads=1319, - num_shots=8, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/manual/quant/test_nvfp4_gemm_archived.py b/test/manual/quant/test_nvfp4_gemm_archived.py index 8468e754a6..96e690fd5d 100644 --- a/test/manual/quant/test_nvfp4_gemm_archived.py +++ b/test/manual/quant/test_nvfp4_gemm_archived.py @@ -10,7 +10,7 @@ from types import SimpleNamespace from urllib.parse import urlparse from sglang.srt.utils import get_device_sm, kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -54,12 +54,11 @@ class FP4GemmBase: base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1319, num_threads=200, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.64) diff --git a/test/manual/test_mla_tp.py b/test/manual/test_mla_tp.py index b971f59486..2e7b388cc5 100644 --- a/test/manual/test_mla_tp.py +++ b/test/manual/test_mla_tp.py @@ -4,7 +4,7 @@ from types import SimpleNamespace import torch from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -45,12 +45,11 @@ class TestDeepseekTP2(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.62) def test_gsm8k_bs1(self): @@ -59,12 +58,11 @@ class TestDeepseekTP2(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=10, num_threads=1, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) self.assertGreater(metrics["score"], 0.62) diff --git a/test/manual/test_torch_flex_attention_backend.py b/test/manual/test_torch_flex_attention_backend.py index 891471bae3..a9b565c694 100644 --- a/test/manual/test_torch_flex_attention_backend.py +++ b/test/manual/test_torch_flex_attention_backend.py @@ -7,7 +7,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -32,13 +32,11 @@ class TestTorchFlexAttnBackend(CustomTestCase): args = SimpleNamespace( base_url=base_url, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=100, num_threads=10, - num_shots=8, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.62) finally: diff --git a/test/manual/test_w4a8_deepseek_v3.py b/test/manual/test_w4a8_deepseek_v3.py index 4440d47923..e005679854 100644 --- a/test/manual/test_w4a8_deepseek_v3.py +++ b/test/manual/test_w4a8_deepseek_v3.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -41,12 +41,11 @@ class TestDeepseekV3W4afp8(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1200, num_threads=1200, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") self.assertGreater(metrics["score"], 0.92) @@ -95,12 +94,11 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/server_info") @@ -163,12 +161,11 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") self.assertGreater(metrics["score"], 0.92) @@ -232,12 +229,11 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") self.assertGreater(metrics["score"], 0.92) diff --git a/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py b/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py index 40f3d6f66b..482ade9ae5 100644 --- a/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py +++ b/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py @@ -8,7 +8,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -62,15 +62,14 @@ class TestDeepseekV3FP4CuteDSLMoEEP4(CustomTestCase): def test_a_gsm8k(self): args = SimpleNamespace( - num_shots=8, - data_path=None, - num_questions=1319, - parallel=1319, - max_new_tokens=512, + eval_name="gsm8k", + num_examples=1319, + num_threads=1319, + max_tokens=512, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3-fp4-cutedsl-moe-ep4)\n" diff --git a/test/registered/cp/test_deepseek_v3_cp_single_node.py b/test/registered/cp/test_deepseek_v3_cp_single_node.py index f6259096ad..863de76196 100644 --- a/test/registered/cp/test_deepseek_v3_cp_single_node.py +++ b/test/registered/cp/test_deepseek_v3_cp_single_node.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -70,13 +70,11 @@ class TestDeepseekV3CPInSeqSplit(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=500, num_threads=32, - num_shots=20, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/registered/cp/test_gqa_prefill_cp.py b/test/registered/cp/test_gqa_prefill_cp.py index 0b0764982e..99f0bf542a 100644 --- a/test/registered/cp/test_gqa_prefill_cp.py +++ b/test/registered/cp/test_gqa_prefill_cp.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -59,7 +59,6 @@ class TestGQACP2TP2EP2(CustomTestCase): args = SimpleNamespace( model=self.model, eval_name="gsm8k", - num_shots=5, num_examples=200, max_tokens=16000, num_threads=128, @@ -71,7 +70,7 @@ class TestGQACP2TP2EP2(CustomTestCase): host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["score"], GSM8K_BASELINE_ACCURACY) @@ -117,7 +116,6 @@ class TestGQACPTP2CP2EP4(CustomTestCase): args = SimpleNamespace( model=self.model, eval_name="gsm8k", - num_shots=5, num_examples=200, max_tokens=16000, num_threads=128, @@ -129,7 +127,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase): host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["score"], GSM8K_BASELINE_ACCURACY) @@ -177,7 +175,6 @@ class TestGQACPCP4EP4(CustomTestCase): args = SimpleNamespace( model=self.model, eval_name="gsm8k", - num_shots=5, num_examples=200, max_tokens=16000, num_threads=128, @@ -189,7 +186,7 @@ class TestGQACPCP4EP4(CustomTestCase): host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreaterEqual(metrics["score"], GSM8K_BASELINE_ACCURACY) diff --git a/test/registered/disaggregation/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py index 8fdec84038..5981bed011 100644 --- a/test/registered/disaggregation/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -16,11 +16,11 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.pause_generation_kit import PauseResumeInPlaceMixin from sglang.test.kits.spec_server_kits import SpecGrammarKit -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, assert_process_healthy, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, DEFAULT_MODEL_NAME_FOR_TEST, @@ -43,12 +43,11 @@ class TestDisaggregationAccuracy(PauseResumeInPlaceMixin, PDDisaggregationServer args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.62) @@ -200,7 +199,6 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase): args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, @@ -208,7 +206,7 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase): # Expect lots of failure but the server cannot crash try: - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") except Exception as e: print(f"Test encountered expected errors: {e}") @@ -378,11 +376,10 @@ class TestDisaggregationMooncakeSpec( # correct afterwards — a leaked host slot or a damaged neighbour shows up as # a wrong answer rather than merely a 200. assert_process_healthy(self, "decode", self.process_decode, self.decode_url) - metrics = run_eval( + metrics = run_sgl_eval( SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=64, num_threads=32, @@ -397,12 +394,11 @@ class TestDisaggregationMooncakeSpec( args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.74) @@ -425,12 +421,11 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase): args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.62) diff --git a/test/registered/disaggregation/test_disaggregation_decode_radix_cache.py b/test/registered/disaggregation/test_disaggregation_decode_radix_cache.py index a78a7a9334..3c70b5b9c9 100644 --- a/test/registered/disaggregation/test_disaggregation_decode_radix_cache.py +++ b/test/registered/disaggregation/test_disaggregation_decode_radix_cache.py @@ -20,6 +20,7 @@ from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, is_in_ci, @@ -100,14 +101,12 @@ class DisaggregationDecodeRadixCacheTestMixin: base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=500, num_threads=100, - num_shots=6, ) - metrics_first = run_eval(args) + metrics_first = run_sgl_eval(args) print(f"First run metrics: {metrics_first}") metrics_second = run_eval(args) diff --git a/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py b/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py index e632d3bf14..4ea69b8ea6 100644 --- a/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py +++ b/test/registered/disaggregation/test_disaggregation_dwdp_gpt_oss.py @@ -2,10 +2,10 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval register_cuda_ci(est_time=170, stage="extra-b", runner_config="4-gpu-b200") @@ -50,13 +50,11 @@ class TestDisaggregationDWDPGptOss(PDDisaggregationServerBase): cls.launch_all() def test_gsm8k(self): - metrics = run_eval( + metrics = run_sgl_eval( SimpleNamespace( base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="chat", - num_shots=5, num_examples=100, max_tokens=4096, num_threads=8, diff --git a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py index 66f19afee6..3a47277b76 100644 --- a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py +++ b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py @@ -2,10 +2,10 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, is_in_ci, @@ -77,12 +77,11 @@ class TestDisaggregationHybridAttentionGDN(PDDisaggregationServerBase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=2048, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.93) @@ -153,12 +152,11 @@ class TestDisaggregationHybridAttentionGDNExtraBuffer(PDDisaggregationServerBase base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=2048, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") # TODO: Fix PD disaggregation accuracy issue (https://github.com/sgl-project/sglang/issues/21744) and increase the threshold back to 0.93. @@ -232,12 +230,11 @@ class TestDisaggregationHybridAttentionGDNDPDecode(PDDisaggregationServerBase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=2048, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") # TODO: Fix PD disaggregation accuracy issue (https://github.com/sgl-project/sglang/issues/21744) and increase the threshold back to 0.93. @@ -305,12 +302,13 @@ class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=8192, + temperature=0.6, + top_p=0.95, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.87) @@ -381,12 +379,13 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=8192, + temperature=0.6, + top_p=0.95, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.87) diff --git a/test/registered/disaggregation/test_disaggregation_nixl.py b/test/registered/disaggregation/test_disaggregation_nixl.py index 04850bc323..940f8fc01d 100644 --- a/test/registered/disaggregation/test_disaggregation_nixl.py +++ b/test/registered/disaggregation/test_disaggregation_nixl.py @@ -8,12 +8,12 @@ import requests from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, assert_process_healthy, configure_nixl_pd_backend, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -260,15 +260,13 @@ class TestDisaggregationNixlBasic(NixlPDDisaggregationServerBase): args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, - num_shots=5, num_threads=128, temperature=0.0, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") self.assertGreater( metrics["score"], @@ -307,7 +305,6 @@ class TestDisaggregationNixlFailure(NixlPDDisaggregationServerBase): args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=50, num_threads=128, @@ -316,7 +313,7 @@ class TestDisaggregationNixlFailure(NixlPDDisaggregationServerBase): # Tolerate eval/request errors; the gate is that workers stay healthy # after the injected transfer failures, not the score itself. try: - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Evaluation metrics: {metrics}") except Exception as e: print(f"Test encountered expected errors: {e}") diff --git a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py index a6290444d0..b8be096f45 100644 --- a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py +++ b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py @@ -35,10 +35,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase register_cuda_ci(est_time=300, stage="base-b", runner_config="2-gpu-large") @@ -129,12 +129,11 @@ class TestOptimisticPrefill( args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = self.assert_retry_counter_increases(lambda: run_eval(args)) + metrics = self.assert_retry_counter_increases(lambda: run_sgl_eval(args)) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.62) time.sleep(1) # trigger memory check @@ -302,12 +301,11 @@ class TestOptimisticPrefillL3BufferWriteThrough( args = SimpleNamespace( base_url=f"http://{self.base_host}:{self.lb_port}", eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = self.assert_retry_counter_increases(lambda: run_eval(args)) + metrics = self.assert_retry_counter_increases(lambda: run_sgl_eval(args)) print(f"Evaluation metrics: {metrics}") self.assertGreater(metrics["score"], 0.62) # Write-through published prefixes to L3; report what retries fetched back. diff --git a/test/registered/disaggregation/test_disaggregation_pp.py b/test/registered/disaggregation/test_disaggregation_pp.py index 47be3e2235..23590b9c53 100644 --- a/test/registered/disaggregation/test_disaggregation_pp.py +++ b/test/registered/disaggregation/test_disaggregation_pp.py @@ -3,10 +3,10 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -82,12 +82,11 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.24) @@ -161,12 +160,11 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.24) diff --git a/test/registered/e2e/models/test_transformers_models.py b/test/registered/e2e/models/test_transformers_models.py index f8223f0545..826cf6e528 100644 --- a/test/registered/e2e/models/test_transformers_models.py +++ b/test/registered/e2e/models/test_transformers_models.py @@ -12,6 +12,7 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -59,12 +60,11 @@ class TestTransformersFallbackEndpoint(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], self.gsm8k_lower_bound) diff --git a/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py b/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py index c3fabac069..8e6365fec8 100644 --- a/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py +++ b/test/registered/e2e/models_large/test_deepseek_v32_indexcache.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -48,15 +48,14 @@ class TestDeepseekV32IndexTopkPattern(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, + eval_name="gsm8k", + num_examples=1400, + num_threads=1400, + max_tokens=512, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): @@ -95,15 +94,14 @@ class TestDeepseekV32IndexFreq(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( - num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, + eval_name="gsm8k", + num_examples=1400, + num_threads=1400, + max_tokens=512, host="127.0.0.1", port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_ci(): diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index 24c68344a4..5acfafa770 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -4,8 +4,8 @@ from types import SimpleNamespace import requests from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -78,12 +78,11 @@ class TestDeepseekMTP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1200, num_threads=1200, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") self.assertGreater(metrics["score"], 0.92) @@ -138,12 +137,11 @@ class TestDeepseekV32TBO(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=1200, num_threads=1200, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.92) diff --git a/test/registered/ep/test_tbo_shared_experts_fusion.py b/test/registered/ep/test_tbo_shared_experts_fusion.py index b31ed171c6..dd9cc0a713 100644 --- a/test/registered/ep/test_tbo_shared_experts_fusion.py +++ b/test/registered/ep/test_tbo_shared_experts_fusion.py @@ -3,7 +3,7 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -58,12 +58,11 @@ class TestTBOWithSharedExpertsFusion(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(metrics) self.assertGreater(metrics["score"], 0.60) diff --git a/test/registered/pp/test_pp_single_node.py b/test/registered/pp/test_pp_single_node.py index d2b6f2e219..a652048c8c 100644 --- a/test/registered/pp/test_pp_single_node.py +++ b/test/registered/pp/test_pp_single_node.py @@ -19,7 +19,7 @@ from sglang.bench_one_batch_server import BenchArgs as OneBatchBenchArgs from sglang.srt.server_args import ServerArgs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST, @@ -62,12 +62,11 @@ class TestPPAccuracy(unittest.TestCase): base_url=self.base_url, model=DEFAULT_MODEL_NAME_FOR_TEST, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_amd_ci(): @@ -137,7 +136,7 @@ class TestDPAttentionDP2PP2(CustomTestCase): num_threads=1024, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["score"], 0.8) @@ -172,12 +171,11 @@ class TestPPMixedChunk(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") if is_in_amd_ci(): diff --git a/test/registered/quant/test_nvfp4_gemm_sm120.py b/test/registered/quant/test_nvfp4_gemm_sm120.py index af1328a446..38823dbd18 100644 --- a/test/registered/quant/test_nvfp4_gemm_sm120.py +++ b/test/registered/quant/test_nvfp4_gemm_sm120.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -49,15 +49,14 @@ class FP4GemmSM120Base: def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=200, + eval_name="gsm8k", + num_examples=1319, + max_tokens=512, + num_threads=200, host=parsed_url.hostname, port=parsed_url.port, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") self.assertGreater(metrics["accuracy"], 0.64) diff --git a/test/registered/spec/eagle/test_adaptive_speculative.py b/test/registered/spec/eagle/test_adaptive_speculative.py index 1250369a01..7b123d0f31 100644 --- a/test/registered/spec/eagle/test_adaptive_speculative.py +++ b/test/registered/spec/eagle/test_adaptive_speculative.py @@ -8,10 +8,12 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, + DEFAULT_DRAFT_MODEL_EAGLE3, DEFAULT_TARGET_MODEL_EAGLE, + DEFAULT_TARGET_MODEL_EAGLE3, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -39,8 +41,8 @@ MAX_DOWNSHIFT_ATTEMPTS = 6 class TestAdaptiveSpeculativeServer(CustomTestCase): """Test adaptive speculative decoding with state switching and GSM8K accuracy.""" - model = DEFAULT_TARGET_MODEL_EAGLE - draft_model = DEFAULT_DRAFT_MODEL_EAGLE + model = DEFAULT_TARGET_MODEL_EAGLE3 + draft_model = DEFAULT_DRAFT_MODEL_EAGLE3 base_url = DEFAULT_URL_FOR_TEST @classmethod @@ -67,10 +69,13 @@ class TestAdaptiveSpeculativeServer(CustomTestCase): timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=[ "--trust-remote-code", + # Target embeddings are shared with the FP16-default draft. + "--dtype", + "bfloat16", "--attention-backend", "triton", "--speculative-algorithm", - "EAGLE", + "EAGLE3", "--speculative-draft-model-path", cls.draft_model, "--speculative-adaptive", @@ -168,12 +173,11 @@ class TestAdaptiveSpeculativeServer(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=2048, num_examples=100, num_threads=64, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"GSM8K after adaptive switches: {metrics}") self.assertGreater(metrics["score"], 0.20) diff --git a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py index 3e608ccda9..40f14939c0 100644 --- a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py +++ b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py @@ -5,8 +5,8 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -72,12 +72,11 @@ class TestDeepseekV3FP4MTP(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/server_info").json() diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index 111429a6fe..1669fd58db 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -5,8 +5,8 @@ import requests from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, DEFAULT_TARGET_MODEL_EAGLE_DP_ATTN, @@ -77,12 +77,12 @@ class TestEAGLE3EngineDPAttention(CustomTestCase): base_url=self.base_url, model=self.model, eval_name="gsm8k", - api="completion", - max_tokens=512, + max_tokens=2048, + sgl_eval_thinking=False, num_examples=200, num_threads=128, ) - metrics = run_eval(args) + metrics = run_sgl_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/server_info") diff --git a/test/registered/spec/test_gemma4_dflash_31b_extra.py b/test/registered/spec/test_gemma4_dflash_31b_extra.py index 6b0d0d6ec9..60872db52f 100644 --- a/test/registered/spec/test_gemma4_dflash_31b_extra.py +++ b/test/registered/spec/test_gemma4_dflash_31b_extra.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -98,11 +98,9 @@ class TestGemma4DFlash31B(CustomTestCase): base_url=cls.base_url, model=TARGET_PATH, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=GSM8K_NUM_EXAMPLES, num_threads=GSM8K_NUM_THREADS, - num_shots=5, ) @staticmethod @@ -144,7 +142,7 @@ class TestGemma4DFlash31B(CustomTestCase): f"{MODEL_NAME}: CUDA graph is disabled", ) - metrics = run_eval(self._gsm8k_args()) + metrics = run_sgl_eval(self._gsm8k_args()) dflash_score = float(metrics["score"]) avg_accept = get_avg_spec_accept_length(self.base_url) finally: diff --git a/test/registered/spec/test_gemma4_mtp_31b_extra.py b/test/registered/spec/test_gemma4_mtp_31b_extra.py index ea8aecae7d..ff53e8dbd6 100644 --- a/test/registered/spec/test_gemma4_mtp_31b_extra.py +++ b/test/registered/spec/test_gemma4_mtp_31b_extra.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval +from sglang.test.sgl_eval_utils import run_sgl_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -99,11 +99,9 @@ class TestGemma4MTP31B(CustomTestCase): base_url=cls.base_url, model=TARGET_PATH, eval_name="gsm8k", - api="completion", max_tokens=512, num_examples=GSM8K_NUM_EXAMPLES, num_threads=GSM8K_NUM_THREADS, - num_shots=5, ) @staticmethod @@ -135,7 +133,7 @@ class TestGemma4MTP31B(CustomTestCase): f"{MODEL_NAME}/topk{topk}: CUDA graph is disabled", ) - metrics = run_eval(self._gsm8k_args()) + metrics = run_sgl_eval(self._gsm8k_args()) mtp_score = float(metrics["score"]) avg_accept = get_avg_spec_accept_length(self.base_url) finally: