diff --git a/python/sglang/test/accuracy_test_runner.py b/python/sglang/test/accuracy_test_runner.py index 83bc6c524..cc780622c 100644 --- a/python/sglang/test/accuracy_test_runner.py +++ b/python/sglang/test/accuracy_test_runner.py @@ -432,56 +432,6 @@ def _run_nemo_skills_eval( kill_process_tree(process.pid) -def _run_few_shot_eval( - model: ModelLaunchSettings, - base_url: str, - num_questions: Optional[int] = None, - num_shots: int = 8, - max_tokens: int = 512, -) -> Tuple[bool, Optional[str], Optional[dict]]: - """Run evaluation using few_shot backend (few_shot_gsm8k.py). - - Returns: - Tuple of (success, error_message, metrics_dict) - """ - from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval - - process = None - try: - process = popen_launch_server( - model.model_path, - base_url, - other_args=model.extra_args, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - env=model.env, - ) - - args = SimpleNamespace( - num_shots=num_shots, - data_path=None, - num_questions=num_questions or 200, - max_new_tokens=max_tokens, - parallel=128, - host="http://127.0.0.1", - port=int(base_url.split(":")[-1]), - ) - - metrics = run_few_shot_eval(args) - - # Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score") - if "accuracy" in metrics and "score" not in metrics: - metrics["score"] = metrics["accuracy"] - - return True, None, metrics - - except Exception as e: - return False, f"Few-shot evaluation exception: {str(e)}", None - - finally: - if process: - kill_process_tree(process.pid) - - def run_accuracy_test( model: ModelLaunchSettings, params: AccuracyTestParams, @@ -507,12 +457,7 @@ def run_accuracy_test( # Run evaluation based on dataset type # - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval) - # - few_shot_eval: gsm8k (default, backward compatible) - # - simple_eval: everything else (gpqa, mmmu, etc.) - has_extended_params = any( - getattr(params, field) is not None - for field in ("thinking_mode", "temperature", "top_p", "top_k", "repeat") - ) + # - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.) if params.dataset in ("mmmu-pro", "mmmu_pro"): success, error, metrics = _run_nemo_skills_eval( model=model, @@ -523,13 +468,6 @@ def run_accuracy_test( temperature=params.temperature, top_p=params.top_p, ) - elif params.dataset == "gsm8k" and not has_extended_params: - success, error, metrics = _run_few_shot_eval( - model=model, - base_url=base_url, - num_questions=params.num_examples, - max_tokens=params.max_tokens or 512, - ) else: success, error, metrics = _run_simple_eval( model=model, diff --git a/python/sglang/test/few_shot_gsm8k.py b/python/sglang/test/few_shot_gsm8k.py index 5d3992bf6..e3631419b 100644 --- a/python/sglang/test/few_shot_gsm8k.py +++ b/python/sglang/test/few_shot_gsm8k.py @@ -1,6 +1,11 @@ """ Run few-shot GSM-8K evaluation. +.. deprecated:: + This module is deprecated. Use ``sglang.test.run_eval`` with + ``eval_name="gsm8k"`` instead, which routes through the unified + Chat API evaluation framework with dump_metric support. + Usage: python3 -m sglang.test.few_shot_gsm8k --num-questions 200 """ @@ -9,6 +14,7 @@ import argparse import ast import re import time +import warnings import numpy as np @@ -50,6 +56,12 @@ def get_answer_value(answer_str): def run_eval(args): + warnings.warn( + "sglang.test.few_shot_gsm8k is deprecated. " + "Use sglang.test.run_eval with eval_name='gsm8k' instead.", + DeprecationWarning, + stacklevel=2, + ) # Select backend set_default_backend(RuntimeEndpoint(normalize_base_url(args.host, args.port))) diff --git a/python/sglang/test/few_shot_gsm8k_engine.py b/python/sglang/test/few_shot_gsm8k_engine.py index 13a30be1c..d06e15d35 100644 --- a/python/sglang/test/few_shot_gsm8k_engine.py +++ b/python/sglang/test/few_shot_gsm8k_engine.py @@ -1,8 +1,16 @@ +""" +.. deprecated:: + This module is deprecated. Use ``sglang.test.run_eval`` with + ``eval_name="gsm8k"`` instead, which routes through the unified + Chat API evaluation framework with dump_metric support. +""" + import argparse import ast import asyncio import re import time +import warnings from typing import Optional import numpy as np @@ -49,6 +57,12 @@ async def concurrent_generate(engine, prompts, sampling_param): def run_eval(args): + warnings.warn( + "sglang.test.few_shot_gsm8k_engine is deprecated. " + "Use sglang.test.run_eval with eval_name='gsm8k' instead.", + DeprecationWarning, + stacklevel=2, + ) # Select backend engine = sgl.Engine(model_path=args.model_path, log_level="error") diff --git a/python/sglang/test/kits/eval_accuracy_kit.py b/python/sglang/test/kits/eval_accuracy_kit.py index cab5d4860..25bf58151 100644 --- a/python/sglang/test/kits/eval_accuracy_kit.py +++ b/python/sglang/test/kits/eval_accuracy_kit.py @@ -3,7 +3,6 @@ from typing import Optional import requests -from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k from sglang.test.run_eval import run_eval from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary @@ -19,17 +18,20 @@ def _check_accept_length(test_case, base_url, threshold): class GSM8KMixin: - """Mixin for few-shot GSM8K evaluation. + """Mixin for GSM8K evaluation via OpenAI Chat API. Required attributes on the test class: base_url: str gsm8k_accuracy_thres: float + + Optional attributes: + model: str (if not set, auto-detected from server) """ gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET gsm8k_accept_length_thres: Optional[float] = None gsm8k_num_questions: int = 200 - gsm8k_parallel: int = 128 + gsm8k_num_threads: int = 128 def test_gsm8k(self): assert ( @@ -39,17 +41,21 @@ class GSM8KMixin: requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=self.gsm8k_num_questions, - max_new_tokens=512, - parallel=self.gsm8k_parallel, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=self.gsm8k_num_questions, + num_threads=self.gsm8k_num_threads, ) - metrics = run_eval_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres) + + if is_in_ci(): + write_github_step_summary(f"### test_gsm8k\n{metrics['score']=:.4f}\n") + + self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres) if self.gsm8k_accept_length_thres is not None: _check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres) diff --git a/python/sglang/test/run_eval.py b/python/sglang/test/run_eval.py index 7553242a1..d872966e7 100644 --- a/python/sglang/test/run_eval.py +++ b/python/sglang/test/run_eval.py @@ -62,7 +62,7 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict: extra_body[param_name] = value common_kwargs = dict( - model=args.model, + model=getattr(args, "model", None), max_tokens=getattr(args, "max_tokens", 2048), top_p=getattr(args, "top_p", 1.0), base_url=base_url, @@ -71,7 +71,12 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict: api_mode = getattr(args, "api", "chat") if api_mode == "completion": - sampler = CompletionSampler(**common_kwargs) + # Default stop tokens for completion API (matches few_shot_gsm8k behavior) + stop = getattr(args, "stop", ["Question", "Assistant:", "<|separator|>"]) + sampler = CompletionSampler( + **common_kwargs, + stop=stop, + ) else: sampler = ChatCompletionSampler( **common_kwargs, @@ -143,7 +148,7 @@ def run_eval(args): categories = args.categories.split(",") if args.categories else None eval_obj = LongBenchV2Eval( - model=args.model, + model=getattr(args, "model", None), data_source=data_source, num_examples=args.num_examples, num_threads=args.num_threads, diff --git a/python/sglang/test/server_fixtures/disaggregation_fixture.py b/python/sglang/test/server_fixtures/disaggregation_fixture.py index 53baed4d5..eda4004e9 100644 --- a/python/sglang/test/server_fixtures/disaggregation_fixture.py +++ b/python/sglang/test/server_fixtures/disaggregation_fixture.py @@ -32,6 +32,7 @@ class PDDisaggregationServerBase(CustomTestCase): cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + cls.base_url = cls.lb_url print( f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} {cls.bootstrap_port=}" ) diff --git a/python/sglang/test/simple_eval_common.py b/python/sglang/test/simple_eval_common.py index c667098b7..b9e4057fa 100644 --- a/python/sglang/test/simple_eval_common.py +++ b/python/sglang/test/simple_eval_common.py @@ -185,6 +185,7 @@ class CompletionSampler(SamplerBase): temperature: float = 0.0, top_p: float = 1.0, max_tokens: int = 2048, + stop: Optional[List[str]] = None, ): self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient()) @@ -195,9 +196,10 @@ class CompletionSampler(SamplerBase): self.temperature = temperature self.top_p = top_p self.max_tokens = max_tokens + self.stop = stop self._completion_tokens: list[int] = [] print( - f"CompletionSampler initialized with {self.model=} {self.temperature=} {self.max_tokens=}" + f"CompletionSampler initialized with {self.model=} {self.temperature=} {self.max_tokens=} {self.stop=}" ) def _pack_message(self, role: str, content: Any): @@ -219,6 +221,7 @@ class CompletionSampler(SamplerBase): temperature=self.temperature, top_p=self.top_p, max_tokens=self.max_tokens, + stop=self.stop, ) if response.usage and response.usage.completion_tokens is not None: self._completion_tokens.append(response.usage.completion_tokens) diff --git a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py index e79e87ed4..4781bb9ae 100644 --- a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py +++ b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py @@ -7,7 +7,6 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, @@ -44,18 +43,19 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=64, num_shots=8, - data_path=None, - num_questions=200, - parallel=64, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) def test_mmlu(self): args = SimpleNamespace( diff --git a/test/manual/ep/test_mooncake_expert_backup.py b/test/manual/ep/test_mooncake_expert_backup.py index fc3089d88..c6cec9cbd 100644 --- a/test/manual/ep/test_mooncake_expert_backup.py +++ b/test/manual/ep/test_mooncake_expert_backup.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, @@ -24,6 +24,7 @@ class TestBackup(CustomTestCase): def setUpClass(cls): cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA cls.base_port = 20000 + cls.base_url = f"http://127.0.0.1:{cls.base_port}" cls.num_processes = 2 # TODO (stage 100): in the future, implement a specified multiprocess launcher cls.processes = [ @@ -124,18 +125,18 @@ class TestBackup(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=self.base_port, + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) if __name__ == "__main__": diff --git a/test/manual/ep/test_nixl_ep.py b/test/manual/ep/test_nixl_ep.py index 3a4be3b8c..9be2ff037 100644 --- a/test/manual/ep/test_nixl_ep.py +++ b/test/manual/ep/test_nixl_ep.py @@ -4,7 +4,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, @@ -71,21 +71,21 @@ class _EPTestBase(CustomTestCase): def _run_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) return metrics def test_gsm8k(self): metrics = self._run_gsm8k() - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestNixlEPTP(_EPTestBase): @@ -108,7 +108,7 @@ class TestNixlMoeMooncakeElasticEP(_EPTestBase): def test_gsm8k_fault_1(self): os.system(f"pkill -f {self.pkill_process_1}") metrics = self._run_gsm8k() - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) if __name__ == "__main__": diff --git a/test/manual/hicache/test_pp_with_hicache.py b/test/manual/hicache/test_pp_with_hicache.py index 761c0fe46..9c14d173b 100644 --- a/test/manual/hicache/test_pp_with_hicache.py +++ b/test/manual/hicache/test_pp_with_hicache.py @@ -13,7 +13,7 @@ from urllib.parse import urlparse import requests from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -189,24 +189,24 @@ class TestPPWithHiCache(unittest.TestCase): def test_eval_accuracy(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=40, - max_new_tokens=256, - parallel=24, - host=f"http://{self.base_host}", - port=int(self.base_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=40, + num_threads=24, ) - metrics_initial = run_eval_few_shot_gsm8k(args) - self.assertGreater(metrics_initial["accuracy"], 0.6) + metrics_initial = run_eval(args) + self.assertGreater(metrics_initial["score"], 0.6) self.flush_cache() - metrics_cached = run_eval_few_shot_gsm8k(args) - self.assertGreater(metrics_cached["accuracy"], 0.6) + metrics_cached = run_eval(args) + self.assertGreater(metrics_cached["score"], 0.6) - accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"]) + accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"]) self.assertLess(accuracy_diff, 0.05) diff --git a/test/manual/models/test_falcon_h1_models.py b/test/manual/models/test_falcon_h1_models.py index 1706cc859..4630e1cdb 100644 --- a/test/manual/models/test_falcon_h1_models.py +++ b/test/manual/models/test_falcon_h1_models.py @@ -1,7 +1,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -31,17 +31,17 @@ class TestFalconH1(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.74) + self.assertGreater(metrics["score"], 0.74) class TestFalconH1TP4(CustomTestCase): @@ -65,17 +65,17 @@ class TestFalconH1TP4(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.74) + self.assertGreater(metrics["score"], 0.74) class TestFalconH1NoGatedRMS(CustomTestCase): @@ -99,17 +99,17 @@ class TestFalconH1NoGatedRMS(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.74) + self.assertGreater(metrics["score"], 0.74) class TestFalconH1NoGatedTP4(CustomTestCase): @@ -133,14 +133,14 @@ class TestFalconH1NoGatedTP4(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.74) + self.assertGreater(metrics["score"], 0.74) diff --git a/test/manual/models/test_grok_models.py b/test/manual/models/test_grok_models.py index 625fa1a65..9a3b0e516 100644 --- a/test/manual/models/test_grok_models.py +++ b/test/manual/models/test_grok_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -34,13 +34,13 @@ class TestGrok(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=64, - max_new_tokens=256, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=64, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") diff --git a/test/manual/models/test_kimi_k2_models.py b/test/manual/models/test_kimi_k2_models.py index 6a2fbed71..6e83ef50c 100644 --- a/test/manual/models/test_kimi_k2_models.py +++ b/test/manual/models/test_kimi_k2_models.py @@ -4,7 +4,7 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -48,22 +48,22 @@ class TestKimiK2Thinking(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.95) + self.assertGreater(metrics["score"], 0.95) if __name__ == "__main__": diff --git a/test/manual/models/test_llama4_models.py b/test/manual/models/test_llama4_models.py index cb0c57604..70c4210fe 100644 --- a/test/manual/models/test_llama4_models.py +++ b/test/manual/models/test_llama4_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -44,17 +44,16 @@ class TestLlama4(CustomTestCase): ], ) args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], model.accuracy) + self.assertGreaterEqual(metrics["score"], model.accuracy) except Exception as e: print(f"Error testing {model.model}: {e}") self.fail(f"Test failed for {model.model}: {e}") diff --git a/test/manual/models/test_mistral_large3_basic.py b/test/manual/models/test_mistral_large3_basic.py index 3b173f1ab..2eac4f79b 100644 --- a/test/manual/models/test_mistral_large3_basic.py +++ b/test/manual/models/test_mistral_large3_basic.py @@ -3,7 +3,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -53,22 +53,23 @@ class TestMistralLarge3Basic(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1400, + num_threads=1400, num_shots=8, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (mistral-large-3)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (mistral-large-3)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.90) + self.assertGreater(metrics["score"], 0.90) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) diff --git a/test/manual/models/test_mtp_models.py b/test/manual/models/test_mtp_models.py index 49b53c1e4..c5f3fc5cd 100644 --- a/test/manual/models/test_mtp_models.py +++ b/test/manual/models/test_mtp_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -41,17 +41,17 @@ class TestMiMoMTP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.7) + self.assertGreater(metrics["score"], 0.7) if __name__ == "__main__": diff --git a/test/manual/models/test_unsloth_models.py b/test/manual/models/test_unsloth_models.py index 24660ea34..9f71ff163 100644 --- a/test/manual/models/test_unsloth_models.py +++ b/test/manual/models/test_unsloth_models.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -29,17 +29,17 @@ class TestUnslothPhi4(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.78) + self.assertGreater(metrics["score"], 0.78) class TestUnslothPhi4Bnb4bit(CustomTestCase): @@ -63,17 +63,17 @@ class TestUnslothPhi4Bnb4bit(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.75) + self.assertGreater(metrics["score"], 0.75) class TestUnslothPhi4UnslothBnb4bit(CustomTestCase): @@ -97,17 +97,17 @@ class TestUnslothPhi4UnslothBnb4bit(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.75) + self.assertGreater(metrics["score"], 0.75) class TestUnslothPhi4MiniInstruct(CustomTestCase): @@ -128,17 +128,17 @@ class TestUnslothPhi4MiniInstruct(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.65) + self.assertGreater(metrics["score"], 0.65) class TestUnslothPhi4MiniBnb4bit(CustomTestCase): @@ -162,17 +162,17 @@ class TestUnslothPhi4MiniBnb4bit(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.6) + self.assertGreater(metrics["score"], 0.6) class TestUnslothPhi4MiniUnslothBnb4bit(CustomTestCase): @@ -196,17 +196,17 @@ class TestUnslothPhi4MiniUnslothBnb4bit(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.6) + self.assertGreater(metrics["score"], 0.6) if __name__ == "__main__": diff --git a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py index 086830a90..9f2e7bfa0 100644 --- a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py +++ b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py @@ -1,7 +1,7 @@ import unittest from types import SimpleNamespace -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -70,18 +70,18 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase): def test_gsm8k_accuracy(self): """Verify that piecewise cuda graph works correctly in prefill server""" args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) - print(f"GSM8K accuracy with piecewise cuda graph: {metrics['accuracy']:.3f}") + metrics = run_eval(args) + print(f"GSM8K accuracy with piecewise cuda graph: {metrics['score']:.3f}") - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) if __name__ == "__main__": diff --git a/test/manual/test_mla_tp.py b/test/manual/test_mla_tp.py index e957cf2de..5684e7b50 100644 --- a/test/manual/test_mla_tp.py +++ b/test/manual/test_mla_tp.py @@ -4,7 +4,7 @@ from types import SimpleNamespace import torch from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -36,30 +36,30 @@ class TestDeepseekTP2(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreater(metrics["accuracy"], 0.62) + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.62) def test_gsm8k_bs1(self): # test torch compile accuracy for bs=1 args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=10, - max_new_tokens=512, - parallel=1, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=10, + num_threads=1, ) - metrics = run_eval_few_shot_gsm8k(args) - self.assertGreater(metrics["accuracy"], 0.62) + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.62) if __name__ == "__main__": diff --git a/test/manual/test_torch_flex_attention_backend.py b/test/manual/test_torch_flex_attention_backend.py index 832ac14c4..891471bae 100644 --- a/test/manual/test_torch_flex_attention_backend.py +++ b/test/manual/test_torch_flex_attention_backend.py @@ -7,7 +7,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -30,17 +30,17 @@ class TestTorchFlexAttnBackend(CustomTestCase): try: args = SimpleNamespace( + base_url=base_url, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=10, num_shots=8, - data_path=None, - num_questions=100, - parallel=10, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) finally: kill_process_tree(process.pid) diff --git a/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py index 7babc15bd..6f89bdfaa 100644 --- a/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -72,18 +72,18 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=512, - parallel=512, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=512, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) class TestDummyWithSBO(CustomTestCase): @@ -148,15 +148,16 @@ class TestDummyWithSBO(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=512, num_shots=0, - data_path=None, - num_questions=512, - parallel=512, - max_new_tokens=16, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") diff --git a/test/registered/8-gpu-models/test_deepseek_v32_basic.py b/test/registered/8-gpu-models/test_deepseek_v32_basic.py index b1c3c3cf9..c3c8430de 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32_basic.py +++ b/test/registered/8-gpu-models/test_deepseek_v32_basic.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -49,22 +49,23 @@ class TestDeepseekV32DP(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1400, + num_threads=1400, num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) @@ -106,22 +107,23 @@ class TestDeepseekV32TP(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1400, + num_threads=1400, num_shots=20, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) diff --git a/test/registered/8-gpu-models/test_deepseek_v32_mtp.py b/test/registered/8-gpu-models/test_deepseek_v32_mtp.py index 75498322f..d32cabcaf 100644 --- a/test/registered/8-gpu-models/test_deepseek_v32_mtp.py +++ b/test/registered/8-gpu-models/test_deepseek_v32_mtp.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -64,15 +64,16 @@ class TestDeepseekV32DPMTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -84,10 +85,10 @@ class TestDeepseekV32DPMTP(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(metrics["score"], 0.94) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): @@ -150,15 +151,16 @@ class TestDeepseekV32DPMTPV2(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -170,10 +172,10 @@ class TestDeepseekV32DPMTPV2(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(metrics["score"], 0.94) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): @@ -232,15 +234,16 @@ class TestDeepseekV32TPMTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -252,10 +255,10 @@ class TestDeepseekV32TPMTP(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(metrics["score"], 0.94) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): @@ -315,15 +318,16 @@ class TestDeepseekV32TPMTPV2(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -335,10 +339,10 @@ class TestDeepseekV32TPMTPV2(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(metrics["score"], 0.94) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): diff --git a/test/registered/8-gpu-models/test_deepseek_v3_basic.py b/test/registered/8-gpu-models/test_deepseek_v3_basic.py index 08a683f52..acbde5475 100644 --- a/test/registered/8-gpu-models/test_deepseek_v3_basic.py +++ b/test/registered/8-gpu-models/test_deepseek_v3_basic.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -47,22 +47,23 @@ class TestDeepseekV3Basic(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1400, + num_threads=1400, num_shots=8, - data_path=None, - num_questions=1400, - parallel=1400, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v3)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) diff --git a/test/registered/8-gpu-models/test_deepseek_v3_mtp.py b/test/registered/8-gpu-models/test_deepseek_v3_mtp.py index 31e99ab08..82cb22285 100644 --- a/test/registered/8-gpu-models/test_deepseek_v3_mtp.py +++ b/test/registered/8-gpu-models/test_deepseek_v3_mtp.py @@ -5,7 +5,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -61,15 +61,15 @@ class TestDeepseekV3MTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -81,10 +81,10 @@ class TestDeepseekV3MTP(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) self.assertGreater(avg_spec_accept_length, 2.8) def test_bs_1_speed(self): diff --git a/test/registered/8-gpu-models/test_mimo_models.py b/test/registered/8-gpu-models/test_mimo_models.py index f9dc8165d..a32fd3ac5 100644 --- a/test/registered/8-gpu-models/test_mimo_models.py +++ b/test/registered/8-gpu-models/test_mimo_models.py @@ -11,7 +11,7 @@ register_cuda_ci(est_time=200, suite="stage-c-test-8-gpu-h200") class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): gsm8k_accuracy_thres = 0.75 gsm8k_num_questions = 1319 - gsm8k_parallel = 1319 + gsm8k_num_threads = 1319 model = "XiaomiMiMo/MiMo-V2-Flash" other_args = [ diff --git a/test/registered/attention/test_fa3.py b/test/registered/attention/test_fa3.py index 56969b9c8..472a2099a 100644 --- a/test/registered/attention/test_fa3.py +++ b/test/registered/attention/test_fa3.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE3, DEFAULT_MODEL_NAME_FOR_TEST, @@ -36,7 +36,6 @@ OFFLINE_PATH_DICT = { GSM_DATASET_PATH: "/shared/public/data/gsm8k/test.jsonl", } - if OFFLINE_MODE: DEFAULT_MODEL_NAME_FOR_TEST = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST] DEFAULT_DRAFT_MODEL_EAGLE3 = OFFLINE_PATH_DICT[DEFAULT_DRAFT_MODEL_EAGLE3] @@ -46,7 +45,6 @@ if OFFLINE_MODE: ] GSM_DATASET_PATH = OFFLINE_PATH_DICT[GSM_DATASET_PATH] - # Default server arguments shared across all tests DEFAULT_SERVER_ARGS = [ "--trust-remote-code", @@ -99,19 +97,21 @@ class BaseFlashAttentionTest(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, num_shots=4, - num_questions=100, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - data_path=GSM_DATASET_PATH, + gsm8k_data_path=GSM_DATASET_PATH, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") # Use the appropriate metric key based on the test class - metric_key = "accuracy" + metric_key = "score" self.assertGreater(metrics[metric_key], self.accuracy_threshold) if self.speculative_decode: diff --git a/test/registered/attention/test_flash_attention_4.py b/test/registered/attention/test_flash_attention_4.py index 656309820..3c9c4242b 100644 --- a/test/registered/attention/test_flash_attention_4.py +++ b/test/registered/attention/test_flash_attention_4.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -42,18 +42,18 @@ class TestFlashAttention4(unittest.TestCase): def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=200, - host=f"{parsed_url.scheme}://{parsed_url.hostname}", - port=parsed_url.port, + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=200, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.89) + self.assertGreater(metrics["score"], 0.89) if __name__ == "__main__": diff --git a/test/registered/attention/test_hybrid_attn_backend.py b/test/registered/attention/test_hybrid_attn_backend.py index 1c70ee032..22889b568 100644 --- a/test/registered/attention/test_hybrid_attn_backend.py +++ b/test/registered/attention/test_hybrid_attn_backend.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, DEFAULT_MODEL_NAME_FOR_TEST, @@ -76,21 +76,20 @@ class TestHybridAttnBackendBase(CustomTestCase): def test_gsm8k(self): requests.get(self.base_url + "/flush_cache") + model = DEFAULT_TARGET_MODEL_EAGLE if self.speculative_decode else self.model args = SimpleNamespace( - num_shots=4, - num_questions=100, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - data_path=GSM_DATASET_PATH, + base_url=self.base_url, + model=model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") - # Use the appropriate metric key based on the test class - metric_key = "accuracy" - self.assertGreater(metrics[metric_key], self.accuracy_threshold) + self.assertGreater(metrics["score"], self.accuracy_threshold) if self.speculative_decode: server_info = requests.get(self.base_url + "/get_server_info") diff --git a/test/registered/attention/test_local_attn.py b/test/registered/attention/test_local_attn.py index d1abe1b75..229260c19 100644 --- a/test/registered/attention/test_local_attn.py +++ b/test/registered/attention/test_local_attn.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -56,19 +56,20 @@ class TestFlashAttention3LocalAttn(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, num_shots=4, - num_questions=100, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - data_path=None, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") # Use the appropriate metric key based on the test class - metric_key = "accuracy" + metric_key = "score" self.assertGreater(metrics[metric_key], self.accuracy_threshold) diff --git a/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py b/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py index b822bf3a4..74e89060f 100644 --- a/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py +++ b/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -72,18 +72,18 @@ class TestDeepseekR1Fp8Flashinfer(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=512, - parallel=512, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=512, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) if __name__ == "__main__": diff --git a/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py b/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py index c3a509efa..b547409c1 100644 --- a/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py +++ b/test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -52,23 +52,24 @@ class TestDeepseekV3FP4CutlassMoE(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, num_shots=8, - data_path=None, - num_questions=1319, - parallel=1319, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) if __name__ == "__main__": diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py index 42164bc2b..11aed30fa 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -48,17 +48,17 @@ class TestFlashinferTrtllmGenAttnBackend(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) if __name__ == "__main__": diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index b8e76570c..b63447a60 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -49,17 +49,17 @@ class FlashinferTrtllmGenMoeBackendFP8Base: def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) class FlashinferTrtllmGenMoeBackendBF16Base: @@ -97,17 +97,17 @@ class FlashinferTrtllmGenMoeBackendBF16Base: def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) class FlashinferTrtllmGenMoeBackendMXFP8Base: @@ -144,17 +144,17 @@ class FlashinferTrtllmGenMoeBackendMXFP8Base: def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) class TestFlashinferTrtllmGenMoeBackendFP8( diff --git a/test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py b/test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py index f215af49b..4011d3b2a 100644 --- a/test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py +++ b/test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -51,17 +51,18 @@ class TestFlashinferTrtllmGenMoeBackend(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, num_shots=8, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=1319, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.88) + self.assertGreater(metrics["score"], 0.88) if __name__ == "__main__": diff --git a/test/registered/cp/test_deepseek_v32_cp_single_node.py b/test/registered/cp/test_deepseek_v32_cp_single_node.py index d590130d1..89a15ce14 100644 --- a/test/registered/cp/test_deepseek_v32_cp_single_node.py +++ b/test/registered/cp/test_deepseek_v32_cp_single_node.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -68,23 +68,24 @@ class TestDeepseekV32CPInSeqSplit(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=32, num_shots=20, - data_path=None, - num_questions=500, - parallel=32, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( f"### test_a_gsm8k (deepseek-v32-cp-in-seq-split)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) class TestDeepseekV32CPRoundRobinSplit(CustomTestCase): @@ -134,23 +135,24 @@ class TestDeepseekV32CPRoundRobinSplit(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=32, num_shots=20, - data_path=None, - num_questions=500, - parallel=32, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( f"### test_a_gsm8k (deepseek-v32-cp-in-seq-split)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) if __name__ == "__main__": diff --git a/test/registered/disaggregation/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py index aaaffdee2..6b03753ef 100644 --- a/test/registered/disaggregation/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -8,7 +8,7 @@ import requests from transformers import AutoTokenizer from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -81,18 +81,17 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=f"http://{self.base_host}:{self.lb_port}", + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) def test_logprob(self): prompt = "The capital of france is " @@ -260,18 +259,17 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=f"http://{self.base_host}:{self.lb_port}", + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) # Expect lots of failure but the server cannot crash try: - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") except Exception as e: print(f"Test encountered expected errors: {e}") @@ -362,18 +360,17 @@ class TestDisaggregationMooncakeSpec(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=f"http://{self.base_host}:{self.lb_port}", + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.74) + self.assertGreater(metrics["score"], 0.74) class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase): @@ -440,18 +437,17 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=f"http://{self.base_host}:{self.lb_port}", + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) if __name__ == "__main__": diff --git a/test/registered/distributed/test_disaggregation_aarch64.py b/test/registered/distributed/test_disaggregation_aarch64.py index be9ad8b58..c5a6f3e93 100644 --- a/test/registered/distributed/test_disaggregation_aarch64.py +++ b/test/registered/distributed/test_disaggregation_aarch64.py @@ -3,7 +3,7 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -82,18 +82,18 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) if __name__ == "__main__": diff --git a/test/registered/distributed/test_disaggregation_different_tp.py b/test/registered/distributed/test_disaggregation_different_tp.py index 4f5654f70..bbf9f75aa 100644 --- a/test/registered/distributed/test_disaggregation_different_tp.py +++ b/test/registered/distributed/test_disaggregation_different_tp.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -79,18 +79,18 @@ class TestDisaggregationMooncakePrefillLargerTP(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase): @@ -154,18 +154,18 @@ class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase): @@ -229,18 +229,18 @@ class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase): @@ -304,18 +304,18 @@ class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) if __name__ == "__main__": diff --git a/test/registered/distributed/test_disaggregation_dp_attention.py b/test/registered/distributed/test_disaggregation_dp_attention.py index c851650c4..ba88a48d7 100644 --- a/test/registered/distributed/test_disaggregation_dp_attention.py +++ b/test/registered/distributed/test_disaggregation_dp_attention.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.bench_serving import run_benchmark from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -94,18 +94,18 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1400, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1400, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestDisaggregationDPAttentionRoundRobin(TestDisaggregationDPAttention): diff --git a/test/registered/distributed/test_disaggregation_hybrid_attention.py b/test/registered/distributed/test_disaggregation_hybrid_attention.py index 4bbcb2914..87eb64f2f 100644 --- a/test/registered/distributed/test_disaggregation_hybrid_attention.py +++ b/test/registered/distributed/test_disaggregation_hybrid_attention.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -74,18 +74,18 @@ class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBase): @@ -150,19 +150,19 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") # TODO: Fix PD disaggregation accuracy issue (https://github.com/sgl-project/sglang/issues/21744) and increase the threshold back to 0.93. - self.assertGreater(metrics["accuracy"], 0.90) + self.assertGreater(metrics["score"], 0.90) class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase): @@ -229,19 +229,19 @@ class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase) def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Evaluation metrics: {metrics}") # TODO: Fix PD disaggregation accuracy issue (https://github.com/sgl-project/sglang/issues/21744) and increase the threshold back to 0.93. - self.assertGreater(metrics["accuracy"], 0.90) + self.assertGreater(metrics["score"], 0.90) if __name__ == "__main__": diff --git a/test/registered/distributed/test_disaggregation_pp.py b/test/registered/distributed/test_disaggregation_pp.py index 7c441b3c8..683da93a6 100644 --- a/test/registered/distributed/test_disaggregation_pp.py +++ b/test/registered/distributed/test_disaggregation_pp.py @@ -3,7 +3,7 @@ import unittest from types import SimpleNamespace from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, ) @@ -78,18 +78,18 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.24) + self.assertGreater(metrics["score"], 0.24) # Wait a little bit so that the memory check happens. time.sleep(5) @@ -156,18 +156,18 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.24) + self.assertGreater(metrics["score"], 0.24) # Wait a little bit so that the memory check happens. time.sleep(5) @@ -235,18 +235,18 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host=f"http://{self.base_host}", - port=int(self.lb_port), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.24) + self.assertGreater(metrics["score"], 0.24) # Wait a little bit so that the memory check happens. time.sleep(5) diff --git a/test/registered/distributed/test_dp_attention.py b/test/registered/distributed/test_dp_attention.py index e523ed679..f991467f8 100644 --- a/test/registered/distributed/test_dp_attention.py +++ b/test/registered/distributed/test_dp_attention.py @@ -7,12 +7,12 @@ from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_IMAGE_URL, DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -154,18 +154,18 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -173,7 +173,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( ] print( f"###test_gsm8k (deepseek-v3 mtp + dp):\n" - f"accuracy={metrics['accuracy']=:.3f}\n" + f"accuracy={metrics['score']=:.3f}\n" f"{avg_spec_accept_length=:.3f}\n" ) self.assertGreater(avg_spec_accept_length, 2.5) diff --git a/test/registered/distributed/test_dp_attention_large.py b/test/registered/distributed/test_dp_attention_large.py index 00e3f18e8..47210e603 100644 --- a/test/registered/distributed/test_dp_attention_large.py +++ b/test/registered/distributed/test_dp_attention_large.py @@ -6,7 +6,6 @@ import requests from sglang.lang.chat_template import get_chat_template_by_model_path from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin @@ -115,18 +114,18 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -134,7 +133,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP( ] print( f"###test_gsm8k (deepseek-v3 mtp + dp):\n" - f"accuracy={metrics['accuracy']=:.3f}\n" + f"accuracy={metrics['score']=:.3f}\n" f"{avg_spec_accept_length=:.3f}\n" ) self.assertGreater(avg_spec_accept_length, 2.5) diff --git a/test/registered/distributed/test_pp_single_node.py b/test/registered/distributed/test_pp_single_node.py index e2512f828..76e1c068d 100644 --- a/test/registered/distributed/test_pp_single_node.py +++ b/test/registered/distributed/test_pp_single_node.py @@ -16,7 +16,6 @@ from sglang.bench_one_batch_server import BenchArgs as OneBatchBenchArgs from sglang.srt.server_args import ServerArgs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, @@ -60,22 +59,22 @@ class TestPPAccuracy(unittest.TestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=DEFAULT_MODEL_NAME_FOR_TEST, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_amd_ci(): # AMD triton backend produces slightly lower accuracy than FA3 on NVIDIA - self.assertGreater(metrics["accuracy"], 0.70) + self.assertGreater(metrics["score"], 0.70) else: - self.assertGreater(metrics["accuracy"], 0.74) + self.assertGreater(metrics["score"], 0.74) # Wait a little bit so that the memory check happens. time.sleep(4) @@ -169,18 +168,18 @@ class TestQwenVLPPAccuracy(unittest.TestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") - self.assertGreaterEqual(metrics["accuracy"], 0.65) + self.assertGreaterEqual(metrics["score"], 0.65) # Wait a little bit so that the memory check happens. time.sleep(4) @@ -223,15 +222,15 @@ class TestQwenPPAccuracy(unittest.TestCase): try: args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=512, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model_name, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) time.sleep(5) return metrics finally: @@ -244,13 +243,13 @@ class TestQwenPPAccuracy(unittest.TestCase): print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") - self.assertGreaterEqual(baseline["accuracy"], 0.74) + self.assertGreaterEqual(baseline["score"], 0.74) self.assertGreaterEqual( - pp_metrics["accuracy"], - baseline["accuracy"] - 0.02, + pp_metrics["score"], + baseline["score"] - 0.02, msg=( f"PP accuracy dropped more than 2% compared to baseline. " - f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}" ), ) @@ -279,15 +278,15 @@ class TestQwenPPTieWeightsAccuracy(unittest.TestCase): try: args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=512, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model_name, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) time.sleep(5) return metrics finally: @@ -299,13 +298,13 @@ class TestQwenPPTieWeightsAccuracy(unittest.TestCase): print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") - self.assertGreaterEqual(baseline["accuracy"], 0.38) + self.assertGreaterEqual(baseline["score"], 0.38) self.assertGreaterEqual( - pp_metrics["accuracy"], - baseline["accuracy"] - 0.02, + pp_metrics["score"], + baseline["score"] - 0.02, msg=( f"PP accuracy dropped more than 2% compared to baseline. " - f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}" ), ) @@ -331,15 +330,15 @@ class TestQwenMoePPAccuracy(unittest.TestCase): try: args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=512, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model_name, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) time.sleep(5) return metrics finally: @@ -351,13 +350,13 @@ class TestQwenMoePPAccuracy(unittest.TestCase): print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") - self.assertGreaterEqual(baseline["accuracy"], 0.74) + self.assertGreaterEqual(baseline["score"], 0.74) self.assertGreaterEqual( - pp_metrics["accuracy"], - baseline["accuracy"] - 0.02, + pp_metrics["score"], + baseline["score"] - 0.02, msg=( f"PP accuracy dropped more than 2% compared to baseline. " - f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}" ), ) @@ -390,15 +389,15 @@ class TestQwen35PPAccuracy(unittest.TestCase): try: args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=512, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model_name, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=512, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) time.sleep(5) return metrics finally: @@ -410,13 +409,13 @@ class TestQwen35PPAccuracy(unittest.TestCase): print(f"[Qwen35 PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") - self.assertGreaterEqual(baseline["accuracy"], 0.83) + self.assertGreaterEqual(baseline["score"], 0.83) self.assertGreaterEqual( - pp_metrics["accuracy"], - baseline["accuracy"] - 0.05, + pp_metrics["score"], + baseline["score"] - 0.05, msg=( f"PP accuracy dropped more than 5% compared to baseline. " - f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}" ), ) diff --git a/test/registered/dllm/test_llada2_mini.py b/test/registered/dllm/test_llada2_mini.py index 3bec211ea..28588b0fb 100644 --- a/test/registered/dllm/test_llada2_mini.py +++ b/test/registered/dllm/test_llada2_mini.py @@ -7,7 +7,7 @@ import unittest from types import SimpleNamespace from sglang.srt.utils import kill_process_tree -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -58,18 +58,18 @@ class TestLLaDA2Mini(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.88) + self.assertGreater(metrics["score"], 0.88) if is_in_amd_ci(): self.assertGreater(metrics["output_throughput"], 80) else: diff --git a/test/registered/dllm/test_llada2_mini_amd.py b/test/registered/dllm/test_llada2_mini_amd.py index 68e0cfec9..396ed1df0 100644 --- a/test/registered/dllm/test_llada2_mini_amd.py +++ b/test/registered/dllm/test_llada2_mini_amd.py @@ -9,7 +9,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -55,19 +55,17 @@ class TestLLaDA2MiniAMD(CustomTestCase): def test_gsm8k(self): """Test GSM8K accuracy with DLLM on AMD.""" args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") # Relaxed thresholds for AMD - may need adjustment - self.assertGreater(metrics["accuracy"], 0.80) + self.assertGreater(metrics["score"], 0.80) self.assertGreater(metrics["output_throughput"], 50) def test_bs_1_speed(self): diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index f107b4c40..493f05581 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -5,7 +5,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST, @@ -67,18 +67,18 @@ class TestDeepseek(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1200, - parallel=1200, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1200, + num_threads=1200, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) class TestDeepseekMTP(CustomTestCase): @@ -135,18 +135,18 @@ class TestDeepseekMTP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1200, - parallel=1200, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1200, + num_threads=1200, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -154,7 +154,7 @@ class TestDeepseekMTP(CustomTestCase): ] print( f"###test_gsm8k:\n" - f"accuracy={metrics['accuracy']=:.3f}\n" + f"accuracy={metrics['score']=:.3f}\n" f"{avg_spec_accept_length=:.3f}\n" ) self.assertGreater(avg_spec_accept_length, 1.85) @@ -195,17 +195,17 @@ class TestDeepseekV32TBO(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1200, - parallel=1200, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1200, + num_threads=1200, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) diff --git a/test/registered/ep/test_deepep_small.py b/test/registered/ep/test_deepep_small.py index 911915aaf..179ed576f 100644 --- a/test/registered/ep/test_deepep_small.py +++ b/test/registered/ep/test_deepep_small.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, @@ -52,18 +52,18 @@ class TestPureDP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestHybridDPTP(CustomTestCase): @@ -97,18 +97,18 @@ class TestHybridDPTP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestTP(CustomTestCase): @@ -139,18 +139,18 @@ class TestTP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) @unittest.skip("covered in test_deepep_large.py") @@ -188,18 +188,18 @@ class TestNoGatherdBuffer(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestTBO(CustomTestCase): @@ -240,18 +240,18 @@ class TestTBO(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestTBOWithTPAttn(CustomTestCase): @@ -289,18 +289,18 @@ class TestTBOWithTPAttn(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) # There exists bug when using MTP + TBO + attn_tp_size > 1, currently skip that case. @@ -342,18 +342,18 @@ class TestTBOWithTPAttnAndDenseDP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) @unittest.skip("covered in TestMTPWithTBO") @@ -399,18 +399,18 @@ class TestMTP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -418,7 +418,7 @@ class TestMTP(CustomTestCase): ] print( f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" - f"accuracy={metrics['accuracy']=:.3f}\n" + f"accuracy={metrics['score']=:.3f}\n" f"{avg_spec_accept_length=:.3f}\n" ) self.assertGreater(avg_spec_accept_length, 2.1) @@ -473,18 +473,18 @@ class TestMTPWithTBO(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -492,7 +492,7 @@ class TestMTPWithTBO(CustomTestCase): ] print( f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" - f"accuracy={metrics['accuracy']=:.3f}\n" + f"accuracy={metrics['score']=:.3f}\n" f"{avg_spec_accept_length=:.3f}\n" ) self.assertGreater(avg_spec_accept_length, 2.1) @@ -549,18 +549,18 @@ class TestMTPWithTPAttnAndTBO(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -568,7 +568,7 @@ class TestMTPWithTPAttnAndTBO(CustomTestCase): ] print( f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" - f"accuracy={metrics['accuracy']=:.3f}\n" + f"accuracy={metrics['score']=:.3f}\n" f"{avg_spec_accept_length=:.3f}\n" ) self.assertGreater(avg_spec_accept_length, 2.1) diff --git a/test/registered/ep/test_mooncake_ep_small.py b/test/registered/ep/test_mooncake_ep_small.py index 44fe4882d..e54381847 100644 --- a/test/registered/ep/test_mooncake_ep_small.py +++ b/test/registered/ep/test_mooncake_ep_small.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, @@ -69,18 +69,18 @@ class TestTP(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) @unittest.skipIf(is_in_ci(), "Skip since mooncake-ep fault-tolerant test is flaky.") diff --git a/test/registered/hicache/test_hicache_storage_file_backend.py b/test/registered/hicache/test_hicache_storage_file_backend.py index d1e1cd4cf..12f779412 100644 --- a/test/registered/hicache/test_hicache_storage_file_backend.py +++ b/test/registered/hicache/test_hicache_storage_file_backend.py @@ -19,7 +19,7 @@ import requests from sglang.benchmark.utils import get_tokenizer from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST, @@ -295,15 +295,14 @@ def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03): # First evaluation - populate cache print("Phase 1: Running initial GSM8K evaluation to populate cache...") args_initial = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=64, - host=f"http://{test_instance.base_host}", - port=int(test_instance.base_port), + base_url=f"http://{test_instance.base_host}:{test_instance.base_port}", + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=64, ) - metrics_initial = run_eval_few_shot_gsm8k(args_initial) + metrics_initial = run_eval(args_initial) # Flush cache to force remote storage access print("Phase 2: Flushing device cache...") @@ -311,18 +310,18 @@ def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03): # Second evaluation - should use remote cache print("Phase 3: Running second GSM8K evaluation using remote cache...") - metrics_cached = run_eval_few_shot_gsm8k(args_initial) + metrics_cached = run_eval(args_initial) # Verify accuracy consistency - accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"]) + accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"]) print(f"Accuracy difference: {accuracy_diff:.4f}") # Assertions test_instance.assertGreater( - metrics_initial["accuracy"], 0.6, "Initial accuracy should be reasonable" + metrics_initial["score"], 0.6, "Initial accuracy should be reasonable" ) test_instance.assertGreater( - metrics_cached["accuracy"], 0.6, "Cached accuracy should be reasonable" + metrics_cached["score"], 0.6, "Cached accuracy should be reasonable" ) test_instance.assertLess( accuracy_diff, diff --git a/test/registered/mla/test_flashmla.py b/test/registered/mla/test_flashmla.py index b270358c8..c5f084e42 100644 --- a/test/registered/mla/test_flashmla.py +++ b/test/registered/mla/test_flashmla.py @@ -11,7 +11,7 @@ import torch from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -53,18 +53,18 @@ class TestFlashMLAAttnBackend(unittest.TestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestFlashMLAMTP(CustomTestCase): @@ -112,18 +112,18 @@ class TestFlashMLAMTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/server_info").json() avg_spec_accept_length = server_info["internal_states"][0][ diff --git a/test/registered/mla/test_mla_deepseek_v3.py b/test/registered/mla/test_mla_deepseek_v3.py index 392154e3d..1ab95e75a 100644 --- a/test/registered/mla/test_mla_deepseek_v3.py +++ b/test/registered/mla/test_mla_deepseek_v3.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import is_cuda, is_hip, kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -45,18 +45,18 @@ class TestMLADeepseekV3(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") @@ -82,18 +82,18 @@ class TestMLADeepseekV3DisableFusedFunc(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) @unittest.skipIf(is_hip(), "FA is not available.") @@ -133,18 +133,18 @@ class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestDeepseekV3MTP(CustomTestCase): @@ -186,18 +186,18 @@ class TestDeepseekV3MTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ diff --git a/test/registered/mla/test_mla_flashinfer.py b/test/registered/mla/test_mla_flashinfer.py index 555a54e5e..773b0a3b1 100644 --- a/test/registered/mla/test_mla_flashinfer.py +++ b/test/registered/mla/test_mla_flashinfer.py @@ -6,7 +6,7 @@ import torch from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -47,18 +47,18 @@ class TestFlashinferMLA(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.615) + self.assertGreater(metrics["score"], 0.615) class TestFlashinferMLAMTP(CustomTestCase): @@ -102,18 +102,18 @@ class TestFlashinferMLAMTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info").json() avg_spec_accept_length = server_info["internal_states"][0][ diff --git a/test/registered/mla/test_mla_int8_deepseek_v3.py b/test/registered/mla/test_mla_int8_deepseek_v3.py index 929d26325..bd1991030 100644 --- a/test/registered/mla/test_mla_int8_deepseek_v3.py +++ b/test/registered/mla/test_mla_int8_deepseek_v3.py @@ -6,7 +6,7 @@ import torch from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -48,18 +48,18 @@ class TestMLADeepseekV3ChannelInt8(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreaterEqual(metrics["accuracy"], 0.61) + self.assertGreaterEqual(metrics["score"], 0.61) @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") @@ -104,18 +104,18 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ @@ -155,18 +155,18 @@ class TestMLADeepseekV3BlockInt8(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.62) + self.assertGreater(metrics["score"], 0.62) class TestDeepseekV3MTPBlockInt8(CustomTestCase): @@ -208,18 +208,18 @@ class TestDeepseekV3MTPBlockInt8(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) server_info = requests.get(self.base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ diff --git a/test/registered/models/test_compressed_tensors_models.py b/test/registered/models/test_compressed_tensors_models.py index 77e751669..3ffc328b2 100644 --- a/test/registered/models/test_compressed_tensors_models.py +++ b/test/registered/models/test_compressed_tensors_models.py @@ -5,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.utils import is_hip, kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -35,21 +35,21 @@ class TestCompressedTensorsLlama3FP8(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") if is_hip(): # Lower threshold for AMD because FP8 dtype differs (fp8_fnuz) - self.assertGreaterEqual(metrics["accuracy"], 0.40) + self.assertGreaterEqual(metrics["score"], 0.40) else: - self.assertGreaterEqual(metrics["accuracy"], 0.45) + self.assertGreaterEqual(metrics["score"], 0.45) if __name__ == "__main__": diff --git a/test/registered/models/test_kimi_linear_models.py b/test/registered/models/test_kimi_linear_models.py index c63f6ac32..f61aad77d 100644 --- a/test/registered/models/test_kimi_linear_models.py +++ b/test/registered/models/test_kimi_linear_models.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -32,17 +32,17 @@ class TestKimiLinear(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.88) + self.assertGreater(metrics["score"], 0.88) if __name__ == "__main__": diff --git a/test/registered/models/test_qwen_models.py b/test/registered/models/test_qwen_models.py index 45b2aeb54..24817c816 100644 --- a/test/registered/models/test_qwen_models.py +++ b/test/registered/models/test_qwen_models.py @@ -5,7 +5,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -35,17 +35,17 @@ class TestQwen2(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.78) + self.assertGreater(metrics["score"], 0.78) class TestQwen2FP8(CustomTestCase): @@ -66,17 +66,17 @@ class TestQwen2FP8(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.78) + self.assertGreater(metrics["score"], 0.78) if __name__ == "__main__": diff --git a/test/registered/models/test_transformers_models.py b/test/registered/models/test_transformers_models.py index 325db6e29..b91fd5831 100644 --- a/test/registered/models/test_transformers_models.py +++ b/test/registered/models/test_transformers_models.py @@ -10,6 +10,7 @@ import torch from sglang.srt.utils import is_hip, kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.run_eval import run_eval from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST, @@ -50,26 +51,22 @@ class TestTransformersFallbackEndpoint(CustomTestCase): num_examples=64, num_threads=32, ) - from sglang.test.run_eval import run_eval - metrics = run_eval(args) self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound) def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - from sglang.test.few_shot_gsm8k import run_eval - metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], self.gsm8k_lower_bound) + self.assertGreater(metrics["score"], self.gsm8k_lower_bound) @unittest.skipIf(is_hip(), "TorchAO int4wo quantization is not supported on AMD GPUs") diff --git a/test/registered/moe/test_glm4_moe_models.py b/test/registered/moe/test_glm4_moe_models.py index 59003f5ba..010556b83 100644 --- a/test/registered/moe/test_glm4_moe_models.py +++ b/test/registered/moe/test_glm4_moe_models.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -35,17 +35,17 @@ class TestGLM4MoE(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=100, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.8) + self.assertGreater(metrics["score"], 0.8) if __name__ == "__main__": diff --git a/test/registered/moe/test_moe_ep.py b/test/registered/moe/test_moe_ep.py index 59848383d..6db68e948 100644 --- a/test/registered/moe/test_moe_ep.py +++ b/test/registered/moe/test_moe_ep.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -39,18 +39,17 @@ class TestEp(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) class TestEpDeepGEMM(CustomTestCase): @@ -81,18 +80,17 @@ class TestEpDeepGEMM(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.60) + self.assertGreater(metrics["score"], 0.60) if __name__ == "__main__": diff --git a/test/registered/quant/test_deepseek_v32_fp4_4gpu.py b/test/registered/quant/test_deepseek_v32_fp4_4gpu.py index 711da8b1a..19b0a1dba 100644 --- a/test/registered/quant/test_deepseek_v32_fp4_4gpu.py +++ b/test/registered/quant/test_deepseek_v32_fp4_4gpu.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, @@ -56,23 +56,24 @@ class TestDeepseekV32FP4DP(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) @@ -123,23 +124,24 @@ class TestDeepseekV32FP4TP(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) diff --git a/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py b/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py index 81c6162af..2480e3b4f 100644 --- a/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py +++ b/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, @@ -72,15 +72,16 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -92,10 +93,10 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): @@ -162,15 +163,16 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=500, + num_threads=500, num_shots=20, - data_path=None, - num_questions=500, - parallel=500, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -182,10 +184,10 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): diff --git a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py index 3658eec44..f54149952 100644 --- a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py +++ b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py @@ -4,7 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, @@ -54,23 +54,24 @@ class TestDeepseekV3FP4(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, num_shots=8, - data_path=None, - num_questions=1319, - parallel=1319, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) def test_bs_1_speed(self): args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) @@ -124,23 +125,24 @@ class TestDeepseekV3FP4CutlassMoE(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, num_shots=8, - data_path=None, - num_questions=1319, - parallel=1319, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) class TestDeepseekV3FP4SymmetricMemory(CustomTestCase): @@ -178,23 +180,24 @@ class TestDeepseekV3FP4SymmetricMemory(CustomTestCase): self, ): # Append an "a" to make this test run first (alphabetically) to warm up the server args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=1319, num_shots=8, - data_path=None, - num_questions=1319, - parallel=1319, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") if is_in_ci(): write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n' + f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' ) - self.assertGreater(metrics["accuracy"], 0.93) + self.assertGreater(metrics["score"], 0.93) if __name__ == "__main__": diff --git a/test/registered/quant/test_fp8_blockwise_gemm.py b/test/registered/quant/test_fp8_blockwise_gemm.py index 48a819051..30a04a1fc 100644 --- a/test/registered/quant/test_fp8_blockwise_gemm.py +++ b/test/registered/quant/test_fp8_blockwise_gemm.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -46,18 +46,19 @@ class FP8BlockwiseGemmBase: def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=200, num_shots=8, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=200, - host=f"{parsed_url.scheme}://{parsed_url.hostname}", - port=parsed_url.port, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreaterEqual(metrics["accuracy"], 0.8) + self.assertGreaterEqual(metrics["score"], 0.8) class MXFP8GemmBase: @@ -88,18 +89,19 @@ class MXFP8GemmBase: def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=200, num_shots=8, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=200, - host=f"{parsed_url.scheme}://{parsed_url.hostname}", - port=parsed_url.port, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreaterEqual(metrics["accuracy"], 0.8) + self.assertGreaterEqual(metrics["score"], 0.8) class TestFP8BlockwiseGemmTriton(FP8BlockwiseGemmBase, unittest.TestCase): diff --git a/test/registered/quant/test_fp8kv_triton.py b/test/registered/quant/test_fp8kv_triton.py index 3158e8e24..218119c98 100644 --- a/test/registered/quant/test_fp8kv_triton.py +++ b/test/registered/quant/test_fp8kv_triton.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -41,17 +41,17 @@ class TestFP8KVCacheTritonBackend(CustomTestCase): def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=200, - host=f"{parsed_url.scheme}://{parsed_url.hostname}", - port=parsed_url.port, + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=200, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.70) + self.assertGreater(metrics["score"], 0.70) if __name__ == "__main__": diff --git a/test/registered/quant/test_int4fp8_moe.py b/test/registered/quant/test_int4fp8_moe.py index f5a8f6dca..c46c50447 100644 --- a/test/registered/quant/test_int4fp8_moe.py +++ b/test/registered/quant/test_int4fp8_moe.py @@ -2,7 +2,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -45,14 +45,15 @@ class TestMixtralAccuracy(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1400, + num_threads=128, num_shots=8, - data_path=None, - num_questions=1400, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.56) + self.assertGreater(metrics["score"], 0.56) diff --git a/test/registered/quant/test_modelopt_fp8.py b/test/registered/quant/test_modelopt_fp8.py index b13adbe26..a65e2e20c 100644 --- a/test/registered/quant/test_modelopt_fp8.py +++ b/test/registered/quant/test_modelopt_fp8.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -35,17 +35,17 @@ class TestModeloptFP8(CustomTestCase): def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=200, - host=f"{parsed_url.scheme}://{parsed_url.hostname}", - port=parsed_url.port, + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=200, ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.70) + self.assertGreater(metrics["score"], 0.70) if __name__ == "__main__": diff --git a/test/registered/quant/test_nvfp4_gemm.py b/test/registered/quant/test_nvfp4_gemm.py index 1a94b6b48..f784973c9 100644 --- a/test/registered/quant/test_nvfp4_gemm.py +++ b/test/registered/quant/test_nvfp4_gemm.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from sglang.srt.utils import get_device_sm, kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -47,18 +47,18 @@ class FP4GemmBase: def test_gsm8k(self): parsed_url = urlparse(self.base_url) args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1319, - max_new_tokens=512, - parallel=200, - host=f"{parsed_url.scheme}://{parsed_url.hostname}", - port=parsed_url.port, + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=200, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], 0.64) + self.assertGreater(metrics["score"], 0.64) @unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") diff --git a/test/registered/quant/test_w4a8_deepseek_v3.py b/test/registered/quant/test_w4a8_deepseek_v3.py index a6c33bea3..90c59fd40 100644 --- a/test/registered/quant/test_w4a8_deepseek_v3.py +++ b/test/registered/quant/test_w4a8_deepseek_v3.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -41,18 +41,18 @@ class TestDeepseekV3W4afp8(CustomTestCase): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1200, - parallel=1200, - max_new_tokens=512, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1200, + num_threads=1200, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) class TestDeepseekV3W4Afp8Mtp(CustomTestCase): @@ -95,15 +95,15 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase): self, ): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -115,10 +115,10 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(metrics["score"], 0.935) self.assertGreater(avg_spec_accept_length, 2.9) @@ -163,18 +163,18 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase): self, ): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase): @@ -231,18 +231,18 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase): self, ): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"Eval accuracy of GSM8K: {metrics=}") - self.assertGreater(metrics["accuracy"], 0.92) + self.assertGreater(metrics["score"], 0.92) if __name__ == "__main__": diff --git a/test/registered/quant/test_w8a8_quantization.py b/test/registered/quant/test_w8a8_quantization.py index 88e344831..a2a2a1cb4 100644 --- a/test/registered/quant/test_w8a8_quantization.py +++ b/test/registered/quant/test_w8a8_quantization.py @@ -6,7 +6,7 @@ import requests from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -51,17 +51,17 @@ class BaseW8A8Test(CustomTestCase): self.skipTest("gsm8k_accuracy_threshold not set for this test") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) metrics = run_eval(args) print(metrics) - self.assertGreater(metrics["accuracy"], self.gsm8k_accuracy_threshold) + self.assertGreater(metrics["score"], self.gsm8k_accuracy_threshold) def run_decode(self, max_new_tokens): response = requests.post( diff --git a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py index 58cba7abb..d1bce0c96 100644 --- a/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py +++ b/test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, @@ -18,7 +18,6 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=900, suite="stage-b-test-4-gpu-b200") - FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" SERVER_LAUNCH_TIMEOUT = 1200 @@ -74,15 +73,15 @@ class TestDeepseekV3FP4MTP(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/server_info").json() @@ -94,11 +93,11 @@ class TestDeepseekV3FP4MTP(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3-fp4 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) - self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(metrics["score"], 0.94) self.assertGreater(avg_spec_accept_length, 2.7) def test_bs_1_speed(self): diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index a25edf588..9758b50ae 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -5,7 +5,7 @@ import requests from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.send_one import BenchArgs, send_one_prompt from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN, @@ -76,15 +76,15 @@ class TestEAGLE3EngineDPAttention(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") server_info = requests.get(self.base_url + "/get_server_info") @@ -104,14 +104,14 @@ class TestEAGLE3EngineDPAttention(CustomTestCase): if is_in_ci(): write_github_step_summary( f"### test_gsm8k (EAGLE3 DP Attention)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) if is_in_amd_ci(): # AMD triton backend produces slightly lower accuracy than FA3 on NVIDIA - self.assertGreater(metrics["accuracy"], 0.88) + self.assertGreater(metrics["score"], 0.88) else: - self.assertGreater(metrics["accuracy"], 0.91) + self.assertGreater(metrics["score"], 0.91) if avg_spec_accept_length is not None: if is_in_amd_ci(): # AMD triton backend produces slightly lower accept length than FA3 on NVIDIA diff --git a/test/registered/spec/eagle/test_eagle_infer_b.py b/test/registered/spec/eagle/test_eagle_infer_b.py index b8ab5c2fb..7c726acfb 100644 --- a/test/registered/spec/eagle/test_eagle_infer_b.py +++ b/test/registered/spec/eagle/test_eagle_infer_b.py @@ -12,13 +12,13 @@ import requests from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval from sglang.test.kits.abort_timeout_kit import ( AbortAllMixin, RunningTimeoutTwoWaveMixin, WaitingTimeoutMixin, ) from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test +from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.eagle_fixture import EagleServerBase from sglang.test.test_utils import DEFAULT_TARGET_MODEL_EAGLE, run_logprob_check @@ -48,18 +48,18 @@ class TestEAGLEServerBasic(EagleServerBase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.target_model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_gsm8k_eval(args) + metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["accuracy"], 0.20) + self.assertGreater(metrics["score"], 0.20) server_info = requests.get(self.base_url + "/server_info").json() avg_spec_accept_length = server_info["internal_states"][0][ @@ -103,16 +103,16 @@ class TestEAGLEServerAdditional(TestEAGLEServerBasic): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=1, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.target_model, + eval_name="gsm8k", + api="completion", + max_tokens=1, + num_examples=200, + num_threads=128, ) - metrics = run_gsm8k_eval(args) + metrics = run_eval(args) self.assertGreater(metrics["output_throughput"], 50) def test_logprob_start_len(self): diff --git a/test/registered/spec/eagle/test_eagle_infer_beta.py b/test/registered/spec/eagle/test_eagle_infer_beta.py index 252062611..faee2ae48 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta.py @@ -7,9 +7,9 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval from sglang.test.kits.matched_stop_kit import MatchedStopMixin from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, DEFAULT_TARGET_MODEL_EAGLE, @@ -86,19 +86,19 @@ class TestEagleServerBase(CustomTestCase, MatchedStopMixin): def test_gsm8k(self): args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=1000, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1000, + num_threads=128, ) metrics = run_eval(args) print(f"TestEagleLargeBS -- {metrics=}") self.assertGreater( - metrics["accuracy"], 0.23 - ) # 0.3333 for 60 questions; 0.234 for 1319 questions + metrics["score"], 0.22 + ) # ~0.227 for 1000 questions via /v1/completions assert self.process.poll() is None def test_logprob_spec_v2_match(self): diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py index 8a6e9779f..50b054cdc 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, @@ -20,19 +20,19 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=300, suite="stage-c-test-4-gpu-b200") -def test_gsm8k(base_url: str): +def test_gsm8k(base_url: str, model: str): requests.get(base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(base_url.split(":")[-1]), + base_url=base_url, + model=model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) server_info = requests.get(base_url + "/get_server_info") avg_spec_accept_length = server_info.json()["internal_states"][0][ "avg_spec_accept_length" @@ -84,8 +84,8 @@ class TestEagleDPAttnServerSmall(CustomTestCase): kill_process_tree(cls.process.pid) def test_a_gsm8k(self): - metrics, avg_spec_accept_length = test_gsm8k(self.base_url) - self.assertGreater(metrics["accuracy"], 0.62) + metrics, avg_spec_accept_length = test_gsm8k(self.base_url, self.model) + self.assertGreater(metrics["score"], 0.62) self.assertGreater(avg_spec_accept_length, 2.7) diff --git a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py index 8a7fcd00d..c875e995c 100644 --- a/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py +++ b/test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py @@ -6,7 +6,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -21,19 +21,19 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True) -def test_gsm8k(base_url: str): +def test_gsm8k(base_url: str, model: str): requests.get(base_url + "/flush_cache") args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(base_url.split(":")[-1]), + base_url=base_url, + model=model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) server_info = requests.get(base_url + "/server_info").json() avg_spec_accept_length = server_info["internal_states"][0]["avg_spec_accept_length"] @@ -92,14 +92,14 @@ class TestEagleDPAttnServerLarge(CustomTestCase): kill_process_tree(cls.process.pid) def test_a_gsm8k(self): - metrics, avg_spec_accept_length = test_gsm8k(self.base_url) + metrics, avg_spec_accept_length = test_gsm8k(self.base_url, self.model) - self.assertGreater(metrics["accuracy"], 0.94) + self.assertGreater(metrics["score"], 0.94) self.assertGreater(avg_spec_accept_length, 2.7) if is_in_ci(): write_github_step_summary( f"### test_gsm8k (deepseek-v3-fp4 mtp)\n" - f'{metrics["accuracy"]=:.3f}\n' + f'{metrics["score"]=:.3f}\n' f"{avg_spec_accept_length=:.2f}\n" ) diff --git a/test/registered/spec/test_standalone_speculative_decoding.py b/test/registered/spec/test_standalone_speculative_decoding.py index 1a3cc0647..2b3a1d8a1 100644 --- a/test/registered/spec/test_standalone_speculative_decoding.py +++ b/test/registered/spec/test_standalone_speculative_decoding.py @@ -7,7 +7,7 @@ import requests from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_STANDALONE, DEFAULT_TARGET_MODEL_STANDALONE, @@ -22,7 +22,6 @@ register_cuda_ci(est_time=308, suite="stage-b-test-1-gpu-large") GSM_DATASET_PATH = None - # Default server arguments shared across all tests DEFAULT_SERVER_ARGS = [ "--trust-remote-code", @@ -97,19 +96,21 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, num_shots=4, - num_questions=100, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - data_path=GSM_DATASET_PATH, + gsm8k_data_path=GSM_DATASET_PATH, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") # Use the appropriate metric key based on the test class - metric_key = "accuracy" + metric_key = "score" self.assertGreater(metrics[metric_key], self.accuracy_threshold) server_info = requests.get(self.base_url + "/get_server_info") @@ -158,19 +159,21 @@ class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase): requests.get(self.base_url + "/flush_cache") args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=100, + num_threads=128, num_shots=4, - num_questions=100, - max_new_tokens=512, - parallel=128, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - data_path=GSM_DATASET_PATH, + gsm8k_data_path=GSM_DATASET_PATH, ) - metrics = run_eval_few_shot_gsm8k(args) + metrics = run_eval(args) print(f"{metrics=}") # Use the appropriate metric key based on the test class - metric_key = "accuracy" + metric_key = "score" self.assertGreater(metrics[metric_key], self.accuracy_threshold) server_info = requests.get(self.base_url + "/get_server_info")