Unify GSM8K eval path to Chat API for regression CI readiness (#21667)

python/sglang/test/accuracy_test_runner.py

@@ -432,56 +432,6 @@ def _run_nemo_skills_eval(
             kill_process_tree(process.pid)
 
 
-def _run_few_shot_eval(
-    model: ModelLaunchSettings,
-    base_url: str,
-    num_questions: Optional[int] = None,
-    num_shots: int = 8,
-    max_tokens: int = 512,
-) -> Tuple[bool, Optional[str], Optional[dict]]:
-    """Run evaluation using few_shot backend (few_shot_gsm8k.py).
-
-    Returns:
-        Tuple of (success, error_message, metrics_dict)
-    """
-    from sglang.test.few_shot_gsm8k import run_eval as run_few_shot_eval
-
-    process = None
-    try:
-        process = popen_launch_server(
-            model.model_path,
-            base_url,
-            other_args=model.extra_args,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            env=model.env,
-        )
-
-        args = SimpleNamespace(
-            num_shots=num_shots,
-            data_path=None,
-            num_questions=num_questions or 200,
-            max_new_tokens=max_tokens,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(base_url.split(":")[-1]),
-        )
-
-        metrics = run_few_shot_eval(args)
-
-        # Normalize metrics format (few_shot returns "accuracy", simple_eval returns "score")
-        if "accuracy" in metrics and "score" not in metrics:
-            metrics["score"] = metrics["accuracy"]
-
-        return True, None, metrics
-
-    except Exception as e:
-        return False, f"Few-shot evaluation exception: {str(e)}", None
-
-    finally:
-        if process:
-            kill_process_tree(process.pid)
-
-
 def run_accuracy_test(
     model: ModelLaunchSettings,
     params: AccuracyTestParams,
@@ -507,12 +457,7 @@ def run_accuracy_test(
 
     # Run evaluation based on dataset type
     # - NeMo Skills: mmmu-pro (and other VLM evals needing ns eval)
-    # - few_shot_eval: gsm8k (default, backward compatible)
-    # - simple_eval: everything else (gpqa, mmmu, etc.)
-    has_extended_params = any(
-        getattr(params, field) is not None
-        for field in ("thinking_mode", "temperature", "top_p", "top_k", "repeat")
-    )
+    # - simple_eval: everything else (gsm8k, gpqa, mmlu, mmmu, etc.)
     if params.dataset in ("mmmu-pro", "mmmu_pro"):
         success, error, metrics = _run_nemo_skills_eval(
             model=model,
@@ -523,13 +468,6 @@ def run_accuracy_test(
             temperature=params.temperature,
             top_p=params.top_p,
         )
-    elif params.dataset == "gsm8k" and not has_extended_params:
-        success, error, metrics = _run_few_shot_eval(
-            model=model,
-            base_url=base_url,
-            num_questions=params.num_examples,
-            max_tokens=params.max_tokens or 512,
-        )
     else:
         success, error, metrics = _run_simple_eval(
             model=model,

python/sglang/test/few_shot_gsm8k.py

@@ -1,6 +1,11 @@
 """
 Run few-shot GSM-8K evaluation.
 
+.. deprecated::
+    This module is deprecated. Use ``sglang.test.run_eval`` with
+    ``eval_name="gsm8k"`` instead, which routes through the unified
+    Chat API evaluation framework with dump_metric support.
+
 Usage:
 python3 -m sglang.test.few_shot_gsm8k --num-questions 200
 """
@@ -9,6 +14,7 @@ import argparse
 import ast
 import re
 import time
+import warnings
 
 import numpy as np
 
@@ -50,6 +56,12 @@ def get_answer_value(answer_str):
 
 
 def run_eval(args):
+    warnings.warn(
+        "sglang.test.few_shot_gsm8k is deprecated. "
+        "Use sglang.test.run_eval with eval_name='gsm8k' instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     # Select backend
     set_default_backend(RuntimeEndpoint(normalize_base_url(args.host, args.port)))
 

python/sglang/test/few_shot_gsm8k_engine.py

@@ -1,8 +1,16 @@
+"""
+.. deprecated::
+    This module is deprecated. Use ``sglang.test.run_eval`` with
+    ``eval_name="gsm8k"`` instead, which routes through the unified
+    Chat API evaluation framework with dump_metric support.
+"""
+
 import argparse
 import ast
 import asyncio
 import re
 import time
+import warnings
 from typing import Optional
 
 import numpy as np
@@ -49,6 +57,12 @@ async def concurrent_generate(engine, prompts, sampling_param):
 
 
 def run_eval(args):
+    warnings.warn(
+        "sglang.test.few_shot_gsm8k_engine is deprecated. "
+        "Use sglang.test.run_eval with eval_name='gsm8k' instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     # Select backend
     engine = sgl.Engine(model_path=args.model_path, log_level="error")
 

python/sglang/test/kits/eval_accuracy_kit.py

@@ -3,7 +3,6 @@ from typing import Optional
 
 import requests
 
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_gsm8k
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
 
@@ -19,17 +18,20 @@ def _check_accept_length(test_case, base_url, threshold):
 
 
 class GSM8KMixin:
-    """Mixin for few-shot GSM8K evaluation.
+    """Mixin for GSM8K evaluation via OpenAI Chat API.
 
     Required attributes on the test class:
         base_url: str
         gsm8k_accuracy_thres: float
+
+    Optional attributes:
+        model: str (if not set, auto-detected from server)
     """
 
     gsm8k_accuracy_thres: float = _THRESHOLD_NOT_SET
     gsm8k_accept_length_thres: Optional[float] = None
     gsm8k_num_questions: int = 200
-    gsm8k_parallel: int = 128
+    gsm8k_num_threads: int = 128
 
     def test_gsm8k(self):
         assert (
@@ -39,17 +41,21 @@ class GSM8KMixin:
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=self.gsm8k_num_questions,
-            max_new_tokens=512,
-            parallel=self.gsm8k_parallel,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=self.gsm8k_num_questions,
+            num_threads=self.gsm8k_num_threads,
         )
-        metrics = run_eval_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreaterEqual(metrics["accuracy"], self.gsm8k_accuracy_thres)
+
+        if is_in_ci():
+            write_github_step_summary(f"### test_gsm8k\n{metrics['score']=:.4f}\n")
+
+        self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
 
         if self.gsm8k_accept_length_thres is not None:
             _check_accept_length(self, self.base_url, self.gsm8k_accept_length_thres)

python/sglang/test/run_eval.py

@@ -62,7 +62,7 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
             extra_body[param_name] = value
 
     common_kwargs = dict(
-        model=args.model,
+        model=getattr(args, "model", None),
         max_tokens=getattr(args, "max_tokens", 2048),
         top_p=getattr(args, "top_p", 1.0),
         base_url=base_url,
@@ -71,7 +71,12 @@ def run_eval_once(args, base_url: str, eval_obj: Eval) -> dict:
 
     api_mode = getattr(args, "api", "chat")
     if api_mode == "completion":
-        sampler = CompletionSampler(**common_kwargs)
+        # Default stop tokens for completion API (matches few_shot_gsm8k behavior)
+        stop = getattr(args, "stop", ["Question", "Assistant:", "<|separator|>"])
+        sampler = CompletionSampler(
+            **common_kwargs,
+            stop=stop,
+        )
     else:
         sampler = ChatCompletionSampler(
             **common_kwargs,
@@ -143,7 +148,7 @@ def run_eval(args):
         categories = args.categories.split(",") if args.categories else None
 
         eval_obj = LongBenchV2Eval(
-            model=args.model,
+            model=getattr(args, "model", None),
             data_source=data_source,
             num_examples=args.num_examples,
             num_threads=args.num_threads,

python/sglang/test/server_fixtures/disaggregation_fixture.py

@@ -32,6 +32,7 @@ class PDDisaggregationServerBase(CustomTestCase):
         cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}"
         cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}"
         cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}"
+        cls.base_url = cls.lb_url
         print(
             f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} {cls.bootstrap_port=}"
         )

python/sglang/test/simple_eval_common.py

@@ -185,6 +185,7 @@ class CompletionSampler(SamplerBase):
         temperature: float = 0.0,
         top_p: float = 1.0,
         max_tokens: int = 2048,
+        stop: Optional[List[str]] = None,
     ):
         self.client = OpenAI(base_url=base_url, http_client=LargerHttpxClient())
 
@@ -195,9 +196,10 @@ class CompletionSampler(SamplerBase):
         self.temperature = temperature
         self.top_p = top_p
         self.max_tokens = max_tokens
+        self.stop = stop
         self._completion_tokens: list[int] = []
         print(
-            f"CompletionSampler initialized with {self.model=} {self.temperature=} {self.max_tokens=}"
+            f"CompletionSampler initialized with {self.model=} {self.temperature=} {self.max_tokens=} {self.stop=}"
         )
 
     def _pack_message(self, role: str, content: Any):
@@ -219,6 +221,7 @@ class CompletionSampler(SamplerBase):
                     temperature=self.temperature,
                     top_p=self.top_p,
                     max_tokens=self.max_tokens,
+                    stop=self.stop,
                 )
                 if response.usage and response.usage.completion_tokens is not None:
                     self._completion_tokens.append(response.usage.completion_tokens)

test/manual/ep/test_moe_deepep_eval_accuracy_large.py

@@ -7,7 +7,6 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST,
@@ -44,18 +43,19 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=64,
             num_shots=8,
-            data_path=None,
-            num_questions=200,
-            parallel=64,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
     def test_mmlu(self):
         args = SimpleNamespace(

test/manual/ep/test_mooncake_expert_backup.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
@@ -24,6 +24,7 @@ class TestBackup(CustomTestCase):
     def setUpClass(cls):
         cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA
         cls.base_port = 20000
+        cls.base_url = f"http://127.0.0.1:{cls.base_port}"
         cls.num_processes = 2
         # TODO (stage 100): in the future, implement a specified multiprocess launcher
         cls.processes = [
@@ -124,18 +125,18 @@ class TestBackup(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=self.base_port,
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 if __name__ == "__main__":

test/manual/ep/test_nixl_ep.py

@@ -4,7 +4,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
@@ -71,21 +71,21 @@ class _EPTestBase(CustomTestCase):
 
     def _run_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
         return metrics
 
     def test_gsm8k(self):
         metrics = self._run_gsm8k()
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestNixlEPTP(_EPTestBase):
@@ -108,7 +108,7 @@ class TestNixlMoeMooncakeElasticEP(_EPTestBase):
     def test_gsm8k_fault_1(self):
         os.system(f"pkill -f {self.pkill_process_1}")
         metrics = self._run_gsm8k()
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 if __name__ == "__main__":

test/manual/hicache/test_pp_with_hicache.py

@@ -13,7 +13,7 @@ from urllib.parse import urlparse
 import requests
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -189,24 +189,24 @@ class TestPPWithHiCache(unittest.TestCase):
 
     def test_eval_accuracy(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=40,
-            max_new_tokens=256,
-            parallel=24,
-            host=f"http://{self.base_host}",
-            port=int(self.base_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=40,
+            num_threads=24,
         )
 
-        metrics_initial = run_eval_few_shot_gsm8k(args)
-        self.assertGreater(metrics_initial["accuracy"], 0.6)
+        metrics_initial = run_eval(args)
+        self.assertGreater(metrics_initial["score"], 0.6)
 
         self.flush_cache()
 
-        metrics_cached = run_eval_few_shot_gsm8k(args)
-        self.assertGreater(metrics_cached["accuracy"], 0.6)
+        metrics_cached = run_eval(args)
+        self.assertGreater(metrics_cached["score"], 0.6)
 
-        accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"])
+        accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"])
         self.assertLess(accuracy_diff, 0.05)
 
 

test/manual/models/test_falcon_h1_models.py

@@ -1,7 +1,7 @@
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -31,17 +31,17 @@ class TestFalconH1(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.74)
+        self.assertGreater(metrics["score"], 0.74)
 
 
 class TestFalconH1TP4(CustomTestCase):
@@ -65,17 +65,17 @@ class TestFalconH1TP4(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.74)
+        self.assertGreater(metrics["score"], 0.74)
 
 
 class TestFalconH1NoGatedRMS(CustomTestCase):
@@ -99,17 +99,17 @@ class TestFalconH1NoGatedRMS(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.74)
+        self.assertGreater(metrics["score"], 0.74)
 
 
 class TestFalconH1NoGatedTP4(CustomTestCase):
@@ -133,14 +133,14 @@ class TestFalconH1NoGatedTP4(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.74)
+        self.assertGreater(metrics["score"], 0.74)

test/manual/models/test_grok_models.py

@@ -2,7 +2,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -34,13 +34,13 @@ class TestGrok(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=64,
-            max_new_tokens=256,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=64,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")

test/manual/models/test_kimi_k2_models.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 import requests
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -48,22 +48,22 @@ class TestKimiK2Thinking(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (Kimi-K2-Thinking)\n" f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.95)
+            self.assertGreater(metrics["score"], 0.95)
 
 
 if __name__ == "__main__":

test/manual/models/test_llama4_models.py

@@ -2,7 +2,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -44,17 +44,16 @@ class TestLlama4(CustomTestCase):
                     ],
                 )
                 args = SimpleNamespace(
-                    num_shots=5,
-                    data_path=None,
-                    num_questions=200,
-                    max_new_tokens=512,
-                    parallel=128,
-                    host="http://127.0.0.1",
-                    port=int(self.base_url.split(":")[-1]),
+                    base_url=self.base_url,
+                    eval_name="gsm8k",
+                    api="completion",
+                    max_tokens=512,
+                    num_examples=200,
+                    num_threads=128,
                 )
                 metrics = run_eval(args)
                 print(f"{metrics=}")
-                self.assertGreaterEqual(metrics["accuracy"], model.accuracy)
+                self.assertGreaterEqual(metrics["score"], model.accuracy)
             except Exception as e:
                 print(f"Error testing {model.model}: {e}")
                 self.fail(f"Test failed for {model.model}: {e}")

test/manual/models/test_mistral_large3_basic.py

@@ -3,7 +3,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -53,22 +53,23 @@ class TestMistralLarge3Basic(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1400,
+            num_threads=1400,
             num_shots=8,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (mistral-large-3)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (mistral-large-3)\n" f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.90)
+            self.assertGreater(metrics["score"], 0.90)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)

test/manual/models/test_mtp_models.py

@@ -2,7 +2,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -41,17 +41,17 @@ class TestMiMoMTP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.7)
+        self.assertGreater(metrics["score"], 0.7)
 
 
 if __name__ == "__main__":

test/manual/models/test_unsloth_models.py

@@ -2,7 +2,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -29,17 +29,17 @@ class TestUnslothPhi4(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.78)
+        self.assertGreater(metrics["score"], 0.78)
 
 
 class TestUnslothPhi4Bnb4bit(CustomTestCase):
@@ -63,17 +63,17 @@ class TestUnslothPhi4Bnb4bit(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.75)
+        self.assertGreater(metrics["score"], 0.75)
 
 
 class TestUnslothPhi4UnslothBnb4bit(CustomTestCase):
@@ -97,17 +97,17 @@ class TestUnslothPhi4UnslothBnb4bit(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.75)
+        self.assertGreater(metrics["score"], 0.75)
 
 
 class TestUnslothPhi4MiniInstruct(CustomTestCase):
@@ -128,17 +128,17 @@ class TestUnslothPhi4MiniInstruct(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.65)
+        self.assertGreater(metrics["score"], 0.65)
 
 
 class TestUnslothPhi4MiniBnb4bit(CustomTestCase):
@@ -162,17 +162,17 @@ class TestUnslothPhi4MiniBnb4bit(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.6)
+        self.assertGreater(metrics["score"], 0.6)
 
 
 class TestUnslothPhi4MiniUnslothBnb4bit(CustomTestCase):
@@ -196,17 +196,17 @@ class TestUnslothPhi4MiniUnslothBnb4bit(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.6)
+        self.assertGreater(metrics["score"], 0.6)
 
 
 if __name__ == "__main__":

test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py

@@ -1,7 +1,7 @@
 import unittest
 from types import SimpleNamespace
 
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -70,18 +70,18 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase):
     def test_gsm8k_accuracy(self):
         """Verify that piecewise cuda graph works correctly in prefill server"""
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"GSM8K accuracy with piecewise cuda graph: {metrics['accuracy']:.3f}")
+        metrics = run_eval(args)
+        print(f"GSM8K accuracy with piecewise cuda graph: {metrics['score']:.3f}")
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 if __name__ == "__main__":

test/manual/test_mla_tp.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 import torch
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -36,30 +36,30 @@ class TestDeepseekTP2(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
-        self.assertGreater(metrics["accuracy"], 0.62)
+        metrics = run_eval(args)
+        self.assertGreater(metrics["score"], 0.62)
 
     def test_gsm8k_bs1(self):
         # test torch compile accuracy for bs=1
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=10,
-            max_new_tokens=512,
-            parallel=1,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=10,
+            num_threads=1,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
-        self.assertGreater(metrics["accuracy"], 0.62)
+        metrics = run_eval(args)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 if __name__ == "__main__":

test/manual/test_torch_flex_attention_backend.py

@@ -7,7 +7,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -30,17 +30,17 @@ class TestTorchFlexAttnBackend(CustomTestCase):
 
         try:
             args = SimpleNamespace(
+                base_url=base_url,
+                eval_name="gsm8k",
+                api="completion",
+                max_tokens=512,
+                num_examples=100,
+                num_threads=10,
                 num_shots=8,
-                data_path=None,
-                num_questions=100,
-                parallel=10,
-                max_new_tokens=512,
-                host="http://127.0.0.1",
-                port=int(base_url.split(":")[-1]),
             )
-            metrics = run_eval_few_shot_gsm8k(args)
+            metrics = run_eval(args)
             print(f"{metrics=}")
-            self.assertGreater(metrics["accuracy"], 0.62)
+            self.assertGreater(metrics["score"], 0.62)
         finally:
             kill_process_tree(process.pid)
 

test/registered/4-gpu-models/test_deepseek_v3_cutedsl_4gpu.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -72,18 +72,18 @@ class TestDeepseekR1Nvfp4CuteDSLDeepEP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=512,
-            parallel=512,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=512,
+            num_threads=512,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
 
 class TestDummyWithSBO(CustomTestCase):
@@ -148,15 +148,16 @@ class TestDummyWithSBO(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=512,
+            num_threads=512,
             num_shots=0,
-            data_path=None,
-            num_questions=512,
-            parallel=512,
-            max_new_tokens=16,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
 

test/registered/8-gpu-models/test_deepseek_v32_basic.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -49,22 +49,23 @@ class TestDeepseekV32DP(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1400,
+            num_threads=1400,
             num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
@@ -106,22 +107,23 @@ class TestDeepseekV32TP(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1400,
+            num_threads=1400,
             num_shots=20,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v32)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v32)\n" f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)

test/registered/8-gpu-models/test_deepseek_v32_mtp.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -64,15 +64,16 @@ class TestDeepseekV32DPMTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -84,10 +85,10 @@ class TestDeepseekV32DPMTP(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v32 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.94)
+            self.assertGreater(metrics["score"], 0.94)
             self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):
@@ -150,15 +151,16 @@ class TestDeepseekV32DPMTPV2(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -170,10 +172,10 @@ class TestDeepseekV32DPMTPV2(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v32 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.94)
+            self.assertGreater(metrics["score"], 0.94)
             self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):
@@ -232,15 +234,16 @@ class TestDeepseekV32TPMTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -252,10 +255,10 @@ class TestDeepseekV32TPMTP(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v32 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.94)
+            self.assertGreater(metrics["score"], 0.94)
             self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):
@@ -315,15 +318,16 @@ class TestDeepseekV32TPMTPV2(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -335,10 +339,10 @@ class TestDeepseekV32TPMTPV2(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v32 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.94)
+            self.assertGreater(metrics["score"], 0.94)
             self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):

test/registered/8-gpu-models/test_deepseek_v3_basic.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -47,22 +47,23 @@ class TestDeepseekV3Basic(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1400,
+            num_threads=1400,
             num_shots=8,
-            data_path=None,
-            num_questions=1400,
-            parallel=1400,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v3)\n" f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)

test/registered/8-gpu-models/test_deepseek_v3_mtp.py

@@ -5,7 +5,7 @@ import requests
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -61,15 +61,15 @@ class TestDeepseekV3MTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -81,10 +81,10 @@ class TestDeepseekV3MTP(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v3 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
             self.assertGreater(avg_spec_accept_length, 2.8)
 
     def test_bs_1_speed(self):

test/registered/8-gpu-models/test_mimo_models.py

@@ -11,7 +11,7 @@ register_cuda_ci(est_time=200, suite="stage-c-test-8-gpu-h200")
 class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase):
     gsm8k_accuracy_thres = 0.75
     gsm8k_num_questions = 1319
-    gsm8k_parallel = 1319
+    gsm8k_num_threads = 1319
     model = "XiaomiMiMo/MiMo-V2-Flash"
 
     other_args = [

test/registered/attention/test_fa3.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE3,
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -36,7 +36,6 @@ OFFLINE_PATH_DICT = {
     GSM_DATASET_PATH: "/shared/public/data/gsm8k/test.jsonl",
 }
 
-
 if OFFLINE_MODE:
     DEFAULT_MODEL_NAME_FOR_TEST = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST]
     DEFAULT_DRAFT_MODEL_EAGLE3 = OFFLINE_PATH_DICT[DEFAULT_DRAFT_MODEL_EAGLE3]
@@ -46,7 +45,6 @@ if OFFLINE_MODE:
     ]
     GSM_DATASET_PATH = OFFLINE_PATH_DICT[GSM_DATASET_PATH]
 
-
 # Default server arguments shared across all tests
 DEFAULT_SERVER_ARGS = [
     "--trust-remote-code",
@@ -99,19 +97,21 @@ class BaseFlashAttentionTest(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
             num_shots=4,
-            num_questions=100,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-            data_path=GSM_DATASET_PATH,
+            gsm8k_data_path=GSM_DATASET_PATH,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         # Use the appropriate metric key based on the test class
-        metric_key = "accuracy"
+        metric_key = "score"
         self.assertGreater(metrics[metric_key], self.accuracy_threshold)
 
         if self.speculative_decode:

test/registered/attention/test_flash_attention_4.py

@@ -4,7 +4,7 @@ from urllib.parse import urlparse
 
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -42,18 +42,18 @@ class TestFlashAttention4(unittest.TestCase):
     def test_gsm8k(self):
         parsed_url = urlparse(self.base_url)
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1319,
-            max_new_tokens=512,
-            parallel=200,
-            host=f"{parsed_url.scheme}://{parsed_url.hostname}",
-            port=parsed_url.port,
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=200,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.89)
+        self.assertGreater(metrics["score"], 0.89)
 
 
 if __name__ == "__main__":

test/registered/attention/test_hybrid_attn_backend.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -76,21 +76,20 @@ class TestHybridAttnBackendBase(CustomTestCase):
     def test_gsm8k(self):
         requests.get(self.base_url + "/flush_cache")
 
+        model = DEFAULT_TARGET_MODEL_EAGLE if self.speculative_decode else self.model
         args = SimpleNamespace(
-            num_shots=4,
-            num_questions=100,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-            data_path=GSM_DATASET_PATH,
+            base_url=self.base_url,
+            model=model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
-        # Use the appropriate metric key based on the test class
-        metric_key = "accuracy"
-        self.assertGreater(metrics[metric_key], self.accuracy_threshold)
+        self.assertGreater(metrics["score"], self.accuracy_threshold)
 
         if self.speculative_decode:
             server_info = requests.get(self.base_url + "/get_server_info")

test/registered/attention/test_local_attn.py

@@ -6,7 +6,7 @@ import requests
 
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -56,19 +56,20 @@ class TestFlashAttention3LocalAttn(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
             num_shots=4,
-            num_questions=100,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-            data_path=None,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         # Use the appropriate metric key based on the test class
-        metric_key = "accuracy"
+        metric_key = "score"
         self.assertGreater(metrics[metric_key], self.accuracy_threshold)
 
 

test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
@@ -72,18 +72,18 @@ class TestDeepseekR1Fp8Flashinfer(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=512,
-            parallel=512,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=512,
+            num_threads=512,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
 
 if __name__ == "__main__":

test/registered/backends/test_deepseek_v3_fp4_cutlass_moe.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
@@ -52,23 +52,24 @@ class TestDeepseekV3FP4CutlassMoE(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=1319,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            parallel=1319,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
 
 
 if __name__ == "__main__":

test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -48,17 +48,17 @@ class TestFlashinferTrtllmGenAttnBackend(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
 
 if __name__ == "__main__":

test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -49,17 +49,17 @@ class FlashinferTrtllmGenMoeBackendFP8Base:
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
 
 class FlashinferTrtllmGenMoeBackendBF16Base:
@@ -97,17 +97,17 @@ class FlashinferTrtllmGenMoeBackendBF16Base:
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
 
 class FlashinferTrtllmGenMoeBackendMXFP8Base:
@@ -144,17 +144,17 @@ class FlashinferTrtllmGenMoeBackendMXFP8Base:
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
 
 class TestFlashinferTrtllmGenMoeBackendFP8(

test/registered/backends/test_qwen3_fp4_trtllm_gen_moe.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -51,17 +51,18 @@ class TestFlashinferTrtllmGenMoeBackend(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=1319,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            max_new_tokens=512,
-            parallel=1319,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.88)
+        self.assertGreater(metrics["score"], 0.88)
 
 
 if __name__ == "__main__":

test/registered/cp/test_deepseek_v32_cp_single_node.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -68,23 +68,24 @@ class TestDeepseekV32CPInSeqSplit(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=32,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=32,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
                 f"### test_a_gsm8k (deepseek-v32-cp-in-seq-split)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
 
 
 class TestDeepseekV32CPRoundRobinSplit(CustomTestCase):
@@ -134,23 +135,24 @@ class TestDeepseekV32CPRoundRobinSplit(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=32,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=32,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
                 f"### test_a_gsm8k (deepseek-v32-cp-in-seq-split)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
 
 
 if __name__ == "__main__":

test/registered/disaggregation/test_disaggregation_basic.py

@@ -8,7 +8,7 @@ import requests
 from transformers import AutoTokenizer
 
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -81,18 +81,17 @@ class TestDisaggregationAccuracy(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=f"http://{self.base_host}:{self.lb_port}",
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
     def test_logprob(self):
         prompt = "The capital of france is "
@@ -260,18 +259,17 @@ class TestDisaggregationMooncakeFailure(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=f"http://{self.base_host}:{self.lb_port}",
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
 
         # Expect lots of failure but the server cannot crash
         try:
-            metrics = run_eval_few_shot_gsm8k(args)
+            metrics = run_eval(args)
             print(f"Evaluation metrics: {metrics}")
         except Exception as e:
             print(f"Test encountered expected errors: {e}")
@@ -362,18 +360,17 @@ class TestDisaggregationMooncakeSpec(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=f"http://{self.base_host}:{self.lb_port}",
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.74)
+        self.assertGreater(metrics["score"], 0.74)
 
 
 class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase):
@@ -440,18 +437,17 @@ class TestDisaggregationSimulatedRetract(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=f"http://{self.base_host}:{self.lb_port}",
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 if __name__ == "__main__":

test/registered/distributed/test_disaggregation_aarch64.py

@@ -3,7 +3,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -82,18 +82,18 @@ class TestDisaggregationMooncakeAARCH64Accuracy(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 if __name__ == "__main__":

test/registered/distributed/test_disaggregation_different_tp.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.environ import envs
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -79,18 +79,18 @@ class TestDisaggregationMooncakePrefillLargerTP(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase):
@@ -154,18 +154,18 @@ class TestDisaggregationMooncakeDecodeLargerTP(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase):
@@ -229,18 +229,18 @@ class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase):
@@ -304,18 +304,18 @@ class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 if __name__ == "__main__":

test/registered/distributed/test_disaggregation_dp_attention.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 from sglang.bench_serving import run_benchmark
 from sglang.srt.environ import envs
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -94,18 +94,18 @@ class TestDisaggregationDPAttention(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1400,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1400,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestDisaggregationDPAttentionRoundRobin(TestDisaggregationDPAttention):

test/registered/distributed/test_disaggregation_hybrid_attention.py

@@ -2,7 +2,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -74,18 +74,18 @@ class TestDisaggregationHybridAttentionMamba(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
 
 class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBase):
@@ -150,19 +150,19 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
         # TODO: Fix PD disaggregation accuracy issue (https://github.com/sgl-project/sglang/issues/21744) and increase the threshold back to 0.93.
-        self.assertGreater(metrics["accuracy"], 0.90)
+        self.assertGreater(metrics["score"], 0.90)
 
 
 class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase):
@@ -229,19 +229,19 @@ class TestDisaggregationHybridAttentionMambaDPDecode(PDDisaggregationServerBase)
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Evaluation metrics: {metrics}")
 
         # TODO: Fix PD disaggregation accuracy issue (https://github.com/sgl-project/sglang/issues/21744) and increase the threshold back to 0.93.
-        self.assertGreater(metrics["accuracy"], 0.90)
+        self.assertGreater(metrics["score"], 0.90)
 
 
 if __name__ == "__main__":

test/registered/distributed/test_disaggregation_pp.py

@@ -3,7 +3,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import (
     PDDisaggregationServerBase,
 )
@@ -78,18 +78,18 @@ class TestDisaggregationPrefillPPAccuracy(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.24)
+        self.assertGreater(metrics["score"], 0.24)
         # Wait a little bit so that the memory check happens.
         time.sleep(5)
 
@@ -156,18 +156,18 @@ class TestDisaggregationPrefillPPDynamicChunkAccuracy(PDDisaggregationServerBase
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.24)
+        self.assertGreater(metrics["score"], 0.24)
         # Wait a little bit so that the memory check happens.
         time.sleep(5)
 
@@ -235,18 +235,18 @@ class TestDisaggregationDecodePPAccuracy(PDDisaggregationServerBase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host=f"http://{self.base_host}",
-            port=int(self.lb_port),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.24)
+        self.assertGreater(metrics["score"], 0.24)
         # Wait a little bit so that the memory check happens.
         time.sleep(5)
 

test/registered/distributed/test_dp_attention.py

@@ -7,12 +7,12 @@ from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
 from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
 from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
 from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
 from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
 from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_IMAGE_URL,
     DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -154,18 +154,18 @@ class TestDPAttentionDP2TP2DeepseekV3MTP(
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -173,7 +173,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP(
         ]
         print(
             f"###test_gsm8k (deepseek-v3 mtp + dp):\n"
-            f"accuracy={metrics['accuracy']=:.3f}\n"
+            f"accuracy={metrics['score']=:.3f}\n"
             f"{avg_spec_accept_length=:.3f}\n"
         )
         self.assertGreater(avg_spec_accept_length, 2.5)

test/registered/distributed/test_dp_attention_large.py

@@ -6,7 +6,6 @@ import requests
 from sglang.lang.chat_template import get_chat_template_by_model_path
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
 from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin
 from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin
 from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin
@@ -115,18 +114,18 @@ class TestDPAttentionDP2TP2DeepseekV3MTP(
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -134,7 +133,7 @@ class TestDPAttentionDP2TP2DeepseekV3MTP(
         ]
         print(
             f"###test_gsm8k (deepseek-v3 mtp + dp):\n"
-            f"accuracy={metrics['accuracy']=:.3f}\n"
+            f"accuracy={metrics['score']=:.3f}\n"
             f"{avg_spec_accept_length=:.3f}\n"
         )
         self.assertGreater(avg_spec_accept_length, 2.5)

test/registered/distributed/test_pp_single_node.py

@@ -16,7 +16,6 @@ from sglang.bench_one_batch_server import BenchArgs as OneBatchBenchArgs
 from sglang.srt.server_args import ServerArgs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
 from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MLA_MODEL_NAME_FOR_TEST,
@@ -60,22 +59,22 @@ class TestPPAccuracy(unittest.TestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=DEFAULT_MODEL_NAME_FOR_TEST,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_amd_ci():
             # AMD triton backend produces slightly lower accuracy than FA3 on NVIDIA
-            self.assertGreater(metrics["accuracy"], 0.70)
+            self.assertGreater(metrics["score"], 0.70)
         else:
-            self.assertGreater(metrics["accuracy"], 0.74)
+            self.assertGreater(metrics["score"], 0.74)
         # Wait a little bit so that the memory check happens.
         time.sleep(4)
 
@@ -169,18 +168,18 @@ class TestQwenVLPPAccuracy(unittest.TestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
-        self.assertGreaterEqual(metrics["accuracy"], 0.65)
+        self.assertGreaterEqual(metrics["score"], 0.65)
         # Wait a little bit so that the memory check happens.
         time.sleep(4)
 
@@ -223,15 +222,15 @@ class TestQwenPPAccuracy(unittest.TestCase):
 
         try:
             args = SimpleNamespace(
-                num_shots=5,
-                data_path=None,
-                num_questions=512,
-                max_new_tokens=512,
-                parallel=128,
-                host="http://127.0.0.1",
-                port=int(self.base_url.split(":")[-1]),
+                base_url=self.base_url,
+                model=self.model_name,
+                eval_name="gsm8k",
+                api="completion",
+                max_tokens=512,
+                num_examples=512,
+                num_threads=128,
             )
-            metrics = run_eval_few_shot_gsm8k(args)
+            metrics = run_eval(args)
             time.sleep(5)
             return metrics
         finally:
@@ -244,13 +243,13 @@ class TestQwenPPAccuracy(unittest.TestCase):
 
         print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
 
-        self.assertGreaterEqual(baseline["accuracy"], 0.74)
+        self.assertGreaterEqual(baseline["score"], 0.74)
         self.assertGreaterEqual(
-            pp_metrics["accuracy"],
-            baseline["accuracy"] - 0.02,
+            pp_metrics["score"],
+            baseline["score"] - 0.02,
             msg=(
                 f"PP accuracy dropped more than 2% compared to baseline. "
-                f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}"
+                f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
             ),
         )
 
@@ -279,15 +278,15 @@ class TestQwenPPTieWeightsAccuracy(unittest.TestCase):
 
         try:
             args = SimpleNamespace(
-                num_shots=5,
-                data_path=None,
-                num_questions=512,
-                max_new_tokens=512,
-                parallel=128,
-                host="http://127.0.0.1",
-                port=int(self.base_url.split(":")[-1]),
+                base_url=self.base_url,
+                model=self.model_name,
+                eval_name="gsm8k",
+                api="completion",
+                max_tokens=512,
+                num_examples=512,
+                num_threads=128,
             )
-            metrics = run_eval_few_shot_gsm8k(args)
+            metrics = run_eval(args)
             time.sleep(5)
             return metrics
         finally:
@@ -299,13 +298,13 @@ class TestQwenPPTieWeightsAccuracy(unittest.TestCase):
 
         print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
 
-        self.assertGreaterEqual(baseline["accuracy"], 0.38)
+        self.assertGreaterEqual(baseline["score"], 0.38)
         self.assertGreaterEqual(
-            pp_metrics["accuracy"],
-            baseline["accuracy"] - 0.02,
+            pp_metrics["score"],
+            baseline["score"] - 0.02,
             msg=(
                 f"PP accuracy dropped more than 2% compared to baseline. "
-                f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}"
+                f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
             ),
         )
 
@@ -331,15 +330,15 @@ class TestQwenMoePPAccuracy(unittest.TestCase):
 
         try:
             args = SimpleNamespace(
-                num_shots=5,
-                data_path=None,
-                num_questions=512,
-                max_new_tokens=512,
-                parallel=128,
-                host="http://127.0.0.1",
-                port=int(self.base_url.split(":")[-1]),
+                base_url=self.base_url,
+                model=self.model_name,
+                eval_name="gsm8k",
+                api="completion",
+                max_tokens=512,
+                num_examples=512,
+                num_threads=128,
             )
-            metrics = run_eval_few_shot_gsm8k(args)
+            metrics = run_eval(args)
             time.sleep(5)
             return metrics
         finally:
@@ -351,13 +350,13 @@ class TestQwenMoePPAccuracy(unittest.TestCase):
 
         print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
 
-        self.assertGreaterEqual(baseline["accuracy"], 0.74)
+        self.assertGreaterEqual(baseline["score"], 0.74)
         self.assertGreaterEqual(
-            pp_metrics["accuracy"],
-            baseline["accuracy"] - 0.02,
+            pp_metrics["score"],
+            baseline["score"] - 0.02,
             msg=(
                 f"PP accuracy dropped more than 2% compared to baseline. "
-                f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}"
+                f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
             ),
         )
 
@@ -390,15 +389,15 @@ class TestQwen35PPAccuracy(unittest.TestCase):
 
         try:
             args = SimpleNamespace(
-                num_shots=5,
-                data_path=None,
-                num_questions=512,
-                max_new_tokens=512,
-                parallel=128,
-                host="http://127.0.0.1",
-                port=int(self.base_url.split(":")[-1]),
+                base_url=self.base_url,
+                model=self.model_name,
+                eval_name="gsm8k",
+                api="completion",
+                max_tokens=512,
+                num_examples=512,
+                num_threads=128,
             )
-            metrics = run_eval_few_shot_gsm8k(args)
+            metrics = run_eval(args)
             time.sleep(5)
             return metrics
         finally:
@@ -410,13 +409,13 @@ class TestQwen35PPAccuracy(unittest.TestCase):
 
         print(f"[Qwen35 PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
 
-        self.assertGreaterEqual(baseline["accuracy"], 0.83)
+        self.assertGreaterEqual(baseline["score"], 0.83)
         self.assertGreaterEqual(
-            pp_metrics["accuracy"],
-            baseline["accuracy"] - 0.05,
+            pp_metrics["score"],
+            baseline["score"] - 0.05,
             msg=(
                 f"PP accuracy dropped more than 5% compared to baseline. "
-                f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}"
+                f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
             ),
         )
 

test/registered/dllm/test_llada2_mini.py

@@ -7,7 +7,7 @@ import unittest
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -58,18 +58,18 @@ class TestLLaDA2Mini(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.88)
+        self.assertGreater(metrics["score"], 0.88)
         if is_in_amd_ci():
             self.assertGreater(metrics["output_throughput"], 80)
         else:

test/registered/dllm/test_llada2_mini_amd.py

@@ -9,7 +9,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -55,19 +55,17 @@ class TestLLaDA2MiniAMD(CustomTestCase):
     def test_gsm8k(self):
         """Test GSM8K accuracy with DLLM on AMD."""
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         # Relaxed thresholds for AMD - may need adjustment
-        self.assertGreater(metrics["accuracy"], 0.80)
+        self.assertGreater(metrics["score"], 0.80)
         self.assertGreater(metrics["output_throughput"], 50)
 
     def test_bs_1_speed(self):

test/registered/ep/test_deepep_large.py

@@ -5,7 +5,7 @@ import requests
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST,
@@ -67,18 +67,18 @@ class TestDeepseek(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1200,
-            parallel=1200,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1200,
+            num_threads=1200,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
 
 class TestDeepseekMTP(CustomTestCase):
@@ -135,18 +135,18 @@ class TestDeepseekMTP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1200,
-            parallel=1200,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1200,
+            num_threads=1200,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -154,7 +154,7 @@ class TestDeepseekMTP(CustomTestCase):
         ]
         print(
             f"###test_gsm8k:\n"
-            f"accuracy={metrics['accuracy']=:.3f}\n"
+            f"accuracy={metrics['score']=:.3f}\n"
             f"{avg_spec_accept_length=:.3f}\n"
         )
         self.assertGreater(avg_spec_accept_length, 1.85)
@@ -195,17 +195,17 @@ class TestDeepseekV32TBO(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1200,
-            parallel=1200,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1200,
+            num_threads=1200,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)

test/registered/ep/test_deepep_small.py

@@ -6,7 +6,7 @@ import requests
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
     DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
@@ -52,18 +52,18 @@ class TestPureDP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestHybridDPTP(CustomTestCase):
@@ -97,18 +97,18 @@ class TestHybridDPTP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestTP(CustomTestCase):
@@ -139,18 +139,18 @@ class TestTP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 @unittest.skip("covered in test_deepep_large.py")
@@ -188,18 +188,18 @@ class TestNoGatherdBuffer(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestTBO(CustomTestCase):
@@ -240,18 +240,18 @@ class TestTBO(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestTBOWithTPAttn(CustomTestCase):
@@ -289,18 +289,18 @@ class TestTBOWithTPAttn(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 # There exists bug when using MTP + TBO + attn_tp_size > 1, currently skip that case.
@@ -342,18 +342,18 @@ class TestTBOWithTPAttnAndDenseDP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 @unittest.skip("covered in TestMTPWithTBO")
@@ -399,18 +399,18 @@ class TestMTP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -418,7 +418,7 @@ class TestMTP(CustomTestCase):
         ]
         print(
             f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n"
-            f"accuracy={metrics['accuracy']=:.3f}\n"
+            f"accuracy={metrics['score']=:.3f}\n"
             f"{avg_spec_accept_length=:.3f}\n"
         )
         self.assertGreater(avg_spec_accept_length, 2.1)
@@ -473,18 +473,18 @@ class TestMTPWithTBO(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -492,7 +492,7 @@ class TestMTPWithTBO(CustomTestCase):
         ]
         print(
             f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n"
-            f"accuracy={metrics['accuracy']=:.3f}\n"
+            f"accuracy={metrics['score']=:.3f}\n"
             f"{avg_spec_accept_length=:.3f}\n"
         )
         self.assertGreater(avg_spec_accept_length, 2.1)
@@ -549,18 +549,18 @@ class TestMTPWithTPAttnAndTBO(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -568,7 +568,7 @@ class TestMTPWithTPAttnAndTBO(CustomTestCase):
         ]
         print(
             f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n"
-            f"accuracy={metrics['accuracy']=:.3f}\n"
+            f"accuracy={metrics['score']=:.3f}\n"
             f"{avg_spec_accept_length=:.3f}\n"
         )
         self.assertGreater(avg_spec_accept_length, 2.1)

test/registered/ep/test_mooncake_ep_small.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.disaggregation_fixture import get_rdma_devices_args
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
@@ -69,18 +69,18 @@ class TestTP(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 @unittest.skipIf(is_in_ci(), "Skip since mooncake-ep fault-tolerant test is flaky.")

test/registered/hicache/test_hicache_storage_file_backend.py

@@ -19,7 +19,7 @@ import requests
 from sglang.benchmark.utils import get_tokenizer
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MLA_MODEL_NAME_FOR_TEST,
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -295,15 +295,14 @@ def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03):
     # First evaluation - populate cache
     print("Phase 1: Running initial GSM8K evaluation to populate cache...")
     args_initial = SimpleNamespace(
-        num_shots=5,
-        data_path=None,
-        num_questions=200,
-        max_new_tokens=512,
-        parallel=64,
-        host=f"http://{test_instance.base_host}",
-        port=int(test_instance.base_port),
+        base_url=f"http://{test_instance.base_host}:{test_instance.base_port}",
+        eval_name="gsm8k",
+        api="completion",
+        max_tokens=512,
+        num_examples=200,
+        num_threads=64,
     )
-    metrics_initial = run_eval_few_shot_gsm8k(args_initial)
+    metrics_initial = run_eval(args_initial)
 
     # Flush cache to force remote storage access
     print("Phase 2: Flushing device cache...")
@@ -311,18 +310,18 @@ def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03):
 
     # Second evaluation - should use remote cache
     print("Phase 3: Running second GSM8K evaluation using remote cache...")
-    metrics_cached = run_eval_few_shot_gsm8k(args_initial)
+    metrics_cached = run_eval(args_initial)
 
     # Verify accuracy consistency
-    accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"])
+    accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"])
     print(f"Accuracy difference: {accuracy_diff:.4f}")
 
     # Assertions
     test_instance.assertGreater(
-        metrics_initial["accuracy"], 0.6, "Initial accuracy should be reasonable"
+        metrics_initial["score"], 0.6, "Initial accuracy should be reasonable"
     )
     test_instance.assertGreater(
-        metrics_cached["accuracy"], 0.6, "Cached accuracy should be reasonable"
+        metrics_cached["score"], 0.6, "Cached accuracy should be reasonable"
     )
     test_instance.assertLess(
         accuracy_diff,

test/registered/mla/test_flashmla.py

@@ -11,7 +11,7 @@ import torch
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -53,18 +53,18 @@ class TestFlashMLAAttnBackend(unittest.TestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestFlashMLAMTP(CustomTestCase):
@@ -112,18 +112,18 @@ class TestFlashMLAMTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/server_info").json()
         avg_spec_accept_length = server_info["internal_states"][0][

test/registered/mla/test_mla_deepseek_v3.py

@@ -6,7 +6,7 @@ import requests
 
 from sglang.srt.utils import is_cuda, is_hip, kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -45,18 +45,18 @@ class TestMLADeepseekV3(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
@@ -82,18 +82,18 @@ class TestMLADeepseekV3DisableFusedFunc(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 @unittest.skipIf(is_hip(), "FA is not available.")
@@ -133,18 +133,18 @@ class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestDeepseekV3MTP(CustomTestCase):
@@ -186,18 +186,18 @@ class TestDeepseekV3MTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][

test/registered/mla/test_mla_flashinfer.py

@@ -6,7 +6,7 @@ import torch
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -47,18 +47,18 @@ class TestFlashinferMLA(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.615)
+        self.assertGreater(metrics["score"], 0.615)
 
 
 class TestFlashinferMLAMTP(CustomTestCase):
@@ -102,18 +102,18 @@ class TestFlashinferMLAMTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info").json()
         avg_spec_accept_length = server_info["internal_states"][0][

test/registered/mla/test_mla_int8_deepseek_v3.py

@@ -6,7 +6,7 @@ import torch
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -48,18 +48,18 @@ class TestMLADeepseekV3ChannelInt8(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreaterEqual(metrics["accuracy"], 0.61)
+        self.assertGreaterEqual(metrics["score"], 0.61)
 
 
 @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
@@ -104,18 +104,18 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][
@@ -155,18 +155,18 @@ class TestMLADeepseekV3BlockInt8(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.62)
+        self.assertGreater(metrics["score"], 0.62)
 
 
 class TestDeepseekV3MTPBlockInt8(CustomTestCase):
@@ -208,18 +208,18 @@ class TestDeepseekV3MTPBlockInt8(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
         server_info = requests.get(self.base_url + "/get_server_info")
         avg_spec_accept_length = server_info.json()["internal_states"][0][

test/registered/models/test_compressed_tensors_models.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import is_hip, kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -35,21 +35,21 @@ class TestCompressedTensorsLlama3FP8(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
         if is_hip():
             # Lower threshold for AMD because FP8 dtype differs (fp8_fnuz)
-            self.assertGreaterEqual(metrics["accuracy"], 0.40)
+            self.assertGreaterEqual(metrics["score"], 0.40)
         else:
-            self.assertGreaterEqual(metrics["accuracy"], 0.45)
+            self.assertGreaterEqual(metrics["score"], 0.45)
 
 
 if __name__ == "__main__":

test/registered/models/test_kimi_linear_models.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -32,17 +32,17 @@ class TestKimiLinear(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.88)
+        self.assertGreater(metrics["score"], 0.88)
 
 
 if __name__ == "__main__":

test/registered/models/test_qwen_models.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -35,17 +35,17 @@ class TestQwen2(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.78)
+        self.assertGreater(metrics["score"], 0.78)
 
 
 class TestQwen2FP8(CustomTestCase):
@@ -66,17 +66,17 @@ class TestQwen2FP8(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.78)
+        self.assertGreater(metrics["score"], 0.78)
 
 
 if __name__ == "__main__":

test/registered/models/test_transformers_models.py

@@ -10,6 +10,7 @@ import torch
 
 from sglang.srt.utils import is_hip, kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.run_eval import run_eval
 from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST,
@@ -50,26 +51,22 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
             num_examples=64,
             num_threads=32,
         )
-        from sglang.test.run_eval import run_eval
-
         metrics = run_eval(args)
         self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound)
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        from sglang.test.few_shot_gsm8k import run_eval
-
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], self.gsm8k_lower_bound)
+        self.assertGreater(metrics["score"], self.gsm8k_lower_bound)
 
 
 @unittest.skipIf(is_hip(), "TorchAO int4wo quantization is not supported on AMD GPUs")

test/registered/moe/test_glm4_moe_models.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -35,17 +35,17 @@ class TestGLM4MoE(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=100,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.8)
+        self.assertGreater(metrics["score"], 0.8)
 
 
 if __name__ == "__main__":

test/registered/moe/test_moe_ep.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -39,18 +39,17 @@ class TestEp(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 class TestEpDeepGEMM(CustomTestCase):
@@ -81,18 +80,17 @@ class TestEpDeepGEMM(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.60)
+        self.assertGreater(metrics["score"], 0.60)
 
 
 if __name__ == "__main__":

test/registered/quant/test_deepseek_v32_fp4_4gpu.py

@@ -3,7 +3,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
@@ -56,23 +56,24 @@ class TestDeepseekV32FP4DP(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
             )
 
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
@@ -123,23 +124,24 @@ class TestDeepseekV32FP4TP(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
             )
 
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)

test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
@@ -72,15 +72,16 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -92,10 +93,10 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v32 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.93)
+            self.assertGreater(metrics["score"], 0.93)
             self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):
@@ -162,15 +163,16 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=500,
+            num_threads=500,
             num_shots=20,
-            data_path=None,
-            num_questions=500,
-            parallel=500,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -182,10 +184,10 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v32 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.93)
+            self.assertGreater(metrics["score"], 0.93)
             self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):

test/registered/quant/test_deepseek_v3_fp4_4gpu.py

@@ -4,7 +4,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
@@ -54,23 +54,24 @@ class TestDeepseekV3FP4(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=1319,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            parallel=1319,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
             )
 
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
@@ -124,23 +125,24 @@ class TestDeepseekV3FP4CutlassMoE(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=1319,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            parallel=1319,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
             )
-            self.assertGreater(metrics["accuracy"], 0.93)
+            self.assertGreater(metrics["score"], 0.93)
 
 
 class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
@@ -178,23 +180,24 @@ class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
         self,
     ):  # Append an "a" to make this test run first (alphabetically) to warm up the server
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=1319,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            parallel=1319,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         if is_in_ci():
             write_github_step_summary(
-                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
+                f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
             )
 
-        self.assertGreater(metrics["accuracy"], 0.93)
+        self.assertGreater(metrics["score"], 0.93)
 
 
 if __name__ == "__main__":

test/registered/quant/test_fp8_blockwise_gemm.py

@@ -4,7 +4,7 @@ from urllib.parse import urlparse
 
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -46,18 +46,19 @@ class FP8BlockwiseGemmBase:
     def test_gsm8k(self):
         parsed_url = urlparse(self.base_url)
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=200,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            max_new_tokens=512,
-            parallel=200,
-            host=f"{parsed_url.scheme}://{parsed_url.hostname}",
-            port=parsed_url.port,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreaterEqual(metrics["accuracy"], 0.8)
+        self.assertGreaterEqual(metrics["score"], 0.8)
 
 
 class MXFP8GemmBase:
@@ -88,18 +89,19 @@ class MXFP8GemmBase:
     def test_gsm8k(self):
         parsed_url = urlparse(self.base_url)
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=200,
             num_shots=8,
-            data_path=None,
-            num_questions=1319,
-            max_new_tokens=512,
-            parallel=200,
-            host=f"{parsed_url.scheme}://{parsed_url.hostname}",
-            port=parsed_url.port,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreaterEqual(metrics["accuracy"], 0.8)
+        self.assertGreaterEqual(metrics["score"], 0.8)
 
 
 class TestFP8BlockwiseGemmTriton(FP8BlockwiseGemmBase, unittest.TestCase):

test/registered/quant/test_fp8kv_triton.py

@@ -4,7 +4,7 @@ from urllib.parse import urlparse
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -41,17 +41,17 @@ class TestFP8KVCacheTritonBackend(CustomTestCase):
     def test_gsm8k(self):
         parsed_url = urlparse(self.base_url)
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=200,
-            host=f"{parsed_url.scheme}://{parsed_url.hostname}",
-            port=parsed_url.port,
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=200,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.70)
+        self.assertGreater(metrics["score"], 0.70)
 
 
 if __name__ == "__main__":

test/registered/quant/test_int4fp8_moe.py

@@ -2,7 +2,7 @@ from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_amd_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
@@ -45,14 +45,15 @@ class TestMixtralAccuracy(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1400,
+            num_threads=128,
             num_shots=8,
-            data_path=None,
-            num_questions=1400,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.56)
+        self.assertGreater(metrics["score"], 0.56)

test/registered/quant/test_modelopt_fp8.py

@@ -4,7 +4,7 @@ from urllib.parse import urlparse
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -35,17 +35,17 @@ class TestModeloptFP8(CustomTestCase):
     def test_gsm8k(self):
         parsed_url = urlparse(self.base_url)
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=200,
-            host=f"{parsed_url.scheme}://{parsed_url.hostname}",
-            port=parsed_url.port,
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=200,
         )
         metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.70)
+        self.assertGreater(metrics["score"], 0.70)
 
 
 if __name__ == "__main__":

test/registered/quant/test_nvfp4_gemm.py

@@ -4,7 +4,7 @@ from urllib.parse import urlparse
 
 from sglang.srt.utils import get_device_sm, kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -47,18 +47,18 @@ class FP4GemmBase:
     def test_gsm8k(self):
         parsed_url = urlparse(self.base_url)
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1319,
-            max_new_tokens=512,
-            parallel=200,
-            host=f"{parsed_url.scheme}://{parsed_url.hostname}",
-            port=parsed_url.port,
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1319,
+            num_threads=200,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(metrics)
 
-        self.assertGreater(metrics["accuracy"], 0.64)
+        self.assertGreater(metrics["score"], 0.64)
 
 
 @unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")

test/registered/quant/test_w4a8_deepseek_v3.py

@@ -6,7 +6,7 @@ import requests
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -41,18 +41,18 @@ class TestDeepseekV3W4afp8(CustomTestCase):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1200,
-            parallel=1200,
-            max_new_tokens=512,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1200,
+            num_threads=1200,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
 
 class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
@@ -95,15 +95,15 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
         self,
     ):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -115,10 +115,10 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v3 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            self.assertGreater(metrics["score"], 0.935)
             self.assertGreater(avg_spec_accept_length, 2.9)
 
 
@@ -163,18 +163,18 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase):
         self,
     ):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
 
 class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase):
@@ -231,18 +231,18 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase):
         self,
     ):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"Eval accuracy of GSM8K: {metrics=}")
 
-        self.assertGreater(metrics["accuracy"], 0.92)
+        self.assertGreater(metrics["score"], 0.92)
 
 
 if __name__ == "__main__":

test/registered/quant/test_w8a8_quantization.py

@@ -6,7 +6,7 @@ import requests
 
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -51,17 +51,17 @@ class BaseW8A8Test(CustomTestCase):
             self.skipTest("gsm8k_accuracy_threshold not set for this test")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(metrics)
-        self.assertGreater(metrics["accuracy"], self.gsm8k_accuracy_threshold)
+        self.assertGreater(metrics["score"], self.gsm8k_accuracy_threshold)
 
     def run_decode(self, max_new_tokens):
         response = requests.post(

test/registered/spec/eagle/test_deepseek_v3_fp4_mtp_small.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_URL_FOR_TEST,
@@ -18,7 +18,6 @@ from sglang.test.test_utils import (
 
 register_cuda_ci(est_time=900, suite="stage-b-test-4-gpu-b200")
 
-
 FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
 SERVER_LAUNCH_TIMEOUT = 1200
 
@@ -74,15 +73,15 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/server_info").json()
@@ -94,11 +93,11 @@ class TestDeepseekV3FP4MTP(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v3-fp4 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
 
-        self.assertGreater(metrics["accuracy"], 0.94)
+        self.assertGreater(metrics["score"], 0.94)
         self.assertGreater(avg_spec_accept_length, 2.7)
 
     def test_bs_1_speed(self):

test/registered/spec/eagle/test_eagle_dp_attention.py

@@ -5,7 +5,7 @@ import requests
 
 from sglang.srt.environ import envs
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE_DP_ATTN,
@@ -76,15 +76,15 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -104,14 +104,14 @@ class TestEAGLE3EngineDPAttention(CustomTestCase):
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (EAGLE3 DP Attention)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
             if is_in_amd_ci():
                 # AMD triton backend produces slightly lower accuracy than FA3 on NVIDIA
-                self.assertGreater(metrics["accuracy"], 0.88)
+                self.assertGreater(metrics["score"], 0.88)
             else:
-                self.assertGreater(metrics["accuracy"], 0.91)
+                self.assertGreater(metrics["score"], 0.91)
             if avg_spec_accept_length is not None:
                 if is_in_amd_ci():
                     # AMD triton backend produces slightly lower accept length than FA3 on NVIDIA

test/registered/spec/eagle/test_eagle_infer_b.py

@@ -12,13 +12,13 @@ import requests
 
 from sglang.srt.environ import envs
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval
 from sglang.test.kits.abort_timeout_kit import (
     AbortAllMixin,
     RunningTimeoutTwoWaveMixin,
     WaitingTimeoutMixin,
 )
 from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
+from sglang.test.run_eval import run_eval
 from sglang.test.server_fixtures.eagle_fixture import EagleServerBase
 from sglang.test.test_utils import DEFAULT_TARGET_MODEL_EAGLE, run_logprob_check
 
@@ -48,18 +48,18 @@ class TestEAGLEServerBasic(EagleServerBase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.target_model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=200,
+            num_threads=128,
         )
 
-        metrics = run_gsm8k_eval(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
-        self.assertGreater(metrics["accuracy"], 0.20)
+        self.assertGreater(metrics["score"], 0.20)
 
         server_info = requests.get(self.base_url + "/server_info").json()
         avg_spec_accept_length = server_info["internal_states"][0][
@@ -103,16 +103,16 @@ class TestEAGLEServerAdditional(TestEAGLEServerBasic):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=1,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.target_model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=1,
+            num_examples=200,
+            num_threads=128,
         )
 
-        metrics = run_gsm8k_eval(args)
+        metrics = run_eval(args)
         self.assertGreater(metrics["output_throughput"], 50)
 
     def test_logprob_start_len(self):

test/registered/spec/eagle/test_eagle_infer_beta.py

@@ -7,9 +7,9 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval
 from sglang.test.kits.matched_stop_kit import MatchedStopMixin
 from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_EAGLE,
     DEFAULT_TARGET_MODEL_EAGLE,
@@ -86,19 +86,19 @@ class TestEagleServerBase(CustomTestCase, MatchedStopMixin):
 
     def test_gsm8k(self):
         args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=1000,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=1000,
+            num_threads=128,
         )
         metrics = run_eval(args)
         print(f"TestEagleLargeBS -- {metrics=}")
         self.assertGreater(
-            metrics["accuracy"], 0.23
-        )  # 0.3333 for 60 questions; 0.234 for 1319 questions
+            metrics["score"], 0.22
+        )  # ~0.227 for 1000 questions via /v1/completions
         assert self.process.poll() is None
 
     def test_logprob_spec_v2_match(self):

test/registered/spec/eagle/test_eagle_infer_beta_dp_attention.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_TEST_MLA,
     DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN,
@@ -20,19 +20,19 @@ from sglang.test.test_utils import (
 register_cuda_ci(est_time=300, suite="stage-c-test-4-gpu-b200")
 
 
-def test_gsm8k(base_url: str):
+def test_gsm8k(base_url: str, model: str):
     requests.get(base_url + "/flush_cache")
 
     args = SimpleNamespace(
-        num_shots=5,
-        data_path=None,
-        num_questions=200,
-        max_new_tokens=512,
-        parallel=128,
-        host="http://127.0.0.1",
-        port=int(base_url.split(":")[-1]),
+        base_url=base_url,
+        model=model,
+        eval_name="gsm8k",
+        api="completion",
+        max_tokens=512,
+        num_examples=200,
+        num_threads=128,
     )
-    metrics = run_eval_few_shot_gsm8k(args)
+    metrics = run_eval(args)
     server_info = requests.get(base_url + "/get_server_info")
     avg_spec_accept_length = server_info.json()["internal_states"][0][
         "avg_spec_accept_length"
@@ -84,8 +84,8 @@ class TestEagleDPAttnServerSmall(CustomTestCase):
         kill_process_tree(cls.process.pid)
 
     def test_a_gsm8k(self):
-        metrics, avg_spec_accept_length = test_gsm8k(self.base_url)
-        self.assertGreater(metrics["accuracy"], 0.62)
+        metrics, avg_spec_accept_length = test_gsm8k(self.base_url, self.model)
+        self.assertGreater(metrics["score"], 0.62)
         self.assertGreater(avg_spec_accept_length, 2.7)
 
 

test/registered/spec/eagle/test_eagle_infer_beta_dp_attention_large.py

@@ -6,7 +6,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DEEPSEEK_NVFP4_MODEL_FOR_TEST,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -21,19 +21,19 @@ from sglang.test.test_utils import (
 register_cuda_ci(est_time=600, suite="nightly-8-gpu-b200", nightly=True)
 
 
-def test_gsm8k(base_url: str):
+def test_gsm8k(base_url: str, model: str):
     requests.get(base_url + "/flush_cache")
 
     args = SimpleNamespace(
-        num_shots=5,
-        data_path=None,
-        num_questions=200,
-        max_new_tokens=512,
-        parallel=128,
-        host="http://127.0.0.1",
-        port=int(base_url.split(":")[-1]),
+        base_url=base_url,
+        model=model,
+        eval_name="gsm8k",
+        api="completion",
+        max_tokens=512,
+        num_examples=200,
+        num_threads=128,
     )
-    metrics = run_eval_few_shot_gsm8k(args)
+    metrics = run_eval(args)
     server_info = requests.get(base_url + "/server_info").json()
     avg_spec_accept_length = server_info["internal_states"][0]["avg_spec_accept_length"]
 
@@ -92,14 +92,14 @@ class TestEagleDPAttnServerLarge(CustomTestCase):
         kill_process_tree(cls.process.pid)
 
     def test_a_gsm8k(self):
-        metrics, avg_spec_accept_length = test_gsm8k(self.base_url)
+        metrics, avg_spec_accept_length = test_gsm8k(self.base_url, self.model)
 
-        self.assertGreater(metrics["accuracy"], 0.94)
+        self.assertGreater(metrics["score"], 0.94)
         self.assertGreater(avg_spec_accept_length, 2.7)
         if is_in_ci():
             write_github_step_summary(
                 f"### test_gsm8k (deepseek-v3-fp4 mtp)\n"
-                f'{metrics["accuracy"]=:.3f}\n'
+                f'{metrics["score"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
 

test/registered/spec/test_standalone_speculative_decoding.py

@@ -7,7 +7,7 @@ import requests
 from sglang.srt.environ import envs
 from sglang.srt.utils import kill_process_tree
 from sglang.test.ci.ci_register import register_cuda_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
+from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_DRAFT_MODEL_STANDALONE,
     DEFAULT_TARGET_MODEL_STANDALONE,
@@ -22,7 +22,6 @@ register_cuda_ci(est_time=308, suite="stage-b-test-1-gpu-large")
 
 GSM_DATASET_PATH = None
 
-
 # Default server arguments shared across all tests
 DEFAULT_SERVER_ARGS = [
     "--trust-remote-code",
@@ -97,19 +96,21 @@ class TestStandaloneSpeculativeDecodingBase(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
             num_shots=4,
-            num_questions=100,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-            data_path=GSM_DATASET_PATH,
+            gsm8k_data_path=GSM_DATASET_PATH,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         # Use the appropriate metric key based on the test class
-        metric_key = "accuracy"
+        metric_key = "score"
         self.assertGreater(metrics[metric_key], self.accuracy_threshold)
 
         server_info = requests.get(self.base_url + "/get_server_info")
@@ -158,19 +159,21 @@ class TestStandaloneV2SpeculativeDecodingBase(CustomTestCase):
         requests.get(self.base_url + "/flush_cache")
 
         args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="gsm8k",
+            api="completion",
+            max_tokens=512,
+            num_examples=100,
+            num_threads=128,
             num_shots=4,
-            num_questions=100,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-            data_path=GSM_DATASET_PATH,
+            gsm8k_data_path=GSM_DATASET_PATH,
         )
-        metrics = run_eval_few_shot_gsm8k(args)
+        metrics = run_eval(args)
         print(f"{metrics=}")
 
         # Use the appropriate metric key based on the test class
-        metric_key = "accuracy"
+        metric_key = "score"
         self.assertGreater(metrics[metric_key], self.accuracy_threshold)
 
         server_info = requests.get(self.base_url + "/get_server_info")