[Test] Split the serving perf tests by topic into basic_perf/ and route their thresholds through a kit (#40505)

python/sglang/test/kits/perf_bench_kit.py

@@ -0,0 +1,82 @@
+"""Report and bound the numbers a serving benchmark produces.
+
+Bounds are tuned per CI runner, so a local run prints them without asserting.
+"""
+
+from dataclasses import dataclass
+from typing import Callable, Optional, Sequence
+
+from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary
+
+
+@dataclass
+class Metric:
+    name: str
+    value: float
+    unit: str
+    bound: Optional[float] = None
+    amd_bound: Optional[float] = None
+    # unittest method that enforces `bound`, e.g. "assertLessEqual".
+    assertion: Optional[str] = None
+
+    def line(self) -> str:
+        return f"{self.name}: {self.value:.2f}" + (f" {self.unit}" if self.unit else "")
+
+    def check(self, test_case) -> None:
+        if self.bound is None:
+            return
+        limit = self.bound
+        if is_in_amd_ci() and self.amd_bound is not None:
+            limit = self.amd_bound
+        getattr(test_case, self.assertion)(self.value, limit)
+
+
+def at_least(name, value, bound, *, amd=None, unit="") -> Metric:
+    """A throughput-like number: the run passes when it reaches `bound`."""
+    return Metric(name, value, unit, bound, amd, "assertGreaterEqual")
+
+
+def at_most(name, value, bound, *, amd=None, unit="") -> Metric:
+    """A latency-like number: the run passes when it stays under `bound`."""
+    return Metric(name, value, unit, bound, amd, "assertLessEqual")
+
+
+def reported(name, value, *, unit="") -> Metric:
+    """A number worth printing that no threshold is attached to."""
+    return Metric(name, value, unit)
+
+
+def check_perf(test_case, *metrics: Metric, suffix: str = "") -> None:
+    """Report every metric under the caller's test name, then enforce in CI."""
+    label = test_case._testMethodName + suffix
+    report = f"### {label}\n" + "".join(m.line() + "\n" for m in metrics)
+    print(report, end="")
+    if not is_in_ci():
+        return
+    write_github_step_summary(report)
+    for m in metrics:
+        m.check(test_case)
+
+
+def check_batch_scaling(
+    test_case,
+    run_all: Callable[[Sequence[int]], Sequence[dict]],
+    bounds: Sequence[tuple],
+) -> None:
+    """Bound the latency at each batch size of one sweep.
+
+    `run_all` takes every size at once so they share one server. Each `bounds`
+    entry is `(batch_size, avg_ms, p95_ms, amd_avg_ms, amd_p95_ms)`.
+    """
+    results = run_all([b[0] for b in bounds])
+    for (batch_size, avg_ms, p95_ms, amd_avg_ms, amd_p95_ms), res in zip(
+        bounds, results
+    ):
+        test_case.assertEqual(res["successful_requests"], res["total_requests"])
+        check_perf(
+            test_case,
+            at_most("avg_latency_ms", res["avg_latency_ms"], avg_ms, amd=amd_avg_ms),
+            at_most("p95_latency_ms", res["p95_latency_ms"], p95_ms, amd=amd_p95_ms),
+            reported("throughput", res["throughput"], unit="req/s"),
+            suffix=f"_size_{batch_size}",
+        )

python/sglang/test/kits/vlm_perf_kit.py

@@ -0,0 +1,94 @@
+"""One VLM serving benchmark, run against a named attention backend.
+
+Pinned rather than inherited from `get_default_attn_backend`, so the file name
+keeps naming the right kernel after the default moves.
+"""
+
+import os
+from typing import Optional
+
+from sglang.test.kits.perf_bench_kit import at_least, at_most, check_perf, reported
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
+    DEFAULT_URL_FOR_TEST,
+    auto_config_device,
+    get_benchmark_args,
+    run_bench_serving_multi,
+)
+
+
+def _local_tokenizer_path():
+    # The HF Hub API call can stall for minutes in CI; prefer a local snapshot.
+    try:
+        from sglang.srt.utils import find_local_repo_dir
+
+        local_dir = find_local_repo_dir(
+            DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, revision=None
+        )
+        if local_dir and os.path.isdir(local_dir):
+            return local_dir
+    except Exception:
+        pass
+    return None
+
+
+def check_vlm_serving_perf(
+    test_case,
+    attention_backend: str,
+    *,
+    e2e_ms: float,
+    ttft_ms: float,
+    itl_ms: float,
+    output_throughput: Optional[float] = None,
+):
+    """Offline then online against one server; bound both phases.
+
+    `output_throughput` unset means the offline number is reported, not bounded.
+    """
+    common = dict(
+        base_url=DEFAULT_URL_FOR_TEST,
+        dataset_name="mmmu",
+        dataset_path="",
+        tokenizer=_local_tokenizer_path(),
+        random_input_len=4096,
+        random_output_len=2048,
+        sharegpt_context_len=None,
+        disable_stream=False,
+        disable_ignore_eos=False,
+        seed=0,
+        device=auto_config_device(),
+        lora_name=None,
+    )
+    offline = get_benchmark_args(num_prompts=200, request_rate=float("inf"), **common)
+    # 50 is enough for a stable median against these loose ceilings.
+    online = get_benchmark_args(num_prompts=50, request_rate=1, **common)
+
+    (_, res_offline), (_, res_online) = run_bench_serving_multi(
+        DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
+        DEFAULT_URL_FOR_TEST,
+        other_server_args=[
+            "--mem-fraction-static",
+            "0.7",
+            "--attention-backend",
+            attention_backend,
+        ],
+        benchmark_args=[offline, online],
+    )
+
+    throughput = res_offline["output_throughput"]
+    check_perf(
+        test_case,
+        (
+            at_least("output_throughput", throughput, output_throughput, unit="token/s")
+            if output_throughput is not None
+            else reported("output_throughput", throughput, unit="token/s")
+        ),
+        at_most(
+            "median_e2e_latency_ms",
+            res_online["median_e2e_latency_ms"],
+            e2e_ms,
+            unit="ms",
+        ),
+        at_most("median_ttft_ms", res_online["median_ttft_ms"], ttft_ms, unit="ms"),
+        at_most("median_itl_ms", res_online["median_itl_ms"], itl_ms, unit="ms"),
+    )

python/sglang/test/test_utils.py

@@ -61,7 +61,6 @@ DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B"
 DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat"
 
 # MLA test models
-DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct"
 DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST = "cross-encoder/ms-marco-MiniLM-L6-v2"
 DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"
 DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8"
@@ -1167,6 +1166,29 @@ def run_score_benchmark(
     device="auto",
 ):
     """Score API benchmark function compatible with run_bench_serving pattern"""
+    return run_score_benchmark_multi(
+        model,
+        [batch_size],
+        num_requests=num_requests,
+        other_server_args=other_server_args,
+        need_warmup=need_warmup,
+        device=device,
+    )[0]
+
+
+def run_score_benchmark_multi(
+    model,
+    batch_sizes,
+    num_requests=100,
+    other_server_args=None,
+    need_warmup=False,
+    device="auto",
+):
+    """One server, one benchmark per batch size.
+
+    Batch size is a property of the request, not of the server, so the launch
+    is shared rather than repeated per size.
+    """
     if other_server_args is None:
         other_server_args = []
 
@@ -1182,7 +1204,7 @@ def run_score_benchmark(
         other_args=other_server_args,
     )
 
-    async def _run_benchmark():
+    async def _run_benchmark(batch_size, warmup):
         # Load tokenizer for generating test data
         from sglang.srt.utils.hf_transformers_utils import get_tokenizer
 
@@ -1205,7 +1227,7 @@ def run_score_benchmark(
                 )
             return text
 
-        if need_warmup:
+        if warmup:
             warmup_data = {
                 "query": generate_text_with_token_count(score_query_tokens),
                 "items": [
@@ -1253,12 +1275,16 @@ def run_score_benchmark(
         )
 
     try:
-        res = asyncio.run(_run_benchmark())
+        results = [
+            asyncio.run(_run_benchmark(bs, need_warmup and i == 0))
+            for i, bs in enumerate(batch_sizes)
+        ]
     finally:
         kill_process_tree(process.pid)
 
-    assert res["completed"] == res["successful_requests"]
+    for res in results:
-    return res
+        assert res["completed"] == res["successful_requests"]
+    return results
 
 
 def run_embeddings_benchmark(
@@ -1271,6 +1297,27 @@ def run_embeddings_benchmark(
     device="auto",
 ):
     """Embeddings API benchmark function compatible with run_bench_serving pattern"""
+    return run_embeddings_benchmark_multi(
+        model,
+        [batch_size],
+        num_requests=num_requests,
+        input_tokens=input_tokens,
+        other_server_args=other_server_args,
+        need_warmup=need_warmup,
+        device=device,
+    )[0]
+
+
+def run_embeddings_benchmark_multi(
+    model,
+    batch_sizes,
+    num_requests=100,
+    input_tokens=500,
+    other_server_args=None,
+    need_warmup=False,
+    device="auto",
+):
+    """One server, one benchmark per batch size. See run_score_benchmark_multi."""
     if other_server_args is None:
         other_server_args = []
 
@@ -1289,7 +1336,7 @@ def run_embeddings_benchmark(
         other_args=server_args,
     )
 
-    async def _run_benchmark():
+    async def _run_benchmark(batch_size, warmup):
 
         def generate_text_with_token_count(num_tokens):
             """Generate text with precise token count using special tokens."""
@@ -1300,7 +1347,7 @@ def run_embeddings_benchmark(
         # Generate input text
         input_text = generate_text_with_token_count(input_tokens)
 
-        if need_warmup:
+        if warmup:
             warmup_data = {
                 "input": input_text,
                 "model": model,
@@ -1340,12 +1387,16 @@ def run_embeddings_benchmark(
         )
 
     try:
-        res = asyncio.run(_run_benchmark())
+        results = [
+            asyncio.run(_run_benchmark(bs, need_warmup and i == 0))
+            for i, bs in enumerate(batch_sizes)
+        ]
     finally:
         kill_process_tree(process.pid)
 
-    assert res["completed"] == res["successful_requests"]
+    for res in results:
-    return res
+        assert res["completed"] == res["successful_requests"]
+    return results
 
 
 def run_bench_serving_multi(

test/registered/basic_perf/test_eagle3_latency.py

@@ -0,0 +1,60 @@
+"""The only test in the tree that bounds speculative decoding LATENCY; every
+other one bounds accept length. CUDA only -- AMD bounds are unmeasured.
+"""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_least, at_most, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_DRAFT_MODEL_EAGLE3,
+    DEFAULT_TARGET_MODEL_EAGLE3,
+    CustomTestCase,
+    run_bench_serving,
+)
+
+register_cuda_ci(est_time=145, stage="extra-a", runner_config="1-gpu-large")
+
+
+class TestEagle3Latency(CustomTestCase):
+    def test_online_latency_eagle3(self):
+        res = run_bench_serving(
+            model=DEFAULT_TARGET_MODEL_EAGLE3,
+            num_prompts=300,
+            request_rate=8,
+            sharegpt_context_len=3072,
+            disable_ignore_eos=True,
+            dataset_name="sharegpt",
+            other_server_args=[
+                "--speculative-algorithm",
+                "EAGLE3",
+                "--speculative-draft-model-path",
+                DEFAULT_DRAFT_MODEL_EAGLE3,
+                "--speculative-num-steps",
+                "5",
+                "--speculative-eagle-topk",
+                "4",
+                "--speculative-num-draft-tokens",
+                "16",
+                "--mem-fraction-static",
+                "0.7",
+                # The draft checkpoint ships fp16 and the target bf16; the CUDA
+                # rmsnorm path rejects a weight and activation pair that disagree.
+                "--dtype",
+                "float16",
+            ],
+            need_warmup=True,
+            seed=42,
+        )
+
+        check_perf(
+            self,
+            at_most(
+                "median_e2e_latency_ms", res["median_e2e_latency_ms"], 1150, unit="ms"
+            ),
+            at_least("accept_length", res["accept_length"], 2.3),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_embeddings_api.py

@@ -0,0 +1,61 @@
+"""Latency and throughput of the /v1/embeddings endpoint."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import (
+    at_least,
+    at_most,
+    check_batch_scaling,
+    check_perf,
+)
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    run_embeddings_benchmark,
+    run_embeddings_benchmark_multi,
+)
+
+register_cuda_ci(est_time=245, stage="extra-a", runner_config="1-gpu-large")
+register_amd_ci(est_time=240, suite="stage-b-test-1-gpu-large-amd")
+
+
+class TestEmbeddingsAPI(CustomTestCase):
+    def test_embeddings_api_latency_throughput(self):
+        res = run_embeddings_benchmark(
+            model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
+            num_requests=1000,
+            batch_size=1,
+            input_tokens=500,
+            other_server_args=[],
+            need_warmup=True,
+        )
+
+        self.assertEqual(res["successful_requests"], res["total_requests"])
+        check_perf(
+            self,
+            at_most("avg_latency_ms", res["avg_latency_ms"], 21, amd=35, unit="ms"),
+            at_most("p95_latency_ms", res["p95_latency_ms"], 26, amd=40, unit="ms"),
+            at_least("throughput", res["throughput"], 48, amd=30, unit="req/s"),
+        )
+
+    def test_embeddings_api_batch_scaling(self):
+        check_batch_scaling(
+            self,
+            lambda batch_sizes: run_embeddings_benchmark_multi(
+                DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
+                batch_sizes,
+                num_requests=500,
+                input_tokens=500,
+            ),
+            # batch size, avg ms, p95 ms, then the same two relaxed for mi300x
+            [
+                (10, 43, 49, 80, 90),
+                (25, 70, 78, 140, 150),
+                (50, 122, 158, 230, 240),
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_lora_latency.py

@@ -0,0 +1,134 @@
+"""Latency of the LoRA serving path, with and without adapter churn."""
+
+import asyncio
+import itertools
+import unittest
+
+import requests
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_most, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    run_bench_serving,
+)
+
+register_cuda_ci(est_time=490, stage="extra-a", runner_config="1-gpu-large")
+register_amd_ci(est_time=430, suite="stage-b-test-1-gpu-large-amd")
+
+
+class TestLoRALatency(CustomTestCase):
+    def test_online_lora_latency(self):
+        res = self._run_lora_latency_test(enable_background_task=False)
+
+        check_perf(
+            self,
+            at_most(
+                "median_e2e_latency_ms",
+                res["median_e2e_latency_ms"],
+                2270,
+                amd=3320,
+                unit="ms",
+            ),
+            # mi300x is about twice as slow as mi325 on LoRA TTFT.
+            at_most("median_ttft_ms", res["median_ttft_ms"], 51, amd=100, unit="ms"),
+        )
+
+    def test_online_lora_latency_with_concurrent_adapter_updates(self):
+        res = self._run_lora_latency_test(enable_background_task=True)
+
+        check_perf(
+            self,
+            at_most(
+                "median_e2e_latency_ms",
+                res["median_e2e_latency_ms"],
+                3170,
+                amd=6000,
+                unit="ms",
+            ),
+            at_most("median_ttft_ms", res["median_ttft_ms"], 55, amd=130, unit="ms"),
+        )
+
+    def _run_lora_latency_test(self, enable_background_task: bool):
+        async def lora_loader_unloader_task(
+            base_url: str,
+            start_event: asyncio.Event,
+            stop_event: asyncio.Event,
+        ):
+            """
+            A background task that repeatedly loads and unloads a LoRA adapter.
+            """
+            await start_event.wait()
+
+            path_cycler = itertools.cycle(
+                [
+                    "pbevan11/llama-3.1-8b-ocr-correction",
+                    "faridlazuarda/valadapt-llama-3.1-8B-it-chinese",
+                    "philschmid/code-llama-3-1-8b-text-to-sql-lora",
+                ]
+            )
+            load_url = f"{base_url}/load_lora_adapter"
+            unload_url = f"{base_url}/unload_lora_adapter"
+            num_updates = 0
+
+            while not stop_event.is_set():
+                lora_path = next(path_cycler)
+                response = await asyncio.to_thread(
+                    requests.post,
+                    load_url,
+                    json={"lora_name": lora_path, "lora_path": lora_path},
+                )
+                self.assertTrue(
+                    response.ok, f"Failed to load LoRA adapter: {response.text}"
+                )
+                num_updates += 1
+
+                if stop_event.is_set():
+                    break
+
+                await asyncio.sleep(1)
+
+                response = await asyncio.to_thread(
+                    requests.post,
+                    unload_url,
+                    json={"lora_name": lora_path},
+                )
+                self.assertTrue(
+                    response.ok, f"Failed to unload LoRA adapter: {response.text}"
+                )
+                num_updates += 1
+
+                await asyncio.sleep(1)
+
+        background_task = lora_loader_unloader_task if enable_background_task else None
+        res = run_bench_serving(
+            model=DEFAULT_MODEL_NAME_FOR_TEST,
+            num_prompts=400,
+            request_rate=8,
+            other_server_args=[
+                "--enable-lora",
+                "--max-loras-per-batch",
+                "1",
+                "--disable-radix-cache",
+                "--random-seed",
+                "42",
+                "--mem-fraction-static",
+                "0.8",
+                "--lora-paths",
+                "nvidia/llama-3.1-nemoguard-8b-topic-control",
+                "--max-lora-rank",
+                "256",
+            ],
+            dataset_name="random",
+            random_input_len=256,
+            random_output_len=256,
+            lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
+            background_task=background_task,
+        )
+
+        return res
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_moe_throughput.py

@@ -0,0 +1,53 @@
+"""Throughput of the MoE model on two GPUs, batched and at batch size one."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_least, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_MOE_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    run_bench_offline_throughput,
+    run_bench_serving,
+)
+
+register_cuda_ci(est_time=290, stage="extra-a", runner_config="2-gpu-large")
+register_amd_ci(est_time=770, suite="stage-b-test-2-gpu-large-amd")
+
+
+class TestMoEThroughput(CustomTestCase):
+    def test_moe_offline_throughput_default(self):
+        res = run_bench_serving(
+            model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
+            num_prompts=300,
+            request_rate=float("inf"),
+            other_server_args=["--tp", "2"],
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput",
+                res["output_throughput"],
+                2670,
+                amd=2100,
+                unit="token/s",
+            ),
+        )
+
+    def test_moe_tp2_bs1(self):
+        output_throughput = run_bench_offline_throughput(
+            DEFAULT_MOE_MODEL_NAME_FOR_TEST,
+            ["--tp", "2", "--cuda-graph-max-bs-decode", "2"],
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput", output_throughput, 139, amd=85, unit="token/s"
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_pp_throughput.py

@@ -0,0 +1,70 @@
+"""Throughput of pipeline parallelism on two GPUs, decode and long prefill."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_least, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_MOE_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    is_in_amd_ci,
+    run_bench_serving,
+)
+
+register_cuda_ci(est_time=490, stage="extra-a", runner_config="2-gpu-large")
+register_amd_ci(est_time=1030, suite="stage-b-test-2-gpu-large-amd")
+
+
+class TestPPThroughput(CustomTestCase):
+    def test_pp_offline_throughput_default_decode(self):
+        res = run_bench_serving(
+            model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
+            num_prompts=1000,
+            request_rate=float("inf"),
+            random_input_len=1,
+            random_output_len=1024,
+            other_server_args=["--pp-size", "2"],
+            need_warmup=True,
+            seed=42,
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput", res["output_throughput"], 6250, unit="token/s"
+            ),
+        )
+
+    def test_pp_long_context_prefill(self):
+        res = run_bench_serving(
+            model="meta-llama/Llama-3.3-70B-Instruct",
+            num_prompts=4,
+            request_rate=float("inf"),
+            random_input_len=128000,
+            random_output_len=1,
+            dataset_name="random",
+            other_server_args=[
+                "--quantization",
+                "fp8",
+                "--pp-size",
+                "2",
+            ]
+            + (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []),
+            need_warmup=False,
+            seed=42,
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "input_throughput",
+                res["input_throughput"],
+                4380,
+                amd=3000,
+                unit="token/s",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_score_api.py

@@ -0,0 +1,55 @@
+"""Latency and throughput of the /v1/score endpoint."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import (
+    at_least,
+    at_most,
+    check_batch_scaling,
+    check_perf,
+)
+from sglang.test.test_utils import (
+    DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
+    CustomTestCase,
+    run_score_benchmark,
+    run_score_benchmark_multi,
+)
+
+register_cuda_ci(est_time=215, stage="extra-a", runner_config="1-gpu-large")
+register_amd_ci(est_time=210, suite="stage-b-test-1-gpu-large-amd")
+
+
+class TestScoreAPI(CustomTestCase):
+    def test_score_api_latency_throughput(self):
+        res = run_score_benchmark(
+            model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
+            num_requests=1000,
+            batch_size=10,
+            other_server_args=[],
+            need_warmup=True,
+        )
+
+        self.assertEqual(res["successful_requests"], res["total_requests"])
+        check_perf(
+            self,
+            at_most("avg_latency_ms", res["avg_latency_ms"], 30, amd=60, unit="ms"),
+            at_most("p95_latency_ms", res["p95_latency_ms"], 32, amd=65, unit="ms"),
+            at_least("throughput", res["throughput"], 34, amd=16, unit="req/s"),
+        )
+
+    def test_score_api_batch_scaling(self):
+        check_batch_scaling(
+            self,
+            lambda batch_sizes: run_score_benchmark_multi(
+                DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
+                batch_sizes,
+                num_requests=500,
+            ),
+            # batch size, avg ms, p95 ms, then the same two relaxed for mi300x
+            [(10, 30, 34, 60, 65), (25, 35, 39, 70, 80), (50, 51, 59, 80, 90)],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_serving_latency.py

@@ -0,0 +1,40 @@
+"""Latency of the default serving path on one large GPU."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_most, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    run_bench_serving,
+)
+
+register_cuda_ci(est_time=190, stage="extra-a", runner_config="1-gpu-large")
+register_amd_ci(est_time=165, suite="stage-b-test-1-gpu-large-amd")
+
+
+class TestServingLatency(CustomTestCase):
+    def test_online_latency_default(self):
+        res = run_bench_serving(
+            model=DEFAULT_MODEL_NAME_FOR_TEST,
+            num_prompts=100,
+            request_rate=1,
+            other_server_args=[],
+        )
+
+        check_perf(
+            self,
+            at_most(
+                "median_e2e_latency_ms",
+                res["median_e2e_latency_ms"],
+                9100,
+                unit="ms",
+            ),
+            at_most("median_ttft_ms", res["median_ttft_ms"], 80, amd=115, unit="ms"),
+            at_most("median_itl_ms", res["median_itl_ms"], 9, unit="ms"),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_serving_throughput.py

@@ -0,0 +1,107 @@
+"""Offline throughput of the default serving path on one large GPU."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_least, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_TEST,
+    DEFAULT_MODEL_NAME_FOR_TEST_FP8,
+    CustomTestCase,
+    run_bench_serving,
+)
+
+register_cuda_ci(est_time=710, stage="extra-a", runner_config="1-gpu-large")
+register_amd_ci(est_time=810, suite="stage-b-test-1-gpu-large-amd")
+
+
+class TestServingThroughput(CustomTestCase):
+    def test_offline_throughput_default(self):
+        res = run_bench_serving(
+            model=DEFAULT_MODEL_NAME_FOR_TEST,
+            num_prompts=500,
+            request_rate=float("inf"),
+            other_server_args=[],
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput",
+                res["output_throughput"],
+                4000,
+                amd=3050,
+                unit="token/s",
+            ),
+        )
+
+    def test_offline_throughput_non_stream_small_batch_size(self):
+        res = run_bench_serving(
+            model=DEFAULT_MODEL_NAME_FOR_TEST,
+            num_prompts=200,
+            request_rate=float("inf"),
+            other_server_args=["--max-running-requests", "10"],
+            dataset_name="sharegpt",
+            random_input_len=None,
+            random_output_len=None,
+            disable_stream=True,
+            need_warmup=True,
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput",
+                res["output_throughput"],
+                1110,
+                amd=1000,
+                unit="token/s",
+            ),
+        )
+
+    def test_offline_throughput_with_triton_attention_backend(self):
+        res = run_bench_serving(
+            model=DEFAULT_MODEL_NAME_FOR_TEST,
+            num_prompts=500,
+            request_rate=float("inf"),
+            other_server_args=[
+                "--attention-backend",
+                "triton",
+                "--context-length",
+                "8192",
+            ],
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput",
+                res["output_throughput"],
+                3730,
+                amd=2700,
+                unit="token/s",
+            ),
+        )
+
+    def test_offline_throughput_default_fp8(self):
+        res = run_bench_serving(
+            model=DEFAULT_MODEL_NAME_FOR_TEST_FP8,
+            num_prompts=500,
+            request_rate=float("inf"),
+            other_server_args=[],
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput",
+                res["output_throughput"],
+                4870,
+                amd=3500,
+                unit="token/s",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_torch_compile_throughput.py

@@ -0,0 +1,33 @@
+"""Throughput of torch.compile at batch size one across two GPUs."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
+from sglang.test.kits.perf_bench_kit import at_least, check_perf
+from sglang.test.test_utils import (
+    DEFAULT_MODEL_NAME_FOR_TEST,
+    CustomTestCase,
+    run_bench_offline_throughput,
+)
+
+register_cuda_ci(est_time=75, stage="extra-a", runner_config="2-gpu-large")
+register_amd_ci(est_time=280, suite="stage-b-test-2-gpu-large-amd")
+
+
+class TestTorchCompileThroughput(CustomTestCase):
+    def test_torch_compile_tp2_bs1(self):
+        output_throughput = run_bench_offline_throughput(
+            DEFAULT_MODEL_NAME_FOR_TEST,
+            ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"],
+        )
+
+        check_perf(
+            self,
+            at_least(
+                "output_throughput", output_throughput, 255, amd=200, unit="token/s"
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_vlm_serving_aiter.py

@@ -0,0 +1,25 @@
+"""VLM serving perf on the aiter attention backend."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_amd_ci
+from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf
+from sglang.test.test_utils import CustomTestCase
+
+register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
+
+
+class TestVLMServingAiter(CustomTestCase):
+    def test_vlm_serving_aiter(self):
+        check_vlm_serving_perf(
+            self,
+            "aiter",
+            output_throughput=2000,
+            e2e_ms=16500,
+            ttft_ms=150,
+            itl_ms=8,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_vlm_serving_fa3.py

@@ -0,0 +1,26 @@
+"""VLM serving perf on the fa3 attention backend."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf
+from sglang.test.test_utils import CustomTestCase
+
+register_cuda_ci(est_time=150, stage="extra-a", runner_config="1-gpu-large")
+
+
+class TestVLMServingFa3(CustomTestCase):
+    def test_vlm_serving_fa3(self):
+        check_vlm_serving_perf(
+            self,
+            "fa3",
+            # No offline bound: never measured on this lane.
+            output_throughput=16700,
+            e2e_ms=11000,
+            ttft_ms=84,
+            itl_ms=5.2,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/basic_perf/test_vlm_serving_flashinfer.py

@@ -0,0 +1,25 @@
+"""VLM serving perf on the flashinfer attention backend."""
+
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf
+from sglang.test.test_utils import CustomTestCase
+
+register_cuda_ci(est_time=195, stage="extra-a", runner_config="1-gpu-small")
+
+
+class TestVLMServingFlashinfer(CustomTestCase):
+    def test_vlm_serving_flashinfer(self):
+        check_vlm_serving_perf(
+            self,
+            "flashinfer",
+            output_throughput=6900,
+            e2e_ms=17300,
+            ttft_ms=76,
+            itl_ms=8.3,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/perf/test_bench_one_batch_2gpu.py

@@ -1,53 +0,0 @@
-import unittest
-
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.test_utils import (
-    DEFAULT_MODEL_NAME_FOR_TEST,
-    DEFAULT_MOE_MODEL_NAME_FOR_TEST,
-    CustomTestCase,
-    is_in_amd_ci,
-    is_in_ci,
-    run_bench_offline_throughput,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=162, stage="extra-a", runner_config="2-gpu-large")
-register_amd_ci(est_time=630, suite="stage-b-test-2-gpu-large-amd")
-
-
-class TestBenchOneBatch2GPU(CustomTestCase):
-    def test_moe_tp2_bs1(self):
-        output_throughput = run_bench_offline_throughput(
-            DEFAULT_MOE_MODEL_NAME_FOR_TEST,
-            ["--tp", "2", "--cuda-graph-max-bs-decode", "2"],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n"
-                f"output_throughput: {output_throughput:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(output_throughput, 85)
-            else:
-                self.assertGreater(output_throughput, 125)
-
-    def test_torch_compile_tp2_bs1(self):
-        output_throughput = run_bench_offline_throughput(
-            DEFAULT_MODEL_NAME_FOR_TEST,
-            ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n"
-                f"output_throughput: {output_throughput:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(output_throughput, 200)
-            else:
-                self.assertGreater(output_throughput, 220)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/perf/test_bench_serving_1gpu_large.py

@@ -1,84 +0,0 @@
-"""
-Performance tests for single GPU that need H200 (80GB) - FP8 and EAGLE tests.
-"""
-
-import unittest
-
-from sglang.srt.utils import is_hip
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.test_utils import (
-    DEFAULT_DRAFT_MODEL_EAGLE,
-    DEFAULT_MODEL_NAME_FOR_TEST_FP8,
-    DEFAULT_TARGET_MODEL_EAGLE,
-    CustomTestCase,
-    is_in_amd_ci,
-    is_in_ci,
-    run_bench_serving,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=275, stage="extra-a", runner_config="1-gpu-large")
-register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-large-amd")
-
-
-class TestBenchServing1GPULarge(CustomTestCase):
-    def test_offline_throughput_default_fp8(self):
-        res = run_bench_serving(
-            model=DEFAULT_MODEL_NAME_FOR_TEST_FP8,
-            num_prompts=500,
-            request_rate=float("inf"),
-            other_server_args=[],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_offline_throughput_default_fp8\n"
-                f"Output throughput: {res['output_throughput']:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(res["output_throughput"], 3500)
-            else:
-                self.assertGreater(res["output_throughput"], 4300)
-
-    @unittest.skipIf(is_hip(), "Skip Eagle test for ROCm")
-    def test_online_latency_eagle(self):
-        res = run_bench_serving(
-            model=DEFAULT_TARGET_MODEL_EAGLE,
-            num_prompts=300,
-            request_rate=8,
-            sharegpt_context_len=3072,
-            disable_ignore_eos=True,
-            dataset_name="sharegpt",
-            other_server_args=[
-                "--speculative-algorithm",
-                "EAGLE",
-                "--speculative-draft-model-path",
-                DEFAULT_DRAFT_MODEL_EAGLE,
-                "--speculative-num-steps",
-                "5",
-                "--speculative-eagle-topk",
-                "4",
-                "--speculative-num-draft-tokens",
-                "16",
-                "--mem-fraction-static",
-                "0.7",
-            ],
-            need_warmup=True,
-            seed=42,
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_online_latency_eagle\n"
-                f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
-                f"accept_length: {res['accept_length']:.2f} \n"
-            )
-            if is_in_amd_ci():
-                self.assertLess(res["median_e2e_latency_ms"], 1800)
-            else:
-                self.assertLess(res["median_e2e_latency_ms"], 900)
-            self.assertGreater(res["accept_length"], 3.0)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/perf/test_bench_serving_1gpu_part1.py

@@ -1,234 +0,0 @@
-"""
-Performance tests for single GPU - LLM throughput/latency and LoRA tests.
-Works on 5090 (32GB).
-"""
-
-import asyncio
-import itertools
-import unittest
-
-import requests
-
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.test_utils import (
-    DEFAULT_MODEL_NAME_FOR_TEST,
-    CustomTestCase,
-    is_in_amd_ci,
-    is_in_ci,
-    run_bench_serving,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=1264, stage="extra-a", runner_config="1-gpu-large")
-register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-large-amd")
-
-
-class TestBenchServing1GPUPart1(CustomTestCase):
-    def test_offline_throughput_default(self):
-        res = run_bench_serving(
-            model=DEFAULT_MODEL_NAME_FOR_TEST,
-            num_prompts=500,
-            request_rate=float("inf"),
-            other_server_args=[],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_offline_throughput_default\n"
-                f"Output throughput: {res['output_throughput']:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(res["output_throughput"], 3050)
-            else:
-                self.assertGreater(res["output_throughput"], 3800)
-
-    def test_offline_throughput_non_stream_small_batch_size(self):
-        res = run_bench_serving(
-            model=DEFAULT_MODEL_NAME_FOR_TEST,
-            num_prompts=200,
-            request_rate=float("inf"),
-            other_server_args=["--max-running-requests", "10"],
-            dataset_name="sharegpt",
-            random_input_len=None,
-            random_output_len=None,
-            disable_stream=True,
-            need_warmup=True,
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_offline_throughput_non_stream_small_batch_size\n"
-                f"Output throughput: {res['output_throughput']:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(res["output_throughput"], 1000)
-            else:
-                self.assertGreater(res["output_throughput"], 1050)
-
-    def test_offline_throughput_with_triton_attention_backend(self):
-        res = run_bench_serving(
-            model=DEFAULT_MODEL_NAME_FOR_TEST,
-            num_prompts=500,
-            request_rate=float("inf"),
-            other_server_args=[
-                "--attention-backend",
-                "triton",
-                "--context-length",
-                "8192",
-            ],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_offline_throughput_with_triton_attention_backend\n"
-                f"Output throughput: {res['output_throughput']:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(res["output_throughput"], 2700)
-            else:
-                self.assertGreater(res["output_throughput"], 3700)
-
-    def test_online_latency_default(self):
-        res = run_bench_serving(
-            model=DEFAULT_MODEL_NAME_FOR_TEST,
-            num_prompts=100,
-            request_rate=1,
-            other_server_args=[],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_online_latency_default\n"
-                f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
-            )
-            self.assertLess(res["median_e2e_latency_ms"], 11000)
-            if is_in_amd_ci():
-                self.assertLess(res["median_ttft_ms"], 115)
-            else:
-                self.assertLess(res["median_ttft_ms"], 86)
-            self.assertLess(res["median_itl_ms"], 10)
-
-    def test_online_lora_latency(self):
-        res = self._run_lora_latency_test(enable_background_task=False)
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_online_lora_latency\n"
-                f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
-                f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
-            )
-            if is_in_amd_ci():
-                self.assertLess(res["median_e2e_latency_ms"], 3320)
-            else:
-                self.assertLess(res["median_e2e_latency_ms"], 2400)
-            # relax for mi300x (LoRA TTFT ~2x slower than mi325)
-            if is_in_amd_ci():
-                self.assertLess(res["median_ttft_ms"], 100)
-            else:
-                self.assertLess(res["median_ttft_ms"], 58)
-
-    def test_online_lora_latency_with_concurrent_adapter_updates(self):
-        res = self._run_lora_latency_test(enable_background_task=True)
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_online_lora_latency_with_concurrent_adapter_updates\n"
-                f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
-                f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
-            )
-            if is_in_amd_ci():
-                self.assertLess(res["median_e2e_latency_ms"], 6000)
-            else:
-                self.assertLess(res["median_e2e_latency_ms"], 4000)
-            # relax for mi300x (LoRA TTFT ~2x slower than mi325)
-            if is_in_amd_ci():
-                self.assertLess(res["median_ttft_ms"], 130)
-            else:
-                self.assertLess(res["median_ttft_ms"], 80)
-
-    def _run_lora_latency_test(self, enable_background_task: bool):
-        """
-        Run a latency test for LoRA with the specified background task setting.
-        """
-
-        async def lora_loader_unloader_task(
-            base_url: str,
-            start_event: asyncio.Event,
-            stop_event: asyncio.Event,
-        ):
-            """
-            A background task that repeatedly loads and unloads a LoRA adapter.
-            """
-            await start_event.wait()
-
-            path_cycler = itertools.cycle(
-                [
-                    "pbevan11/llama-3.1-8b-ocr-correction",
-                    "faridlazuarda/valadapt-llama-3.1-8B-it-chinese",
-                    "philschmid/code-llama-3-1-8b-text-to-sql-lora",
-                ]
-            )
-            load_url = f"{base_url}/load_lora_adapter"
-            unload_url = f"{base_url}/unload_lora_adapter"
-            num_updates = 0
-
-            while not stop_event.is_set():
-                lora_path = next(path_cycler)
-                response = await asyncio.to_thread(
-                    requests.post,
-                    load_url,
-                    json={"lora_name": lora_path, "lora_path": lora_path},
-                )
-                self.assertTrue(
-                    response.ok, f"Failed to load LoRA adapter: {response.text}"
-                )
-                num_updates += 1
-
-                if stop_event.is_set():
-                    break
-
-                await asyncio.sleep(1)
-
-                response = await asyncio.to_thread(
-                    requests.post,
-                    unload_url,
-                    json={"lora_name": lora_path},
-                )
-                self.assertTrue(
-                    response.ok, f"Failed to unload LoRA adapter: {response.text}"
-                )
-                num_updates += 1
-
-                await asyncio.sleep(1)
-
-        background_task = lora_loader_unloader_task if enable_background_task else None
-        res = run_bench_serving(
-            model=DEFAULT_MODEL_NAME_FOR_TEST,
-            num_prompts=400,
-            request_rate=8,
-            other_server_args=[
-                "--enable-lora",
-                "--max-loras-per-batch",
-                "1",
-                "--disable-radix-cache",
-                "--random-seed",
-                "42",
-                "--mem-fraction-static",
-                "0.8",
-                "--lora-paths",
-                "nvidia/llama-3.1-nemoguard-8b-topic-control",
-                "--max-lora-rank",
-                "256",
-            ],
-            dataset_name="random",
-            random_input_len=256,
-            random_output_len=256,
-            lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
-            background_task=background_task,
-        )
-
-        return res
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/perf/test_bench_serving_1gpu_part2.py

@@ -1,185 +0,0 @@
-"""
-Performance tests for single GPU - VLM, Score API, and Embeddings API tests.
-Works on 5090 (32GB).
-"""
-
-import unittest
-
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.test_utils import (
-    DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
-    DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
-    DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
-    CustomTestCase,
-    is_in_amd_ci,
-    is_in_ci,
-    run_bench_serving,
-    run_embeddings_benchmark,
-    run_score_benchmark,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=909, stage="extra-a", runner_config="1-gpu-large")
-register_amd_ci(est_time=900, suite="stage-b-test-1-gpu-large-amd")
-
-
-class TestBenchServing1GPUPart2(CustomTestCase):
-    def test_vlm_online_latency(self):
-        res = run_bench_serving(
-            model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
-            num_prompts=250,
-            request_rate=1,
-            other_server_args=[
-                "--mem-fraction-static",
-                "0.7",
-            ],
-            dataset_name="mmmu",
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_vlm_online_latency\n"
-                f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
-            )
-            self.assertLess(res["median_e2e_latency_ms"], 16500)
-            if is_in_amd_ci():
-                self.assertLess(res["median_ttft_ms"], 150)
-            else:
-                self.assertLess(res["median_ttft_ms"], 100)
-            self.assertLess(res["median_itl_ms"], 8)
-
-    def test_score_api_latency_throughput(self):
-        """Test score API latency and throughput performance"""
-        res = run_score_benchmark(
-            model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
-            num_requests=1000,
-            batch_size=10,
-            other_server_args=[],
-            need_warmup=True,
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_score_api_throughput\n"
-                f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
-                f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
-                f"Score API throughput: {res['throughput']:.2f} req/s\n"
-                f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
-            )
-
-        self.assertEqual(res["successful_requests"], res["total_requests"])
-        # relax for mi300x
-        if is_in_amd_ci():
-            self.assertLess(res["avg_latency_ms"], 60)
-            self.assertLess(res["p95_latency_ms"], 65)
-            self.assertGreater(res["throughput"], 16)
-        else:
-            self.assertLess(res["avg_latency_ms"], 48)
-            self.assertLess(res["p95_latency_ms"], 50)
-            self.assertGreater(res["throughput"], 20)
-
-    def test_score_api_batch_scaling(self):
-        """Test score API performance with different batch sizes"""
-        batch_sizes = [10, 25, 50]
-
-        for batch_size in batch_sizes:
-            res = run_score_benchmark(
-                model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
-                num_requests=500,
-                batch_size=batch_size,
-            )
-
-            if is_in_ci():
-                write_github_step_summary(
-                    f"### test_score_api_batch_scaling_size_{batch_size}\n"
-                    f"Batch size: {batch_size}\n"
-                    f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
-                    f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
-                    f"Throughput: {res['throughput']:.2f} req/s\n"
-                    f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
-                )
-
-            self.assertEqual(res["successful_requests"], res["total_requests"])
-            # relax for mi300x
-            if is_in_amd_ci():
-                bounds = {10: (60, 65), 25: (70, 80), 50: (80, 90)}
-                default_bounds = (90, 90)
-            else:
-                bounds = {10: (45, 50), 25: (50, 60), 50: (60, 65)}
-                default_bounds = (60, 65)
-            avg_latency_bound, p95_latency_bound = bounds.get(
-                batch_size, default_bounds
-            )
-            self.assertLess(res["avg_latency_ms"], avg_latency_bound)
-            self.assertLess(res["p95_latency_ms"], p95_latency_bound)
-
-    def test_embeddings_api_latency_throughput(self):
-        """Test embeddings API latency and throughput performance"""
-        res = run_embeddings_benchmark(
-            model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
-            num_requests=1000,
-            batch_size=1,
-            input_tokens=500,
-            other_server_args=[],
-            need_warmup=True,
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_embeddings_api_throughput\n"
-                f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
-                f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
-                f"Embeddings API throughput: {res['throughput']:.2f} req/s\n"
-                f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
-            )
-
-        self.assertEqual(res["successful_requests"], res["total_requests"])
-        # relax for mi300x
-        if is_in_amd_ci():
-            self.assertLess(res["avg_latency_ms"], 35)
-            self.assertLess(res["p95_latency_ms"], 40)
-            self.assertGreater(res["throughput"], 30)
-        else:
-            self.assertLess(res["avg_latency_ms"], 20)
-            self.assertLess(res["p95_latency_ms"], 25)
-            self.assertGreater(res["throughput"], 60)
-
-    def test_embeddings_api_batch_scaling(self):
-        """Test embeddings API performance with different batch sizes"""
-        batch_sizes = [10, 25, 50]
-
-        for batch_size in batch_sizes:
-            res = run_embeddings_benchmark(
-                model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
-                num_requests=500,
-                batch_size=batch_size,
-                input_tokens=500,
-            )
-
-            if is_in_ci():
-                write_github_step_summary(
-                    f"### test_embeddings_api_batch_scaling_size_{batch_size}\n"
-                    f"Batch size: {batch_size}\n"
-                    f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
-                    f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
-                    f"Throughput: {res['throughput']:.2f} req/s\n"
-                    f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
-                )
-
-            self.assertEqual(res["successful_requests"], res["total_requests"])
-            # relax for mi300x
-            if is_in_amd_ci():
-                bounds = {10: (80, 90), 25: (140, 150), 50: (230, 240)}
-                default_bounds = (300, 300)
-            else:
-                bounds = {10: (60, 65), 25: (115, 120), 50: (190, 195)}
-                default_bounds = (250, 250)
-            avg_latency_bound, p95_latency_bound = bounds.get(
-                batch_size, default_bounds
-            )
-            self.assertLess(res["avg_latency_ms"], avg_latency_bound)
-            self.assertLess(res["p95_latency_ms"], p95_latency_bound)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/perf/test_bench_serving_2gpu.py

@@ -1,90 +0,0 @@
-"""
-Performance tests for 2-GPU that need large GPUs (H200 80GB) - MoE and Pipeline Parallel tests.
-"""
-
-import unittest
-
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.test_utils import (
-    DEFAULT_MOE_MODEL_NAME_FOR_TEST,
-    CustomTestCase,
-    is_in_amd_ci,
-    is_in_ci,
-    run_bench_serving,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=687, stage="extra-a", runner_config="2-gpu-large")
-register_amd_ci(est_time=1450, suite="stage-b-test-2-gpu-large-amd")
-
-
-class TestBenchServing2GPU(CustomTestCase):
-    def test_moe_offline_throughput_default(self):
-        res = run_bench_serving(
-            model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
-            num_prompts=300,
-            request_rate=float("inf"),
-            other_server_args=["--tp", "2"],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_moe_offline_throughput_default\n"
-                f"Output throughput: {res['output_throughput']:.2f} token/s\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(res["output_throughput"], 2100)
-            else:
-                self.assertGreater(res["output_throughput"], 2200)
-
-    def test_pp_offline_throughput_default_decode(self):
-        res = run_bench_serving(
-            model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
-            num_prompts=1000,
-            request_rate=float("inf"),
-            random_input_len=1,
-            random_output_len=1024,
-            other_server_args=["--pp-size", "2"],
-            need_warmup=True,
-            seed=42,
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_pp_offline_throughput_default_decode\n"
-                f"Output throughput: {res['output_throughput']:.2f} token/s\n"
-            )
-            self.assertGreater(res["output_throughput"], 6700)
-
-    def test_pp_long_context_prefill(self):
-        res = run_bench_serving(
-            model="meta-llama/Llama-3.3-70B-Instruct",
-            num_prompts=4,
-            request_rate=float("inf"),
-            random_input_len=128000,
-            random_output_len=1,
-            dataset_name="random",
-            other_server_args=[
-                "--quantization",
-                "fp8",
-                "--pp-size",
-                "2",
-            ]
-            + (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []),
-            need_warmup=False,
-            seed=42,
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_pp_long_context_latency_prefill\n"
-                f"input_throughput: {res['input_throughput']:.2f} ms\n"
-            )
-            if is_in_amd_ci():
-                self.assertGreater(res["input_throughput"], 3000)
-            else:
-                self.assertGreater(res["input_throughput"], 4000)
-
-
-if __name__ == "__main__":
-    unittest.main()

test/registered/perf/test_vlm_perf_5090.py

@@ -1,83 +0,0 @@
-"""
-VLM Performance tests that work on 5090 (32GB) - VLM offline throughput and online latency tests.
-"""
-
-import os
-import unittest
-
-from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
-from sglang.test.test_utils import (
-    DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
-    DEFAULT_URL_FOR_TEST,
-    CustomTestCase,
-    auto_config_device,
-    get_benchmark_args,
-    is_in_ci,
-    run_bench_serving_multi,
-    write_github_step_summary,
-)
-
-register_cuda_ci(est_time=200, stage="extra-a", runner_config="1-gpu-small")
-register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
-
-
-def _local_tokenizer_path():
-    # Prefer the local snapshot so the benchmark client's AutoTokenizer does
-    # not call the HF Hub API, which can stall for minutes in CI.
-    try:
-        from sglang.srt.utils import find_local_repo_dir
-
-        local_dir = find_local_repo_dir(
-            DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, revision=None
-        )
-        if local_dir and os.path.isdir(local_dir):
-            return local_dir
-    except Exception:
-        pass
-    return None
-
-
-class TestVLMPerf5090(CustomTestCase):
-    def test_vlm_perf(self):
-        common = dict(
-            base_url=DEFAULT_URL_FOR_TEST,
-            dataset_name="mmmu",
-            dataset_path="",
-            tokenizer=_local_tokenizer_path(),
-            random_input_len=4096,
-            random_output_len=2048,
-            sharegpt_context_len=None,
-            disable_stream=False,
-            disable_ignore_eos=False,
-            seed=0,
-            device=auto_config_device(),
-            lora_name=None,
-        )
-        offline = get_benchmark_args(
-            num_prompts=200, request_rate=float("inf"), **common
-        )
-        # 50 prompts at 1 req/s keeps the online phase ~1 min; medians are
-        # stable at this sample size and the thresholds are loose ceilings.
-        online = get_benchmark_args(num_prompts=50, request_rate=1, **common)
-
-        (_, res_offline), (_, res_online) = run_bench_serving_multi(
-            DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
-            DEFAULT_URL_FOR_TEST,
-            other_server_args=["--mem-fraction-static", "0.7"],
-            benchmark_args=[offline, online],
-        )
-
-        if is_in_ci():
-            write_github_step_summary(
-                f"### test_vlm_perf (5090)\n"
-                f"Output throughput: {res_offline['output_throughput']:.2f} token/s\n"
-                f"median_e2e_latency_ms: {res_online['median_e2e_latency_ms']:.2f} ms\n"
-            )
-            self.assertGreater(res_offline["output_throughput"], 2000)
-            self.assertLess(res_online["median_e2e_latency_ms"], 16500)
-            self.assertLess(res_online["median_ttft_ms"], 150)
-            self.assertLess(res_online["median_itl_ms"], 8)
-
-
-if __name__ == "__main__":
-    unittest.main()