diff --git a/python/sglang/test/kits/perf_bench_kit.py b/python/sglang/test/kits/perf_bench_kit.py new file mode 100644 index 000000000..ba782bfd5 --- /dev/null +++ b/python/sglang/test/kits/perf_bench_kit.py @@ -0,0 +1,82 @@ +"""Report and bound the numbers a serving benchmark produces. + +Bounds are tuned per CI runner, so a local run prints them without asserting. +""" + +from dataclasses import dataclass +from typing import Callable, Optional, Sequence + +from sglang.test.test_utils import is_in_amd_ci, is_in_ci, write_github_step_summary + + +@dataclass +class Metric: + name: str + value: float + unit: str + bound: Optional[float] = None + amd_bound: Optional[float] = None + # unittest method that enforces `bound`, e.g. "assertLessEqual". + assertion: Optional[str] = None + + def line(self) -> str: + return f"{self.name}: {self.value:.2f}" + (f" {self.unit}" if self.unit else "") + + def check(self, test_case) -> None: + if self.bound is None: + return + limit = self.bound + if is_in_amd_ci() and self.amd_bound is not None: + limit = self.amd_bound + getattr(test_case, self.assertion)(self.value, limit) + + +def at_least(name, value, bound, *, amd=None, unit="") -> Metric: + """A throughput-like number: the run passes when it reaches `bound`.""" + return Metric(name, value, unit, bound, amd, "assertGreaterEqual") + + +def at_most(name, value, bound, *, amd=None, unit="") -> Metric: + """A latency-like number: the run passes when it stays under `bound`.""" + return Metric(name, value, unit, bound, amd, "assertLessEqual") + + +def reported(name, value, *, unit="") -> Metric: + """A number worth printing that no threshold is attached to.""" + return Metric(name, value, unit) + + +def check_perf(test_case, *metrics: Metric, suffix: str = "") -> None: + """Report every metric under the caller's test name, then enforce in CI.""" + label = test_case._testMethodName + suffix + report = f"### {label}\n" + "".join(m.line() + "\n" for m in metrics) + print(report, end="") + if not is_in_ci(): + return + write_github_step_summary(report) + for m in metrics: + m.check(test_case) + + +def check_batch_scaling( + test_case, + run_all: Callable[[Sequence[int]], Sequence[dict]], + bounds: Sequence[tuple], +) -> None: + """Bound the latency at each batch size of one sweep. + + `run_all` takes every size at once so they share one server. Each `bounds` + entry is `(batch_size, avg_ms, p95_ms, amd_avg_ms, amd_p95_ms)`. + """ + results = run_all([b[0] for b in bounds]) + for (batch_size, avg_ms, p95_ms, amd_avg_ms, amd_p95_ms), res in zip( + bounds, results + ): + test_case.assertEqual(res["successful_requests"], res["total_requests"]) + check_perf( + test_case, + at_most("avg_latency_ms", res["avg_latency_ms"], avg_ms, amd=amd_avg_ms), + at_most("p95_latency_ms", res["p95_latency_ms"], p95_ms, amd=amd_p95_ms), + reported("throughput", res["throughput"], unit="req/s"), + suffix=f"_size_{batch_size}", + ) diff --git a/python/sglang/test/kits/vlm_perf_kit.py b/python/sglang/test/kits/vlm_perf_kit.py new file mode 100644 index 000000000..9e0a8591c --- /dev/null +++ b/python/sglang/test/kits/vlm_perf_kit.py @@ -0,0 +1,94 @@ +"""One VLM serving benchmark, run against a named attention backend. + +Pinned rather than inherited from `get_default_attn_backend`, so the file name +keeps naming the right kernel after the default moves. +""" + +import os +from typing import Optional + +from sglang.test.kits.perf_bench_kit import at_least, at_most, check_perf, reported +from sglang.test.test_utils import ( + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + auto_config_device, + get_benchmark_args, + run_bench_serving_multi, +) + + +def _local_tokenizer_path(): + # The HF Hub API call can stall for minutes in CI; prefer a local snapshot. + try: + from sglang.srt.utils import find_local_repo_dir + + local_dir = find_local_repo_dir( + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, revision=None + ) + if local_dir and os.path.isdir(local_dir): + return local_dir + except Exception: + pass + return None + + +def check_vlm_serving_perf( + test_case, + attention_backend: str, + *, + e2e_ms: float, + ttft_ms: float, + itl_ms: float, + output_throughput: Optional[float] = None, +): + """Offline then online against one server; bound both phases. + + `output_throughput` unset means the offline number is reported, not bounded. + """ + common = dict( + base_url=DEFAULT_URL_FOR_TEST, + dataset_name="mmmu", + dataset_path="", + tokenizer=_local_tokenizer_path(), + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + disable_stream=False, + disable_ignore_eos=False, + seed=0, + device=auto_config_device(), + lora_name=None, + ) + offline = get_benchmark_args(num_prompts=200, request_rate=float("inf"), **common) + # 50 is enough for a stable median against these loose ceilings. + online = get_benchmark_args(num_prompts=50, request_rate=1, **common) + + (_, res_offline), (_, res_online) = run_bench_serving_multi( + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + other_server_args=[ + "--mem-fraction-static", + "0.7", + "--attention-backend", + attention_backend, + ], + benchmark_args=[offline, online], + ) + + throughput = res_offline["output_throughput"] + check_perf( + test_case, + ( + at_least("output_throughput", throughput, output_throughput, unit="token/s") + if output_throughput is not None + else reported("output_throughput", throughput, unit="token/s") + ), + at_most( + "median_e2e_latency_ms", + res_online["median_e2e_latency_ms"], + e2e_ms, + unit="ms", + ), + at_most("median_ttft_ms", res_online["median_ttft_ms"], ttft_ms, unit="ms"), + at_most("median_itl_ms", res_online["median_itl_ms"], itl_ms, unit="ms"), + ) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index 38bdfb614..86165aaac 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -61,7 +61,6 @@ DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B" DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat" # MLA test models -DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct" DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST = "cross-encoder/ms-marco-MiniLM-L6-v2" DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8" @@ -1167,6 +1166,29 @@ def run_score_benchmark( device="auto", ): """Score API benchmark function compatible with run_bench_serving pattern""" + return run_score_benchmark_multi( + model, + [batch_size], + num_requests=num_requests, + other_server_args=other_server_args, + need_warmup=need_warmup, + device=device, + )[0] + + +def run_score_benchmark_multi( + model, + batch_sizes, + num_requests=100, + other_server_args=None, + need_warmup=False, + device="auto", +): + """One server, one benchmark per batch size. + + Batch size is a property of the request, not of the server, so the launch + is shared rather than repeated per size. + """ if other_server_args is None: other_server_args = [] @@ -1182,7 +1204,7 @@ def run_score_benchmark( other_args=other_server_args, ) - async def _run_benchmark(): + async def _run_benchmark(batch_size, warmup): # Load tokenizer for generating test data from sglang.srt.utils.hf_transformers_utils import get_tokenizer @@ -1205,7 +1227,7 @@ def run_score_benchmark( ) return text - if need_warmup: + if warmup: warmup_data = { "query": generate_text_with_token_count(score_query_tokens), "items": [ @@ -1253,12 +1275,16 @@ def run_score_benchmark( ) try: - res = asyncio.run(_run_benchmark()) + results = [ + asyncio.run(_run_benchmark(bs, need_warmup and i == 0)) + for i, bs in enumerate(batch_sizes) + ] finally: kill_process_tree(process.pid) - assert res["completed"] == res["successful_requests"] - return res + for res in results: + assert res["completed"] == res["successful_requests"] + return results def run_embeddings_benchmark( @@ -1271,6 +1297,27 @@ def run_embeddings_benchmark( device="auto", ): """Embeddings API benchmark function compatible with run_bench_serving pattern""" + return run_embeddings_benchmark_multi( + model, + [batch_size], + num_requests=num_requests, + input_tokens=input_tokens, + other_server_args=other_server_args, + need_warmup=need_warmup, + device=device, + )[0] + + +def run_embeddings_benchmark_multi( + model, + batch_sizes, + num_requests=100, + input_tokens=500, + other_server_args=None, + need_warmup=False, + device="auto", +): + """One server, one benchmark per batch size. See run_score_benchmark_multi.""" if other_server_args is None: other_server_args = [] @@ -1289,7 +1336,7 @@ def run_embeddings_benchmark( other_args=server_args, ) - async def _run_benchmark(): + async def _run_benchmark(batch_size, warmup): def generate_text_with_token_count(num_tokens): """Generate text with precise token count using special tokens.""" @@ -1300,7 +1347,7 @@ def run_embeddings_benchmark( # Generate input text input_text = generate_text_with_token_count(input_tokens) - if need_warmup: + if warmup: warmup_data = { "input": input_text, "model": model, @@ -1340,12 +1387,16 @@ def run_embeddings_benchmark( ) try: - res = asyncio.run(_run_benchmark()) + results = [ + asyncio.run(_run_benchmark(bs, need_warmup and i == 0)) + for i, bs in enumerate(batch_sizes) + ] finally: kill_process_tree(process.pid) - assert res["completed"] == res["successful_requests"] - return res + for res in results: + assert res["completed"] == res["successful_requests"] + return results def run_bench_serving_multi( diff --git a/test/registered/basic_perf/test_eagle3_latency.py b/test/registered/basic_perf/test_eagle3_latency.py new file mode 100644 index 000000000..53738018c --- /dev/null +++ b/test/registered/basic_perf/test_eagle3_latency.py @@ -0,0 +1,60 @@ +"""The only test in the tree that bounds speculative decoding LATENCY; every +other one bounds accept length. CUDA only -- AMD bounds are unmeasured. +""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_least, at_most, check_perf +from sglang.test.test_utils import ( + DEFAULT_DRAFT_MODEL_EAGLE3, + DEFAULT_TARGET_MODEL_EAGLE3, + CustomTestCase, + run_bench_serving, +) + +register_cuda_ci(est_time=145, stage="extra-a", runner_config="1-gpu-large") + + +class TestEagle3Latency(CustomTestCase): + def test_online_latency_eagle3(self): + res = run_bench_serving( + model=DEFAULT_TARGET_MODEL_EAGLE3, + num_prompts=300, + request_rate=8, + sharegpt_context_len=3072, + disable_ignore_eos=True, + dataset_name="sharegpt", + other_server_args=[ + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_DRAFT_MODEL_EAGLE3, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "16", + "--mem-fraction-static", + "0.7", + # The draft checkpoint ships fp16 and the target bf16; the CUDA + # rmsnorm path rejects a weight and activation pair that disagree. + "--dtype", + "float16", + ], + need_warmup=True, + seed=42, + ) + + check_perf( + self, + at_most( + "median_e2e_latency_ms", res["median_e2e_latency_ms"], 1150, unit="ms" + ), + at_least("accept_length", res["accept_length"], 2.3), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_embeddings_api.py b/test/registered/basic_perf/test_embeddings_api.py new file mode 100644 index 000000000..22bd51871 --- /dev/null +++ b/test/registered/basic_perf/test_embeddings_api.py @@ -0,0 +1,61 @@ +"""Latency and throughput of the /v1/embeddings endpoint.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import ( + at_least, + at_most, + check_batch_scaling, + check_perf, +) +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_embeddings_benchmark, + run_embeddings_benchmark_multi, +) + +register_cuda_ci(est_time=245, stage="extra-a", runner_config="1-gpu-large") +register_amd_ci(est_time=240, suite="stage-b-test-1-gpu-large-amd") + + +class TestEmbeddingsAPI(CustomTestCase): + def test_embeddings_api_latency_throughput(self): + res = run_embeddings_benchmark( + model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + num_requests=1000, + batch_size=1, + input_tokens=500, + other_server_args=[], + need_warmup=True, + ) + + self.assertEqual(res["successful_requests"], res["total_requests"]) + check_perf( + self, + at_most("avg_latency_ms", res["avg_latency_ms"], 21, amd=35, unit="ms"), + at_most("p95_latency_ms", res["p95_latency_ms"], 26, amd=40, unit="ms"), + at_least("throughput", res["throughput"], 48, amd=30, unit="req/s"), + ) + + def test_embeddings_api_batch_scaling(self): + check_batch_scaling( + self, + lambda batch_sizes: run_embeddings_benchmark_multi( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + batch_sizes, + num_requests=500, + input_tokens=500, + ), + # batch size, avg ms, p95 ms, then the same two relaxed for mi300x + [ + (10, 43, 49, 80, 90), + (25, 70, 78, 140, 150), + (50, 122, 158, 230, 240), + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_lora_latency.py b/test/registered/basic_perf/test_lora_latency.py new file mode 100644 index 000000000..249e11046 --- /dev/null +++ b/test/registered/basic_perf/test_lora_latency.py @@ -0,0 +1,134 @@ +"""Latency of the LoRA serving path, with and without adapter churn.""" + +import asyncio +import itertools +import unittest + +import requests + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_most, check_perf +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_serving, +) + +register_cuda_ci(est_time=490, stage="extra-a", runner_config="1-gpu-large") +register_amd_ci(est_time=430, suite="stage-b-test-1-gpu-large-amd") + + +class TestLoRALatency(CustomTestCase): + def test_online_lora_latency(self): + res = self._run_lora_latency_test(enable_background_task=False) + + check_perf( + self, + at_most( + "median_e2e_latency_ms", + res["median_e2e_latency_ms"], + 2270, + amd=3320, + unit="ms", + ), + # mi300x is about twice as slow as mi325 on LoRA TTFT. + at_most("median_ttft_ms", res["median_ttft_ms"], 51, amd=100, unit="ms"), + ) + + def test_online_lora_latency_with_concurrent_adapter_updates(self): + res = self._run_lora_latency_test(enable_background_task=True) + + check_perf( + self, + at_most( + "median_e2e_latency_ms", + res["median_e2e_latency_ms"], + 3170, + amd=6000, + unit="ms", + ), + at_most("median_ttft_ms", res["median_ttft_ms"], 55, amd=130, unit="ms"), + ) + + def _run_lora_latency_test(self, enable_background_task: bool): + async def lora_loader_unloader_task( + base_url: str, + start_event: asyncio.Event, + stop_event: asyncio.Event, + ): + """ + A background task that repeatedly loads and unloads a LoRA adapter. + """ + await start_event.wait() + + path_cycler = itertools.cycle( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "faridlazuarda/valadapt-llama-3.1-8B-it-chinese", + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ] + ) + load_url = f"{base_url}/load_lora_adapter" + unload_url = f"{base_url}/unload_lora_adapter" + num_updates = 0 + + while not stop_event.is_set(): + lora_path = next(path_cycler) + response = await asyncio.to_thread( + requests.post, + load_url, + json={"lora_name": lora_path, "lora_path": lora_path}, + ) + self.assertTrue( + response.ok, f"Failed to load LoRA adapter: {response.text}" + ) + num_updates += 1 + + if stop_event.is_set(): + break + + await asyncio.sleep(1) + + response = await asyncio.to_thread( + requests.post, + unload_url, + json={"lora_name": lora_path}, + ) + self.assertTrue( + response.ok, f"Failed to unload LoRA adapter: {response.text}" + ) + num_updates += 1 + + await asyncio.sleep(1) + + background_task = lora_loader_unloader_task if enable_background_task else None + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=400, + request_rate=8, + other_server_args=[ + "--enable-lora", + "--max-loras-per-batch", + "1", + "--disable-radix-cache", + "--random-seed", + "42", + "--mem-fraction-static", + "0.8", + "--lora-paths", + "nvidia/llama-3.1-nemoguard-8b-topic-control", + "--max-lora-rank", + "256", + ], + dataset_name="random", + random_input_len=256, + random_output_len=256, + lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"], + background_task=background_task, + ) + + return res + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_moe_throughput.py b/test/registered/basic_perf/test_moe_throughput.py new file mode 100644 index 000000000..f4a881ba6 --- /dev/null +++ b/test/registered/basic_perf/test_moe_throughput.py @@ -0,0 +1,53 @@ +"""Throughput of the MoE model on two GPUs, batched and at batch size one.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_least, check_perf +from sglang.test.test_utils import ( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_offline_throughput, + run_bench_serving, +) + +register_cuda_ci(est_time=290, stage="extra-a", runner_config="2-gpu-large") +register_amd_ci(est_time=770, suite="stage-b-test-2-gpu-large-amd") + + +class TestMoEThroughput(CustomTestCase): + def test_moe_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2"], + ) + + check_perf( + self, + at_least( + "output_throughput", + res["output_throughput"], + 2670, + amd=2100, + unit="token/s", + ), + ) + + def test_moe_tp2_bs1(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + ["--tp", "2", "--cuda-graph-max-bs-decode", "2"], + ) + + check_perf( + self, + at_least( + "output_throughput", output_throughput, 139, amd=85, unit="token/s" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_pp_throughput.py b/test/registered/basic_perf/test_pp_throughput.py new file mode 100644 index 000000000..abf8d1e79 --- /dev/null +++ b/test/registered/basic_perf/test_pp_throughput.py @@ -0,0 +1,70 @@ +"""Throughput of pipeline parallelism on two GPUs, decode and long prefill.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_least, check_perf +from sglang.test.test_utils import ( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + run_bench_serving, +) + +register_cuda_ci(est_time=490, stage="extra-a", runner_config="2-gpu-large") +register_amd_ci(est_time=1030, suite="stage-b-test-2-gpu-large-amd") + + +class TestPPThroughput(CustomTestCase): + def test_pp_offline_throughput_default_decode(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=1000, + request_rate=float("inf"), + random_input_len=1, + random_output_len=1024, + other_server_args=["--pp-size", "2"], + need_warmup=True, + seed=42, + ) + + check_perf( + self, + at_least( + "output_throughput", res["output_throughput"], 6250, unit="token/s" + ), + ) + + def test_pp_long_context_prefill(self): + res = run_bench_serving( + model="meta-llama/Llama-3.3-70B-Instruct", + num_prompts=4, + request_rate=float("inf"), + random_input_len=128000, + random_output_len=1, + dataset_name="random", + other_server_args=[ + "--quantization", + "fp8", + "--pp-size", + "2", + ] + + (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []), + need_warmup=False, + seed=42, + ) + + check_perf( + self, + at_least( + "input_throughput", + res["input_throughput"], + 4380, + amd=3000, + unit="token/s", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_score_api.py b/test/registered/basic_perf/test_score_api.py new file mode 100644 index 000000000..3c9da5f1c --- /dev/null +++ b/test/registered/basic_perf/test_score_api.py @@ -0,0 +1,55 @@ +"""Latency and throughput of the /v1/score endpoint.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import ( + at_least, + at_most, + check_batch_scaling, + check_perf, +) +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, + CustomTestCase, + run_score_benchmark, + run_score_benchmark_multi, +) + +register_cuda_ci(est_time=215, stage="extra-a", runner_config="1-gpu-large") +register_amd_ci(est_time=210, suite="stage-b-test-1-gpu-large-amd") + + +class TestScoreAPI(CustomTestCase): + def test_score_api_latency_throughput(self): + res = run_score_benchmark( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, + num_requests=1000, + batch_size=10, + other_server_args=[], + need_warmup=True, + ) + + self.assertEqual(res["successful_requests"], res["total_requests"]) + check_perf( + self, + at_most("avg_latency_ms", res["avg_latency_ms"], 30, amd=60, unit="ms"), + at_most("p95_latency_ms", res["p95_latency_ms"], 32, amd=65, unit="ms"), + at_least("throughput", res["throughput"], 34, amd=16, unit="req/s"), + ) + + def test_score_api_batch_scaling(self): + check_batch_scaling( + self, + lambda batch_sizes: run_score_benchmark_multi( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, + batch_sizes, + num_requests=500, + ), + # batch size, avg ms, p95 ms, then the same two relaxed for mi300x + [(10, 30, 34, 60, 65), (25, 35, 39, 70, 80), (50, 51, 59, 80, 90)], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_serving_latency.py b/test/registered/basic_perf/test_serving_latency.py new file mode 100644 index 000000000..d612ef3c3 --- /dev/null +++ b/test/registered/basic_perf/test_serving_latency.py @@ -0,0 +1,40 @@ +"""Latency of the default serving path on one large GPU.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_most, check_perf +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_serving, +) + +register_cuda_ci(est_time=190, stage="extra-a", runner_config="1-gpu-large") +register_amd_ci(est_time=165, suite="stage-b-test-1-gpu-large-amd") + + +class TestServingLatency(CustomTestCase): + def test_online_latency_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=100, + request_rate=1, + other_server_args=[], + ) + + check_perf( + self, + at_most( + "median_e2e_latency_ms", + res["median_e2e_latency_ms"], + 9100, + unit="ms", + ), + at_most("median_ttft_ms", res["median_ttft_ms"], 80, amd=115, unit="ms"), + at_most("median_itl_ms", res["median_itl_ms"], 9, unit="ms"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_serving_throughput.py b/test/registered/basic_perf/test_serving_throughput.py new file mode 100644 index 000000000..e8c1ca571 --- /dev/null +++ b/test/registered/basic_perf/test_serving_throughput.py @@ -0,0 +1,107 @@ +"""Offline throughput of the default serving path on one large GPU.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_least, check_perf +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_FP8, + CustomTestCase, + run_bench_serving, +) + +register_cuda_ci(est_time=710, stage="extra-a", runner_config="1-gpu-large") +register_amd_ci(est_time=810, suite="stage-b-test-1-gpu-large-amd") + + +class TestServingThroughput(CustomTestCase): + def test_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + check_perf( + self, + at_least( + "output_throughput", + res["output_throughput"], + 4000, + amd=3050, + unit="token/s", + ), + ) + + def test_offline_throughput_non_stream_small_batch_size(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=200, + request_rate=float("inf"), + other_server_args=["--max-running-requests", "10"], + dataset_name="sharegpt", + random_input_len=None, + random_output_len=None, + disable_stream=True, + need_warmup=True, + ) + + check_perf( + self, + at_least( + "output_throughput", + res["output_throughput"], + 1110, + amd=1000, + unit="token/s", + ), + ) + + def test_offline_throughput_with_triton_attention_backend(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[ + "--attention-backend", + "triton", + "--context-length", + "8192", + ], + ) + + check_perf( + self, + at_least( + "output_throughput", + res["output_throughput"], + 3730, + amd=2700, + unit="token/s", + ), + ) + + def test_offline_throughput_default_fp8(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST_FP8, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + check_perf( + self, + at_least( + "output_throughput", + res["output_throughput"], + 4870, + amd=3500, + unit="token/s", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_torch_compile_throughput.py b/test/registered/basic_perf/test_torch_compile_throughput.py new file mode 100644 index 000000000..53ab7db8b --- /dev/null +++ b/test/registered/basic_perf/test_torch_compile_throughput.py @@ -0,0 +1,33 @@ +"""Throughput of torch.compile at batch size one across two GPUs.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kits.perf_bench_kit import at_least, check_perf +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_offline_throughput, +) + +register_cuda_ci(est_time=75, stage="extra-a", runner_config="2-gpu-large") +register_amd_ci(est_time=280, suite="stage-b-test-2-gpu-large-amd") + + +class TestTorchCompileThroughput(CustomTestCase): + def test_torch_compile_tp2_bs1(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"], + ) + + check_perf( + self, + at_least( + "output_throughput", output_throughput, 255, amd=200, unit="token/s" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_vlm_serving_aiter.py b/test/registered/basic_perf/test_vlm_serving_aiter.py new file mode 100644 index 000000000..0a57ae1d4 --- /dev/null +++ b/test/registered/basic_perf/test_vlm_serving_aiter.py @@ -0,0 +1,25 @@ +"""VLM serving perf on the aiter attention backend.""" + +import unittest + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd") + + +class TestVLMServingAiter(CustomTestCase): + def test_vlm_serving_aiter(self): + check_vlm_serving_perf( + self, + "aiter", + output_throughput=2000, + e2e_ms=16500, + ttft_ms=150, + itl_ms=8, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_vlm_serving_fa3.py b/test/registered/basic_perf/test_vlm_serving_fa3.py new file mode 100644 index 000000000..fc2fb7101 --- /dev/null +++ b/test/registered/basic_perf/test_vlm_serving_fa3.py @@ -0,0 +1,26 @@ +"""VLM serving perf on the fa3 attention backend.""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=150, stage="extra-a", runner_config="1-gpu-large") + + +class TestVLMServingFa3(CustomTestCase): + def test_vlm_serving_fa3(self): + check_vlm_serving_perf( + self, + "fa3", + # No offline bound: never measured on this lane. + output_throughput=16700, + e2e_ms=11000, + ttft_ms=84, + itl_ms=5.2, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/basic_perf/test_vlm_serving_flashinfer.py b/test/registered/basic_perf/test_vlm_serving_flashinfer.py new file mode 100644 index 000000000..6d5af9741 --- /dev/null +++ b/test/registered/basic_perf/test_vlm_serving_flashinfer.py @@ -0,0 +1,25 @@ +"""VLM serving perf on the flashinfer attention backend.""" + +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=195, stage="extra-a", runner_config="1-gpu-small") + + +class TestVLMServingFlashinfer(CustomTestCase): + def test_vlm_serving_flashinfer(self): + check_vlm_serving_perf( + self, + "flashinfer", + output_throughput=6900, + e2e_ms=17300, + ttft_ms=76, + itl_ms=8.3, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/perf/test_bench_one_batch_2gpu.py b/test/registered/perf/test_bench_one_batch_2gpu.py deleted file mode 100644 index 212ae9625..000000000 --- a/test/registered/perf/test_bench_one_batch_2gpu.py +++ /dev/null @@ -1,53 +0,0 @@ -import unittest - -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - DEFAULT_MOE_MODEL_NAME_FOR_TEST, - CustomTestCase, - is_in_amd_ci, - is_in_ci, - run_bench_offline_throughput, - write_github_step_summary, -) - -register_cuda_ci(est_time=162, stage="extra-a", runner_config="2-gpu-large") -register_amd_ci(est_time=630, suite="stage-b-test-2-gpu-large-amd") - - -class TestBenchOneBatch2GPU(CustomTestCase): - def test_moe_tp2_bs1(self): - output_throughput = run_bench_offline_throughput( - DEFAULT_MOE_MODEL_NAME_FOR_TEST, - ["--tp", "2", "--cuda-graph-max-bs-decode", "2"], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n" - f"output_throughput: {output_throughput:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(output_throughput, 85) - else: - self.assertGreater(output_throughput, 125) - - def test_torch_compile_tp2_bs1(self): - output_throughput = run_bench_offline_throughput( - DEFAULT_MODEL_NAME_FOR_TEST, - ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n" - f"output_throughput: {output_throughput:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(output_throughput, 200) - else: - self.assertGreater(output_throughput, 220) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/perf/test_bench_serving_1gpu_large.py b/test/registered/perf/test_bench_serving_1gpu_large.py deleted file mode 100644 index 69b1c8da5..000000000 --- a/test/registered/perf/test_bench_serving_1gpu_large.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Performance tests for single GPU that need H200 (80GB) - FP8 and EAGLE tests. -""" - -import unittest - -from sglang.srt.utils import is_hip -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_DRAFT_MODEL_EAGLE, - DEFAULT_MODEL_NAME_FOR_TEST_FP8, - DEFAULT_TARGET_MODEL_EAGLE, - CustomTestCase, - is_in_amd_ci, - is_in_ci, - run_bench_serving, - write_github_step_summary, -) - -register_cuda_ci(est_time=275, stage="extra-a", runner_config="1-gpu-large") -register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-large-amd") - - -class TestBenchServing1GPULarge(CustomTestCase): - def test_offline_throughput_default_fp8(self): - res = run_bench_serving( - model=DEFAULT_MODEL_NAME_FOR_TEST_FP8, - num_prompts=500, - request_rate=float("inf"), - other_server_args=[], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_offline_throughput_default_fp8\n" - f"Output throughput: {res['output_throughput']:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(res["output_throughput"], 3500) - else: - self.assertGreater(res["output_throughput"], 4300) - - @unittest.skipIf(is_hip(), "Skip Eagle test for ROCm") - def test_online_latency_eagle(self): - res = run_bench_serving( - model=DEFAULT_TARGET_MODEL_EAGLE, - num_prompts=300, - request_rate=8, - sharegpt_context_len=3072, - disable_ignore_eos=True, - dataset_name="sharegpt", - other_server_args=[ - "--speculative-algorithm", - "EAGLE", - "--speculative-draft-model-path", - DEFAULT_DRAFT_MODEL_EAGLE, - "--speculative-num-steps", - "5", - "--speculative-eagle-topk", - "4", - "--speculative-num-draft-tokens", - "16", - "--mem-fraction-static", - "0.7", - ], - need_warmup=True, - seed=42, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_online_latency_eagle\n" - f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" - f"accept_length: {res['accept_length']:.2f} \n" - ) - if is_in_amd_ci(): - self.assertLess(res["median_e2e_latency_ms"], 1800) - else: - self.assertLess(res["median_e2e_latency_ms"], 900) - self.assertGreater(res["accept_length"], 3.0) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/perf/test_bench_serving_1gpu_part1.py b/test/registered/perf/test_bench_serving_1gpu_part1.py deleted file mode 100644 index b2b30cada..000000000 --- a/test/registered/perf/test_bench_serving_1gpu_part1.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -Performance tests for single GPU - LLM throughput/latency and LoRA tests. -Works on 5090 (32GB). -""" - -import asyncio -import itertools -import unittest - -import requests - -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST, - CustomTestCase, - is_in_amd_ci, - is_in_ci, - run_bench_serving, - write_github_step_summary, -) - -register_cuda_ci(est_time=1264, stage="extra-a", runner_config="1-gpu-large") -register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-large-amd") - - -class TestBenchServing1GPUPart1(CustomTestCase): - def test_offline_throughput_default(self): - res = run_bench_serving( - model=DEFAULT_MODEL_NAME_FOR_TEST, - num_prompts=500, - request_rate=float("inf"), - other_server_args=[], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_offline_throughput_default\n" - f"Output throughput: {res['output_throughput']:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(res["output_throughput"], 3050) - else: - self.assertGreater(res["output_throughput"], 3800) - - def test_offline_throughput_non_stream_small_batch_size(self): - res = run_bench_serving( - model=DEFAULT_MODEL_NAME_FOR_TEST, - num_prompts=200, - request_rate=float("inf"), - other_server_args=["--max-running-requests", "10"], - dataset_name="sharegpt", - random_input_len=None, - random_output_len=None, - disable_stream=True, - need_warmup=True, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_offline_throughput_non_stream_small_batch_size\n" - f"Output throughput: {res['output_throughput']:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(res["output_throughput"], 1000) - else: - self.assertGreater(res["output_throughput"], 1050) - - def test_offline_throughput_with_triton_attention_backend(self): - res = run_bench_serving( - model=DEFAULT_MODEL_NAME_FOR_TEST, - num_prompts=500, - request_rate=float("inf"), - other_server_args=[ - "--attention-backend", - "triton", - "--context-length", - "8192", - ], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_offline_throughput_with_triton_attention_backend\n" - f"Output throughput: {res['output_throughput']:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(res["output_throughput"], 2700) - else: - self.assertGreater(res["output_throughput"], 3700) - - def test_online_latency_default(self): - res = run_bench_serving( - model=DEFAULT_MODEL_NAME_FOR_TEST, - num_prompts=100, - request_rate=1, - other_server_args=[], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_online_latency_default\n" - f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" - ) - self.assertLess(res["median_e2e_latency_ms"], 11000) - if is_in_amd_ci(): - self.assertLess(res["median_ttft_ms"], 115) - else: - self.assertLess(res["median_ttft_ms"], 86) - self.assertLess(res["median_itl_ms"], 10) - - def test_online_lora_latency(self): - res = self._run_lora_latency_test(enable_background_task=False) - - if is_in_ci(): - write_github_step_summary( - f"### test_online_lora_latency\n" - f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" - f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" - ) - if is_in_amd_ci(): - self.assertLess(res["median_e2e_latency_ms"], 3320) - else: - self.assertLess(res["median_e2e_latency_ms"], 2400) - # relax for mi300x (LoRA TTFT ~2x slower than mi325) - if is_in_amd_ci(): - self.assertLess(res["median_ttft_ms"], 100) - else: - self.assertLess(res["median_ttft_ms"], 58) - - def test_online_lora_latency_with_concurrent_adapter_updates(self): - res = self._run_lora_latency_test(enable_background_task=True) - - if is_in_ci(): - write_github_step_summary( - f"### test_online_lora_latency_with_concurrent_adapter_updates\n" - f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" - f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" - ) - if is_in_amd_ci(): - self.assertLess(res["median_e2e_latency_ms"], 6000) - else: - self.assertLess(res["median_e2e_latency_ms"], 4000) - # relax for mi300x (LoRA TTFT ~2x slower than mi325) - if is_in_amd_ci(): - self.assertLess(res["median_ttft_ms"], 130) - else: - self.assertLess(res["median_ttft_ms"], 80) - - def _run_lora_latency_test(self, enable_background_task: bool): - """ - Run a latency test for LoRA with the specified background task setting. - """ - - async def lora_loader_unloader_task( - base_url: str, - start_event: asyncio.Event, - stop_event: asyncio.Event, - ): - """ - A background task that repeatedly loads and unloads a LoRA adapter. - """ - await start_event.wait() - - path_cycler = itertools.cycle( - [ - "pbevan11/llama-3.1-8b-ocr-correction", - "faridlazuarda/valadapt-llama-3.1-8B-it-chinese", - "philschmid/code-llama-3-1-8b-text-to-sql-lora", - ] - ) - load_url = f"{base_url}/load_lora_adapter" - unload_url = f"{base_url}/unload_lora_adapter" - num_updates = 0 - - while not stop_event.is_set(): - lora_path = next(path_cycler) - response = await asyncio.to_thread( - requests.post, - load_url, - json={"lora_name": lora_path, "lora_path": lora_path}, - ) - self.assertTrue( - response.ok, f"Failed to load LoRA adapter: {response.text}" - ) - num_updates += 1 - - if stop_event.is_set(): - break - - await asyncio.sleep(1) - - response = await asyncio.to_thread( - requests.post, - unload_url, - json={"lora_name": lora_path}, - ) - self.assertTrue( - response.ok, f"Failed to unload LoRA adapter: {response.text}" - ) - num_updates += 1 - - await asyncio.sleep(1) - - background_task = lora_loader_unloader_task if enable_background_task else None - res = run_bench_serving( - model=DEFAULT_MODEL_NAME_FOR_TEST, - num_prompts=400, - request_rate=8, - other_server_args=[ - "--enable-lora", - "--max-loras-per-batch", - "1", - "--disable-radix-cache", - "--random-seed", - "42", - "--mem-fraction-static", - "0.8", - "--lora-paths", - "nvidia/llama-3.1-nemoguard-8b-topic-control", - "--max-lora-rank", - "256", - ], - dataset_name="random", - random_input_len=256, - random_output_len=256, - lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"], - background_task=background_task, - ) - - return res - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/perf/test_bench_serving_1gpu_part2.py b/test/registered/perf/test_bench_serving_1gpu_part2.py deleted file mode 100644 index 561326682..000000000 --- a/test/registered/perf/test_bench_serving_1gpu_part2.py +++ /dev/null @@ -1,185 +0,0 @@ -""" -Performance tests for single GPU - VLM, Score API, and Embeddings API tests. -Works on 5090 (32GB). -""" - -import unittest - -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, - DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, - DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, - CustomTestCase, - is_in_amd_ci, - is_in_ci, - run_bench_serving, - run_embeddings_benchmark, - run_score_benchmark, - write_github_step_summary, -) - -register_cuda_ci(est_time=909, stage="extra-a", runner_config="1-gpu-large") -register_amd_ci(est_time=900, suite="stage-b-test-1-gpu-large-amd") - - -class TestBenchServing1GPUPart2(CustomTestCase): - def test_vlm_online_latency(self): - res = run_bench_serving( - model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, - num_prompts=250, - request_rate=1, - other_server_args=[ - "--mem-fraction-static", - "0.7", - ], - dataset_name="mmmu", - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_vlm_online_latency\n" - f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" - ) - self.assertLess(res["median_e2e_latency_ms"], 16500) - if is_in_amd_ci(): - self.assertLess(res["median_ttft_ms"], 150) - else: - self.assertLess(res["median_ttft_ms"], 100) - self.assertLess(res["median_itl_ms"], 8) - - def test_score_api_latency_throughput(self): - """Test score API latency and throughput performance""" - res = run_score_benchmark( - model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, - num_requests=1000, - batch_size=10, - other_server_args=[], - need_warmup=True, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_score_api_throughput\n" - f"Average latency: {res['avg_latency_ms']:.2f} ms\n" - f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" - f"Score API throughput: {res['throughput']:.2f} req/s\n" - f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" - ) - - self.assertEqual(res["successful_requests"], res["total_requests"]) - # relax for mi300x - if is_in_amd_ci(): - self.assertLess(res["avg_latency_ms"], 60) - self.assertLess(res["p95_latency_ms"], 65) - self.assertGreater(res["throughput"], 16) - else: - self.assertLess(res["avg_latency_ms"], 48) - self.assertLess(res["p95_latency_ms"], 50) - self.assertGreater(res["throughput"], 20) - - def test_score_api_batch_scaling(self): - """Test score API performance with different batch sizes""" - batch_sizes = [10, 25, 50] - - for batch_size in batch_sizes: - res = run_score_benchmark( - model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE, - num_requests=500, - batch_size=batch_size, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_score_api_batch_scaling_size_{batch_size}\n" - f"Batch size: {batch_size}\n" - f"Average latency: {res['avg_latency_ms']:.2f} ms\n" - f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" - f"Throughput: {res['throughput']:.2f} req/s\n" - f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" - ) - - self.assertEqual(res["successful_requests"], res["total_requests"]) - # relax for mi300x - if is_in_amd_ci(): - bounds = {10: (60, 65), 25: (70, 80), 50: (80, 90)} - default_bounds = (90, 90) - else: - bounds = {10: (45, 50), 25: (50, 60), 50: (60, 65)} - default_bounds = (60, 65) - avg_latency_bound, p95_latency_bound = bounds.get( - batch_size, default_bounds - ) - self.assertLess(res["avg_latency_ms"], avg_latency_bound) - self.assertLess(res["p95_latency_ms"], p95_latency_bound) - - def test_embeddings_api_latency_throughput(self): - """Test embeddings API latency and throughput performance""" - res = run_embeddings_benchmark( - model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, - num_requests=1000, - batch_size=1, - input_tokens=500, - other_server_args=[], - need_warmup=True, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_embeddings_api_throughput\n" - f"Average latency: {res['avg_latency_ms']:.2f} ms\n" - f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" - f"Embeddings API throughput: {res['throughput']:.2f} req/s\n" - f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" - ) - - self.assertEqual(res["successful_requests"], res["total_requests"]) - # relax for mi300x - if is_in_amd_ci(): - self.assertLess(res["avg_latency_ms"], 35) - self.assertLess(res["p95_latency_ms"], 40) - self.assertGreater(res["throughput"], 30) - else: - self.assertLess(res["avg_latency_ms"], 20) - self.assertLess(res["p95_latency_ms"], 25) - self.assertGreater(res["throughput"], 60) - - def test_embeddings_api_batch_scaling(self): - """Test embeddings API performance with different batch sizes""" - batch_sizes = [10, 25, 50] - - for batch_size in batch_sizes: - res = run_embeddings_benchmark( - model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, - num_requests=500, - batch_size=batch_size, - input_tokens=500, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_embeddings_api_batch_scaling_size_{batch_size}\n" - f"Batch size: {batch_size}\n" - f"Average latency: {res['avg_latency_ms']:.2f} ms\n" - f"P95 latency: {res['p95_latency_ms']:.2f} ms\n" - f"Throughput: {res['throughput']:.2f} req/s\n" - f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n" - ) - - self.assertEqual(res["successful_requests"], res["total_requests"]) - # relax for mi300x - if is_in_amd_ci(): - bounds = {10: (80, 90), 25: (140, 150), 50: (230, 240)} - default_bounds = (300, 300) - else: - bounds = {10: (60, 65), 25: (115, 120), 50: (190, 195)} - default_bounds = (250, 250) - avg_latency_bound, p95_latency_bound = bounds.get( - batch_size, default_bounds - ) - self.assertLess(res["avg_latency_ms"], avg_latency_bound) - self.assertLess(res["p95_latency_ms"], p95_latency_bound) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/perf/test_bench_serving_2gpu.py b/test/registered/perf/test_bench_serving_2gpu.py deleted file mode 100644 index 907d9d5c1..000000000 --- a/test/registered/perf/test_bench_serving_2gpu.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Performance tests for 2-GPU that need large GPUs (H200 80GB) - MoE and Pipeline Parallel tests. -""" - -import unittest - -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_MOE_MODEL_NAME_FOR_TEST, - CustomTestCase, - is_in_amd_ci, - is_in_ci, - run_bench_serving, - write_github_step_summary, -) - -register_cuda_ci(est_time=687, stage="extra-a", runner_config="2-gpu-large") -register_amd_ci(est_time=1450, suite="stage-b-test-2-gpu-large-amd") - - -class TestBenchServing2GPU(CustomTestCase): - def test_moe_offline_throughput_default(self): - res = run_bench_serving( - model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, - num_prompts=300, - request_rate=float("inf"), - other_server_args=["--tp", "2"], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_moe_offline_throughput_default\n" - f"Output throughput: {res['output_throughput']:.2f} token/s\n" - ) - if is_in_amd_ci(): - self.assertGreater(res["output_throughput"], 2100) - else: - self.assertGreater(res["output_throughput"], 2200) - - def test_pp_offline_throughput_default_decode(self): - res = run_bench_serving( - model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, - num_prompts=1000, - request_rate=float("inf"), - random_input_len=1, - random_output_len=1024, - other_server_args=["--pp-size", "2"], - need_warmup=True, - seed=42, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_pp_offline_throughput_default_decode\n" - f"Output throughput: {res['output_throughput']:.2f} token/s\n" - ) - self.assertGreater(res["output_throughput"], 6700) - - def test_pp_long_context_prefill(self): - res = run_bench_serving( - model="meta-llama/Llama-3.3-70B-Instruct", - num_prompts=4, - request_rate=float("inf"), - random_input_len=128000, - random_output_len=1, - dataset_name="random", - other_server_args=[ - "--quantization", - "fp8", - "--pp-size", - "2", - ] - + (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []), - need_warmup=False, - seed=42, - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_pp_long_context_latency_prefill\n" - f"input_throughput: {res['input_throughput']:.2f} ms\n" - ) - if is_in_amd_ci(): - self.assertGreater(res["input_throughput"], 3000) - else: - self.assertGreater(res["input_throughput"], 4000) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/perf/test_vlm_perf_5090.py b/test/registered/perf/test_vlm_perf_5090.py deleted file mode 100644 index 3193ab547..000000000 --- a/test/registered/perf/test_vlm_perf_5090.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -VLM Performance tests that work on 5090 (32GB) - VLM offline throughput and online latency tests. -""" - -import os -import unittest - -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import ( - DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - auto_config_device, - get_benchmark_args, - is_in_ci, - run_bench_serving_multi, - write_github_step_summary, -) - -register_cuda_ci(est_time=200, stage="extra-a", runner_config="1-gpu-small") -register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd") - - -def _local_tokenizer_path(): - # Prefer the local snapshot so the benchmark client's AutoTokenizer does - # not call the HF Hub API, which can stall for minutes in CI. - try: - from sglang.srt.utils import find_local_repo_dir - - local_dir = find_local_repo_dir( - DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, revision=None - ) - if local_dir and os.path.isdir(local_dir): - return local_dir - except Exception: - pass - return None - - -class TestVLMPerf5090(CustomTestCase): - def test_vlm_perf(self): - common = dict( - base_url=DEFAULT_URL_FOR_TEST, - dataset_name="mmmu", - dataset_path="", - tokenizer=_local_tokenizer_path(), - random_input_len=4096, - random_output_len=2048, - sharegpt_context_len=None, - disable_stream=False, - disable_ignore_eos=False, - seed=0, - device=auto_config_device(), - lora_name=None, - ) - offline = get_benchmark_args( - num_prompts=200, request_rate=float("inf"), **common - ) - # 50 prompts at 1 req/s keeps the online phase ~1 min; medians are - # stable at this sample size and the thresholds are loose ceilings. - online = get_benchmark_args(num_prompts=50, request_rate=1, **common) - - (_, res_offline), (_, res_online) = run_bench_serving_multi( - DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, - DEFAULT_URL_FOR_TEST, - other_server_args=["--mem-fraction-static", "0.7"], - benchmark_args=[offline, online], - ) - - if is_in_ci(): - write_github_step_summary( - f"### test_vlm_perf (5090)\n" - f"Output throughput: {res_offline['output_throughput']:.2f} token/s\n" - f"median_e2e_latency_ms: {res_online['median_e2e_latency_ms']:.2f} ms\n" - ) - self.assertGreater(res_offline["output_throughput"], 2000) - self.assertLess(res_online["median_e2e_latency_ms"], 16500) - self.assertLess(res_online["median_ttft_ms"], 150) - self.assertLess(res_online["median_itl_ms"], 8) - - -if __name__ == "__main__": - unittest.main()