[Test] Split the serving perf tests by topic into basic_perf/ and route their thresholds through a kit (#40505)
This commit is contained in:
Liangsheng Yin
@@ -0,0 +1,60 @@
|
||||
"""The only test in the tree that bounds speculative decoding LATENCY; every
|
||||
other one bounds accept length. CUDA only -- AMD bounds are unmeasured.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_least, at_most, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||
DEFAULT_TARGET_MODEL_EAGLE3,
|
||||
CustomTestCase,
|
||||
run_bench_serving,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=145, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestEagle3Latency(CustomTestCase):
|
||||
def test_online_latency_eagle3(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_TARGET_MODEL_EAGLE3,
|
||||
num_prompts=300,
|
||||
request_rate=8,
|
||||
sharegpt_context_len=3072,
|
||||
disable_ignore_eos=True,
|
||||
dataset_name="sharegpt",
|
||||
other_server_args=[
|
||||
"--speculative-algorithm",
|
||||
"EAGLE3",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||
"--speculative-num-steps",
|
||||
"5",
|
||||
"--speculative-eagle-topk",
|
||||
"4",
|
||||
"--speculative-num-draft-tokens",
|
||||
"16",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
# The draft checkpoint ships fp16 and the target bf16; the CUDA
|
||||
# rmsnorm path rejects a weight and activation pair that disagree.
|
||||
"--dtype",
|
||||
"float16",
|
||||
],
|
||||
need_warmup=True,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_most(
|
||||
"median_e2e_latency_ms", res["median_e2e_latency_ms"], 1150, unit="ms"
|
||||
),
|
||||
at_least("accept_length", res["accept_length"], 2.3),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Latency and throughput of the /v1/embeddings endpoint."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import (
|
||||
at_least,
|
||||
at_most,
|
||||
check_batch_scaling,
|
||||
check_perf,
|
||||
)
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
run_embeddings_benchmark,
|
||||
run_embeddings_benchmark_multi,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=245, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=240, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestEmbeddingsAPI(CustomTestCase):
|
||||
def test_embeddings_api_latency_throughput(self):
|
||||
res = run_embeddings_benchmark(
|
||||
model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
num_requests=1000,
|
||||
batch_size=1,
|
||||
input_tokens=500,
|
||||
other_server_args=[],
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
check_perf(
|
||||
self,
|
||||
at_most("avg_latency_ms", res["avg_latency_ms"], 21, amd=35, unit="ms"),
|
||||
at_most("p95_latency_ms", res["p95_latency_ms"], 26, amd=40, unit="ms"),
|
||||
at_least("throughput", res["throughput"], 48, amd=30, unit="req/s"),
|
||||
)
|
||||
|
||||
def test_embeddings_api_batch_scaling(self):
|
||||
check_batch_scaling(
|
||||
self,
|
||||
lambda batch_sizes: run_embeddings_benchmark_multi(
|
||||
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
batch_sizes,
|
||||
num_requests=500,
|
||||
input_tokens=500,
|
||||
),
|
||||
# batch size, avg ms, p95 ms, then the same two relaxed for mi300x
|
||||
[
|
||||
(10, 43, 49, 80, 90),
|
||||
(25, 70, 78, 140, 150),
|
||||
(50, 122, 158, 230, 240),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Latency of the LoRA serving path, with and without adapter churn."""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_most, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
run_bench_serving,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=490, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=430, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestLoRALatency(CustomTestCase):
|
||||
def test_online_lora_latency(self):
|
||||
res = self._run_lora_latency_test(enable_background_task=False)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_most(
|
||||
"median_e2e_latency_ms",
|
||||
res["median_e2e_latency_ms"],
|
||||
2270,
|
||||
amd=3320,
|
||||
unit="ms",
|
||||
),
|
||||
# mi300x is about twice as slow as mi325 on LoRA TTFT.
|
||||
at_most("median_ttft_ms", res["median_ttft_ms"], 51, amd=100, unit="ms"),
|
||||
)
|
||||
|
||||
def test_online_lora_latency_with_concurrent_adapter_updates(self):
|
||||
res = self._run_lora_latency_test(enable_background_task=True)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_most(
|
||||
"median_e2e_latency_ms",
|
||||
res["median_e2e_latency_ms"],
|
||||
3170,
|
||||
amd=6000,
|
||||
unit="ms",
|
||||
),
|
||||
at_most("median_ttft_ms", res["median_ttft_ms"], 55, amd=130, unit="ms"),
|
||||
)
|
||||
|
||||
def _run_lora_latency_test(self, enable_background_task: bool):
|
||||
async def lora_loader_unloader_task(
|
||||
base_url: str,
|
||||
start_event: asyncio.Event,
|
||||
stop_event: asyncio.Event,
|
||||
):
|
||||
"""
|
||||
A background task that repeatedly loads and unloads a LoRA adapter.
|
||||
"""
|
||||
await start_event.wait()
|
||||
|
||||
path_cycler = itertools.cycle(
|
||||
[
|
||||
"pbevan11/llama-3.1-8b-ocr-correction",
|
||||
"faridlazuarda/valadapt-llama-3.1-8B-it-chinese",
|
||||
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
|
||||
]
|
||||
)
|
||||
load_url = f"{base_url}/load_lora_adapter"
|
||||
unload_url = f"{base_url}/unload_lora_adapter"
|
||||
num_updates = 0
|
||||
|
||||
while not stop_event.is_set():
|
||||
lora_path = next(path_cycler)
|
||||
response = await asyncio.to_thread(
|
||||
requests.post,
|
||||
load_url,
|
||||
json={"lora_name": lora_path, "lora_path": lora_path},
|
||||
)
|
||||
self.assertTrue(
|
||||
response.ok, f"Failed to load LoRA adapter: {response.text}"
|
||||
)
|
||||
num_updates += 1
|
||||
|
||||
if stop_event.is_set():
|
||||
break
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
response = await asyncio.to_thread(
|
||||
requests.post,
|
||||
unload_url,
|
||||
json={"lora_name": lora_path},
|
||||
)
|
||||
self.assertTrue(
|
||||
response.ok, f"Failed to unload LoRA adapter: {response.text}"
|
||||
)
|
||||
num_updates += 1
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
background_task = lora_loader_unloader_task if enable_background_task else None
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=400,
|
||||
request_rate=8,
|
||||
other_server_args=[
|
||||
"--enable-lora",
|
||||
"--max-loras-per-batch",
|
||||
"1",
|
||||
"--disable-radix-cache",
|
||||
"--random-seed",
|
||||
"42",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--lora-paths",
|
||||
"nvidia/llama-3.1-nemoguard-8b-topic-control",
|
||||
"--max-lora-rank",
|
||||
"256",
|
||||
],
|
||||
dataset_name="random",
|
||||
random_input_len=256,
|
||||
random_output_len=256,
|
||||
lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
|
||||
background_task=background_task,
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Throughput of the MoE model on two GPUs, batched and at batch size one."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_least, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
run_bench_offline_throughput,
|
||||
run_bench_serving,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=290, stage="extra-a", runner_config="2-gpu-large")
|
||||
register_amd_ci(est_time=770, suite="stage-b-test-2-gpu-large-amd")
|
||||
|
||||
|
||||
class TestMoEThroughput(CustomTestCase):
|
||||
def test_moe_offline_throughput_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=300,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--tp", "2"],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput",
|
||||
res["output_throughput"],
|
||||
2670,
|
||||
amd=2100,
|
||||
unit="token/s",
|
||||
),
|
||||
)
|
||||
|
||||
def test_moe_tp2_bs1(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
["--tp", "2", "--cuda-graph-max-bs-decode", "2"],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput", output_throughput, 139, amd=85, unit="token/s"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Throughput of pipeline parallelism on two GPUs, decode and long prefill."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_least, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
run_bench_serving,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=490, stage="extra-a", runner_config="2-gpu-large")
|
||||
register_amd_ci(est_time=1030, suite="stage-b-test-2-gpu-large-amd")
|
||||
|
||||
|
||||
class TestPPThroughput(CustomTestCase):
|
||||
def test_pp_offline_throughput_default_decode(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=1000,
|
||||
request_rate=float("inf"),
|
||||
random_input_len=1,
|
||||
random_output_len=1024,
|
||||
other_server_args=["--pp-size", "2"],
|
||||
need_warmup=True,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput", res["output_throughput"], 6250, unit="token/s"
|
||||
),
|
||||
)
|
||||
|
||||
def test_pp_long_context_prefill(self):
|
||||
res = run_bench_serving(
|
||||
model="meta-llama/Llama-3.3-70B-Instruct",
|
||||
num_prompts=4,
|
||||
request_rate=float("inf"),
|
||||
random_input_len=128000,
|
||||
random_output_len=1,
|
||||
dataset_name="random",
|
||||
other_server_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--pp-size",
|
||||
"2",
|
||||
]
|
||||
+ (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []),
|
||||
need_warmup=False,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"input_throughput",
|
||||
res["input_throughput"],
|
||||
4380,
|
||||
amd=3000,
|
||||
unit="token/s",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Latency and throughput of the /v1/score endpoint."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import (
|
||||
at_least,
|
||||
at_most,
|
||||
check_batch_scaling,
|
||||
check_perf,
|
||||
)
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
CustomTestCase,
|
||||
run_score_benchmark,
|
||||
run_score_benchmark_multi,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=215, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=210, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestScoreAPI(CustomTestCase):
|
||||
def test_score_api_latency_throughput(self):
|
||||
res = run_score_benchmark(
|
||||
model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
num_requests=1000,
|
||||
batch_size=10,
|
||||
other_server_args=[],
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
check_perf(
|
||||
self,
|
||||
at_most("avg_latency_ms", res["avg_latency_ms"], 30, amd=60, unit="ms"),
|
||||
at_most("p95_latency_ms", res["p95_latency_ms"], 32, amd=65, unit="ms"),
|
||||
at_least("throughput", res["throughput"], 34, amd=16, unit="req/s"),
|
||||
)
|
||||
|
||||
def test_score_api_batch_scaling(self):
|
||||
check_batch_scaling(
|
||||
self,
|
||||
lambda batch_sizes: run_score_benchmark_multi(
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
batch_sizes,
|
||||
num_requests=500,
|
||||
),
|
||||
# batch size, avg ms, p95 ms, then the same two relaxed for mi300x
|
||||
[(10, 30, 34, 60, 65), (25, 35, 39, 70, 80), (50, 51, 59, 80, 90)],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Latency of the default serving path on one large GPU."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_most, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
run_bench_serving,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=190, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=165, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestServingLatency(CustomTestCase):
|
||||
def test_online_latency_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=100,
|
||||
request_rate=1,
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_most(
|
||||
"median_e2e_latency_ms",
|
||||
res["median_e2e_latency_ms"],
|
||||
9100,
|
||||
unit="ms",
|
||||
),
|
||||
at_most("median_ttft_ms", res["median_ttft_ms"], 80, amd=115, unit="ms"),
|
||||
at_most("median_itl_ms", res["median_itl_ms"], 9, unit="ms"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Offline throughput of the default serving path on one large GPU."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_least, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||
CustomTestCase,
|
||||
run_bench_serving,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=710, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=810, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestServingThroughput(CustomTestCase):
|
||||
def test_offline_throughput_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput",
|
||||
res["output_throughput"],
|
||||
4000,
|
||||
amd=3050,
|
||||
unit="token/s",
|
||||
),
|
||||
)
|
||||
|
||||
def test_offline_throughput_non_stream_small_batch_size(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=200,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--max-running-requests", "10"],
|
||||
dataset_name="sharegpt",
|
||||
random_input_len=None,
|
||||
random_output_len=None,
|
||||
disable_stream=True,
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput",
|
||||
res["output_throughput"],
|
||||
1110,
|
||||
amd=1000,
|
||||
unit="token/s",
|
||||
),
|
||||
)
|
||||
|
||||
def test_offline_throughput_with_triton_attention_backend(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--context-length",
|
||||
"8192",
|
||||
],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput",
|
||||
res["output_throughput"],
|
||||
3730,
|
||||
amd=2700,
|
||||
unit="token/s",
|
||||
),
|
||||
)
|
||||
|
||||
def test_offline_throughput_default_fp8(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput",
|
||||
res["output_throughput"],
|
||||
4870,
|
||||
amd=3500,
|
||||
unit="token/s",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Throughput of torch.compile at batch size one across two GPUs."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.kits.perf_bench_kit import at_least, check_perf
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
run_bench_offline_throughput,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=75, stage="extra-a", runner_config="2-gpu-large")
|
||||
register_amd_ci(est_time=280, suite="stage-b-test-2-gpu-large-amd")
|
||||
|
||||
|
||||
class TestTorchCompileThroughput(CustomTestCase):
|
||||
def test_torch_compile_tp2_bs1(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"],
|
||||
)
|
||||
|
||||
check_perf(
|
||||
self,
|
||||
at_least(
|
||||
"output_throughput", output_throughput, 255, amd=200, unit="token/s"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""VLM serving perf on the aiter attention backend."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
class TestVLMServingAiter(CustomTestCase):
|
||||
def test_vlm_serving_aiter(self):
|
||||
check_vlm_serving_perf(
|
||||
self,
|
||||
"aiter",
|
||||
output_throughput=2000,
|
||||
e2e_ms=16500,
|
||||
ttft_ms=150,
|
||||
itl_ms=8,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""VLM serving perf on the fa3 attention backend."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=150, stage="extra-a", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestVLMServingFa3(CustomTestCase):
|
||||
def test_vlm_serving_fa3(self):
|
||||
check_vlm_serving_perf(
|
||||
self,
|
||||
"fa3",
|
||||
# No offline bound: never measured on this lane.
|
||||
output_throughput=16700,
|
||||
e2e_ms=11000,
|
||||
ttft_ms=84,
|
||||
itl_ms=5.2,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""VLM serving perf on the flashinfer attention backend."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.vlm_perf_kit import check_vlm_serving_perf
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=195, stage="extra-a", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestVLMServingFlashinfer(CustomTestCase):
|
||||
def test_vlm_serving_flashinfer(self):
|
||||
check_vlm_serving_perf(
|
||||
self,
|
||||
"flashinfer",
|
||||
output_throughput=6900,
|
||||
e2e_ms=17300,
|
||||
ttft_ms=76,
|
||||
itl_ms=8.3,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,53 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_offline_throughput,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=162, stage="extra-a", runner_config="2-gpu-large")
|
||||
register_amd_ci(est_time=630, suite="stage-b-test-2-gpu-large-amd")
|
||||
|
||||
|
||||
class TestBenchOneBatch2GPU(CustomTestCase):
|
||||
def test_moe_tp2_bs1(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
["--tp", "2", "--cuda-graph-max-bs-decode", "2"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n"
|
||||
f"output_throughput: {output_throughput:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(output_throughput, 85)
|
||||
else:
|
||||
self.assertGreater(output_throughput, 125)
|
||||
|
||||
def test_torch_compile_tp2_bs1(self):
|
||||
output_throughput = run_bench_offline_throughput(
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n"
|
||||
f"output_throughput: {output_throughput:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(output_throughput, 200)
|
||||
else:
|
||||
self.assertGreater(output_throughput, 220)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,84 +0,0 @@
|
||||
"""
|
||||
Performance tests for single GPU that need H200 (80GB) - FP8 and EAGLE tests.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||
DEFAULT_TARGET_MODEL_EAGLE,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_serving,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=275, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestBenchServing1GPULarge(CustomTestCase):
|
||||
def test_offline_throughput_default_fp8(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST_FP8,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_default_fp8\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 3500)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 4300)
|
||||
|
||||
@unittest.skipIf(is_hip(), "Skip Eagle test for ROCm")
|
||||
def test_online_latency_eagle(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_TARGET_MODEL_EAGLE,
|
||||
num_prompts=300,
|
||||
request_rate=8,
|
||||
sharegpt_context_len=3072,
|
||||
disable_ignore_eos=True,
|
||||
dataset_name="sharegpt",
|
||||
other_server_args=[
|
||||
"--speculative-algorithm",
|
||||
"EAGLE",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_DRAFT_MODEL_EAGLE,
|
||||
"--speculative-num-steps",
|
||||
"5",
|
||||
"--speculative-eagle-topk",
|
||||
"4",
|
||||
"--speculative-num-draft-tokens",
|
||||
"16",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
],
|
||||
need_warmup=True,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_online_latency_eagle\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
f"accept_length: {res['accept_length']:.2f} \n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_e2e_latency_ms"], 1800)
|
||||
else:
|
||||
self.assertLess(res["median_e2e_latency_ms"], 900)
|
||||
self.assertGreater(res["accept_length"], 3.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,234 +0,0 @@
|
||||
"""
|
||||
Performance tests for single GPU - LLM throughput/latency and LoRA tests.
|
||||
Works on 5090 (32GB).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_serving,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=1264, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=1100, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestBenchServing1GPUPart1(CustomTestCase):
|
||||
def test_offline_throughput_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_default\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 3050)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 3800)
|
||||
|
||||
def test_offline_throughput_non_stream_small_batch_size(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=200,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--max-running-requests", "10"],
|
||||
dataset_name="sharegpt",
|
||||
random_input_len=None,
|
||||
random_output_len=None,
|
||||
disable_stream=True,
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_non_stream_small_batch_size\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 1000)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 1050)
|
||||
|
||||
def test_offline_throughput_with_triton_attention_backend(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=500,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=[
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--context-length",
|
||||
"8192",
|
||||
],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_offline_throughput_with_triton_attention_backend\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 2700)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 3700)
|
||||
|
||||
def test_online_latency_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=100,
|
||||
request_rate=1,
|
||||
other_server_args=[],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_online_latency_default\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 11000)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 115)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 86)
|
||||
self.assertLess(res["median_itl_ms"], 10)
|
||||
|
||||
def test_online_lora_latency(self):
|
||||
res = self._run_lora_latency_test(enable_background_task=False)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_online_lora_latency\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_e2e_latency_ms"], 3320)
|
||||
else:
|
||||
self.assertLess(res["median_e2e_latency_ms"], 2400)
|
||||
# relax for mi300x (LoRA TTFT ~2x slower than mi325)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 100)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 58)
|
||||
|
||||
def test_online_lora_latency_with_concurrent_adapter_updates(self):
|
||||
res = self._run_lora_latency_test(enable_background_task=True)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_online_lora_latency_with_concurrent_adapter_updates\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_e2e_latency_ms"], 6000)
|
||||
else:
|
||||
self.assertLess(res["median_e2e_latency_ms"], 4000)
|
||||
# relax for mi300x (LoRA TTFT ~2x slower than mi325)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 130)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 80)
|
||||
|
||||
def _run_lora_latency_test(self, enable_background_task: bool):
|
||||
"""
|
||||
Run a latency test for LoRA with the specified background task setting.
|
||||
"""
|
||||
|
||||
async def lora_loader_unloader_task(
|
||||
base_url: str,
|
||||
start_event: asyncio.Event,
|
||||
stop_event: asyncio.Event,
|
||||
):
|
||||
"""
|
||||
A background task that repeatedly loads and unloads a LoRA adapter.
|
||||
"""
|
||||
await start_event.wait()
|
||||
|
||||
path_cycler = itertools.cycle(
|
||||
[
|
||||
"pbevan11/llama-3.1-8b-ocr-correction",
|
||||
"faridlazuarda/valadapt-llama-3.1-8B-it-chinese",
|
||||
"philschmid/code-llama-3-1-8b-text-to-sql-lora",
|
||||
]
|
||||
)
|
||||
load_url = f"{base_url}/load_lora_adapter"
|
||||
unload_url = f"{base_url}/unload_lora_adapter"
|
||||
num_updates = 0
|
||||
|
||||
while not stop_event.is_set():
|
||||
lora_path = next(path_cycler)
|
||||
response = await asyncio.to_thread(
|
||||
requests.post,
|
||||
load_url,
|
||||
json={"lora_name": lora_path, "lora_path": lora_path},
|
||||
)
|
||||
self.assertTrue(
|
||||
response.ok, f"Failed to load LoRA adapter: {response.text}"
|
||||
)
|
||||
num_updates += 1
|
||||
|
||||
if stop_event.is_set():
|
||||
break
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
response = await asyncio.to_thread(
|
||||
requests.post,
|
||||
unload_url,
|
||||
json={"lora_name": lora_path},
|
||||
)
|
||||
self.assertTrue(
|
||||
response.ok, f"Failed to unload LoRA adapter: {response.text}"
|
||||
)
|
||||
num_updates += 1
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
background_task = lora_loader_unloader_task if enable_background_task else None
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=400,
|
||||
request_rate=8,
|
||||
other_server_args=[
|
||||
"--enable-lora",
|
||||
"--max-loras-per-batch",
|
||||
"1",
|
||||
"--disable-radix-cache",
|
||||
"--random-seed",
|
||||
"42",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--lora-paths",
|
||||
"nvidia/llama-3.1-nemoguard-8b-topic-control",
|
||||
"--max-lora-rank",
|
||||
"256",
|
||||
],
|
||||
dataset_name="random",
|
||||
random_input_len=256,
|
||||
random_output_len=256,
|
||||
lora_name=["nvidia/llama-3.1-nemoguard-8b-topic-control"],
|
||||
background_task=background_task,
|
||||
)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,185 +0,0 @@
|
||||
"""
|
||||
Performance tests for single GPU - VLM, Score API, and Embeddings API tests.
|
||||
Works on 5090 (32GB).
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_serving,
|
||||
run_embeddings_benchmark,
|
||||
run_score_benchmark,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=909, stage="extra-a", runner_config="1-gpu-large")
|
||||
register_amd_ci(est_time=900, suite="stage-b-test-1-gpu-large-amd")
|
||||
|
||||
|
||||
class TestBenchServing1GPUPart2(CustomTestCase):
|
||||
def test_vlm_online_latency(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=250,
|
||||
request_rate=1,
|
||||
other_server_args=[
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
],
|
||||
dataset_name="mmmu",
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_vlm_online_latency\n"
|
||||
f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertLess(res["median_e2e_latency_ms"], 16500)
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["median_ttft_ms"], 150)
|
||||
else:
|
||||
self.assertLess(res["median_ttft_ms"], 100)
|
||||
self.assertLess(res["median_itl_ms"], 8)
|
||||
|
||||
def test_score_api_latency_throughput(self):
|
||||
"""Test score API latency and throughput performance"""
|
||||
res = run_score_benchmark(
|
||||
model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
num_requests=1000,
|
||||
batch_size=10,
|
||||
other_server_args=[],
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_score_api_throughput\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Score API throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["avg_latency_ms"], 60)
|
||||
self.assertLess(res["p95_latency_ms"], 65)
|
||||
self.assertGreater(res["throughput"], 16)
|
||||
else:
|
||||
self.assertLess(res["avg_latency_ms"], 48)
|
||||
self.assertLess(res["p95_latency_ms"], 50)
|
||||
self.assertGreater(res["throughput"], 20)
|
||||
|
||||
def test_score_api_batch_scaling(self):
|
||||
"""Test score API performance with different batch sizes"""
|
||||
batch_sizes = [10, 25, 50]
|
||||
|
||||
for batch_size in batch_sizes:
|
||||
res = run_score_benchmark(
|
||||
model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST_SCORE,
|
||||
num_requests=500,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_score_api_batch_scaling_size_{batch_size}\n"
|
||||
f"Batch size: {batch_size}\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
bounds = {10: (60, 65), 25: (70, 80), 50: (80, 90)}
|
||||
default_bounds = (90, 90)
|
||||
else:
|
||||
bounds = {10: (45, 50), 25: (50, 60), 50: (60, 65)}
|
||||
default_bounds = (60, 65)
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(
|
||||
batch_size, default_bounds
|
||||
)
|
||||
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
|
||||
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
|
||||
|
||||
def test_embeddings_api_latency_throughput(self):
|
||||
"""Test embeddings API latency and throughput performance"""
|
||||
res = run_embeddings_benchmark(
|
||||
model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
num_requests=1000,
|
||||
batch_size=1,
|
||||
input_tokens=500,
|
||||
other_server_args=[],
|
||||
need_warmup=True,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_embeddings_api_throughput\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Embeddings API throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
self.assertLess(res["avg_latency_ms"], 35)
|
||||
self.assertLess(res["p95_latency_ms"], 40)
|
||||
self.assertGreater(res["throughput"], 30)
|
||||
else:
|
||||
self.assertLess(res["avg_latency_ms"], 20)
|
||||
self.assertLess(res["p95_latency_ms"], 25)
|
||||
self.assertGreater(res["throughput"], 60)
|
||||
|
||||
def test_embeddings_api_batch_scaling(self):
|
||||
"""Test embeddings API performance with different batch sizes"""
|
||||
batch_sizes = [10, 25, 50]
|
||||
|
||||
for batch_size in batch_sizes:
|
||||
res = run_embeddings_benchmark(
|
||||
model=DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST,
|
||||
num_requests=500,
|
||||
batch_size=batch_size,
|
||||
input_tokens=500,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_embeddings_api_batch_scaling_size_{batch_size}\n"
|
||||
f"Batch size: {batch_size}\n"
|
||||
f"Average latency: {res['avg_latency_ms']:.2f} ms\n"
|
||||
f"P95 latency: {res['p95_latency_ms']:.2f} ms\n"
|
||||
f"Throughput: {res['throughput']:.2f} req/s\n"
|
||||
f"Successful requests: {res['successful_requests']}/{res['total_requests']}\n"
|
||||
)
|
||||
|
||||
self.assertEqual(res["successful_requests"], res["total_requests"])
|
||||
# relax for mi300x
|
||||
if is_in_amd_ci():
|
||||
bounds = {10: (80, 90), 25: (140, 150), 50: (230, 240)}
|
||||
default_bounds = (300, 300)
|
||||
else:
|
||||
bounds = {10: (60, 65), 25: (115, 120), 50: (190, 195)}
|
||||
default_bounds = (250, 250)
|
||||
avg_latency_bound, p95_latency_bound = bounds.get(
|
||||
batch_size, default_bounds
|
||||
)
|
||||
self.assertLess(res["avg_latency_ms"], avg_latency_bound)
|
||||
self.assertLess(res["p95_latency_ms"], p95_latency_bound)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,90 +0,0 @@
|
||||
"""
|
||||
Performance tests for 2-GPU that need large GPUs (H200 80GB) - MoE and Pipeline Parallel tests.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_amd_ci,
|
||||
is_in_ci,
|
||||
run_bench_serving,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=687, stage="extra-a", runner_config="2-gpu-large")
|
||||
register_amd_ci(est_time=1450, suite="stage-b-test-2-gpu-large-amd")
|
||||
|
||||
|
||||
class TestBenchServing2GPU(CustomTestCase):
|
||||
def test_moe_offline_throughput_default(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=300,
|
||||
request_rate=float("inf"),
|
||||
other_server_args=["--tp", "2"],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_moe_offline_throughput_default\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["output_throughput"], 2100)
|
||||
else:
|
||||
self.assertGreater(res["output_throughput"], 2200)
|
||||
|
||||
def test_pp_offline_throughput_default_decode(self):
|
||||
res = run_bench_serving(
|
||||
model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
|
||||
num_prompts=1000,
|
||||
request_rate=float("inf"),
|
||||
random_input_len=1,
|
||||
random_output_len=1024,
|
||||
other_server_args=["--pp-size", "2"],
|
||||
need_warmup=True,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_pp_offline_throughput_default_decode\n"
|
||||
f"Output throughput: {res['output_throughput']:.2f} token/s\n"
|
||||
)
|
||||
self.assertGreater(res["output_throughput"], 6700)
|
||||
|
||||
def test_pp_long_context_prefill(self):
|
||||
res = run_bench_serving(
|
||||
model="meta-llama/Llama-3.3-70B-Instruct",
|
||||
num_prompts=4,
|
||||
request_rate=float("inf"),
|
||||
random_input_len=128000,
|
||||
random_output_len=1,
|
||||
dataset_name="random",
|
||||
other_server_args=[
|
||||
"--quantization",
|
||||
"fp8",
|
||||
"--pp-size",
|
||||
"2",
|
||||
]
|
||||
+ (["--mem-fraction-static", "0.7"] if is_in_amd_ci() else []),
|
||||
need_warmup=False,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_pp_long_context_latency_prefill\n"
|
||||
f"input_throughput: {res['input_throughput']:.2f} ms\n"
|
||||
)
|
||||
if is_in_amd_ci():
|
||||
self.assertGreater(res["input_throughput"], 3000)
|
||||
else:
|
||||
self.assertGreater(res["input_throughput"], 4000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,83 +0,0 @@
|
||||
"""
|
||||
VLM Performance tests that work on 5090 (32GB) - VLM offline throughput and online latency tests.
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
auto_config_device,
|
||||
get_benchmark_args,
|
||||
is_in_ci,
|
||||
run_bench_serving_multi,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=200, stage="extra-a", runner_config="1-gpu-small")
|
||||
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
def _local_tokenizer_path():
|
||||
# Prefer the local snapshot so the benchmark client's AutoTokenizer does
|
||||
# not call the HF Hub API, which can stall for minutes in CI.
|
||||
try:
|
||||
from sglang.srt.utils import find_local_repo_dir
|
||||
|
||||
local_dir = find_local_repo_dir(
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, revision=None
|
||||
)
|
||||
if local_dir and os.path.isdir(local_dir):
|
||||
return local_dir
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class TestVLMPerf5090(CustomTestCase):
|
||||
def test_vlm_perf(self):
|
||||
common = dict(
|
||||
base_url=DEFAULT_URL_FOR_TEST,
|
||||
dataset_name="mmmu",
|
||||
dataset_path="",
|
||||
tokenizer=_local_tokenizer_path(),
|
||||
random_input_len=4096,
|
||||
random_output_len=2048,
|
||||
sharegpt_context_len=None,
|
||||
disable_stream=False,
|
||||
disable_ignore_eos=False,
|
||||
seed=0,
|
||||
device=auto_config_device(),
|
||||
lora_name=None,
|
||||
)
|
||||
offline = get_benchmark_args(
|
||||
num_prompts=200, request_rate=float("inf"), **common
|
||||
)
|
||||
# 50 prompts at 1 req/s keeps the online phase ~1 min; medians are
|
||||
# stable at this sample size and the thresholds are loose ceilings.
|
||||
online = get_benchmark_args(num_prompts=50, request_rate=1, **common)
|
||||
|
||||
(_, res_offline), (_, res_online) = run_bench_serving_multi(
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
other_server_args=["--mem-fraction-static", "0.7"],
|
||||
benchmark_args=[offline, online],
|
||||
)
|
||||
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### test_vlm_perf (5090)\n"
|
||||
f"Output throughput: {res_offline['output_throughput']:.2f} token/s\n"
|
||||
f"median_e2e_latency_ms: {res_online['median_e2e_latency_ms']:.2f} ms\n"
|
||||
)
|
||||
self.assertGreater(res_offline["output_throughput"], 2000)
|
||||
self.assertLess(res_online["median_e2e_latency_ms"], 16500)
|
||||
self.assertLess(res_online["median_ttft_ms"], 150)
|
||||
self.assertLess(res_online["median_itl_ms"], 8)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user