minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

Update test cases and performance testing framework (#40392)

Browse Source
This commit is contained in:
ZY Y
2026-09-20 22:42:00 +08:00
committed by GitHub
parent 404dee10c0
commit 5f017ffabb
3 changed files with 162 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/ascend/e2e/test_npu_performance_utils.py
+64
View File
@@ -228,6 +228,20 @@ TTFT_TOLERANCE = 1.02 # +2%
E2E_TOLERANCE = 1.02 # +2%
OUTPUT_TOKEN_THROUGHPUT_TOLERANCE = 0.98 # -2%
def _get_spec_num_draft_tokens(other_args):
"""Extract the value of --speculative-num-draft-tokens from server args."""
if not other_args:
return None
args = [str(arg) for arg in other_args]
if "--speculative-num-draft-tokens" not in args:
return None
idx = args.index("--speculative-num-draft-tokens")
if idx + 1 >= len(args):
return None
return int(args[idx + 1])
# Package filtering keywords
PACKAGE_FILTER_KEYWORDS = [
"sglang",
@@ -570,6 +584,11 @@ def run_bench_serving(
parts = stripped_line.split()
if len(parts) >= 5:
metrics["mean_e2e_latency"] = parts[4]
elif "Accept length" in stripped_line:
# Format: "Accept length: 4.35"
parts = stripped_line.split()
if len(parts) >= 3:
metrics["accept_length"] = parts[2]
reader_done.set()
process.wait()
if process.returncode != 0:
@@ -919,6 +938,29 @@ def assert_metrics(self, metrics):
labels={"test_case": tc_name, "type": "perf"},
)
spec_num_draft_tokens = _get_spec_num_draft_tokens(
getattr(self, "other_args", None)
)
# accept_rate = accept_length / --speculative-num-draft-tokens.
# Dump only when all inputs are present; the mandatory checks below
# decide pass/fail when the baseline is set.
if (
getattr(self, "accept_rate", None)
and metrics.get("accept_length")
and spec_num_draft_tokens
):
accept_rate = float(metrics["accept_length"]) / spec_num_draft_tokens
dump_metric(
"accept_rate",
accept_rate,
labels={"test_case": tc_name, "type": "perf"},
)
dump_metric(
"accept_rate_baseline",
float(self.accept_rate),
labels={"test_case": tc_name, "type": "perf"},
)
if self.tpot:
if self.tpot < TPOT_THRESHOLD:
self.assertLessEqual(
@@ -945,6 +987,25 @@ def assert_metrics(self, metrics):
float(metrics["mean_e2e_latency"]),
self.mean_e2e_latency * E2E_TOLERANCE,
)
# Once an accept_rate baseline is set, a missing "Accept length" line
# (e.g. server_info request failed or spec decoding inactive) or a
# missing --speculative-num-draft-tokens arg must fail the test
# instead of being silently skipped.
if getattr(self, "accept_rate", None):
self.assertIsNotNone(
metrics.get("accept_length"),
"accept_length not found in bench_serving output "
"while accept_rate baseline is set",
)
self.assertIsNotNone(
spec_num_draft_tokens,
"--speculative-num-draft-tokens not found in other_args "
"while accept_rate baseline is set",
)
self.assertGreaterEqual(
float(metrics["accept_length"]) / spec_num_draft_tokens,
self.accept_rate,
)
class TestNpuPerformanceTestCaseBase(CustomTestCase):
@@ -975,6 +1036,9 @@ class TestNpuPerformanceTestCaseBase(CustomTestCase):
tpot = None
mean_e2e_latency = None
output_token_throughput = None
# Baseline for accept_length / speculative-num-draft-tokens; None disables
# the assertion.
accept_rate = None
dp = None
generation_kwargs = None
test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py
+46 -10
View File
@@ -1,3 +1,5 @@
import os
import subprocess
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
@@ -8,6 +10,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
from sglang.utils import wait_for_server
register_npu_ci(
est_time=3600,
@@ -29,6 +33,7 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = {
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_NPU_USE_MULTI_STREAM": "1",
# skip gpu branch
"SGLANG_OPT_FP8_WO_A_GEMM": "0",
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False",
@@ -43,10 +48,13 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = {
# DSPARK
"SGLANG_RAGGED_VERIFY_MODE": "static",
"SGLANG_DSPARK_FAST_KERNEL": "0",
# mtp
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
# deepep
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "2048",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
"DEEPEP_HCCL_BUFFSIZE": "2500",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96",
"DEEPEP_HYBRID_DEPLOYMENT": "1",
# war barrier
"SGLANG_ENABLE_WAR_BARRIER": "1",
@@ -66,15 +74,15 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.62,
0.68,
"--prefill-max-requests",
32,
192,
"--max-prefill-tokens",
131072,
80000,
"--chunked-prefill-size",
131072,
"--max-running-requests",
96,
192,
"--dp-size",
16,
"--enable-dp-attention",
@@ -87,6 +95,8 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"--load-balance-method",
"round_robin",
"--speculative-algorithm",
"DSPARK",
"--speculative-draft-model-path",
@@ -96,16 +106,17 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [
"--speculative-draft-attention-backend",
"ascend",
"--speculative-num-draft-tokens",
6,
7,
"--speculative-dspark-block-size",
5,
"--skip-server-warmup",
6,
"--cuda-graph-bs-decode",
1,
2,
4,
5,
6,
8,
10,
"--disable-radix-cache",
]
@@ -136,6 +147,31 @@ class TestNPUDeepSeekV4FlashW8A88PGPQA(TestNpuAccuracyTestCaseBase):
timeout = 6000
seed = 1
@classmethod
def setUpClass(cls):
"""Launch server via `python3 -m sglang.launch_server` instead of `sglang serve`."""
cls._setup_per_case_output()
cls.base_url = DEFAULT_URL_FOR_TEST
env = os.environ.copy()
if cls.envs:
env.update(cls.envs)
_, host, port = cls.base_url.split(":")
command = [
"python3",
"-m",
"sglang.launch_server",
"--model-path",
cls.model,
*[str(x) for x in cls.other_args],
"--host",
host[2:],
"--port",
port,
]
cls.process = subprocess.Popen(command, env=env)
wait_for_server(cls.base_url, timeout=cls.server_timeout, process=cls.process)
def test_npu_deepseek_v4_flash_w8a8_8p_gpqa(self):
"""Run NPU accuracy test for DeepSeek-V4-Flash W8A8 8p DSPARK GPQA."""
self.run_accuracy()
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
+52 -18
View File
@@ -1,3 +1,5 @@
import os
import subprocess
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
@@ -7,6 +9,8 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
from sglang.utils import wait_for_server
register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3", nightly=True)
register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3-cann910", nightly=True)
@@ -23,9 +27,11 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
"SGLANG_NPU_USE_MULTI_STREAM": "1",
# deepep
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "2048",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "35",
"DEEPEP_HCCL_BUFFSIZE": "2500",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96",
"DEEPEP_HYBRID_DEPLOYMENT": "1",
"SGLANG_RAGGED_VERIFY_MODE": "static",
"SGLANG_DSPARK_FAST_KERNEL": "0",
# war barrier
"SGLANG_ENABLE_WAR_BARRIER": "1",
"SGLANG_FORCE_COARSE_WAR_BARRIER": "1",
@@ -55,7 +61,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"--device",
"npu",
"--prefill-max-requests",
160,
192,
"--max-prefill-tokens",
80000,
"--attention-backend",
@@ -67,7 +73,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"--chunked-prefill-size",
131072,
"--max-running-requests",
160,
192,
"--dp-size",
16,
"--enable-dp-attention",
@@ -80,24 +86,28 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"--skip-server-warmup",
"--load-balance-method",
"round_robin",
"--cuda-graph-bs-decode",
1,
2,
4,
6,
8,
10,
# MTP (EAGLE) configuration.
# DSPARK configuration.
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
2,
"--speculative-eagle-topk",
1,
"DSPARK",
"--speculative-draft-model-path",
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
"--speculative-draft-model-quantization",
"modelslim",
"--speculative-draft-attention-backend",
"ascend",
"--speculative-num-draft-tokens",
3,
"--ep-size",
16,
7,
"--speculative-dspark-block-size",
6,
"--disable-radix-cache",
]
@@ -111,18 +121,42 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase):
other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS
envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS
dataset_name = "random"
dataset_path = "/root/.cache/modelscope/hub/datasets/gsm8k_deepseekv4/cache0_8000/formal_run1_160_8000_cache0.json"
input_len = 8000
output_len = 1000
num_prompts = 160
max_concurrency = 160
random_range_ratio = 1
warmup_requests = 16
warmup_requests = 32
request_rate = float("inf")
seed = 1
tpot = 50
max_attempts = 3
output_token_throughput = 2825
output_token_throughput = 3100
accept_rate = 0.5
@classmethod
def setUpClass(cls):
"""Launch server via `python3 -m sglang.launch_server` instead of `sglang serve`."""
cls._setup_per_case_output()
cls.base_url = DEFAULT_URL_FOR_TEST
env = os.environ.copy()
if cls.envs:
env.update(cls.envs)
_, host, port = cls.base_url.split(":")
command = [
"python3",
"-m",
"sglang.launch_server",
"--model-path",
cls.model,
*[str(x) for x in cls.other_args],
"--host",
host[2:],
"--port",
port,
]
cls.process = subprocess.Popen(command, env=env)
wait_for_server(cls.base_url, timeout=cls.timeout, process=cls.process)
def test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms(self):
"""Run NPU performance test for DeepSeek-V4-Flash W8A8 8p in8k out1k."""