minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add three new test cases (#35502)

Browse Source 
Co-authored-by: HeYao <heyao@example.com>
This commit is contained in:
HeYao
2026-08-19 17:59:45 +08:00
committed by GitHub
co-authored by HeYao
parent adca19c497
commit f22442d3a4
6 changed files with 391 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-npu.yml
+13
View File
@@ -368,6 +368,14 @@ jobs:
test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py
test_type: 'perf'
prefill_decode_deployment: 'separation'
# deepseek_v4_flash performance tests
- name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
prefill_size: 1
decode_size: 1
router_size: 1
test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
test_type: 'perf'
prefill_decode_deployment: 'separation'
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
with:
runner: linux-amd64-cpu-4
@@ -401,6 +409,11 @@ jobs:
node_size: 2
test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
test_type: 'accuracy'
# glm_5_2 accuracy tests
- name: glm_5_2_w4a8_16p_gpqa
node_size: 2
test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
test_type: 'accuracy'
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
with:
runner: linux-amd64-cpu-4
python/sglang/test/ascend/e2e/test_npu_performance_utils.py
+1
View File
@@ -164,6 +164,7 @@ GLM5_TOP64_PRUNED_GSM8K_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/yzgjhdxf/GLM-5-top64-pruned-gsm8k"
)
GLM_5_1_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.1-w4a8"
GLM_5_2_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.2-w4a8"
MINIMAX_M2_5_W8A8_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/Eco-Tech/MiniMax-M2.5-w8a8-QuaRot"
)
test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
+106
View File
@@ -0,0 +1,106 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
BENCHMARK_TOOL_DEFAULT,
TestNpuAccuracyMultiNodePdMixTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
GLM_5_2_W4A8_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
GLM_5_2_W4A8_16P_TWO_NODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"TRANSFORMERS_VERBOSITY": "error",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "2500",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
32,
"--nnodes",
2,
# "--dp-size",
# 8,
# "--enable-dp-attention",
"--chunked-prefill-size",
65536,
"--max-prefill-tokens",
280000,
"--trust-remote-code",
"--mem-fraction-static",
0.70,
"--served-model-name",
"glm-5",
"--cuda-graph-max-bs",
32,
"--max-running-requests",
32,
"--quantization",
"modelslim",
# "--speculative-draft-model-quantization",
# "unquant",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--load-balance-method",
"round_robin",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
]
GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG = {
"model_path": GLM_5_2_W4A8_MODEL_PATH,
"other_args": GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS,
"node_envs": GLM_5_2_W4A8_16P_TWO_NODE_ENVS,
}
class TestNPUGLM_5_2_W4A8_16P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase):
"""Test NPU accuracy for GLM-5.2-w4a8 16p two nodes on gpqa_diamond"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model_config = GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG
accuracy = 0.912
datasets = ["gpqa_diamond"]
eval_batch_size = 32
generation_config = {"max_tokens": 65536, "temperature": 1.0}
def test_npu_glm_5_2_w4a8_16p_gpqa(self):
"""Run NPU accuracy test for GLM-5.2-w4a8 16p two nodes on gpqa_diamond"""
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
+234
View File
@@ -0,0 +1,234 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
# Prefill node environment variables for DSV4-Flash PD-Sep deployment.
DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"INF_NAN_MODE_FORCE_DISABLE": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
# skip gpu branch
"SGLANG_OPT_FP8_WO_A_GEMM": "0",
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"SGLANG_DSV4_FP4_EXPERTS": "False",
"SGLANG_OPT_FUSE_WQA_WKV": "0",
"SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch",
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "False",
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "False",
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False",
"SGLANG_OPT_USE_TILELANG_MHC_POST": "False",
# ZBAL
"HCCL_BUFFSIZE": "8",
"SGLANG_ZBAL_LOCAL_MEM_SIZE": "62084",
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
"ZBAL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True",
"SGLANG_ZBAL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669",
"ZBAL_ENABLE_GRAPH": "1",
# PD disagg
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "60",
# MTP
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
}
# Decode node environment variables for DSV4-Flash PD-Sep deployment.
DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"INF_NAN_MODE_FORCE_DISABLE": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
# deepep
"HCCL_BUFFSIZE": "1200",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "8",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048",
"DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256",
# skip gpu branch
"SGLANG_OPT_FP8_WO_A_GEMM": "0",
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"SGLANG_DSV4_FP4_EXPERTS": "False",
"SGLANG_OPT_FUSE_WQA_WKV": "0",
"SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch",
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "False",
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "False",
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False",
"SGLANG_OPT_USE_TILELANG_MHC_POST": "False",
# MTP
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
}
# Prefill node launch arguments for DSV4-Flash PD-Sep.
DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ARGS = [
"--page-size",
128,
"--tp-size",
16,
"--trust-remote-code",
"--device",
"npu",
"--attention-backend",
"dsv4",
"--watchdog-timeout",
9000,
"--disaggregation-mode",
"prefill",
"--disaggregation-transfer-backend",
"ascend",
"--disaggregation-bootstrap-port",
8998,
"--mem-fraction-static",
0.62,
"--prefill-max-requests",
6,
"--max-prefill-tokens",
70000,
"--chunked-prefill-size",
-1,
"--max-running-requests",
112,
"--dp-size",
16,
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"normal",
"--quantization",
"modelslim",
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"--disable-cuda-graph",
"--disable-radix-cache",
"--load-balance-method",
"round_robin",
"--ep-dispatch-algorithm",
"static",
"--init-expert-location",
"/root/.cache/modelscope/hub/models/hot_map/pd_prefill_0720.pt",
]
# Decode node launch arguments for DSV4-Flash PD-Sep.
DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [
"--page-size",
128,
"--tp-size",
16,
"--trust-remote-code",
"--device",
"npu",
"--attention-backend",
"dsv4",
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.75,
"--prefill-max-requests",
1,
"--disable-radix-cache",
"--chunked-prefill-size",
32768,
"--disaggregation-mode",
"decode",
"--disaggregation-transfer-backend",
"ascend",
"--max-running-requests",
896,
"--dp-size",
16,
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--quantization",
"modelslim",
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"--cuda-graph-bs",
1,
2,
4,
8,
16,
24,
36,
40,
48,
56,
# MTP (EAGLE) configuration.
"--speculative-algorithm",
"EAGLE",
"--speculative-num-steps",
2,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
3,
]
# Model config for DSV4-Flash W8A8 1P+1D PD-Sep deployment.
DEEPSEEK_V4_FLASH_W8A8_1P1D_MODEL_CONFIG = {
"model_path": DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH,
"prefill_args": DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ARGS,
"decode_args": DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS,
"prefill_envs": DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ENVS,
"decode_envs": DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ENVS,
"router_args": ["--policy", "cache_aware"],
"router_envs": {},
}
class TestNPUDeepSeekV4FlashW8A81P1D16PIn8kOut1k50ms(
TestNpuPerfMultiNodePdSepTestCaseBase
):
"""Test NPU performance for DeepSeek-V4-Flash W8A8 PD-Sep 1P+1D 16p in8k out1k."""
model_config = DEEPSEEK_V4_FLASH_W8A8_1P1D_MODEL_CONFIG
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_name = "random"
input_len = 8000
output_len = 1000
num_prompts = 2400
max_concurrency = 800
random_range_ratio = 1
warmup_requests = 0
request_rate = float("inf")
seed = 1
tpot = 50
output_token_throughput = 7046
def test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms(self):
"""Run NPU performance test for DeepSeek-V4-Flash W8A8 1P+1D 16p in8k out1k."""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py
+1 -1
View File
@@ -8,7 +8,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=3600, suite="nightly-perf-16-npu-a3", nightly=True)
register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3", nightly=True)
# Environment variables for DSV4-Flash single-node PD-mix deployment.
DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
+36 -16
View File
@@ -8,24 +8,34 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import (
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=1200, suite="base-c-test-perf-16-npu-a3")
register_npu_ci(est_time=1200, suite="nightly-perf-16-npu-a3", nightly=True)
register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3", nightly=True)
# Environment variables for DSV4-Flash single-node PD-mix deployment.
DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"INF_NAN_MODE_FORCE_DISABLE": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"USE_NPU_MOE_GATING_TOP_K": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "1",
# deepep
"DEEPEP_HCCL_BUFFSIZE": "1000",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "16",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048",
"DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
# zbal
"HCCL_BUFFSIZE": "8",
"SGLANG_ZBAL_LOCAL_MEM_SIZE": "61000",
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
"ZBAL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True",
"SGLANG_ZBAL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669",
"ZBAL_ENABLE_GRAPH": "1",
# dsv4
"IS_DEEPSEEK_V4": "1",
"SGLANG_DEBUG_LAYER_NORM": "1",
"SGLANG_DEBUG_FWD_INPUT": "1",
"USE_FUSED_HC_PRE_ASCENDC": "1",
"SGLANG_DSV4_NPU_FUSED_COMPRESSOR": "1",
"SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL": "1",
# skip gpu branch
"SGLANG_OPT_FP8_WO_A_GEMM": "0",
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False",
@@ -37,9 +47,14 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = {
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "False",
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False",
"SGLANG_OPT_USE_TILELANG_MHC_POST": "False",
# MTP (EAGLE) related envs
# mtp
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_DEBUG_MTP_VERIFY": "0",
"SGLANG_DEBUG_MTP_VERIFY_LIMIT": "8",
"SGLANG_DEBUG_MTP_VERIFY_ROWS": "4",
"SGLANG_DISABLE_DRAFT_EXTEND_GRAPH": "1",
}
# Server launch arguments for DSV4-Flash W8A8 single-node 8p PD-mix.
@@ -51,17 +66,16 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"--trust-remote-code",
"--device",
"npu",
"--prefill-max-requests",
160,
"--attention-backend",
"dsv4",
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.6,
"--prefill-max-requests",
2,
"--disable-radix-cache",
0.7,
"--chunked-prefill-size",
-1,
131072,
"--max-running-requests",
160,
"--dp-size",
@@ -75,7 +89,8 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
"modelslim",
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"auto",
"--skip-server-warmup",
"--cuda-graph-bs",
1,
2,
@@ -91,6 +106,9 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
3,
"--ep-size",
16,
"--disable-radix-cache",
]
@@ -103,6 +121,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase):
other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS
envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS
dataset_name = "random"
dataset_path = "/root/.cache/modelscope/hub/datasets/gsm8k_deepseekv4/cache0_8000/formal_run1_160_8000_cache0.json"
input_len = 8000
output_len = 1000
num_prompts = 160
@@ -112,7 +131,8 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase):
request_rate = float("inf")
seed = 1
tpot = 50
output_token_throughput = 1708
max_attempts = 3
output_token_throughput = 2825
def test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms(self):
"""Run NPU performance test for DeepSeek-V4-Flash W8A8 8p in8k out1k."""