minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[NPU] Nightly CI refactor and enhancement (#27433)

Browse Source
This commit is contained in:
hhhh1252023
2026-06-26 08:52:06 +08:00
committed by GitHub
parent f83cbc2516
commit 781537b61d
58 changed files with 10919 additions and 584 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py
+46
View File
@@ -0,0 +1,46 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.test_ascend_utils import DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_PATH
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
OTHER_ARGS = [
"--trust-remote-code",
"--mem-fraction-static",
"0.9",
"--attention-backend",
"ascend",
"--disable-cuda-graph",
"--tp-size",
"16",
"--quantization",
"modelslim",
"--disable-radix-cache",
]
class TestNPUDeepSeek_V3_2_8P_AIME2025(TestNpuAccuracyTestCaseBase):
model = DEEPSEEK_V3_2_EXP_W8A8_WEIGHTS_PATH
other_args = OTHER_ARGS
accuracy = 0.931
datasets = ["aime25"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 64
def test_aime2025(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py
+71
View File
@@ -0,0 +1,71 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_6V_FLASH_MODEL_PATH
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "1000",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_SET_CPU_AFFINITY": "1",
}
OTHER_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
2,
"--chunked-prefill-size",
16384,
"--max-prefill-tokens",
150000,
"--dtype",
"bfloat16",
"--max-running-requests",
32,
"--trust-remote-code",
"--mem-fraction-static",
0.75,
"--cuda-graph-bs",
1,
2,
4,
8,
16,
32,
"--watchdog-timeout",
9000,
]
class TestQwen3(TestNpuAccuracyTestCaseBase):
model = GLM_4_6V_FLASH_MODEL_PATH
envs = ENVS
other_args = OTHER_ARGS
accuracy = 0.711
datasets = ["mmmu"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 64
def test_mmmu(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py
+72
View File
@@ -0,0 +1,72 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import GLM_4_7_FLASH_MODEL_PATH
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "1000",
"HCCL_OP_EXPANSION_MODE": "AIV",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"SGLANG_SET_CPU_AFFINITY": "1",
}
OTHER_ARGS = [
"--attention-backend",
"ascend",
"--device",
"npu",
"--tp-size",
2,
"--chunked-prefill-size",
16384,
"--max-prefill-tokens",
150000,
"--dtype",
"bfloat16",
"--max-running-requests",
32,
"--trust-remote-code",
"--mem-fraction-static",
0.75,
"--cuda-graph-bs",
1,
2,
4,
8,
16,
32,
"--watchdog-timeout",
9000,
]
class TestNPUDeepSeek_V3_2_8P_AIME2025(TestNpuAccuracyTestCaseBase):
model = GLM_4_7_FLASH_MODEL_PATH
envs = ENVS
other_args = OTHER_ARGS
accuracy = 0.916
datasets = ["aime25"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 64
def test_aime2025(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py
+86
View File
@@ -0,0 +1,86 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
64,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
65536,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--tp-size",
16,
"--mem-fraction-static",
0.72,
"--cuda-graph-bs",
64,
"--dtype",
"bfloat16",
]
class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase):
model = QWEN3_32B_MODEL_PATH
envs = QWEN3_32B_ENVS
other_args = QWEN3_32B_OTHER_ARGS
accuracy = 0.516
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 40000, "temperature": 1.0}
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py
+98
View File
@@ -0,0 +1,98 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_6_27B_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_64K_PREFIX_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
}
QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
32768,
"--max-prefill-tokens",
32768,
"--mamba-scheduler-strategy",
"extra_buffer",
"--trust-remote-code",
"--max-running-requests",
20,
"--max-mamba-cache-size",
120,
"--mem-fraction-static",
0.8,
"--cuda-graph-bs",
1,
2,
4,
8,
10,
12,
16,
18,
20,
"--enable-prefill-delayer",
"--prefill-delayer-queue-min-ratio",
0.5,
"--prefill-delayer-max-delay-ms",
30000,
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_gpqa(TestNpuAccuracyTestCaseBase):
model = QWEN3_6_27B_MODEL_PATH
envs = QWEN3_6_27B_64K_PREFIX_ENVS
other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS
accuracy = 0.878
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 81920, "temperature": 1.0}
def test_gpqa(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py
+96
View File
@@ -0,0 +1,96 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_6_35B_A3B_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "100",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
}
QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
131072,
"--disable-radix-cache",
"--trust-remote-code",
"--enable-prefill-delayer",
"--max-running-requests",
4,
"--max-mamba-cache-size",
4,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
1,
2,
3,
4,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_aime26(TestNpuAccuracyTestCaseBase):
model = QWEN3_6_35B_A3B_MODEL_PATH
envs = QWEN3_6_35B_A3B_3K5_1K5_ENVS
other_args = QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS
accuracy = 0.927
datasets = ["aime26"]
few_shot_num = 0
eval_batch_size = 4
generation_config = {
"max_tokens": 131072,
"temperature": 0.2,
"repetition_penalty": 1.08,
}
def test_aime26(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py
+104
View File
@@ -0,0 +1,104 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400",
}
OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
162,
"--disable-radix-cache",
# "--speculative-draft-model-quantization",
# "unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
# "--speculative-algorithm",
# "EAGLE3",
# "--speculative-draft-model-path",
# QWEN3_A3B_EAGLE_MODEL_PATH,
# "--speculative-num-steps",
# 3,
# "--speculative-eagle-topk",
# 1,
# "--speculative-num-draft-tokens",
# 4,
"--tp-size",
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
1,
5,
15,
40,
70,
100,
120,
130,
140,
146,
150,
154,
156,
158,
160,
162,
"--dtype",
"bfloat16",
]
class TestQwen3(TestNpuAccuracyTestCaseBase):
model = QWEN3_OMNI_30B_A3B_THINKING_MODEL_PATH
envs = ENVS
other_args = OTHER_ARGS
accuracy = 0.576
datasets = ["mmmu"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 64
def test_mmmu(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py
+102
View File
@@ -0,0 +1,102 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_VL_30B_A3B_THINKING_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400",
}
OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
162,
"--disable-radix-cache",
# "--speculative-draft-model-quantization",
# "unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
# "--speculative-algorithm",
# "EAGLE3",
# "--speculative-draft-model-path",
# QWEN3_A3B_EAGLE_MODEL_PATH,
# "--speculative-num-steps",
# 3,
# "--speculative-eagle-topk",
# 1,
# "--speculative-num-draft-tokens",
# 4,
"--tp-size",
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
1,
5,
15,
40,
70,
100,
120,
130,
140,
146,
150,
154,
156,
158,
160,
162,
"--dtype",
"bfloat16",
]
class TestQwen3(TestNpuAccuracyTestCaseBase):
model = QWEN3_VL_30B_A3B_THINKING_MODEL_PATH
envs = ENVS
other_args = OTHER_ARGS
accuracy = 0.76
datasets = ["mmmu"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 64
def test_mmmu(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py
+86
View File
@@ -0,0 +1,86 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_VL_8B_THINKING_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
}
OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
16,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
-1,
"--tp-size",
2,
"--mem-fraction-static",
0.894,
"--cuda-graph-bs",
1,
5,
15,
16,
"--dtype",
"bfloat16",
# "--speculative-draft-model-quantization",
# "unquant",
# "--speculative-algorithm",
# "EAGLE3",
# "--speculative-draft-model-path",
# QWEN3_8B_EAGLE_MODEL_PATH,
# "--speculative-num-steps",
# 4,
# "--speculative-eagle-topk",
# 1,
# "--speculative-num-draft-tokens",
# 5,
]
class TestQwen3(TestNpuAccuracyTestCaseBase):
model = QWEN3_VL_8B_THINKING_MODEL_PATH
envs = ENVS
other_args = OTHER_ARGS
accuracy = 0.741
datasets = ["mmmu"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
eval_batch_size = 16
def test_mmmu(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
+198
View File
@@ -0,0 +1,198 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyMultiNodePdSepTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
GLM_5_1_W4A8_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
GLM_5_1_PD_SEP_PREFILL_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200",
"HCCL_BUFFSIZE": "1200",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "72",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "1024",
"DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"TASK_QUEUE_ENABLE": "2",
"ENABLE_PROFILING": "0",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
GLM_5_1_PD_SEP_DECODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200",
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200",
"SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"HCCL_BUFFSIZE": "200",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16",
"TASK_QUEUE_ENABLE": "0",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
GLM_5_1_PD_SEP_PREFILL_ARGS = [
"--disaggregation-mode",
"prefill",
"--tp-size",
4,
"--nnodes",
2,
"--mem-fraction-static",
0.72,
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--disaggregation-transfer-backend",
"ascend",
"--max-running-requests",
16,
"--served-model-name",
"glm-5",
"--chunked-prefill-size",
16384,
"--max-prefill-tokens",
180000,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"normal",
"--disable-shared-experts-fusion",
"--disable-cuda-graph",
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--enable-nsa-prefill-context-parallel",
"--nsa-prefill-cp-mode",
"in-seq-split",
"--attn-cp-size",
4,
"--enable-dp-lm-head",
"--moe-dense-tp",
1,
"--pp-size",
8,
]
GLM_5_1_PD_SEP_DECODE_ARGS = [
"--disaggregation-mode",
"decode",
"--tp-size",
32,
"--nnodes",
2,
"--dp-size",
32,
"--enable-dp-attention",
"--ep-size",
32,
"--mem-fraction-static",
0.85,
"--max-running-requests",
32,
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--served-model-name",
"glm-5",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"low_latency",
"--cuda-graph-bs",
1,
2,
3,
"--disaggregation-transfer-backend",
"ascend",
"--watchdog-timeout",
9000,
"--context-length",
180000,
"--tokenizer-worker-num",
16,
"--prefill-round-robin-balance",
"--disable-shared-experts-fusion",
"--dtype",
"bfloat16",
"--load-balance-method",
"round_robin",
"--speculative-draft-model-quantization",
"unquant",
]
GLM_5_1_PD_SEP_MODEL_CONFIG = {
"model_path": GLM_5_1_W4A8_MODEL_PATH,
"prefill_args": GLM_5_1_PD_SEP_PREFILL_ARGS,
"decode_args": GLM_5_1_PD_SEP_DECODE_ARGS,
"prefill_envs": GLM_5_1_PD_SEP_PREFILL_ENVS,
"decode_envs": GLM_5_1_PD_SEP_DECODE_ENVS,
"router_args": ["--policy", "round_robin"],
"router_envs": {},
}
class TestNPUGLM5_1_W4A8_PD_SEP_AIME2026(TestNpuAccuracyMultiNodePdSepTestCaseBase):
"""Test NPU accuracy for GLM-5.1-w4a8 PD separation on AIME2026"""
model_config = GLM_5_1_PD_SEP_MODEL_CONFIG
accuracy = 0.953
datasets = ["aime26"]
eval_batch_size = 64
generation_config = {"max_tokens": 65536, "temperature": 1.0}
def test_npu_glm5_1_w4a8_pd_sep_aime2026(self):
"""Run NPU accuracy test for GLM-5.1-w4a8 PD separation on AIME2026"""
self.run_accuracy()
class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBase):
"""Test NPU performance for GLM-5.1-w4a8 PD separation 4 nodes in3k5 out1k5"""
model_config = GLM_5_1_PD_SEP_MODEL_CONFIG
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
dataset_name = "random"
max_concurrency = 1
num_prompts = 1
input_len = 65536
output_len = 1024
random_range_ratio = 1
tpot = 50
output_token_throughput = 16
def test_npu_glm5_1_w4a8_pd_sep_in3k5_out1k5(self):
"""Run NPU performance test for GLM-5.1-w4a8 PD separation"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
+125
View File
@@ -0,0 +1,125 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyMultiNodePdMixTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
KIMI_K2_6_EAGLE3_MODEL_PATH,
KIMI_K2_6_W4A8_MODEL_PATH,
TestNpuPerfMultiNodePdMixTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=1800,
suite="nightly-8-npu-a3",
nightly=True,
disabled="Currently it is executed by the npu performance workflow.",
)
ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
"HCCL_BUFFSIZE": "4400",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--tp-size",
32,
"--nnodes",
2,
"--mem-fraction-static",
0.55,
"--max-running-requests",
32,
"--chunked-prefill-size",
262144,
"--context-length",
75000,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--sampling-backend",
"ascend",
"--enable-dp-attention",
"--dp-size",
32,
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--cuda-graph-bs",
1,
"--disable-radix-cache",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
KIMI_K2_6_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
]
MODEL_CONFIG = {
"model_path": KIMI_K2_6_W4A8_MODEL_PATH,
"other_args": OTHER_ARGS,
"node_envs": ENVS,
}
class TestNPUKimiK2_6_W4A8_16P_AIME2025(TestNpuAccuracyMultiNodePdMixTestCaseBase):
model_config = MODEL_CONFIG
accuracy = 0.961
datasets = ["aime25"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 65536, "temperature": 1.0}
def test_aime2025(self):
self.run_accuracy()
class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model_config = MODEL_CONFIG
dataset_name = "random"
max_concurrency = 32
num_prompts = 32
input_len = 64000
output_len = 1000
random_range_ratio = 1
tpot = 100
output_token_throughput = 160
def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py
+132
View File
@@ -0,0 +1,132 @@
import os
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
BENCHMARK_TOOL_DEFAULT,
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
MINIMAX_M2_5_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="npu-performance",
nightly=True,
)
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_FUSED_MOE_MODE": "2",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"HCCL_BUFFSIZE": "1024",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
}
MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
"--tp-size",
8,
"--mem-fraction-static",
0.63,
"--max-running-requests",
26,
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
"--enable-prefill-delayer",
"--prefill-max-requests",
10,
"--chunked-prefill-size",
67072,
"--max-prefill-token",
67000,
"--cuda-graph-bs",
2,
4,
8,
12,
16,
18,
20,
22,
24,
26,
"--moe-a2a-backend",
"ascend_fuseep",
"--deepep-mode",
"auto",
"--quantization",
"modelslim",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--dtype",
"bfloat16",
"--trust-remote-code",
]
class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms(
TestNpuPerformanceTestCaseBase
):
"""MiniMax-M2.5-w8a8 4p (4 cards) 64k input 1k output with 90% prefix cache performance test"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS
envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 26
num_prompts = 104
input_len = 65536
output_len = 1024
random_range_ratio = 1
repeat_rate = 0.9
tpot = 50
output_token_throughput = 390.5839
request_rate = float("inf")
def test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms(self):
"""Run MiniMax-M2.5-w8a8 4p 64k/1k prefix90 performance test"""
self.run_throughput()
class TestNPUMiniMaxM2_5_W8A8_4P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase):
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS
envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS
accuracy = 0.852
datasets = ["gpqa_diamond"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
max_concurrency = 64
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py
+132
View File
@@ -0,0 +1,132 @@
import os
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
MINIMAX_M2_5_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-16-npu-a3",
nightly=True,
disabled="performance testcase",
)
MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"TASK_QUEUE_ENABLE": "1",
"HCCL_BUFFSIZE": "1024",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_FUSED_MOE_MODE": "2",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
}
MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [
"--tp-size",
16,
"--enable-dp-attention",
"--dp-size",
16,
"--mem-fraction-static",
0.75,
"--max-running-requests",
320,
"--disable-radix-cache",
"--reasoning-parser",
"minimax-append-think",
"--tool-call-parser",
"minimax-m2",
"--prefill-delayer-max-delay-passes",
500,
"--enable-prefill-delayer",
"--chunked-prefill-size",
-1,
"--max-prefill-token",
8192,
"--cuda-graph-bs",
1,
2,
4,
8,
12,
16,
20,
"--moe-a2a-backend",
"ascend_fuseep",
"--deepep-mode",
"auto",
"--quantization",
"modelslim",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
MINIMAX_M2_5_EAGLE3_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--dtype",
"bfloat16",
]
class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_HighThroughput(
TestNpuPerformanceTestCaseBase
):
"""Test NPU performance for MiniMax-M2.5-w8a8 8p single node high throughput in3k5 out1k5"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = MINIMAX_M2_5_W8A8_MODEL_PATH
other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS
envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS
dataset_name = "random"
max_concurrency = 320
num_prompts = 1280
input_len = 3500
output_len = 1500
random_range_ratio = 1
tpot = 50
output_token_throughput = 5717.58
def test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_high_throughput(self):
"""Run NPU performance test for MiniMax-M2.5-w8a8 high throughput"""
self.run_throughput()
class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase):
model = MINIMAX_M2_5_W8A8_MODEL_PATH
envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS
other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS
accuracy = 0.852
datasets = ["gpqa_diamond"]
few_shot_num = 0
generation_config = {"max_tokens": 65536, "temperature": 1.0}
max_concurrency = 64
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+122
View File
@@ -0,0 +1,122 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_8B_EAGLE_MODEL_PATH,
QWEN3_8B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
}
QWEN3_8B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
70,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
16384,
"--tp-size",
1,
"--mem-fraction-static",
0.85,
"--cuda-graph-bs",
8,
12,
24,
36,
48,
51,
55,
60,
63,
64,
66,
68,
70,
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestQwen8B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_8B_W8A8_MODEL_PATH
other_args = QWEN3_8B_OTHER_ARGS
envs = QWEN3_8B_ENVS
dataset_name = "random"
max_concurrency = 64
num_prompts = 256
input_len = 3500
output_len = 1500
random_range_ratio = 1
tpot = 37
output_token_throughput = 1586
def test_qwen3_8b(self):
self.run_throughput()
class TestQwen8B_gpqa(TestNpuAccuracyTestCaseBase):
model = QWEN3_8B_W8A8_MODEL_PATH
envs = QWEN3_8B_ENVS
other_args = QWEN3_8B_OTHER_ARGS
accuracy = 0.4444
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 40000, "temperature": 1.0}
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py
+108
View File
@@ -0,0 +1,108 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_8B_EAGLE_MODEL_PATH,
QWEN3_8B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_8B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
}
QWEN3_8B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
16,
"--max-prefill-tokens",
16384,
"--disable-radix-cache",
"--chunked-prefill-size",
-1,
"--tp-size",
2,
"--mem-fraction-static",
0.894,
"--cuda-graph-bs",
1,
5,
15,
16,
"--dtype",
"bfloat16",
"--speculative-draft-model-quantization",
"unquant",
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_8B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
]
class TestQwen8B(TestNpuPerformanceTestCaseBase):
max_attempts = 5
model = QWEN3_8B_W8A8_MODEL_PATH
other_args = QWEN3_8B_OTHER_ARGS
envs = QWEN3_8B_ENVS
dataset_name = "random"
max_concurrency = 16
num_prompts = 16
input_len = 6144
output_len = 1500
random_range_ratio = 1
tpot = 11.79
output_token_throughput = 930
def test_qwen3_8b(self):
self.run_throughput()
class TestQwen8B_gpqa(TestNpuAccuracyTestCaseBase):
model = QWEN3_8B_W8A8_MODEL_PATH
envs = QWEN3_8B_ENVS
other_args = QWEN3_8B_OTHER_ARGS
accuracy = 0.4444
datasets = ["gpqa_diamond"]
few_shot_num = 0
generation_config = {"max_tokens": 32768, "temperature": 1.0}
eval_batch_size = 16
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py
+127
View File
@@ -0,0 +1,127 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH,
QWEN3_A3B_EAGLE_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_30B_A3B_ENVS = {
"ASCEND_LAUNCH_BLOCKING": "0",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400",
}
QWEN3_30B_A3B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
162,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_A3B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
2,
"--mem-fraction-static",
0.87,
"--cuda-graph-bs",
1,
5,
15,
40,
70,
100,
120,
130,
140,
146,
150,
154,
156,
158,
160,
162,
"--dtype",
"bfloat16",
]
class TestQwen30B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH
other_args = QWEN3_30B_A3B_OTHER_ARGS
envs = QWEN3_30B_A3B_ENVS
dataset_name = "random"
max_concurrency = 160
num_prompts = int(max_concurrency) * 4
input_len = 3500
output_len = 1500
random_range_ratio = 1
tpot = 50
output_token_throughput = 3200
def test_qwen3_30b(self):
self.run_throughput()
class TestQwen30B_A3B_aime25(TestNpuAccuracyTestCaseBase):
model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH
envs = QWEN3_30B_A3B_ENVS
other_args = QWEN3_30B_A3B_OTHER_ARGS
accuracy = 0.613
datasets = ["aime25"]
few_shot_num = 0
generation_config = {"max_tokens": 32768, "temperature": 1.0}
eval_batch_size = 16
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py
+91
View File
@@ -0,0 +1,91 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--max-running-requests",
1,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
65536,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
4,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
5,
"--tp-size",
16,
"--mem-fraction-static",
0.72,
"--cuda-graph-bs",
1,
"--dtype",
"bfloat16",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 1
num_prompts = 1
input_len = 18000
output_len = 4000
random_range_ratio = 1
tpot = 6
output_token_throughput = 171
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py
+135
View File
@@ -0,0 +1,135 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
DEFAULT_URL_FOR_TEST,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
101,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
16,
32,
64,
72,
88,
90,
92,
94,
96,
97,
98,
99,
100,
101,
"--dtype",
"bfloat16",
]
class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase):
"""Test NPU accuracy for Qwen3-32B-W8A8 on qpqa"""
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
accuracy = 0.516
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 40000, "temperature": 1.0}
@classmethod
def tearDownClass(cls):
pass
def test_qwen3_32b_qpqa(self):
"""Run NPU accuracy test for Qwen3-32B-W8A8 on qpqa"""
self.run_accuracy()
class TestQwen32B(TestNpuPerformanceTestCaseBase):
base_url = DEFAULT_URL_FOR_TEST
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 100
num_prompts = 400
input_len = 3584
output_len = 1536
random_range_ratio = 1
tpot = 50
output_token_throughput = 1600
@classmethod
def setUpClass(cls):
pass
def test_qwen3_32b(self):
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py
+124
View File
@@ -0,0 +1,124 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_32B_EAGLE_MODEL_PATH,
QWEN3_32B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_32B_ENVS = {
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
QWEN3_32B_OTHER_ARGS = [
"--trust-remote-code",
"--nnodes",
"1",
"--node-rank",
"0",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--max-running-requests",
101,
"--disable-radix-cache",
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
QWEN3_32B_EAGLE_MODEL_PATH,
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--tp-size",
4,
"--mem-fraction-static",
0.845,
"--cuda-graph-bs",
16,
32,
64,
72,
88,
90,
92,
94,
96,
97,
98,
99,
100,
101,
"--dtype",
"bfloat16",
]
class TestQwen32B(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_32B_W8A8_MODEL_PATH
other_args = QWEN3_32B_OTHER_ARGS
envs = QWEN3_32B_ENVS
dataset_name = "random"
max_concurrency = 100
num_prompts = 400
input_len = 3584
output_len = 1536
random_range_ratio = 1
tpot = 50
output_token_throughput = 1600
def test_qwen3_32b(self):
self.run_throughput()
class TestQwen32B_mmlupro(TestNpuAccuracyTestCaseBase):
model = QWEN3_32B_W8A8_MODEL_PATH
envs = QWEN3_32B_ENVS
other_args = QWEN3_32B_OTHER_ARGS
accuracy = 0.4949
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {"max_tokens": 40000, "temperature": 1.0}
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
+111
View File
@@ -0,0 +1,111 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_MM_CUSTOM_GEN,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance case",
)
QWEN3_6_27B_1024_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_NPU_PROFILING": "1",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_1024_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
52000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
50,
"--max-mamba-cache-size",
60,
"--mem-fraction-static",
0.76,
"--cuda-graph-bs",
2,
4,
8,
16,
24,
32,
40,
42,
45,
50,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--mm-enable-dp-encoder",
]
class TestNPUQwen3_6_27B_1P_In1024x1024_30_Out1024_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B 1p in1024x1024 30 out1024 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN
model = QWEN3_6_27B_MODEL_PATH
other_args = QWEN3_6_27B_1024_OTHER_ARGS
envs = QWEN3_6_27B_1024_ENVS
dataset_name = "random"
max_concurrency = 48
num_prompts = 48
input_len = 30
output_len = 1024
random_range_ratio = 1
image_resolution = "1024x1024"
image_count = 1
tpot = 50
output_token_throughput = 800.8
def test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms(self):
"""Run NPU performance test for Qwen3.6-27B in1024x1024 30 out1024 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
+108
View File
@@ -0,0 +1,108 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_MM_CUSTOM_GEN,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_1080P_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_NPU_PROFILING": "1",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_1080P_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
48000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
30,
"--max-mamba-cache-size",
40,
"--mem-fraction-static",
0.76,
"--cuda-graph-bs",
2,
4,
8,
16,
24,
28,
30,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--mm-enable-dp-encoder",
]
class TestNPUQwen3_6_27B_1P_In1080p_30_Out256_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B 1p in1080p 30 out256 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_MM_CUSTOM_GEN
model = QWEN3_6_27B_MODEL_PATH
other_args = QWEN3_6_27B_1080P_OTHER_ARGS
envs = QWEN3_6_27B_1080P_ENVS
dataset_name = "random"
max_concurrency = 30
num_prompts = 120
input_len = 30
output_len = 256
random_range_ratio = 1
image_resolution = "1920x1080"
image_count = 1
tpot = 50
output_token_throughput = 226
def test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms(self):
"""Run NPU performance test for Qwen3.6-27B in1080p 30 out256 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py
+106
View File
@@ -0,0 +1,106 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_64K_PREFIX_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"SGLANG_SET_CPU_AFFINITY": "1",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
}
QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
32768,
"--max-prefill-tokens",
32768,
"--mamba-scheduler-strategy",
"extra_buffer",
"--trust-remote-code",
"--max-running-requests",
20,
"--max-mamba-cache-size",
120,
"--mem-fraction-static",
0.8,
"--cuda-graph-bs",
1,
2,
4,
8,
10,
12,
16,
18,
20,
"--enable-prefill-delayer",
"--prefill-delayer-queue-min-ratio",
0.5,
"--prefill-delayer-max-delay-ms",
30000,
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_Prefix90_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k prefix90 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = QWEN3_6_27B_MODEL_PATH
other_args = QWEN3_6_27B_64K_PREFIX_OTHER_ARGS
envs = QWEN3_6_27B_64K_PREFIX_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 20
num_prompts = 80
input_len = 64000
output_len = 1000
random_range_ratio = 1
repeat_rate = 0.9
request_rate = float("inf")
tpot = 50
output_token_throughput = 225
def test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k prefix90 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py
+120
View File
@@ -0,0 +1,120 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_3K5_1K5_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
60000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
64,
"--max-mamba-cache-size",
74,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
2,
8,
16,
32,
48,
64,
"--enable-multimodal",
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 1p in3k5 out1k5 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS
envs = QWEN3_6_27B_3K5_1K5_ENVS
dataset_name = "random"
max_concurrency = 54
num_prompts = 216
input_len = 3500
output_len = 1500
random_range_ratio = 1
tpot = 50
output_token_throughput = 786.69
def test_npu_qwen3_6_27b_1p_in3k5_out1k5_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in3k5 out1k5 50ms"""
self.run_throughput()
class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase):
model = QWEN3_6_27B_W8A8_MODEL_PATH
envs = QWEN3_6_27B_3K5_1K5_ENVS
other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS
accuracy = 0.855
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 8
generation_config = {"max_tokens": 81920, "temperature": 1.0}
def test_accuracy(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py
+94
View File
@@ -0,0 +1,94 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_128K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_128K_OTHER_ARGS = [
"--tp-size",
4,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
74000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
6,
"--max-mamba-cache-size",
7,
"--mem-fraction-static",
0.63,
"--cuda-graph-bs",
1,
2,
4,
5,
6,
"--enable-multimodal",
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
]
class TestNPUQwen3_6_27B_2P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in128k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_128K_OTHER_ARGS
envs = QWEN3_6_27B_128K_ENVS
dataset_name = "random"
max_concurrency = 4
num_prompts = 16
input_len = 128000
output_len = 1000
random_range_ratio = 1
tpot = 50
output_token_throughput = 41.39
def test_npu_qwen3_6_27b_2p_in128k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in128k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py
+104
View File
@@ -0,0 +1,104 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_16K_1k_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_16K_1k_OTHER_ARGS = [
"--tp-size",
4,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
50000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
28,
"--max-mamba-cache-size",
50,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
2,
8,
12,
16,
20,
24,
28,
"--enable-multimodal",
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_2P_In16k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in16k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_16K_1k_OTHER_ARGS
envs = QWEN3_6_27B_16K_1k_ENVS
dataset_name = "random"
max_concurrency = 28
num_prompts = 112
input_len = 16000
output_len = 1000
random_range_ratio = 1
tpot = 50
output_token_throughput = 426.1
def test_npu_qwen3_6_27b_2p_in16k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in16k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py
+100
View File
@@ -0,0 +1,100 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_27B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_27B_64K_1K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
"ASCEND_USE_FIA": "1",
}
QWEN3_6_27B_64K_1K_OTHER_ARGS = [
"--tp-size",
4,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
50000,
"--disable-radix-cache",
"--trust-remote-code",
"--max-running-requests",
28,
"--max-mamba-cache-size",
50,
"--mem-fraction-static",
0.7,
"--cuda-graph-bs",
2,
4,
6,
"--enable-multimodal",
"--quantization",
"modelslim",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_27B_2P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-27B-w8a8 2p in64k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_27B_W8A8_MODEL_PATH
other_args = QWEN3_6_27B_64K_1K_OTHER_ARGS
envs = QWEN3_6_27B_64K_1K_ENVS
dataset_name = "random"
max_concurrency = 6
num_prompts = 24
input_len = 64000
output_len = 1000
random_range_ratio = 1
tpot = 50
output_token_throughput = 122.6
def test_npu_qwen3_6_27b_2p_in64k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-27B-w8a8 in64k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py
+99
View File
@@ -0,0 +1,99 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_35B_A3B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_35B_A3B_128K_1K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
}
QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-total-tokens",
600000,
"--max-prefill-tokens",
65536,
"--disable-radix-cache",
"--trust-remote-code",
"--enable-prefill-delayer",
"--max-running-requests",
4,
"--max-mamba-cache-size",
12,
"--mem-fraction-static",
0.6,
"--max-mamba-cache-size",
20,
"--disable-cuda-graph",
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-35B-A3B 1p in128k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_35B_A3B_MODEL_PATH
other_args = QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS
envs = QWEN3_6_35B_A3B_128K_1K_ENVS
dataset_name = "random"
max_concurrency = 4
num_prompts = 16
input_len = 128000
output_len = 1000
random_range_ratio = 1
tpot = 50
output_token_throughput = 60.57
def test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-35B-A3B in128k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py
+108
View File
@@ -0,0 +1,108 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_35B_A3B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_35B_A3B_128K_PREFIX_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
}
QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
16384,
"--max-prefill-tokens",
65536,
"--trust-remote-code",
"--enable-prefill-delayer",
"--mamba-scheduler-strategy",
"extra_buffer",
"--max-running-requests",
103,
"--max-mamba-cache-size",
85,
"--mem-fraction-static",
0.85,
"--cuda-graph-bs",
2,
4,
8,
16,
32,
48,
64,
80,
96,
103,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_35BA3B_1P_In128k_Out1k_Prefix90_50ms(
TestNpuPerformanceTestCaseBase
):
"""Test NPU performance for Qwen3.6-35B-A3B 1p in128k out1k prefix90 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = QWEN3_6_35B_A3B_MODEL_PATH
other_args = QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS
envs = QWEN3_6_35B_A3B_128K_PREFIX_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 103
num_prompts = 412
input_len = 64000
output_len = 1000
random_range_ratio = 1
repeat_rate = 0.9
tpot = 50
request_rate = float("inf")
output_token_throughput = 308.2
def test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms(self):
"""Run NPU performance test for Qwen3.6-35B-A3B in128k out1k prefix90 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py
+103
View File
@@ -0,0 +1,103 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_35B_A3B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "800",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
}
QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
35000,
"--disable-radix-cache",
"--trust-remote-code",
"--enable-prefill-delayer",
"--max-running-requests",
110,
"--max-mamba-cache-size",
115,
"--mem-fraction-static",
0.78,
"--cuda-graph-bs",
4,
16,
32,
64,
84,
105,
110,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_35BA3B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-35B-A3B 1p in3k5 out1k5 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_35B_A3B_MODEL_PATH
other_args = QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS
envs = QWEN3_6_35B_A3B_3K5_1K5_ENVS
dataset_name = "random"
max_concurrency = 110
num_prompts = 440
input_len = 3500
output_len = 1500
random_range_ratio = 1
tpot = 50
output_token_throughput = 2031.71
def test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms(self):
"""Run NPU performance test for Qwen3.6-35B-A3B in3k5 out1k5 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py
+103
View File
@@ -0,0 +1,103 @@
import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
QWEN3_6_35B_A3B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="full-2-npu-a3",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_35B_A3B_64K_1K_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "1",
}
QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-total-tokens",
600000,
"--max-prefill-tokens",
65536,
"--disable-radix-cache",
"--trust-remote-code",
"--enable-prefill-delayer",
"--max-running-requests",
10,
"--max-mamba-cache-size",
20,
"--mem-fraction-static",
0.65,
"--cuda-graph-bs",
2,
4,
8,
12,
14,
16,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_50ms(TestNpuPerformanceTestCaseBase):
"""Test NPU performance for Qwen3.6-35B-A3B 1p in64k out1k 50ms"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
aisbench_dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = QWEN3_6_35B_A3B_MODEL_PATH
other_args = QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS
envs = QWEN3_6_35B_A3B_64K_1K_ENVS
dataset_name = "random"
max_concurrency = 10
num_prompts = 40
input_len = 64000
output_len = 1000
random_range_ratio = 1
tpot = 50
output_token_throughput = 141.72
def test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms(self):
"""Run NPU performance test for Qwen3.6-35B-A3B in64k out1k 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py
+141
View File
@@ -0,0 +1,141 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
BENCHMARK_TOOL_DEFAULT,
DEFAULT_URL_FOR_TEST,
QWEN3_6_35B_A3B_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_BUFFSIZE": "300",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
}
QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [
"--tp-size",
2,
"--nnodes",
1,
"--attention-backend",
"ascend",
"--device",
"npu",
"--chunked-prefill-size",
-1,
"--max-prefill-tokens",
65536,
"--trust-remote-code",
"--enable-prefill-delayer",
"--mamba-scheduler-strategy",
"extra_buffer",
"--max-running-requests",
42,
"--max-mamba-cache-size",
210,
"--mem-fraction-static",
0.71,
"--cuda-graph-bs",
2,
8,
16,
24,
32,
36,
40,
42,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",
"--dtype",
"bfloat16",
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
]
class TestNPUQwen3_6_35BA3B_1P_AIME2026(TestNpuAccuracyTestCaseBase):
"""Test NPU accuracy for Qwen3.6-35B-A3B 1p on AIME2026"""
model = QWEN3_6_35B_A3B_MODEL_PATH
other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS
envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS
accuracy = 0.927
datasets = ["aime26"]
few_shot_num = 0
eval_batch_size = 64
generation_config = {
"max_tokens": 65536,
"temperature": 0.2,
"repetition_penalty": 1.08,
}
@classmethod
def tearDownClass(cls):
pass
def test_npu_qwen3_6_35b_a3b_1p_aime2026(self):
"""Run NPU accuracy test for Qwen3.6-35B-A3B on AIME2026"""
self.run_accuracy()
class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_Prefix90_50ms(
TestNpuPerformanceTestCaseBase
):
"""Test NPU performance for Qwen3.6-35B-A3B 1p in64k out1k prefix90 50ms"""
base_url = DEFAULT_URL_FOR_TEST
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = QWEN3_6_35B_A3B_MODEL_PATH
other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS
envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS
dataset_name = "generated-shared-prefix"
max_concurrency = 42
num_prompts = 42
input_len = 65536
output_len = 1024
random_range_ratio = 1
repeat_rate = 0.9
tpot = 50
request_rate = float("inf")
output_token_throughput = 660
@classmethod
def setUpClass(cls):
pass
def test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms(self):
"""Run NPU performance test for Qwen3.6-35B-A3B in64k out1k prefix90 50ms"""
self.run_throughput()
if __name__ == "__main__":
unittest.main()
test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py
+135
View File
@@ -0,0 +1,135 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
QWEN3_NEXT_80B_A3B_MODEL_PATH,
QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="performance testcase",
)
QWEN3_NEXT_80B_A3B_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "400",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "10",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048",
"HCCL_OP_EXPANSION_MODE": "AIV",
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "0",
"SGLANG_WARMUP_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"HCCL_BUFFSIZE": "2000",
"ZBCCL_LOCAL_MEM_SIZE": "60416",
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0",
"ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669",
"ZBCCL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True",
"ZBCCL_ENABLE_GRAPH": "1",
}
QWEN3_NEXT_80B_A3B_OTHER_ARGS = [
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--page-size",
128,
"--tp-size",
4,
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.85,
"--disable-radix-cache",
"--max-prefill-tokens",
28672,
"--context-length",
81920,
"--max-total-tokens",
122304,
"--dp-size",
2,
"--enable-dp-attention",
"--enable-dp-lm-head",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
3,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
4,
"--speculative-draft-model-quantization",
"unquant",
"--chunked-prefill-size",
-1,
"--max-running-requests",
16,
"--cuda-graph-bs",
2,
4,
8,
"--mamba-ssm-dtype",
"bfloat16",
"--speculative-draft-model-path",
QWEN3_NEXT_80B_A3B_MODEL_PATH,
]
class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase):
max_attempts = 5
model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH
other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS
envs = QWEN3_NEXT_80B_A3B_ENVS
dataset_name = "random"
max_concurrency = 16
num_prompts = 16
input_len = 6144
output_len = 1500
random_range_ratio = 1
tpot = 15.62
def test_qwen3_next_80b_a3b(self):
self.run_throughput()
class TestQwen3Next80BA3B_aime25(TestNpuAccuracyTestCaseBase):
model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH
envs = QWEN3_NEXT_80B_A3B_ENVS
other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS
accuracy = 0.695
datasets = ["aime25"]
few_shot_num = 0
generation_config = {
"max_tokens": 65536,
"temperature": 0.7,
"top_p": 0.8,
"top_k": 20,
"extra_body": {"chat_template_kwargs": {"enable_thinking": True}},
}
max_concurrency = 16
def test_aime25(self):
self.run_accuracy()
if __name__ == "__main__":
unittest.main()