minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

test(npu): add DSV4-Flash / GLM-5.2 / Kimi-K3 gpqa accuracy cases (#37431)

Browse Source 
Co-authored-by: Sugar920 <Sugar920@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Sugar920
2026-09-01 22:00:04 +08:00
committed by GitHub
co-authored by Sugar920 Claude
parent 4c2c169e6b
commit 3ae54c6ca2
8 changed files with 315 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-npu.yml
+5
View File
@@ -415,6 +415,11 @@ jobs:
node_size: 2
test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
test_type: 'accuracy'
# kimi_k3 accuracy tests
- name: kimi_k3_w4a8_32p_gpqa
node_size: 4
test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
test_type: 'accuracy'
uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
with:
runner: linux-amd64-cpu-4
python/sglang/test/ascend/e2e/test_npu_performance_utils.py
+4 -2
View File
@@ -179,8 +179,8 @@ MINIMAX_M2_5_EAGLE3_MODEL_PATH = (
QWEN3_5_397B_W8A8_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp"
)
DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp"
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/Eco-Tech/DeepSeek-V4-Flash-0731-w8a8"
)
QWEN3_5_397B_W4A8_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp"
@@ -189,6 +189,8 @@ KIMI_K2_6_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/Kimi-K2
KIMI_K2_6_EAGLE3_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/lightseekorg/kimi-k2.6-eagle3"
)
KIMI_K3_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/sgl-npu/Kimi-K3-W4A8"
KIMI_K3_DSPARK_MODEL_PATH = "/root/.cache/modelscope/hub/models/RadixArk/Kimi-K3-DSpark"
GLM_4_6V_FLASH_MODEL_PATH = "/root/.cache/modelscope/hub/models/ZhipuAI/GLM-4.6V-Flash"
QWEN3_VL_8B_THINKING_MODEL_PATH = (
"/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-8B-Thinking"
test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py
+135
View File
@@ -0,0 +1,135 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
BENCHMARK_TOOL_DEFAULT,
TestNpuAccuracyTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_performance_utils import (
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="nightly-acc-16-npu-a3",
nightly=True,
)
DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = {
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"INF_NAN_MODE_FORCE_DISABLE": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
# skip gpu branch
"SGLANG_OPT_FP8_WO_A_GEMM": "0",
"SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"SGLANG_DSV4_FP4_EXPERTS": "False",
"SGLANG_OPT_FUSE_WQA_WKV": "0",
"SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch",
"SGLANG_OPT_USE_FUSED_HASH_TOPK": "False",
"SGLANG_OPT_USE_TILELANG_MHC_PRE": "False",
"SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False",
"SGLANG_OPT_USE_TILELANG_MHC_POST": "False",
# DSPARK
"SGLANG_RAGGED_VERIFY_MODE": "static",
"SGLANG_DSPARK_FAST_KERNEL": "0",
# deepep
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"HCCL_BUFFSIZE": "1400",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
}
DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [
"--page-size",
128,
"--tp-size",
16,
"--trust-remote-code",
"--device",
"npu",
"--attention-backend",
"dsv4",
"--watchdog-timeout",
9000,
"--mem-fraction-static",
0.62,
"--prefill-max-requests",
32,
"--max-prefill-tokens",
131072,
"--chunked-prefill-size",
131072,
"--max-running-requests",
96,
"--dp-size",
16,
"--enable-dp-attention",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--quantization",
"modelslim",
"--enable-dp-lm-head",
"--kv-cache-dtype",
"bfloat16",
"--speculative-algorithm",
"DSPARK",
"--speculative-draft-model-path",
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
"--speculative-draft-model-quantization",
"modelslim",
"--speculative-draft-attention-backend",
"ascend",
"--speculative-num-draft-tokens",
6,
"--speculative-dspark-block-size",
5,
"--skip-server-warmup",
"--cuda-graph-bs",
1,
2,
4,
5,
6,
]
DEEPSEEK_V4_FLASH_W8A8_GENERATION_CONFIG_HIGH = {
"max_tokens": 125000,
"top_p": 1,
"temperature": 1,
"n": 1,
"extra_body": {
"chat_template_kwargs": {"thinking": True, "reasoning_effort": "high"}
},
}
class TestNPUDeepSeekV4FlashW8A88PGPQA(TestNpuAccuracyTestCaseBase):
"""Test NPU accuracy for DeepSeek-V4-Flash W8A8 8p DSPARK GPQA."""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH
other_args = DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS
envs = DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS
accuracy = 0.874
datasets = ["gpqa_diamond"]
few_shot_num = 0
generation_config = DEEPSEEK_V4_FLASH_W8A8_GENERATION_CONFIG_HIGH
eval_batch_size = 128
stream = True
timeout = 6000
seed = 1
def test_npu_deepseek_v4_flash_w8a8_8p_gpqa(self):
"""Run NPU accuracy test for DeepSeek-V4-Flash W8A8 8p DSPARK GPQA."""
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
+29 -18
View File
@@ -18,15 +18,17 @@ register_npu_ci(
)
GLM_5_2_W4A8_16P_TWO_NODE_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"STREAMS_PER_DEVICE": "32",
"SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"TRANSFORMERS_VERBOSITY": "error",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "2500",
"DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "1024",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "72",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
}
@@ -40,32 +42,38 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
32,
"--nnodes",
2,
# "--dp-size",
# 8,
# "--enable-dp-attention",
"--dp-size",
8,
"--enable-dp-attention",
"--chunked-prefill-size",
65536,
"--max-prefill-tokens",
280000,
"--trust-remote-code",
"--mem-fraction-static",
0.70,
0.76,
"--context-length",
135000,
"--served-model-name",
"glm-5",
"glm-5.2-w4a8",
"--cuda-graph-max-bs",
32,
4,
"--max-running-requests",
32,
"--quantization",
"modelslim",
# "--speculative-draft-model-quantization",
# "unquant",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--disable-shared-experts-fusion",
"--load-balance-method",
"round_robin",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
"--enable-metrics",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
@@ -74,10 +82,6 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [
1,
"--speculative-num-draft-tokens",
4,
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
]
GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG = {
@@ -94,8 +98,15 @@ class TestNPUGLM_5_2_W4A8_16P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase):
model_config = GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG
accuracy = 0.912
datasets = ["gpqa_diamond"]
# eval_batch_size = 16
# generation_config = {"max_tokens": 131072, "temperature": 1.0}
eval_batch_size = 32
generation_config = {"max_tokens": 65536, "temperature": 1.0}
generation_config = {
"max_tokens": 65536,
"temperature": 1.0,
"timeout": 1200,
"stream": True,
}
def test_npu_glm_5_2_w4a8_16p_gpqa(self):
"""Run NPU accuracy test for GLM-5.2-w4a8 16p two nodes on gpqa_diamond"""
test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
+136
View File
@@ -0,0 +1,136 @@
import unittest
from sglang.test.ascend.e2e.test_npu_accuracy_utils import (
BENCHMARK_TOOL_DEFAULT,
TestNpuAccuracyMultiNodePdMixTestCaseBase,
)
from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME
from sglang.test.ascend.e2e.test_npu_performance_utils import (
KIMI_K3_DSPARK_MODEL_PATH,
KIMI_K3_W4A8_MODEL_PATH,
)
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(
est_time=3600,
suite="",
nightly=True,
disabled="accuracy testcase",
)
KIMI_K3_W4A8_32P_ENVS = {
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS": "1",
"SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE": "1",
"TRITON_CACHE_DIR": "/tmp/triton_cache",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_RAGGED_VERIFY_MODE": "static",
"SGLANG_DSPARK_FOLDED_PROPOSAL": "0",
"SGLANG_DSPARK_FOLDED_SAMPLING": "0",
"SGLANG_DSPARK_STACKED_CTX_KV": "0",
"SGLANG_DSPARK_EMBED_IN_GRAPH": "0",
"PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
"STREAMS_PER_DEVICE": "32",
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128",
"HCCL_BUFFSIZE": "2000",
"DEEPEP_NORMAL_LONG_SEQ_ROUND": "64",
"DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "512",
"HCCL_OP_EXPANSION_MODE": "AIV",
}
KIMI_K3_W4A8_32P_OTHER_ARGS = [
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--nnodes",
4,
"--tokenizer-path",
KIMI_K3_W4A8_MODEL_PATH,
"--trust-remote-code",
"--attention-backend",
"ascend",
"--device",
"npu",
"--quantization",
"modelslim",
"--dtype",
"bfloat16",
"--tp-size",
64,
"--enable-dp-attention",
"--dp-size",
4,
"--enable-dp-lm-head",
"--enable-shared-experts-attn-tp",
"--enable-dense-mlp-attn-tp",
"--mem-fraction-static",
0.72,
"--chunked-prefill-size",
8192,
"--cuda-graph-bs",
1,
4,
16,
"--max-running-requests",
64,
"--reasoning-parser",
"kimi_k3",
"--moe-a2a-backend",
"deepep",
"--deepep-mode",
"auto",
"--speculative-algorithm",
"DSPARK",
"--speculative-draft-model-path",
KIMI_K3_DSPARK_MODEL_PATH,
"--speculative-draft-model-quantization",
"unquant",
"--speculative-dspark-block-size",
7,
"--speculative-draft-attention-backend",
"ascend",
"--linear-attn-verify-backend",
"triton",
"--speculative-eagle-topk",
1,
"--disable-radix-cache",
"--disable-custom-all-reduce",
"--watchdog-timeout",
9000,
]
KIMI_K3_W4A8_32P_MODEL_CONFIG = {
"model_path": KIMI_K3_W4A8_MODEL_PATH,
"other_args": KIMI_K3_W4A8_32P_OTHER_ARGS,
"node_envs": KIMI_K3_W4A8_32P_ENVS,
}
class TestNPUKimiK3_W4A8_32P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase):
"""Test NPU accuracy for Kimi-K3-w4a8 32p four nodes on gpqa_diamond"""
benchmark_tool = BENCHMARK_TOOL_DEFAULT
model_config = KIMI_K3_W4A8_32P_MODEL_CONFIG
accuracy = 0.935
datasets = ["gpqa_diamond"]
few_shot_num = 0
eval_batch_size = 32
generation_config = {
"max_tokens": 131072,
"temperature": 1.0,
"top_p": 0.95,
"extra_body": {"reasoning_effort": "max"},
}
timeout = 10000
seed = 42
def test_npu_kimi_k3_w4a8_32p_gpqa(self):
"""Run NPU accuracy test for Kimi-K3-w4a8 32p four nodes on gpqa_diamond"""
self.run_accuracy()
if __name__ == "__main__":
unittest.main()
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
+2 -2
View File
@@ -3,7 +3,7 @@ import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH,
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
TestNpuPerfMultiNodePdSepTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
@@ -195,7 +195,7 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [
# Model config for DSV4-Flash W8A8 1P+1D PD-Sep deployment.
DEEPSEEK_V4_FLASH_W8A8_1P1D_MODEL_CONFIG = {
"model_path": DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH,
"model_path": DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
"prefill_args": DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ARGS,
"decode_args": DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS,
"prefill_envs": DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ENVS,
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py
+2 -2
View File
@@ -3,7 +3,7 @@ import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH,
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
@@ -116,7 +116,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn32kOut1k50ms(TestNpuPerformanceTestCaseBase)
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH
model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH
other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS
envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS
dataset_name = "random"
test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py
+2 -2
View File
@@ -3,7 +3,7 @@ import unittest
from sglang.test.ascend.e2e.test_npu_performance_utils import (
AISBENCHMARK_DATASET_DEFAULT,
BENCHMARK_TOOL_DEFAULT,
DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH,
DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH,
TestNpuPerformanceTestCaseBase,
)
from sglang.test.ci.ci_register import register_npu_ci
@@ -117,7 +117,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase):
benchmark_tool = BENCHMARK_TOOL_DEFAULT
dataset_type = AISBENCHMARK_DATASET_DEFAULT
model = DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH
model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH
other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS
envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS
dataset_name = "random"