diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index 6cde2e662..bff9ac74c 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -415,6 +415,11 @@ jobs: node_size: 2 test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py test_type: 'accuracy' + # kimi_k3 accuracy tests + - name: kimi_k3_w4a8_32p_gpqa + node_size: 4 + test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py + test_type: 'accuracy' uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml with: runner: linux-amd64-cpu-4 diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index fc9736f23..8d8e5a51e 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -179,8 +179,8 @@ MINIMAX_M2_5_EAGLE3_MODEL_PATH = ( QWEN3_5_397B_W8A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp" ) -DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH = ( - "/root/.cache/modelscope/hub/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp" +DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/DeepSeek-V4-Flash-0731-w8a8" ) QWEN3_5_397B_W4A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp" @@ -189,6 +189,8 @@ KIMI_K2_6_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/Kimi-K2 KIMI_K2_6_EAGLE3_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/lightseekorg/kimi-k2.6-eagle3" ) +KIMI_K3_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/sgl-npu/Kimi-K3-W4A8" +KIMI_K3_DSPARK_MODEL_PATH = "/root/.cache/modelscope/hub/models/RadixArk/Kimi-K3-DSpark" GLM_4_6V_FLASH_MODEL_PATH = "/root/.cache/modelscope/hub/models/ZhipuAI/GLM-4.6V-Flash" QWEN3_VL_8B_THINKING_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Qwen/Qwen3-VL-8B-Thinking" diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py new file mode 100644 index 000000000..a3bda90eb --- /dev/null +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -0,0 +1,135 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + BENCHMARK_TOOL_DEFAULT, + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="nightly-acc-16-npu-a3", + nightly=True, +) + +DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "INF_NAN_MODE_FORCE_DISABLE": "1", + "SGLANG_SET_CPU_AFFINITY": "1", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + # skip gpu branch + "SGLANG_OPT_FP8_WO_A_GEMM": "0", + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", + "FORCE_DRAFT_MODEL_NON_QUANT": "1", + "SGLANG_DSV4_FP4_EXPERTS": "False", + "SGLANG_OPT_FUSE_WQA_WKV": "0", + "SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "False", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "False", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "False", + # DSPARK + "SGLANG_RAGGED_VERIFY_MODE": "static", + "SGLANG_DSPARK_FAST_KERNEL": "0", + # deepep + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "HCCL_BUFFSIZE": "1400", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", +} + +DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ + "--page-size", + 128, + "--tp-size", + 16, + "--trust-remote-code", + "--device", + "npu", + "--attention-backend", + "dsv4", + "--watchdog-timeout", + 9000, + "--mem-fraction-static", + 0.62, + "--prefill-max-requests", + 32, + "--max-prefill-tokens", + 131072, + "--chunked-prefill-size", + 131072, + "--max-running-requests", + 96, + "--dp-size", + 16, + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--quantization", + "modelslim", + "--enable-dp-lm-head", + "--kv-cache-dtype", + "bfloat16", + "--speculative-algorithm", + "DSPARK", + "--speculative-draft-model-path", + DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, + "--speculative-draft-model-quantization", + "modelslim", + "--speculative-draft-attention-backend", + "ascend", + "--speculative-num-draft-tokens", + 6, + "--speculative-dspark-block-size", + 5, + "--skip-server-warmup", + "--cuda-graph-bs", + 1, + 2, + 4, + 5, + 6, +] + + +DEEPSEEK_V4_FLASH_W8A8_GENERATION_CONFIG_HIGH = { + "max_tokens": 125000, + "top_p": 1, + "temperature": 1, + "n": 1, + "extra_body": { + "chat_template_kwargs": {"thinking": True, "reasoning_effort": "high"} + }, +} + + +class TestNPUDeepSeekV4FlashW8A88PGPQA(TestNpuAccuracyTestCaseBase): + """Test NPU accuracy for DeepSeek-V4-Flash W8A8 8p DSPARK GPQA.""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH + other_args = DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS + envs = DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS + accuracy = 0.874 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = DEEPSEEK_V4_FLASH_W8A8_GENERATION_CONFIG_HIGH + eval_batch_size = 128 + stream = True + timeout = 6000 + seed = 1 + + def test_npu_deepseek_v4_flash_w8a8_8p_gpqa(self): + """Run NPU accuracy test for DeepSeek-V4-Flash W8A8 8p DSPARK GPQA.""" + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py index 5ddafd66e..044ae58c5 100644 --- a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py +++ b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py @@ -18,15 +18,17 @@ register_npu_ci( ) GLM_5_2_W4A8_16P_TWO_NODE_ENVS = { - "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", - "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", - "TRANSFORMERS_VERBOSITY": "error", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "DEEPEP_HCCL_BUFFSIZE": "2500", + "DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "1024", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "72", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "HCCL_SOCKET_IFNAME": NIC_NAME, "GLOO_SOCKET_IFNAME": NIC_NAME, } @@ -40,32 +42,38 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [ 32, "--nnodes", 2, - # "--dp-size", - # 8, - # "--enable-dp-attention", + "--dp-size", + 8, + "--enable-dp-attention", "--chunked-prefill-size", 65536, "--max-prefill-tokens", 280000, "--trust-remote-code", "--mem-fraction-static", - 0.70, + 0.76, + "--context-length", + 135000, "--served-model-name", - "glm-5", + "glm-5.2-w4a8", "--cuda-graph-max-bs", - 32, + 4, "--max-running-requests", 32, "--quantization", "modelslim", - # "--speculative-draft-model-quantization", - # "unquant", "--moe-a2a-backend", "deepep", "--deepep-mode", "auto", + "--disable-shared-experts-fusion", "--load-balance-method", "round_robin", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--enable-metrics", "--speculative-algorithm", "NEXTN", "--speculative-num-steps", @@ -74,10 +82,6 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [ 1, "--speculative-num-draft-tokens", 4, - "--reasoning-parser", - "glm45", - "--tool-call-parser", - "glm47", ] GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG = { @@ -94,8 +98,15 @@ class TestNPUGLM_5_2_W4A8_16P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase): model_config = GLM_5_2_W4A8_16P_TWO_NODE_MODEL_CONFIG accuracy = 0.912 datasets = ["gpqa_diamond"] + # eval_batch_size = 16 + # generation_config = {"max_tokens": 131072, "temperature": 1.0} eval_batch_size = 32 - generation_config = {"max_tokens": 65536, "temperature": 1.0} + generation_config = { + "max_tokens": 65536, + "temperature": 1.0, + "timeout": 1200, + "stream": True, + } def test_npu_glm_5_2_w4a8_16p_gpqa(self): """Run NPU accuracy test for GLM-5.2-w4a8 16p two nodes on gpqa_diamond""" diff --git a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py new file mode 100644 index 000000000..b3d8ae60d --- /dev/null +++ b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py @@ -0,0 +1,136 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + BENCHMARK_TOOL_DEFAULT, + TestNpuAccuracyMultiNodePdMixTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + KIMI_K3_DSPARK_MODEL_PATH, + KIMI_K3_W4A8_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +KIMI_K3_W4A8_32P_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS": "1", + "SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE": "1", + "TRITON_CACHE_DIR": "/tmp/triton_cache", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_RAGGED_VERIFY_MODE": "static", + "SGLANG_DSPARK_FOLDED_PROPOSAL": "0", + "SGLANG_DSPARK_FOLDED_SAMPLING": "0", + "SGLANG_DSPARK_STACKED_CTX_KV": "0", + "SGLANG_DSPARK_EMBED_IN_GRAPH": "0", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, + "STREAMS_PER_DEVICE": "32", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "HCCL_BUFFSIZE": "2000", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "64", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "512", + "HCCL_OP_EXPANSION_MODE": "AIV", +} + +KIMI_K3_W4A8_32P_OTHER_ARGS = [ + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--nnodes", + 4, + "--tokenizer-path", + KIMI_K3_W4A8_MODEL_PATH, + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--dtype", + "bfloat16", + "--tp-size", + 64, + "--enable-dp-attention", + "--dp-size", + 4, + "--enable-dp-lm-head", + "--enable-shared-experts-attn-tp", + "--enable-dense-mlp-attn-tp", + "--mem-fraction-static", + 0.72, + "--chunked-prefill-size", + 8192, + "--cuda-graph-bs", + 1, + 4, + 16, + "--max-running-requests", + 64, + "--reasoning-parser", + "kimi_k3", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--speculative-algorithm", + "DSPARK", + "--speculative-draft-model-path", + KIMI_K3_DSPARK_MODEL_PATH, + "--speculative-draft-model-quantization", + "unquant", + "--speculative-dspark-block-size", + 7, + "--speculative-draft-attention-backend", + "ascend", + "--linear-attn-verify-backend", + "triton", + "--speculative-eagle-topk", + 1, + "--disable-radix-cache", + "--disable-custom-all-reduce", + "--watchdog-timeout", + 9000, +] + +KIMI_K3_W4A8_32P_MODEL_CONFIG = { + "model_path": KIMI_K3_W4A8_MODEL_PATH, + "other_args": KIMI_K3_W4A8_32P_OTHER_ARGS, + "node_envs": KIMI_K3_W4A8_32P_ENVS, +} + + +class TestNPUKimiK3_W4A8_32P_GPQA(TestNpuAccuracyMultiNodePdMixTestCaseBase): + """Test NPU accuracy for Kimi-K3-w4a8 32p four nodes on gpqa_diamond""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model_config = KIMI_K3_W4A8_32P_MODEL_CONFIG + accuracy = 0.935 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 32 + generation_config = { + "max_tokens": 131072, + "temperature": 1.0, + "top_p": 0.95, + "extra_body": {"reasoning_effort": "max"}, + } + timeout = 10000 + seed = 42 + + def test_npu_kimi_k3_w4a8_32p_gpqa(self): + """Run NPU accuracy test for Kimi-K3-w4a8 32p four nodes on gpqa_diamond""" + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py index 0f8f5f460..bb02ffff1 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py @@ -3,7 +3,7 @@ import unittest from sglang.test.ascend.e2e.test_npu_performance_utils import ( AISBENCHMARK_DATASET_DEFAULT, BENCHMARK_TOOL_DEFAULT, - DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH, + DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, TestNpuPerfMultiNodePdSepTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -195,7 +195,7 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [ # Model config for DSV4-Flash W8A8 1P+1D PD-Sep deployment. DEEPSEEK_V4_FLASH_W8A8_1P1D_MODEL_CONFIG = { - "model_path": DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH, + "model_path": DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, "prefill_args": DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ARGS, "decode_args": DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS, "prefill_envs": DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ENVS, diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py index a4704fbdd..fb560543d 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py @@ -3,7 +3,7 @@ import unittest from sglang.test.ascend.e2e.test_npu_performance_utils import ( AISBENCHMARK_DATASET_DEFAULT, BENCHMARK_TOOL_DEFAULT, - DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH, + DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -116,7 +116,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn32kOut1k50ms(TestNpuPerformanceTestCaseBase) benchmark_tool = BENCHMARK_TOOL_DEFAULT dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH + model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS dataset_name = "random" diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 288fc031a..22f2209d7 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -3,7 +3,7 @@ import unittest from sglang.test.ascend.e2e.test_npu_performance_utils import ( AISBENCHMARK_DATASET_DEFAULT, BENCHMARK_TOOL_DEFAULT, - DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH, + DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -117,7 +117,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase): benchmark_tool = BENCHMARK_TOOL_DEFAULT dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH + model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS dataset_name = "random"