From 4ad418d2c3d43cb3c699bc9419d32673b1fca7d8 Mon Sep 17 00:00:00 2001 From: pllimax Date: Thu, 16 Jul 2026 21:24:01 +0800 Subject: [PATCH] Push test case scripts from test repo to main upstream community repository (#31114) --- .../nightly-test-npu-e2e-single-node.yml | 2 + .github/workflows/nightly-test-npu.yml | 24 ++- .../ascend/e2e/test_npu_performance_utils.py | 6 + ...epseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py | 127 ++++++++++++ ...1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py | 2 +- ..._k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py | 2 +- ...mo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py | 191 ++++++++++++++++++ ...v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py | 187 +++++++++++++++++ ..._w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py | 2 +- ...max_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py | 2 +- ...n3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py | 2 +- ...pu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py | 2 +- ...a3b_1p_in64k_out1k_prefix90_50ms_aime26.py | 2 +- ...ext_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py | 2 +- 14 files changed, 541 insertions(+), 12 deletions(-) create mode 100644 test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py create mode 100644 test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py create mode 100644 test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py diff --git a/.github/workflows/nightly-test-npu-e2e-single-node.yml b/.github/workflows/nightly-test-npu-e2e-single-node.yml index e288d3362..940127cdc 100644 --- a/.github/workflows/nightly-test-npu-e2e-single-node.yml +++ b/.github/workflows/nightly-test-npu-e2e-single-node.yml @@ -164,6 +164,8 @@ jobs: source /usr/local/Ascend/cann/set_env.sh || true source /usr/local/Ascend/nnal/atb/set_env.sh || true + source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true + source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true # Set environment of cann log_path="/root/.cache/tests/logs/log/${current_date}/${tc_name}/${HOSTNAME}" diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index c743732af..aed7af837 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -239,10 +239,6 @@ jobs: - name: glm4_7_flash_1p_aime25 runner: linux-aarch64-a3-2 test_case: test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py - # glm4_6v_flash accuracy tests - - name: glm4_6v_flash_1p_mmmu - runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py # qwen3_vl_8b_thinking accuracy tests - name: qwen3_vl_8b_thinking_1p_mmmu runner: linux-aarch64-a3-2 @@ -251,6 +247,11 @@ jobs: - name: qwen3_vl_30b_a3b_thinking_1p_mmmu runner: linux-aarch64-a3-2 test_case: test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py + # deepseek_v4_flash performance tests + - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py + test_type: 'perf' uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml with: runner: ${{ matrix.test_config.runner }} @@ -278,6 +279,21 @@ jobs: test_case: test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py test_type: 'perf' prefill_decode_deployment: 'separation' + # mimo_v2_flash performance tests + - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s + prefill_size: 1 + decode_size: 1 + router_size: 1 + test_case: test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py + test_type: 'perf' + prefill_decode_deployment: 'separation' + - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms + prefill_size: 1 + decode_size: 1 + router_size: 1 + test_case: test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py + test_type: 'perf' + prefill_decode_deployment: 'separation' uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml with: runner: linux-amd64-cpu-8 diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index 33a90b545..c0b733576 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -154,6 +154,9 @@ GLM_5_1_W4A8_MODEL_PATH = "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.1-w MINIMAX_M2_5_W8A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/MiniMax-M2.5-w8a8-QuaRot" ) +MIMO_V2_FLASH_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/iridiumine/MiMo-V2-Flash-W8A8" +) MINIMAX_M2_5_EAGLE3_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/sgl-npu/MiniMax-M2.5-eagel-model-0318" ) @@ -161,6 +164,9 @@ MINIMAX_M2_5_EAGLE3_MODEL_PATH = ( QWEN3_5_397B_W8A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp" ) +DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp" +) QWEN3_5_397B_W4A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp" ) diff --git a/test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py new file mode 100644 index 000000000..62fbef9a7 --- /dev/null +++ b/test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -0,0 +1,127 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +# Environment variables for DSV4-Flash single-node PD-mix deployment. +DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "INF_NAN_MODE_FORCE_DISABLE": "1", + "SGLANG_SET_CPU_AFFINITY": "1", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + # deepep + "DEEPEP_HCCL_BUFFSIZE": "1000", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "16", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048", + "DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1", + # skip gpu branch + "SGLANG_OPT_FP8_WO_A_GEMM": "0", + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", + "FORCE_DRAFT_MODEL_NON_QUANT": "1", + "SGLANG_DSV4_FP4_EXPERTS": "False", + "SGLANG_OPT_FUSE_WQA_WKV": "0", + "SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "False", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "False", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "False", + # MTP (EAGLE) related envs + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", +} + +# Server launch arguments for DSV4-Flash W8A8 single-node 8p PD-mix. +DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ + "--page-size", + 128, + "--tp-size", + 16, + "--trust-remote-code", + "--device", + "npu", + "--attention-backend", + "dsv4", + "--watchdog-timeout", + 9000, + "--mem-fraction-static", + 0.7, + "--prefill-max-requests", + 2, + "--disable-radix-cache", + "--chunked-prefill-size", + -1, + "--max-running-requests", + 160, + "--dp-size", + 16, + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--quantization", + "modelslim", + "--enable-dp-lm-head", + "--kv-cache-dtype", + "bfloat16", + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 10, + # MTP (EAGLE) configuration. + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + 2, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 3, +] + + +class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for DeepSeek-V4-Flash W8A8 8p in8k out1k.""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = DEEPSEEK_V4_FLASH_W8A8_MTP_MODEL_PATH + other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS + envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS + dataset_name = "random" + input_len = 8000 + output_len = 1000 + num_prompts = 160 + max_concurrency = 160 + random_range_ratio = 1 + warmup_requests = 0 + request_rate = float("inf") + seed = 1 + tpot = 50 + output_token_throughput = 1708 + + def test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms(self): + """Run NPU performance test for DeepSeek-V4-Flash W8A8 8p in8k out1k.""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index b2d2e86e2..708a9f9b0 100644 --- a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -25,7 +25,7 @@ GLM_5_1_PD_SEP_PREFILL_ENVS = { "STREAMS_PER_DEVICE": "32", "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200", - "HCCL_BUFFSIZE": "1200", + "DEEPEP_HCCL_BUFFSIZE": "1200", "DEEPEP_NORMAL_LONG_SEQ_ROUND": "72", "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "1024", "DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1", diff --git a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index 72f6c5111..acf865171 100644 --- a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -26,7 +26,7 @@ ENVS = { "STREAMS_PER_DEVICE": "32", "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", - "HCCL_BUFFSIZE": "4400", + "DEEPEP_HCCL_BUFFSIZE": "4400", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "HCCL_SOCKET_IFNAME": NIC_NAME, "GLOO_SOCKET_IFNAME": NIC_NAME, diff --git a/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py b/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py new file mode 100644 index 000000000..504ce0c82 --- /dev/null +++ b/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py @@ -0,0 +1,191 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + MIMO_V2_FLASH_MODEL_PATH, + TestNpuPerfMultiNodePdSepTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +PREFILL_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", + "DEEPEP_HCCL_BUFFSIZE": "1024", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_NPU_PROFILING": "0", + "SGLANG_NPU_PROFILING_STAGE": "prefill", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_BF16_DISPATCH": "0", + "SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_CONNECT_TIMEOUT": "1800", + "ASCEND_USE_FIA": "1", +} + +DECODE_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "DEEPEP_HCCL_BUFFSIZE": "800", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_NPU_PROFILING": "0", + "SGLANG_NPU_PROFILING_STAGE": "prefill", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_BF16_DISPATCH": "0", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_CONNECT_TIMEOUT": "1800", + "SGLANG_PROFILE_WITH_STACK": "True", + "ASCEND_USE_FIA": "1", +} + +PREFILL_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 8, + "--nnodes", + "1", + "--node-rank", + "0", + "--chunked-prefill-size", + 8192, + "--trust-remote-code", + "--max-running-requests", + 64, + "--mem-fraction-static", + 0.8, + "--swa-full-tokens-ratio", + 0.3, + "--disaggregation-mode", + "prefill", + "--disaggregation-transfer-backend", + "ascend", + "--disable-radix-cache", + "--disable-cuda-graph", + "--disable-piecewise-cuda-graph", + "--dp-size", + "2", +] + +DECODE_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 16, + "--nnodes", + "1", + "--node-rank", + "0", + "--trust-remote-code", + "--max-running-requests", + 64, + "--mem-fraction-static", + 0.8, + "--swa-full-tokens-ratio", + 0.3, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + "--disaggregation-mode", + "decode", + "--disaggregation-transfer-backend", + "ascend", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--enable-multi-layer-eagle", + "--disable-radix-cache", + "--dp-size", + "2", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", +] + +ROUTER_ARGS = [ + "--health-check-interval-secs", + "3600", + "--mini-lb", +] + +MODEL_CONFIG = { + "model_path": MIMO_V2_FLASH_MODEL_PATH, + "prefill_args": PREFILL_ARGS, + "decode_args": DECODE_ARGS, + "prefill_envs": PREFILL_ENVS, + "decode_envs": DECODE_ENVS, + "router_args": ROUTER_ARGS, + "router_envs": {}, +} + + +class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1_TTFT_5s( + TestNpuPerfMultiNodePdSepTestCaseBase +): + """Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1, 0 cache, TTFT=5s""" + + model_config = MODEL_CONFIG + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_name = "random" + max_concurrency = 64 + num_prompts = 128 + request_rate = 0.4 + input_len = 16000 + output_len = 1 + random_range_ratio = 1 + seed = 1 + ttft = 5000 + + def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1_ttft_5s(self): + """Run NPU performance test for 1P+1D 16p with 16k input, 1 output, 0 cache, TTFT=5s""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py b/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py new file mode 100644 index 000000000..ddaeaf7c4 --- /dev/null +++ b/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py @@ -0,0 +1,187 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + MIMO_V2_FLASH_MODEL_PATH, + TestNpuPerfMultiNodePdSepTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +PREFILL_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", + "DEEPEP_HCCL_BUFFSIZE": "1024", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_NPU_PROFILING": "0", + "SGLANG_NPU_PROFILING_STAGE": "prefill", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_BF16_DISPATCH": "0", + "SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_CONNECT_TIMEOUT": "1800", + "ASCEND_USE_FIA": "1", +} + +DECODE_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "DEEPEP_HCCL_BUFFSIZE": "800", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_NPU_PROFILING": "0", + "SGLANG_NPU_PROFILING_STAGE": "prefill", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "32", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "3600", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "3600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_BF16_DISPATCH": "0", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_CONNECT_TIMEOUT": "1800", + "SGLANG_PROFILE_WITH_STACK": "True", + "ASCEND_USE_FIA": "1", +} + +PREFILL_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 8, + "--nnodes", + "1", + "--node-rank", + "0", + "--chunked-prefill-size", + 8192, + "--trust-remote-code", + "--max-running-requests", + 64, + "--mem-fraction-static", + 0.8, + "--swa-full-tokens-ratio", + 0.3, + "--disaggregation-mode", + "prefill", + "--disaggregation-transfer-backend", + "ascend", + "--disable-radix-cache", + "--disable-cuda-graph", + "--disable-piecewise-cuda-graph", + "--dp-size", + "2", +] + +DECODE_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 16, + "--nnodes", + "1", + "--node-rank", + "0", + "--trust-remote-code", + "--max-running-requests", + 32, + "--mem-fraction-static", + 0.8, + "--swa-full-tokens-ratio", + 0.3, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 12, + 16, + "--disaggregation-mode", + "decode", + "--disaggregation-transfer-backend", + "ascend", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--enable-multi-layer-eagle", + "--disable-radix-cache", + "--dp-size", + "2", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", +] + +ROUTER_ARGS = [ + "--health-check-interval-secs", + "3600", + "--mini-lb", +] + +MODEL_CONFIG = { + "model_path": MIMO_V2_FLASH_MODEL_PATH, + "prefill_args": PREFILL_ARGS, + "decode_args": DECODE_ARGS, + "prefill_envs": PREFILL_ENVS, + "decode_envs": DECODE_ENVS, + "router_args": ROUTER_ARGS, + "router_envs": {}, +} + + +class TestNPUMimo_v2_flash_1P1D_16p_In16k_Out1k_TTFT_5s( + TestNpuPerfMultiNodePdSepTestCaseBase +): + """Test NPU performance for mimo_v2_flash 1P+1D 16p: input_len=16000, output_len=1000, 0 cache, TPOT=20ms""" + + model_config = MODEL_CONFIG + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_name = "random" + max_concurrency = 32 + num_prompts = 128 + request_rate = float("inf") + input_len = 16000 + output_len = 1000 + random_range_ratio = 1 + seed = 1 + tpot = 20 + + def test_npu_mimo_v2_flash_1p1d_16p_in16k_out1k_tpot_20ms(self): + """Run NPU performance test for 1P+1D 16p with 16k input, 1k output, 0 cache, TPOT=20ms""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py index a6d5b5be3..6248c435a 100644 --- a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -30,7 +30,7 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { "SGLANG_NPU_FUSED_MOE_MODE": "2", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "HCCL_BUFFSIZE": "1024", + "DEEPEP_HCCL_BUFFSIZE": "1024", "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", } diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index 6a10b1992..5a82dbf79 100644 --- a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -26,7 +26,7 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "TASK_QUEUE_ENABLE": "1", - "HCCL_BUFFSIZE": "1024", + "DEEPEP_HCCL_BUFFSIZE": "1024", "ASCEND_USE_FIA": "1", "SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", diff --git a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py index 7cf641700..a4eaa6303 100644 --- a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py +++ b/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -29,7 +29,7 @@ QWEN3_30B_A3B_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", - "HCCL_BUFFSIZE": "400", + "DEEPEP_HCCL_BUFFSIZE": "400", } QWEN3_30B_A3B_OTHER_ARGS = [ diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py index 846c23657..80edd0221 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py @@ -18,7 +18,7 @@ register_npu_ci( QWEN3_6_35B_A3B_128K_1K_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", - "HCCL_BUFFSIZE": "1600", + "DEEPEP_HCCL_BUFFSIZE": "1600", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py index 462d055c3..de3cbab02 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py +++ b/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py @@ -21,7 +21,7 @@ register_npu_ci( QWEN3_6_35B_A3B_64K_PREFIX_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", - "HCCL_BUFFSIZE": "300", + "DEEPEP_HCCL_BUFFSIZE": "300", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", diff --git a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 802345945..2d3d74f1a 100644 --- a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -33,7 +33,7 @@ QWEN3_NEXT_80B_A3B_ENVS = { "SGLANG_WARMUP_TIMEOUT": "3600", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "FORCE_DRAFT_MODEL_NON_QUANT": "1", - "HCCL_BUFFSIZE": "2000", + "DEEPEP_HCCL_BUFFSIZE": "2000", "ZBCCL_LOCAL_MEM_SIZE": "60416", "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", "ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669",