From 3239baef255d09decc753674a16012d102c947da Mon Sep 17 00:00:00 2001 From: pllimax Date: Thu, 3 Sep 2026 21:57:14 +0800 Subject: [PATCH] [CI][NPU] Fix kimi_k2_6 16p in64k perf test and dsv4-flash testcases (#37760) --- .../test/ascend/e2e/run_npu_testcase.sh | 3 ++ ...test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py | 5 ++- ..._v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py | 25 +++++++------ ...pseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py | 35 +++++++----------- ...epseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py | 36 +++++++------------ ...pu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py | 2 +- 6 files changed, 44 insertions(+), 62 deletions(-) diff --git a/python/sglang/test/ascend/e2e/run_npu_testcase.sh b/python/sglang/test/ascend/e2e/run_npu_testcase.sh index ea1014193..10ed4b3f5 100644 --- a/python/sglang/test/ascend/e2e/run_npu_testcase.sh +++ b/python/sglang/test/ascend/e2e/run_npu_testcase.sh @@ -110,6 +110,9 @@ fi # set environment of cann . /usr/local/Ascend/cann/set_env.sh . /usr/local/Ascend/nnal/atb/set_env.sh +# Adapt Deepseek-V4-Flash test cases with additional environment variables. +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true echo "Running test case ${test_case}" tc_name=${test_case##*/} diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py index a3bda90eb..03d5237e4 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -39,8 +39,11 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = { "SGLANG_DSPARK_FAST_KERNEL": "0", # deepep "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "HCCL_BUFFSIZE": "1400", + "DEEPEP_HCCL_BUFFSIZE": "2048", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", + # war barrier + "SGLANG_ENABLE_WAR_BARRIER": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1", } DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py index bb02ffff1..ce287f020 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py @@ -24,7 +24,13 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ENVS = { "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", "HCCL_OP_EXPANSION_MODE": "AIV", + # deepep "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "DEEPEP_HCCL_BUFFSIZE": "2048", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "35", + # war barrier + "SGLANG_ENABLE_WAR_BARRIER": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1", # skip gpu branch "SGLANG_OPT_FP8_WO_A_GEMM": "0", "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", @@ -36,13 +42,6 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ENVS = { "SGLANG_OPT_USE_TILELANG_MHC_PRE": "False", "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False", "SGLANG_OPT_USE_TILELANG_MHC_POST": "False", - # ZBAL - "HCCL_BUFFSIZE": "8", - "SGLANG_ZBAL_LOCAL_MEM_SIZE": "62084", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "ZBAL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", - "SGLANG_ZBAL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", - "ZBAL_ENABLE_GRAPH": "1", # PD disagg "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "60", # MTP @@ -80,6 +79,7 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ENVS = { # MTP "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_NPU_USE_MULTI_STREAM": "1", } # Prefill node launch arguments for DSV4-Flash PD-Sep. @@ -102,13 +102,13 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_PREFILL_ARGS = [ "--disaggregation-bootstrap-port", 8998, "--mem-fraction-static", - 0.62, + 0.68, "--prefill-max-requests", 6, "--max-prefill-tokens", - 70000, + 80000, "--chunked-prefill-size", - -1, + 131072, "--max-running-requests", 112, "--dp-size", @@ -152,7 +152,7 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [ 1, "--disable-radix-cache", "--chunked-prefill-size", - 32768, + -1, "--disaggregation-mode", "decode", "--disaggregation-transfer-backend", @@ -181,7 +181,6 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [ 36, 40, 48, - 56, # MTP (EAGLE) configuration. "--speculative-algorithm", "EAGLE", @@ -219,7 +218,7 @@ class TestNPUDeepSeekV4FlashW8A81P1D16PIn8kOut1k50ms( num_prompts = 2400 max_concurrency = 800 random_range_ratio = 1 - warmup_requests = 0 + warmup_requests = 16 request_rate = float("inf") seed = 1 tpot = 50 diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py index fb560543d..053239207 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py @@ -15,27 +15,19 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", "INF_NAN_MODE_FORCE_DISABLE": "1", + "SGLANG_SET_CPU_AFFINITY": "1", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", "USE_NPU_MOE_GATING_TOP_K": "1", "SGLANG_NPU_USE_MULTI_STREAM": "1", # deepep "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", - # zbal - "HCCL_BUFFSIZE": "8", - "SGLANG_ZBAL_LOCAL_MEM_SIZE": "61000", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "ZBAL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", - "SGLANG_ZBAL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", - "ZBAL_ENABLE_GRAPH": "1", - # dsv4 - "IS_DEEPSEEK_V4": "1", - "SGLANG_DEBUG_LAYER_NORM": "1", - "SGLANG_DEBUG_FWD_INPUT": "1", - "USE_FUSED_HC_PRE_ASCENDC": "1", - "SGLANG_DSV4_NPU_FUSED_COMPRESSOR": "1", - "SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL": "0", + "DEEPEP_HCCL_BUFFSIZE": "2048", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "35", + # war barrier + "SGLANG_ENABLE_WAR_BARRIER": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1", # skip gpu branch "SGLANG_OPT_FP8_WO_A_GEMM": "0", "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", @@ -50,11 +42,6 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { # mtp "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_DEBUG_MTP_VERIFY": "0", - "SGLANG_DEBUG_MTP_VERIFY_LIMIT": "8", - "SGLANG_DEBUG_MTP_VERIFY_ROWS": "4", - "SGLANG_DISABLE_DRAFT_EXTEND_GRAPH": "1", } # Server launch arguments for DSV4-Flash W8A8 single-node 8p PD-mix. @@ -68,12 +55,14 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "npu", "--prefill-max-requests", 32, + "--max-prefill-tokens", + 80000, "--attention-backend", "dsv4", "--watchdog-timeout", 9000, "--mem-fraction-static", - 0.7, + 0.68, "--chunked-prefill-size", 131072, "--max-running-requests", @@ -89,7 +78,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "modelslim", "--enable-dp-lm-head", "--kv-cache-dtype", - "auto", + "bfloat16", "--skip-server-warmup", "--cuda-graph-bs", 1, @@ -126,7 +115,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn32kOut1k50ms(TestNpuPerformanceTestCaseBase) num_prompts = 64 max_concurrency = 64 random_range_ratio = 1 - warmup_requests = 0 + warmup_requests = 16 request_rate = float("inf") seed = 1 tpot = 50 diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 22f2209d7..7c698dd89 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -15,27 +15,18 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", "INF_NAN_MODE_FORCE_DISABLE": "1", + "SGLANG_SET_CPU_AFFINITY": "1", "HCCL_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo", - "USE_NPU_MOE_GATING_TOP_K": "1", + "HCCL_OP_EXPANSION_MODE": "AIV", "SGLANG_NPU_USE_MULTI_STREAM": "1", # deepep "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", - # zbal - "HCCL_BUFFSIZE": "8", - "SGLANG_ZBAL_LOCAL_MEM_SIZE": "61000", - "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", - "ZBAL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", - "SGLANG_ZBAL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", - "ZBAL_ENABLE_GRAPH": "1", - # dsv4 - "IS_DEEPSEEK_V4": "1", - "SGLANG_DEBUG_LAYER_NORM": "1", - "SGLANG_DEBUG_FWD_INPUT": "1", - "USE_FUSED_HC_PRE_ASCENDC": "1", - "SGLANG_DSV4_NPU_FUSED_COMPRESSOR": "1", - "SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL": "1", + "DEEPEP_HCCL_BUFFSIZE": "2048", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "35", + # war barrier + "SGLANG_ENABLE_WAR_BARRIER": "1", + "SGLANG_FORCE_COARSE_WAR_BARRIER": "1", # skip gpu branch "SGLANG_OPT_FP8_WO_A_GEMM": "0", "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", @@ -50,11 +41,6 @@ DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { # mtp "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_NPU_PROFILING": "0", - "SGLANG_DEBUG_MTP_VERIFY": "0", - "SGLANG_DEBUG_MTP_VERIFY_LIMIT": "8", - "SGLANG_DEBUG_MTP_VERIFY_ROWS": "4", - "SGLANG_DISABLE_DRAFT_EXTEND_GRAPH": "1", } # Server launch arguments for DSV4-Flash W8A8 single-node 8p PD-mix. @@ -68,12 +54,14 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "npu", "--prefill-max-requests", 160, + "--max-prefill-tokens", + 80000, "--attention-backend", "dsv4", "--watchdog-timeout", 9000, "--mem-fraction-static", - 0.7, + 0.68, "--chunked-prefill-size", 131072, "--max-running-requests", @@ -89,7 +77,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "modelslim", "--enable-dp-lm-head", "--kv-cache-dtype", - "auto", + "bfloat16", "--skip-server-warmup", "--cuda-graph-bs", 1, @@ -127,7 +115,7 @@ class TestNPUDeepSeekV4FlashW8A88PIn8kOut1k50ms(TestNpuPerformanceTestCaseBase): num_prompts = 160 max_concurrency = 160 random_range_ratio = 1 - warmup_requests = 0 + warmup_requests = 16 request_rate = float("inf") seed = 1 tpot = 50 diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py index 290e52aa1..693ddc1bb 100644 --- a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py @@ -48,7 +48,7 @@ OTHER_ARGS = [ "--max-running-requests", 32, "--chunked-prefill-size", - 262144, + 32768, "--context-length", 75000, "--enable-multimodal",