From 0e500feae64280d754002ba9d817867f9eb4554e Mon Sep 17 00:00:00 2001 From: hanwlax Date: Mon, 17 Aug 2026 09:50:48 +0800 Subject: [PATCH] fix tpot by adjusting the sliding max-prefill-size window size (#34856) --- ...test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py | 4 +++- .../qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py | 1 + .../test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py | 3 ++- .../test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py | 1 + .../qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py | 1 + .../test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py | 3 ++- .../test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py | 4 +++- 7 files changed, 13 insertions(+), 4 deletions(-) diff --git a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py index 7ef54643f..a948263b8 100644 --- a/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py @@ -25,7 +25,7 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "DEEPEP_HCCL_BUFFSIZE": "1024", "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", } @@ -41,6 +41,8 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ "--tool-call-parser", "minimax-m2", "--enable-prefill-delayer", + "--prefill-delayer-max-delay-passes", + 128, "--prefill-max-requests", 10, "--chunked-prefill-size", diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py index 125c55ac5..f14459056 100644 --- a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py @@ -20,6 +20,7 @@ QWEN3_8B_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", } QWEN3_8B_OTHER_ARGS = [ diff --git a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py index d115b37ff..b4dd856cd 100644 --- a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py @@ -21,7 +21,8 @@ QWEN3_235B_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "16", "SGLANG_NPU_PROFILING": "0", "SGLANG_NPU_PROFILING_BS": "27", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416", diff --git a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py index 50ebda549..4c0f4a029 100644 --- a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py @@ -21,6 +21,7 @@ QWEN3_30B_A3B_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", "DEEPEP_HCCL_BUFFSIZE": "400", } diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py index 7752ee5b5..a3bb52bed 100644 --- a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py @@ -20,6 +20,7 @@ QWEN3_32B_ENVS = { "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", "SGLANG_NPU_USE_DEEPGEMM": "1", } diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py index 0bd8aab8d..a04e3c422 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py @@ -22,7 +22,8 @@ QWEN3_6_27B_1024_ENVS = { "SGLANG_NPU_PROFILING_STAGE": "prefill", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", - "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "300", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", "ASCEND_USE_FIA": "1", } diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py index 01039457a..08e9bc143 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py @@ -25,7 +25,7 @@ QWEN3_6_27B_1080P_ENVS = { "SGLANG_NPU_PROFILING": "0", "SGLANG_NPU_PROFILING_STAGE": "prefill", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64", + "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128", "ASCEND_USE_FIA": "1", } @@ -70,6 +70,8 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [ 0.45, "--prefill-delayer-max-delay-ms", 5500, + "--prefill-delayer-max-delay-passes", + 300, "--enable-multimodal", "--mm-attention-backend", "ascend_attn",