minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix tpot by adjusting the sliding max-prefill-size window size (#34856)

Browse Source
This commit is contained in:
hanwlax
2026-08-17 09:50:48 +08:00
committed by GitHub
parent 8e0499bd50
commit 0e500feae6
7 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
+3 -1
View File
@@ -25,7 +25,7 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "1024", "DEEPEP_HCCL_BUFFSIZE": "1024",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64", "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
} }
@@ -41,6 +41,8 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
"--tool-call-parser", "--tool-call-parser",
"minimax-m2", "minimax-m2",
"--enable-prefill-delayer", "--enable-prefill-delayer",
"--prefill-delayer-max-delay-passes",
128,
"--prefill-max-requests", "--prefill-max-requests",
10, 10,
"--chunked-prefill-size", "--chunked-prefill-size",
test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
+1
View File
@@ -20,6 +20,7 @@ QWEN3_8B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
} }
QWEN3_8B_OTHER_ARGS = [ QWEN3_8B_OTHER_ARGS = [
test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
+2 -1
View File
@@ -21,7 +21,8 @@ QWEN3_235B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "16",
"SGLANG_NPU_PROFILING": "0", "SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_BS": "27", "SGLANG_NPU_PROFILING_BS": "27",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416",
test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
+1
View File
@@ -21,6 +21,7 @@ QWEN3_30B_A3B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"DEEPEP_HCCL_BUFFSIZE": "400", "DEEPEP_HCCL_BUFFSIZE": "400",
} }
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
+1
View File
@@ -20,6 +20,7 @@ QWEN3_32B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"SGLANG_NPU_USE_DEEPGEMM": "1", "SGLANG_NPU_USE_DEEPGEMM": "1",
} }
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
+2 -1
View File
@@ -22,7 +22,8 @@ QWEN3_6_27B_1024_ENVS = {
"SGLANG_NPU_PROFILING_STAGE": "prefill", "SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "300",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
} }
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
+3 -1
View File
@@ -25,7 +25,7 @@ QWEN3_6_27B_1080P_ENVS = {
"SGLANG_NPU_PROFILING": "0", "SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill", "SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64", "SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
} }
@@ -70,6 +70,8 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [
0.45, 0.45,
"--prefill-delayer-max-delay-ms", "--prefill-delayer-max-delay-ms",
5500, 5500,
"--prefill-delayer-max-delay-passes",
300,
"--enable-multimodal", "--enable-multimodal",
"--mm-attention-backend", "--mm-attention-backend",
"ascend_attn", "ascend_attn",