minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix tpot by adjusting the sliding max-prefill-size window size (#34856)

Browse Source
This commit is contained in:
hanwlax
2026-08-17 09:50:48 +08:00
committed by GitHub
parent 8e0499bd50
commit 0e500feae6
7 changed files with 13 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py
+3 -1
View File
@@ -25,7 +25,7 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"DEEPEP_HCCL_BUFFSIZE": "1024",
"SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}",
}
@@ -41,6 +41,8 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [
"--tool-call-parser",
"minimax-m2",
"--enable-prefill-delayer",
"--prefill-delayer-max-delay-passes",
128,
"--prefill-max-requests",
10,
"--chunked-prefill-size",
test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py
+1
View File
@@ -20,6 +20,7 @@ QWEN3_8B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
}
QWEN3_8B_OTHER_ARGS = [
test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py
+2 -1
View File
@@ -21,7 +21,8 @@ QWEN3_235B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "16",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_BS": "27",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416",
test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py
+1
View File
@@ -21,6 +21,7 @@ QWEN3_30B_A3B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"DEEPEP_HCCL_BUFFSIZE": "400",
}
test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py
+1
View File
@@ -20,6 +20,7 @@ QWEN3_32B_ENVS = {
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"SGLANG_NPU_USE_DEEPGEMM": "1",
}
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py
+2 -1
View File
@@ -22,7 +22,8 @@ QWEN3_6_27B_1024_ENVS = {
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "150",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "300",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"ASCEND_USE_FIA": "1",
}
test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py
+3 -1
View File
@@ -25,7 +25,7 @@ QWEN3_6_27B_1080P_ENVS = {
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "64",
"SGLANG_PREFILL_DELAYER_MAX_PREFILL_BS_WINDOW_SIZE": "128",
"ASCEND_USE_FIA": "1",
}
@@ -70,6 +70,8 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [
0.45,
"--prefill-delayer-max-delay-ms",
5500,
"--prefill-delayer-max-delay-passes",
300,
"--enable-multimodal",
"--mm-attention-backend",
"ascend_attn",