minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[NPU] [DOC] update npu best practice (#36476)

Browse Source
This commit is contained in:
amote-i
2026-08-29 09:38:52 +08:00
committed by GitHub
parent 0df69849ba
commit 51c18d9aa8
18 changed files with 1862 additions and 744 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/docs.json
+3
View File
@@ -1111,6 +1111,7 @@
"pages": [
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1",
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2",
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash",
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1",
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2",
"docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3",
@@ -1133,7 +1134,9 @@
"pages": [
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5",
"docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash",
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx
+60 -52
View File
@@ -14,7 +14,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1.5k | 16ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-16ms) |
| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-20ms) |
| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k-19-0ms) |
| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.9k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k9-out1k-19-0ms) |
| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 6k+1.6k | 20.5ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in6k-out1k6-20-5ms) |
@@ -97,9 +97,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=4000
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=3500
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -112,12 +112,12 @@ do
--node-rank 0 \
--nnodes 1 \
--tp-size 16 \
--mem-fraction-static 0.62 \
--mem-fraction-static 0.8 \
--quantization modelslim \
--max-running-requests 32 \
--context-length 8192 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--chunked-prefill-size 20480 \
--max-prefill-tokens 20480 \
--moe-a2a-backend deepep \
--deepep-mode normal \
@@ -131,11 +131,11 @@ do
--dtype bfloat16 \
--reasoning-parser deepseek-r1 \
--tool-call-parser deepseekv3 \
--max-total-tokens 80000 \
--disaggregation-transfer-backend ascend \
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -146,11 +146,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export TASK_QUEUE_ENABLE=1
python3 -m sglang.launch_server \
@@ -161,7 +162,7 @@ do
--nnodes 1 \
--tp-size 16 \
--dp-size 16 \
--mem-fraction-static 0.805 \
--mem-fraction-static 0.9 \
--max-running-requests 416 \
--quantization modelslim \
--moe-a2a-backend deepep \
@@ -182,11 +183,11 @@ do
--load-balance-method round_robin \
--reasoning-parser deepseek-r1 \
--tool-call-parser deepseekv3 \
--max-total-tokens 145792 \
--disaggregation-transfer-backend ascend \
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -218,12 +219,13 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--request-rate 24 \
--max-concurrency 416 \
--num-prompts 1664 \
--random-input-len 3584 \
--random-output-len 1536 \
--num-prompts 1664 \
--random-range-ratio 1 \
--request-rate 24
--seed 1
```
<a id="single-node-pd-mixed" title="Referenced by external docs. Verify before removing."></a>
@@ -269,15 +271,16 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512
export DEEPEP_HCCL_BUFFSIZE=2000
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -294,11 +297,11 @@ python3 -m sglang.launch_server \
--quantization modelslim \
--watchdog-timeout 9000 \
--cuda-graph-bs 4 8 12 14 \
--mem-fraction-static 0.77 \
--mem-fraction-static 0.9 \
--max-running-requests 224 \
--context-length 8188 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--chunked-prefill-size 65536 \
--max-prefill-tokens 3000 \
--moe-a2a-backend deepep \
--deepep-mode auto \
@@ -311,7 +314,8 @@ python3 -m sglang.launch_server \
--speculative-num-draft-tokens 4 \
--dtype bfloat16 \
--reasoning-parser deepseek-r1 \
--tool-call-parser deepseekv3
--tool-call-parser deepseekv3 \
--max-total-tokens 100000
```
#### Benchmark
@@ -325,15 +329,16 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 224 \
--num-prompts 896 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 896 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
<a id="pd-disaggregation" title="Referenced by external docs. Verify before removing."></a>
### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 16ms
### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 20ms
**Model**: DeepSeek-R1
@@ -347,7 +352,7 @@ python -m sglang.bench_serving \
**Dataset**: 3.5k+1.5k
**TPOT**: 16ms
**TPOT**: 20ms
#### Model Deployment
@@ -400,9 +405,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1536
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -439,7 +444,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -450,11 +454,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=650
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -494,7 +499,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -529,12 +533,13 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--request-rate 16 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 32 \
--random-range-ratio 1 \
--request-rate 16
--seed 42
```
### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 50ms
@@ -606,11 +611,12 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=800
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_USE_AG_AFTER_QLORA=1
export TASK_QUEUE_ENABLE=2
@@ -624,12 +630,13 @@ do
--nnodes 1 \
--tp-size 16 \
--mem-fraction-static 0.778 \
--quantization modelslim \
--max-running-requests 16 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 60000 \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--deepep-mode normal \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
--speculative-eagle-topk 1 \
@@ -645,7 +652,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -656,12 +662,14 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=600
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=600
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_LM_HEAD_TP=8
export SGLANG_NPU_FUSED_MOE_MODE=1
export TASK_QUEUE_ENABLE=1
python3 -m sglang.launch_server \
@@ -676,8 +684,8 @@ do
--dp-size 32 \
--mem-fraction-static 0.82 \
--max-running-requests 1024 \
--quantization modelslim \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 1 \
--enable-dp-attention \
--deepep-mode low_latency \
--moe-dense-tp 1 \
@@ -699,7 +707,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -734,12 +741,13 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--request-rate 40 \
--max-concurrency 1024 \
--num-prompts 7168 \
--random-input-len 3584 \
--random-output-len 1536 \
--num-prompts 7168 \
--random-range-ratio 1 \
--request-rate 40
--seed 1
```
### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K 19.0ms
@@ -809,9 +817,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1536
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -848,7 +856,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -859,11 +866,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=650
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -903,7 +911,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -938,12 +945,13 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--request-rate 16 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 3500 \
--random-output-len 1024 \
--num-prompts 32 \
--random-range-ratio 1 \
--request-rate 16
--seed 1
```
### DeepSeek-R1 W8A8 2P1D 32P IN3K9 OUT1K 19.0ms
@@ -1013,9 +1021,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1536
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -1052,7 +1060,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -1063,11 +1070,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=650
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -1107,7 +1115,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -1142,12 +1149,13 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--request-rate 16 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 3900 \
--random-output-len 1024 \
--num-prompts 32 \
--random-range-ratio 1 \
--request-rate 16
--seed 1
```
### DeepSeek-R1 W8A8 2P1D 32P IN6K OUT1K6 20.5ms
@@ -1217,9 +1225,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1536
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -1255,7 +1263,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -1266,11 +1273,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=650
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -1309,7 +1317,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -1344,10 +1351,11 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--request-rate 16 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 6000 \
--random-output-len 1600 \
--num-prompts 32 \
--random-range-ratio 1 \
--request-rate 16
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx
+20 -22
View File
@@ -92,9 +92,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -120,8 +120,8 @@ do
--disaggregation-transfer-backend ascend \
--disable-cuda-graph \
--moe-dense-tp-size 1 \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 32 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
@@ -132,7 +132,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -143,11 +142,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=400
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
@@ -185,7 +185,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -219,10 +218,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 8 \
--num-prompts 8 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 8 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K BS16
@@ -290,9 +290,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -318,8 +318,8 @@ do
--disaggregation-transfer-backend ascend \
--disable-cuda-graph \
--moe-dense-tp-size 1 \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 32 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
@@ -330,7 +330,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -341,11 +340,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=400
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
@@ -383,7 +383,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -417,9 +416,9 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 16 \
--num-prompts 16 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 16 \
--random-range-ratio 1
```
@@ -488,9 +487,9 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -516,8 +515,8 @@ do
--disaggregation-transfer-backend ascend \
--disable-cuda-graph \
--moe-dense-tp-size 1 \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 32 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
@@ -528,7 +527,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -539,11 +537,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=400
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
@@ -581,7 +580,6 @@ do
--trust-remote-code \
--attention-backend ascend \
--device npu
NODE_RANK=$i
break
fi
done
@@ -615,8 +613,8 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 8 \
--num-prompts 8 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 8 \
--random-range-ratio 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx
+486
View File
@@ -0,0 +1,486 @@
---
title: "DeepSeek-V4-Flash"
metatags:
description: "Best Practice for DeepSeek-V4-Flash on Ascend NPU"
---
<Note>
This page focuses on optimal configuration and benchmark results for DeepSeek-V4-Flash on the Ascend NPU.
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
</Note>
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| DeepSeek-V4-Flash | Atlas 800I A3 | 16 | PD Disaggregation | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-1p1d-16p-in8k-out1k-50ms) |
| DeepSeek-V4-Flash | Atlas 800I A3 | 8 | PD Mixed | 32k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in32k-out1k-50ms) |
| DeepSeek-V4-Flash | Atlas 800I A3 | 8 | PD Mixed | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in8k-out1k-50ms) |
## Optimal Configuration
<a id="pd-disaggregation" title="Referenced by external docs. Verify before removing."></a>
### DeepSeek-V4-Flash W8A8 1P1D 16P IN8K OUT1K 50ms
**Model**: DeepSeek-V4-Flash
**Hardware**: Atlas 800I A3
**Cards**: 16
**Deploy Mode**: PD Disaggregation
**Quantization**: W8A8 INT8
**Dataset**: 8k+1k
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# P_IP: prefill node IP address
# D_IP: decode node IP address
# ASCEND_MF_STORE_URL: prefill node IP with port
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export HCCL_OP_EXPANSION_MODE=AIV
export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DSV4_FP4_EXPERTS=False
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False
export SGLANG_OPT_FP8_WO_A_GEMM=0
export SGLANG_OPT_FUSE_WQA_WKV=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=False
export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False
export SGLANG_OPT_USE_TILELANG_MHC_POST=False
export SGLANG_OPT_USE_TILELANG_MHC_PRE=False
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
P_IP=('<your prefill ip>')
D_IP=('<your decode ip>')
export ASCEND_MF_STORE_URL="tcp://<your prefill ip>:24670"
MODEL_PATH=/path/to/model-weights
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=8
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
export SGLANG_ZBAL_LOCAL_MEM_SIZE=62084
export ZBAL_ENABLE_GRAPH=1
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode prefill \
--host ${P_IP[$i]} \
--port 8000 \
--disaggregation-bootstrap-port 8998 \
--page-size 128 \
--tp-size 16 \
--trust-remote-code \
--device npu \
--attention-backend dsv4 \
--watchdog-timeout 9000 \
--disaggregation-transfer-backend ascend \
--mem-fraction-static 0.62 \
--prefill-max-requests 6 \
--max-prefill-tokens 70000 \
--chunked-prefill-size -1 \
--max-running-requests 112 \
--dp-size 16 \
--enable-dp-attention \
--moe-a2a-backend deepep \
--deepep-mode normal \
--quantization modelslim \
--enable-dp-lm-head \
--kv-cache-dtype bfloat16 \
--disable-cuda-graph \
--disable-radix-cache \
--load-balance-method round_robin \
--ep-dispatch-algorithm static
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_LONG_SEQ_ROUND=8
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode decode \
--host ${D_IP[$i]} \
--port 8001 \
--page-size 128 \
--tp-size 16 \
--trust-remote-code \
--device npu \
--attention-backend dsv4 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.75 \
--prefill-max-requests 1 \
--disable-radix-cache \
--chunked-prefill-size 32768 \
--disaggregation-transfer-backend ascend \
--max-running-requests 896 \
--dp-size 16 \
--enable-dp-attention \
--moe-a2a-backend deepep \
--deepep-mode auto \
--quantization modelslim \
--enable-dp-lm-head \
--kv-cache-dtype bfloat16 \
--cuda-graph-bs 1 2 4 8 16 24 36 40 48 56 \
--speculative-algorithm EAGLE \
--speculative-num-steps 2 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 3
break
fi
done
```
```bash Command
# ============================================================
# Before running, replace the following placeholders:
# <your prefill ip>: prefill node IP address
# <your decode ip>: decode node IP address
# ============================================================
python -m sglang_router.launch_router \
--pd-disaggregation \
--prefill http://<your prefill ip>:8000 8998 \
--decode http://<your decode ip>:8001 \
--host 127.0.0.1 \
--port 6688 \
--policy cache_aware
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--random-input-len 8000 \
--random-output-len 1000 \
--num-prompts 1600 \
--max-concurrency 800 \
--random-range-ratio 1 \
--warmup-requests 0 \
--request-rate inf \
--seed 1
```
<a id="single-node-pd-mixed" title="Referenced by external docs. Verify before removing."></a>
### DeepSeek-V4-Flash W8A8 8P IN32K OUT1K 50ms
**Model**: DeepSeek-V4-Flash
**Hardware**: Atlas 800I A3
**Cards**: 8
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 32k+1k
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=8
export HCCL_SOCKET_IFNAME=<network-interface>
export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_DISABLE_DRAFT_EXTEND_GRAPH=1
export SGLANG_DSV4_FP4_EXPERTS=False
export SGLANG_DSV4_NPU_FUSED_COMPRESSOR=1
export SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL=0
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False
export SGLANG_OPT_FP8_WO_A_GEMM=0
export SGLANG_OPT_FUSE_WQA_WKV=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=False
export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False
export SGLANG_OPT_USE_TILELANG_MHC_POST=False
export SGLANG_OPT_USE_TILELANG_MHC_PRE=False
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
export SGLANG_ZBAL_LOCAL_MEM_SIZE=61000
export STREAMS_PER_DEVICE=32
export USE_FUSED_HC_PRE_ASCENDC=1
export USE_NPU_MOE_GATING_TOP_K=1
export ZBAL_ENABLE_GRAPH=1
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--page-size 128 \
--tp-size 16 \
--trust-remote-code \
--device npu \
--prefill-max-requests 32 \
--attention-backend dsv4 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.7 \
--chunked-prefill-size 131072 \
--max-running-requests 64 \
--dp-size 16 \
--enable-dp-attention \
--moe-a2a-backend deepep \
--deepep-mode auto \
--quantization modelslim \
--enable-dp-lm-head \
--kv-cache-dtype auto \
--skip-server-warmup \
--cuda-graph-bs 1 2 4 8 \
--speculative-algorithm EAGLE \
--speculative-num-steps 2 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 3 \
--ep-size 16 \
--disable-radix-cache
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--random-input-len 32000 \
--random-output-len 1000 \
--num-prompts 64 \
--max-concurrency 64 \
--random-range-ratio 1 \
--warmup-requests 0 \
--request-rate inf \
--seed 1 \
--max-attempts 3
```
### DeepSeek-V4-Flash W8A8 8P IN8K OUT1K 50ms
**Model**: DeepSeek-V4-Flash
**Hardware**: Atlas 800I A3
**Cards**: 8
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 8k+1k
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=8
export HCCL_SOCKET_IFNAME=<network-interface>
export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_DISABLE_DRAFT_EXTEND_GRAPH=1
export SGLANG_DSV4_FP4_EXPERTS=False
export SGLANG_DSV4_NPU_FUSED_COMPRESSOR=1
export SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False
export SGLANG_OPT_FP8_WO_A_GEMM=0
export SGLANG_OPT_FUSE_WQA_WKV=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=False
export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False
export SGLANG_OPT_USE_TILELANG_MHC_POST=False
export SGLANG_OPT_USE_TILELANG_MHC_PRE=False
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
export SGLANG_ZBAL_LOCAL_MEM_SIZE=61000
export STREAMS_PER_DEVICE=32
export USE_FUSED_HC_PRE_ASCENDC=1
export USE_NPU_MOE_GATING_TOP_K=1
export ZBAL_ENABLE_GRAPH=1
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--page-size 128 \
--tp-size 16 \
--trust-remote-code \
--device npu \
--prefill-max-requests 160 \
--attention-backend dsv4 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.7 \
--chunked-prefill-size 131072 \
--max-running-requests 160 \
--dp-size 16 \
--enable-dp-attention \
--moe-a2a-backend deepep \
--deepep-mode auto \
--quantization modelslim \
--enable-dp-lm-head \
--kv-cache-dtype auto \
--skip-server-warmup \
--cuda-graph-bs 1 2 4 8 10 \
--speculative-algorithm EAGLE \
--speculative-num-steps 2 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 3 \
--ep-size 16 \
--disable-radix-cache
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--random-input-len 8000 \
--random-output-len 1000 \
--num-prompts 320 \
--max-concurrency 160 \
--random-range-ratio 1 \
--warmup-requests 0 \
--request-rate inf \
--seed 1 \
--max-attempts 3
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx
+78 -272
View File
@@ -25,7 +25,6 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend
| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 16k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) |
| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64k+1k | 55.2ms | 7.58s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-55-2ms) |
| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) |
| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 65k+1.5k (100% prefix cache hit rate) | 33ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-48p-in65k-out1k5-prefix100-33ms) |
| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 128k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) |
| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 64k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) |
@@ -77,13 +76,14 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=2500
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2500
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -142,10 +142,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 128 \
--num-prompts 128 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 128 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
<a id="pd-disaggregation" title="Referenced by external docs. Verify before removing."></a>
@@ -219,15 +220,18 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=72
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699
export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184
export TASK_QUEUE_ENABLE=2
export ZBAL_ENABLE_GRAPH=1
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
@@ -246,7 +250,7 @@ do
--disaggregation-transfer-backend ascend \
--max-running-requests 16 \
--served-model-name glm-5 \
--chunked-prefill-size 8192 \
--chunked-prefill-size 32768 \
--max-prefill-tokens 180000 \
--moe-a2a-backend deepep \
--deepep-mode normal \
@@ -257,13 +261,11 @@ do
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--attn-cp-size 4 \
--disable-radix-cache \
--enable-dp-lm-head \
--moe-dense-tp 1 \
--pp-size 8 \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -279,6 +281,8 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -289,6 +293,7 @@ do
--port 8001 \
--dist-init-addr ${D_IP[0]}:5000 \
--node-rank $i \
--trust-remote-code \
--tp-size 32 \
--nnodes 2 \
--dp-size 32 \
@@ -312,10 +317,10 @@ do
--dtype bfloat16 \
--load-balance-method round_robin \
--speculative-draft-model-quantization unquant \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
break
fi
done
@@ -347,11 +352,12 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### GLM-5.1 W4A8 1P1D 32P IN16K OUT1K 50ms
@@ -420,11 +426,10 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -455,15 +460,14 @@ do
--dp-size 4 \
--enable-dp-attention \
--load-balance-method round_robin \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 8 \
--enable-dp-lm-head \
--moe-dense-tp 1 \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -474,11 +478,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=650
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -519,7 +524,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -552,10 +556,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 128 \
--num-prompts 512 \
--random-input-len 16384 \
--random-output-len 1024 \
--num-prompts 512 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### GLM-5.1 W4A8 1P1D 32P IN64K OUT1K 55.2ms
@@ -627,15 +632,19 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=72
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699
export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184
export TASK_QUEUE_ENABLE=2
export ZBAL_ENABLE_GRAPH=1
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
@@ -645,6 +654,7 @@ do
--dist-init-addr ${P_IP[0]}:5000 \
--disaggregation-bootstrap-port 8998 \
--node-rank $i \
--trust-remote-code \
--tp-size 4 \
--nnodes 2 \
--mem-fraction-static 0.72 \
@@ -654,7 +664,7 @@ do
--disaggregation-transfer-backend ascend \
--max-running-requests 16 \
--served-model-name glm-5 \
--chunked-prefill-size 8192 \
--chunked-prefill-size 32768 \
--max-prefill-tokens 180000 \
--moe-a2a-backend deepep \
--deepep-mode normal \
@@ -665,13 +675,10 @@ do
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--attn-cp-size 4 \
--disable-radix-cache \
--enable-dp-lm-head \
--moe-dense-tp 1 \
--pp-size 8 \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
--pp-size 8
break
fi
done
@@ -687,6 +694,8 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -720,10 +729,11 @@ do
--dtype bfloat16 \
--load-balance-method round_robin \
--speculative-draft-model-quantization unquant \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -755,11 +765,12 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 65536 \
--random-output-len 1024 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### GLM-5.1 W4A8 1P1D 32P IN64K OUT1K 50ms
@@ -829,13 +840,12 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=72
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -873,7 +883,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -925,7 +934,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -958,10 +966,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--random-input-len 65536 \
--random-output-len 1024 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### GLM-5.1 W4A8 1P1D 32P IN65K OUT1K5 PREFIX90 25ms
@@ -1030,11 +1039,10 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -1055,7 +1063,7 @@ do
--disaggregation-transfer-backend ascend \
--max-running-requests 64 \
--served-model-name glm-5 \
--chunked-prefill-size 524288 \
--chunked-prefill-size 53248 \
--max-prefill-tokens 180000 \
--moe-a2a-backend deepep \
--deepep-mode normal \
@@ -1065,15 +1073,14 @@ do
--dp-size 4 \
--enable-dp-attention \
--load-balance-method round_robin \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 8 \
--enable-dp-lm-head \
--moe-dense-tp 1 \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -1084,11 +1091,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=650
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1129,7 +1137,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -1154,8 +1161,8 @@ python -m sglang_router.launch_router \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 59904` = `int(66560 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6656` = `int(66560 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 59904` = `round(66560 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6656` = `round(66560 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -1174,203 +1181,6 @@ python -m sglang.bench_serving \
--request-rate inf
```
### GLM-5.1 W4A8 1P1D 48P IN65K OUT1K5 PREFIX100 33ms
**Model**: GLM-5.1
**Hardware**: Atlas 800I A3
**Cards**: 48
**Deploy Mode**: PD Disaggregation
**Quantization**: W4A8 INT8
**Dataset**: 65k+1.5k (100% prefix cache hit rate)
**TPOT**: 33ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# P_IP: prefill node IP address
# D_IP: decode node IP address
# ASCEND_MF_STORE_URL: prefill node IP with port
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
P_IP=('<your prefill ip1>' '<your prefill ip2>')
D_IP=('<your decode ip1>' '<your decode ip2>' '<your decode ip3>' '<your decode ip4>')
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
MODEL_PATH=/path/to/model-weights
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode prefill \
--host ${P_IP[$i]} \
--port 8000 \
--dist-init-addr ${P_IP[0]}:5000 \
--disaggregation-bootstrap-port 8998 \
--node-rank $i \
--tp-size 32 \
--nnodes 2 \
--mem-fraction-static 0.72 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--disaggregation-transfer-backend ascend \
--max-running-requests 192 \
--served-model-name glm-5 \
--chunked-prefill-size 16384 \
--moe-a2a-backend deepep \
--deepep-mode normal \
--disable-shared-experts-fusion \
--disable-cuda-graph \
--dtype bfloat16 \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=650
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode decode \
--host ${D_IP[$i]} \
--port 8001 \
--dist-init-addr ${D_IP[0]}:5000 \
--node-rank $i \
--tp-size 64 \
--nnodes 4 \
--dp-size 64 \
--ep-size 64 \
--enable-dp-attention \
--mem-fraction-static 0.84 \
--max-running-requests 192 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--served-model-name glm-5 \
--moe-a2a-backend deepep \
--deepep-mode low_latency \
--enable-dp-lm-head \
--moe-dense-tp 1 \
--cuda-graph-bs 1 2 3 \
--disaggregation-transfer-backend ascend \
--watchdog-timeout 9000 \
--context-length 180000 \
--speculative-draft-model-quantization unquant \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tokenizer-worker-num 4 \
--prefill-round-robin-balance \
--disable-shared-experts-fusion \
--dtype bfloat16 \
--load-balance-method round_robin \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
```
```bash Command
# ============================================================
# Before running, replace the following placeholders:
# <your prefill ip>: prefill node IP address
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
# ============================================================
python -m sglang_router.launch_router \
--pd-disaggregation \
--prefill http://<your prefill ip>:8000 8998 \
--decode http://<your decode ip1>:8001 \
--host 127.0.0.1 \
--port 6688 \
--policy round_robin
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 128 \
--random-input-len 66560 \
--random-output-len 1536 \
--num-prompts 512 \
--random-range-ratio 1
```
### GLM-5.1 W4A8 2P1D 48P IN128K OUT1K PREFIX90 50ms
**Model**: GLM-5.1
@@ -1438,13 +1248,12 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=72
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -1473,8 +1282,8 @@ do
--disable-cuda-graph \
--dtype bfloat16 \
--speculative-draft-model-quantization unquant \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 4 \
--enable-dp-lm-head \
--moe-dense-tp 1 \
@@ -1482,7 +1291,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -1498,6 +1306,7 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1538,7 +1347,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -1564,8 +1372,8 @@ python -m sglang_router.launch_router \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion.
`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 117965` = `round(131072 * 0.9)` is the shared prefix portion.
`--gsp-question-len 13107` = `round(131072 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -1576,7 +1384,7 @@ python -m sglang.bench_serving \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 576 \
--gsp-system-prompt-len 117964 \
--gsp-system-prompt-len 117965 \
--gsp-question-len 13107 \
--gsp-output-len 1024 \
--max-concurrency 144 \
@@ -1651,13 +1459,12 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=72
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export ENABLE_PROFILING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
@@ -1685,8 +1492,8 @@ do
--disable-cuda-graph \
--dtype bfloat16 \
--speculative-draft-model-quantization unquant \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--enable-prefill-cp \
--cp-strategy zigzag \
--attn-cp-size 4 \
--enable-dp-lm-head \
--moe-dense-tp 1 \
@@ -1694,7 +1501,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -1705,11 +1511,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=300
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=300
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1750,7 +1557,6 @@ do
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--trust-remote-code
NODE_RANK=$i
break
fi
done
@@ -1778,8 +1584,8 @@ python -m sglang_router.launch_router \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -1791,7 +1597,7 @@ python -m sglang.bench_serving \
--gsp-num-groups 1 \
--gsp-prompts-per-group 1280 \
--gsp-system-prompt-len 58982 \
--gsp-question-len 6553 \
--gsp-question-len 6554 \
--gsp-output-len 1024 \
--max-concurrency 320 \
--num-prompts 1280 \
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx
+226
View File
@@ -0,0 +1,226 @@
---
title: "GLM-5.2"
metatags:
description: "Best Practice for GLM-5.2 on Ascend NPU"
---
<Note>
This page focuses on optimal configuration and benchmark results for GLM-5.2 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [GLM-5.2 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2).
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
</Note>
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| GLM-5.2 | Atlas 800I A3 | 32 | PD Disaggregation | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-2-w4a8-3p1d-32p-in16k-out1k-50ms) |
## Optimal Configuration
<a id="pd-disaggregation" title="Referenced by external docs. Verify before removing."></a>
### GLM-5.2 W4A8 3P1D 32P IN16K OUT1K 50ms
**Model**: GLM-5.2
**Hardware**: Atlas 800I A3
**Cards**: 32
**Deploy Mode**: PD Disaggregation
**Quantization**: W4A8 INT8
**Dataset**: 16k+1k
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# P_IP: prefill node IP address
# D_IP: decode node IP address
# ASCEND_MF_STORE_URL: prefill node IP with port
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=1200
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=1200
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
export TRANSFORMERS_VERBOSITY=error
P_IP=('<your prefill ip1>' '<your prefill ip2>' '<your prefill ip3>')
D_IP=('<your decode ip>')
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
MODEL_PATH=/path/to/model-weights
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export DEEP_USE_ALLTOALL_MODE=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=128
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_PP_LAYER_PARTITION=18,20,24,16
export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184
export TASK_QUEUE_ENABLE=2
export ZBAL_HCCL_OP=send,recv
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode prefill \
--host ${P_IP[$i]} \
--port 8000 \
--disaggregation-bootstrap-port $((8998 + $i)) \
--trust-remote-code \
--tp-size 4 \
--mem-fraction-static 0.72 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--disaggregation-transfer-backend ascend \
--max-running-requests 16 \
--served-model-name glm-5 \
--chunked-prefill-size 4096 \
--max-prefill-tokens 180000 \
--moe-a2a-backend deepep \
--deepep-mode normal \
--disable-shared-experts-fusion \
--disable-cuda-graph \
--dtype bfloat16 \
--speculative-draft-model-quantization unquant \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--attn-cp-size 4 \
--enable-dp-lm-head \
--moe-dense-tp 1 \
--pp-size 4 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 2
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=300
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode decode \
--host ${D_IP[$i]} \
--port 8001 \
--trust-remote-code \
--tp-size 16 \
--dp-size 16 \
--enable-dp-attention \
--ep-size 16 \
--mem-fraction-static 0.895 \
--max-running-requests 128 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--served-model-name glm-5 \
--moe-a2a-backend deepep \
--deepep-mode low_latency \
--cuda-graph-max-bs 8 \
--disaggregation-transfer-backend ascend \
--watchdog-timeout 9000 \
--context-length 180000 \
--tokenizer-worker-num 8 \
--prefill-round-robin-balance \
--disable-shared-experts-fusion \
--dtype bfloat16 \
--load-balance-method round_robin \
--speculative-draft-model-quantization unquant \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
break
fi
done
```
```bash Command
# ============================================================
# Before running, replace the following placeholders:
# <your prefill ip1>, <your prefill ip2>, <your prefill ip3>: prefill node IP addresses
# <your decode ip>: decode node IP address
# ============================================================
python -m sglang_router.launch_router \
--pd-disaggregation \
--prefill http://<your prefill ip1>:8000 8998 \
--prefill http://<your prefill ip2>:8000 8999 \
--prefill http://<your prefill ip3>:8000 9000 \
--decode http://<your decode ip>:8001 \
--host 127.0.0.1 \
--port 6688 \
--policy round_robin
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 64 \
--num-prompts 200 \
--random-input-len 16000 \
--random-output-len 1000 \
--random-range-ratio 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx
+53 -49
View File
@@ -80,9 +80,9 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=4400
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=4400
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
@@ -149,10 +149,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 32 \
--num-prompts 32 \
--random-input-len 64000 \
--random-output-len 1000 \
--num-prompts 32 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
<a id="pd-disaggregation" title="Referenced by external docs. Verify before removing."></a>
@@ -261,7 +262,6 @@ do
--enable-multimodal \
--mm-attention-backend ascend_attn \
--sampling-backend ascend
NODE_RANK=$i
break
fi
done
@@ -272,11 +272,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -313,7 +314,6 @@ do
--speculative-draft-model-quantization unquant \
--reasoning-parser kimi_k2 \
--tool-call-parser kimi_k2
NODE_RANK=$i
break
fi
done
@@ -346,11 +346,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--request-rate inf \
--random-input-len 128000 \
--random-output-len 1000 \
--num-prompts 1 \
--random-range-ratio 1 \
--request-rate inf
--seed 1
```
### Kimi-K2.6 W4A8 1P1D 16P IN128K OUT1K PREFIX90 100ms
@@ -460,7 +461,6 @@ do
--sampling-backend ascend \
--reasoning-parser kimi_k2 \
--tool-call-parser kimi_k2
NODE_RANK=$i
break
fi
done
@@ -471,11 +471,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -512,7 +513,6 @@ do
--speculative-draft-model-quantization unquant \
--reasoning-parser kimi_k2 \
--tool-call-parser kimi_k2
NODE_RANK=$i
break
fi
done
@@ -537,8 +537,8 @@ python -m sglang_router.launch_router \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 115200` = `int(128000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 12800` = `int(128000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 115200` = `round(128000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 12800` = `round(128000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -626,8 +626,8 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1800
export HCCL_SOCKET_IFNAME=<network-interface>
python3 -m sglang.launch_server \
@@ -659,7 +659,6 @@ do
--sampling-backend ascend \
--reasoning-parser kimi_k2 \
--tool-call-parser kimi_k2
NODE_RANK=$i
break
fi
done
@@ -670,11 +669,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -711,7 +711,6 @@ do
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--speculative-draft-model-quantization unquant
NODE_RANK=$i
break
fi
done
@@ -744,11 +743,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--request-rate inf \
--random-input-len 64000 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1 \
--request-rate inf
--seed 1
```
### Kimi-K2.6 W4A8 1P1D 16P IN64K OUT1K5 PREFIX90 100ms
@@ -822,8 +822,8 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1800
export HCCL_SOCKET_IFNAME=<network-interface>
python3 -m sglang.launch_server \
@@ -853,7 +853,6 @@ do
--sampling-backend ascend \
--reasoning-parser kimi_k2 \
--tool-call-parser kimi_k2
NODE_RANK=$i
break
fi
done
@@ -864,11 +863,12 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1200
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -905,7 +905,6 @@ do
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--speculative-draft-model-quantization unquant
NODE_RANK=$i
break
fi
done
@@ -930,8 +929,8 @@ python -m sglang_router.launch_router \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6400` = `int(64000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 57600` = `round(64000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6400` = `round(64000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -977,8 +976,6 @@ python -m sglang.bench_serving \
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
@@ -998,12 +995,13 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=1280
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export HCCL_BUFFSIZE=1500
export HCCL_OP_EXPANSION_MODE=AIV
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -1018,7 +1016,7 @@ python3 -m sglang.launch_server \
--device npu \
--attention-backend ascend \
--tp-size 16 \
--mem-fraction-static 0.872 \
--mem-fraction-static 0.865 \
--max-running-requests 176 \
--chunked-prefill-size 32768 \
--context-length 8192 \
@@ -1054,15 +1052,16 @@ python -m sglang.bench_serving \
--backend sglang-oai-chat \
--host 127.0.0.1 \
--port 6688 \
--image-resolution 1024x1024 \
--image-count 1 \
--max-concurrency 160 \
--num-prompts 640 \
--request-rate inf \
--random-input-len 30 \
--random-output-len 1024 \
--num-prompts 640 \
--random-range-ratio 1 \
--request-rate inf \
--warmup-requests 16 \
--image-count 1 \
--image-resolution 1024x1024
--seed 1
```
### Kimi-K2.6 W4A8 8P IN1080P 30 OUT256 50ms
@@ -1088,8 +1087,6 @@ python -m sglang.bench_serving \
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
@@ -1109,12 +1106,13 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=1024
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export HCCL_BUFFSIZE=2400
export HCCL_OP_EXPANSION_MODE=AIV
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -1129,7 +1127,6 @@ python3 -m sglang.launch_server \
--device npu \
--attention-backend ascend \
--tp-size 16 \
--base-gpu-id 0 \
--mem-fraction-static 0.852 \
--max-running-requests 64 \
--chunked-prefill-size 16384 \
@@ -1166,15 +1163,16 @@ python -m sglang.bench_serving \
--backend sglang-oai-chat \
--host 127.0.0.1 \
--port 6688 \
--image-resolution 1920x1080 \
--image-count 1 \
--max-concurrency 48 \
--num-prompts 196 \
--request-rate inf \
--random-input-len 30 \
--random-output-len 256 \
--num-prompts 196 \
--random-range-ratio 1 \
--request-rate inf \
--warmup-requests 16 \
--image-count 1 \
--image-resolution 1920x1080
--seed 1
```
### Kimi-K2.6 W4A8 8P IN3K5 OUT1K5 20ms
@@ -1221,15 +1219,16 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -1281,12 +1280,14 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-attempts 5 \
--max-concurrency 64 \
--num-prompts 256 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 256 \
--random-range-ratio 1 \
--warmup-requests 0
--warmup-requests 0 \
--seed 1
```
### Kimi-K2.6 W4A8 8P IN3K5 OUT1K5 50ms
@@ -1333,15 +1334,17 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=1200
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_BUFFSIZE=200
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -1366,7 +1369,7 @@ python3 -m sglang.launch_server \
--dp-size 16 \
--moe-a2a-backend deepep \
--deepep-mode auto \
--cuda-graph-bs-decode 1 2 4 8 12 13 \
--cuda-graph-bs-decode 1 2 4 8 11 12 13 \
--disable-radix-cache \
--model-loader-extra-config '{"enable_multithread_load": true}' \
--speculative-algorithm EAGLE3 \
@@ -1375,7 +1378,7 @@ python3 -m sglang.launch_server \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--speculative-draft-model-quantization unquant \
--prefill-delayer-max-delay-passes 200 \
--prefill-delayer-max-delay-passes 50 \
--enable-prefill-delayer \
--reasoning-parser kimi_k2 \
--tool-call-parser kimi_k2
@@ -1392,9 +1395,10 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 192 \
--num-prompts 768 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 768 \
--random-range-ratio 1 \
--warmup-requests 0
--warmup-requests 0 \
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx
+25 -25
View File
@@ -83,6 +83,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -103,8 +104,8 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1024
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1
@@ -130,7 +131,6 @@ do
--disable-cuda-graph \
--disable-piecewise-cuda-graph \
--dp-size 2
NODE_RANK=$i
break
fi
done
@@ -141,8 +141,8 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
@@ -172,7 +172,6 @@ do
--enable-dp-lm-head \
--moe-a2a-backend deepep \
--deepep-mode low_latency
NODE_RANK=$i
break
fi
done
@@ -206,11 +205,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 64 \
--num-prompts 128 \
--request-rate 0.4 \
--random-input-len 16000 \
--random-output-len 1 \
--num-prompts 128 \
--random-range-ratio 1 \
--request-rate 0.4
--seed 1
```
### MiMo-V2-Flash 1P1D 12P IN16K OUT1K TPOT 20ms
@@ -268,6 +268,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -288,8 +289,8 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1024
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1
@@ -315,7 +316,6 @@ do
--disable-cuda-graph \
--disable-piecewise-cuda-graph \
--dp-size 2
NODE_RANK=$i
break
fi
done
@@ -326,8 +326,8 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
@@ -357,7 +357,6 @@ do
--enable-dp-lm-head \
--moe-a2a-backend deepep \
--deepep-mode low_latency
NODE_RANK=$i
break
fi
done
@@ -391,11 +390,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 32 \
--num-prompts 128 \
--request-rate inf \
--random-input-len 16000 \
--random-output-len 1000 \
--num-prompts 128 \
--random-range-ratio 1 \
--request-rate inf
--seed 1
```
### MiMo-V2-Flash 1P1D 12P IN32K OUT1 TTFT 5s
@@ -453,6 +453,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -473,8 +474,8 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1024
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1
@@ -500,7 +501,6 @@ do
--disable-cuda-graph \
--disable-piecewise-cuda-graph \
--dp-size 2
NODE_RANK=$i
break
fi
done
@@ -511,8 +511,8 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
@@ -542,7 +542,6 @@ do
--enable-dp-lm-head \
--moe-a2a-backend deepep \
--deepep-mode low_latency
NODE_RANK=$i
break
fi
done
@@ -576,11 +575,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 64 \
--num-prompts 128 \
--request-rate 0.4 \
--random-input-len 32000 \
--random-output-len 1 \
--num-prompts 128 \
--random-range-ratio 1 \
--request-rate 0.4
--seed 1
```
### MiMo-V2-Flash 1P1D 12P IN32K OUT1K TPOT 20ms
@@ -638,6 +638,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -658,8 +659,8 @@ do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=1024
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1
@@ -685,7 +686,6 @@ do
--disable-cuda-graph \
--disable-piecewise-cuda-graph \
--dp-size 2
NODE_RANK=$i
break
fi
done
@@ -696,8 +696,8 @@ do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export DEEPEP_HCCL_BUFFSIZE=800
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
@@ -727,7 +727,6 @@ do
--enable-dp-lm-head \
--moe-a2a-backend deepep \
--deepep-mode low_latency
NODE_RANK=$i
break
fi
done
@@ -760,10 +759,11 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 64 \
--max-concurrency 48 \
--num-prompts 192 \
--request-rate inf \
--random-input-len 32000 \
--random-output-len 1000 \
--num-prompts 128 \
--random-range-ratio 1 \
--request-rate inf
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx
+40 -26
View File
@@ -85,6 +85,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_SET_CPU_AFFINITY=1
@@ -118,7 +119,8 @@ python3 -m sglang.launch_server \
--trust-remote-code \
--tokenizer-worker-num 4 \
--reasoning-parser minimax-append-think \
--tool-call-parser minimax-m2
--tool-call-parser minimax-m2 \
--device npu
```
#### Benchmark
@@ -132,10 +134,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 18 \
--num-prompts 72 \
--random-input-len 32768 \
--random-output-len 1024 \
--num-prompts 72 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### MiniMax-M2.5 W8A8 4P IN64K OUT1K PREFIX90 50ms
@@ -184,14 +187,15 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=1024
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
@@ -210,7 +214,8 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 67000 \
--cuda-graph-bs 2 4 8 12 16 18 20 22 24 26 \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--deepep-mode auto \
--quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
@@ -218,14 +223,15 @@ python3 -m sglang.launch_server \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16 \
--trust-remote-code
--trust-remote-code \
--device npu
```
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -237,7 +243,7 @@ python -m sglang.bench_serving \
--gsp-num-groups 1 \
--gsp-prompts-per-group 104 \
--gsp-system-prompt-len 58982 \
--gsp-question-len 6553 \
--gsp-question-len 6554 \
--gsp-output-len 1024 \
--max-concurrency 26 \
--num-prompts 104 \
@@ -290,14 +296,16 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=1024
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
@@ -318,7 +326,8 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 80000 \
--cuda-graph-bs 2 4 6 8 \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--deepep-mode auto \
--quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
@@ -326,14 +335,15 @@ python3 -m sglang.launch_server \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--tokenizer-worker-num 4 \
--dtype bfloat16
--dtype bfloat16 \
--device npu
```
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion.
`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 117965` = `round(131072 * 0.9)` is the shared prefix portion.
`--gsp-question-len 13107` = `round(131072 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -344,7 +354,7 @@ python -m sglang.bench_serving \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 16 \
--gsp-system-prompt-len 117964 \
--gsp-system-prompt-len 117965 \
--gsp-question-len 13107 \
--gsp-output-len 1024 \
--max-concurrency 4 \
@@ -398,13 +408,15 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=2048
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2048
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
@@ -427,7 +439,6 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 8192 \
--cuda-graph-bs 1 2 3 4 5 6 \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--deepep-mode auto \
--quantization modelslim \
--speculative-algorithm EAGLE3 \
@@ -436,7 +447,8 @@ python3 -m sglang.launch_server \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16
--dtype bfloat16 \
--device npu
```
#### Benchmark
@@ -450,10 +462,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 112 \
--num-prompts 448 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 448 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### MiniMax-M2.5 W8A8 8P IN3K5 OUT1K5 50ms
@@ -502,8 +515,8 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=1024
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1024
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
@@ -526,12 +539,11 @@ python3 -m sglang.launch_server \
--tool-call-parser minimax-m2 \
--prefill-delayer-max-delay-passes 500 \
--enable-prefill-delayer \
--chunked-prefill-size -1 \
--chunked-prefill-size 196608 \
--max-prefill-tokens 8192 \
--cuda-graph-bs 1 2 4 8 12 16 20 \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--deepep-mode auto \
--quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
@@ -539,7 +551,8 @@ python3 -m sglang.launch_server \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16
--dtype bfloat16 \
--device npu
```
#### Benchmark
@@ -553,8 +566,9 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 320 \
--num-prompts 1280 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1280 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx
+11 -7
View File
@@ -70,13 +70,14 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=1600
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1600
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
@@ -116,10 +117,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--random-input-len 11000 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-235B-A22B W8A8 8P IN3K5 OUT1K5 50.1ms
@@ -166,8 +168,8 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=570
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=570
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
@@ -175,6 +177,7 @@ export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -186,6 +189,7 @@ python3 -m sglang.launch_server \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 432 \
--context-length 8192 \
--dtype bfloat16 \
@@ -195,7 +199,6 @@ python3 -m sglang.launch_server \
--ep-dispatch-algorithm static \
--disable-radix-cache \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
@@ -223,8 +226,9 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 432 \
--num-prompts 1728 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1728 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx
+18 -10
View File
@@ -81,6 +81,7 @@ export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_USE_MAX_DP_ATT=1
@@ -125,10 +126,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 162 \
--num-prompts 624 \
--random-input-len 1000 \
--random-output-len 100 \
--num-prompts 624 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1 \
--max-attempts 4
```
### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 10ms
@@ -176,13 +179,14 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_LAUNCH_BLOCKING=0
export DEEPEP_HCCL_BUFFSIZE=400
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -224,10 +228,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 50ms
@@ -275,8 +280,8 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_LAUNCH_BLOCKING=0
export DEEPEP_HCCL_BUFFSIZE=400
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
@@ -323,10 +328,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 160 \
--num-prompts 640 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 640 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-30B-A3B W8A8 1P IN6K OUT1K5 BS16
@@ -373,13 +379,14 @@ unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export DEEPEP_HCCL_BUFFSIZE=400
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export TRANSFORMERS_VERBOSITY=error
@@ -420,9 +427,10 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-attempts 5 \
--max-concurrency 16 \
--num-prompts 16 \
--random-input-len 6144 \
--random-output-len 1500 \
--num-prompts 16 \
--random-range-ratio 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx
+105 -102
View File
@@ -20,8 +20,8 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms-a2) |
| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) |
| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5k+1.5k | 55ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-55ms-a2) |
## Optimal Configuration
@@ -117,108 +117,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--random-input-len 18000 \
--random-output-len 4000 \
--num-prompts 1 \
--random-range-ratio 1
```
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms A2
**Model**: Qwen3-32B
**Hardware**: Atlas 800I A2
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5k+1.5k
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 101 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tp-size 4 \
--mem-fraction-static 0.845 \
--cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \
--dtype bfloat16 \
--reasoning-parser qwen3 \
--tool-call-parser qwen
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 100 \
--random-input-len 3584 \
--random-output-len 1536 \
--num-prompts 400 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms
@@ -313,8 +216,108 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 100 \
--num-prompts 400 \
--random-input-len 3584 \
--random-output-len 1536 \
--random-range-ratio 1 \
--seed 1
```
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 55ms A2
**Model**: Qwen3-32B
**Hardware**: Atlas 800I A2
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5k+1.5k
**TPOT**: 55ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 101 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tp-size 4 \
--mem-fraction-static 0.845 \
--cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \
--dtype bfloat16 \
--reasoning-parser qwen3 \
--tool-call-parser qwen
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 100 \
--num-prompts 400 \
--random-range-ratio 1
--random-input-len 3584 \
--random-output-len 1536 \
--random-range-ratio 1 \
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx
+105 -66
View File
@@ -1,11 +1,13 @@
---
title: "Qwen3.5-397B-A17B"
metatags:
description: "Best Practice for Qwen3.5-397B on Ascend NPU"
description: "Best Practice for Qwen3.5-397B-A17B on Ascend NPU"
---
<Note>
This page focuses on optimal configuration and benchmark results for Qwen3.5-397B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.5-397B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b).
This page focuses on optimal configuration and benchmark results for Qwen3.5-397B-A17B on the Ascend NPU. For environment
setup, model weight download, feature configuration, and deployment instructions, etc., see
the [Qwen3.5-397B-A17B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b).
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
</Note>
@@ -14,29 +16,29 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in128k-out1k-20ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in16k-out1k-20ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in3k5-out1k5-20ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-20ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-20ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-20ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 22.2ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-22-2ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-20ms) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in128k-out1k-50ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in128k-out1k-prefix90-50ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in16k-out1k-50ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in3k5-out1k5-50ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-50ms) |
| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-prefix90-50ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-50ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-prefix90-50ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-50ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-50ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-50ms) |
| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-prefix90-50ms) |
## Optimal Configuration
<a id="single-node-pd-mixed" title="Referenced by external docs. Verify before removing."></a>
### Qwen3.5-397B W4A8 8P IN128K OUT1K 20ms
### Qwen3.5-397B-A17B W4A8 8P IN128K OUT1K 20ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -87,6 +89,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -135,18 +138,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 2 \
--max-concurrency 3 \
--num-prompts 3 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 3 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 2
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN128K OUT1K 50ms
### Qwen3.5-397B-A17B W4A8 8P IN128K OUT1K 50ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -197,6 +203,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -218,7 +225,6 @@ python3 -m sglang.launch_server \
--trust-remote-code \
--max-running-requests 16 \
--mem-fraction-static 0.6 \
--cuda-graph-bs 2 4 6 8 12 14 16 \
--quantization modelslim \
--enable-multimodal \
--moe-a2a-backend deepep \
@@ -245,18 +251,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 8 \
--max-concurrency 10 \
--num-prompts 10 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 10 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 8
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN128K OUT1K PREFIX90 50ms
### Qwen3.5-397B-A17B W4A8 8P IN128K OUT1K PREFIX90 50ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -296,17 +305,18 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=2200
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096
export DEEPEP_NORMAL_LONG_SEQ_ROUND=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2200
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -320,7 +330,7 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 131072 \
--max-mamba-cache-size 320 \
--prefill-max-requests 10 \
--mamba-radix-cache-strategy extra_buffer \
--mamba-scheduler-strategy extra_buffer \
--trust-remote-code \
--max-running-requests 64 \
--mem-fraction-static 0.6 \
@@ -343,8 +353,8 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion.
`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 117965` = `round(131072 * 0.9)` is the shared prefix portion.
`--gsp-question-len 13107` = `round(131072 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -355,17 +365,19 @@ python -m sglang.bench_serving \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 40 \
--gsp-system-prompt-len 117964 \
--gsp-system-prompt-len 117965 \
--gsp-question-len 13107 \
--gsp-output-len 1024 \
--max-concurrency 40 \
--num-prompts 40 \
--request-rate inf
--request-rate inf \
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN16K OUT1K 20ms
### Qwen3.5-397B-A17B W4A8 8P IN16K OUT1K 20ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -416,6 +428,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -468,18 +481,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 32 \
--max-concurrency 40 \
--num-prompts 40 \
--random-input-len 16384 \
--random-output-len 1024 \
--num-prompts 40 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 32
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN16K OUT1K 50ms
### Qwen3.5-397B-A17B W4A8 8P IN16K OUT1K 50ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -530,6 +546,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -582,18 +599,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 8 \
--max-concurrency 132 \
--num-prompts 132 \
--random-input-len 16384 \
--random-output-len 1024 \
--num-prompts 132 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 8
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN3K5 OUT1K5 20ms
### Qwen3.5-397B-A17B W4A8 8P IN3K5 OUT1K5 22.2ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -605,7 +625,7 @@ python -m sglang.bench_serving \
**Dataset**: 3.5k+1.5k
**TPOT**: 20ms
**TPOT**: 22.2ms
#### Model Deployment
@@ -644,6 +664,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -695,18 +716,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 64 \
--max-concurrency 160 \
--num-prompts 160 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 160 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 64
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN3K5 OUT1K5 50ms
### Qwen3.5-397B-A17B W4A8 8P IN3K5 OUT1K5 50ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -757,6 +781,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -808,18 +833,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 16 \
--max-concurrency 432 \
--num-prompts 432 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 432 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 16
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN64K OUT1K 20ms
### Qwen3.5-397B-A17B W4A8 8P IN64K OUT1K 20ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -870,6 +898,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -922,18 +951,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--warmup-requests 6 \
--max-concurrency 6 \
--num-prompts 6 \
--random-input-len 65536 \
--random-output-len 1024 \
--num-prompts 6 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 6
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN64K OUT1K 50ms
### Qwen3.5-397B-A17B W4A8 8P IN64K OUT1K 50ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -984,6 +1016,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -1036,18 +1069,21 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 28 \
--warmup-requests 8 \
--max-concurrency 24 \
--num-prompts 24 \
--random-input-len 65536 \
--random-output-len 1024 \
--num-prompts 28 \
--random-range-ratio 1 \
--seed 1 \
--request-rate inf \
--warmup-requests 8
--temperature 0.6 \
--top-p 0.95
```
### Qwen3.5-397B W4A8 8P IN64K OUT1K PREFIX90 50ms
### Qwen3.5-397B-A17B W4A8 8P IN64K OUT1K PREFIX90 50ms
**Model**: Qwen3.5-397B
**Model**: Qwen3.5-397B-A17B
**Hardware**: Atlas 800I A3
@@ -1087,17 +1123,18 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=2200
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096
export DEEPEP_NORMAL_LONG_SEQ_ROUND=20
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2200
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -1110,7 +1147,7 @@ python3 -m sglang.launch_server \
--chunked-prefill-size -1 \
--max-prefill-tokens 65536 \
--max-mamba-cache-size 640 \
--mamba-radix-cache-strategy extra_buffer \
--mamba-scheduler-strategy extra_buffer \
--trust-remote-code \
--max-running-requests 128 \
--mem-fraction-static 0.6 \
@@ -1137,8 +1174,8 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -1148,11 +1185,13 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 112 \
--gsp-prompts-per-group 96 \
--gsp-system-prompt-len 58982 \
--gsp-question-len 6553 \
--gsp-question-len 6554 \
--gsp-output-len 1024 \
--max-concurrency 112 \
--num-prompts 112 \
--request-rate inf
--max-concurrency 96 \
--num-prompts 96 \
--request-rate inf \
--temperature 0.6 \
--top-p 0.95
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx
+156 -49
View File
@@ -21,6 +21,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in64k-out1k-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) |
## Optimal Configuration
@@ -75,7 +76,7 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=300
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
@@ -92,10 +93,10 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 52000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 50 \
--max-running-requests 60 \
--max-mamba-cache-size 60 \
--mem-fraction-static 0.76 \
--cuda-graph-bs 2 4 8 16 24 32 40 42 45 50 \
--mem-fraction-static 0.74 \
--cuda-graph-bs 2 4 8 14 16 24 26 32 36 37 40 42 44 45 46 50 52 60 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
@@ -111,19 +112,22 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `RANDOM` dataset.
We tested it based on the `IMAGE` dataset with 1024x1024 resolution.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--dataset-name image \
--backend sglang-oai-chat \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 48 \
--max-concurrency 60 \
--num-prompts 240 \
--random-input-len 30 \
--random-output-len 1024 \
--num-prompts 48 \
--random-range-ratio 1
--random-range-ratio 1 \
--image-resolution 1024x1024 \
--image-count 1 \
--seed 1
```
### Qwen3.6-27B 1P IN1080P 30 OUT256 50ms
@@ -171,10 +175,7 @@ export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
export STREAMS_PER_DEVICE=32
@@ -187,13 +188,16 @@ python3 -m sglang.launch_server \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 48000 \
--max-prefill-tokens 82688 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 30 \
--max-mamba-cache-size 40 \
--mem-fraction-static 0.76 \
--cuda-graph-bs 2 4 8 16 24 28 30 \
--max-running-requests 38 \
--max-mamba-cache-size 38 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 1 2 4 8 10 12 16 20 24 28 30 32 35 38 \
--enable-prefill-delayer \
--prefill-delayer-queue-min-ratio 0.45 \
--prefill-delayer-max-delay-ms 5500 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
@@ -202,26 +206,29 @@ python3 -m sglang.launch_server \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--mm-enable-dp-encoder \
--reasoning-parser qwen3 \
--tool-call-parser qwen3_coder
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
We tested it based on the `IMAGE` dataset with 1920x1080 resolution.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--dataset-name image \
--backend sglang-oai-chat \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 30 \
--warmup-requests 38 \
--max-concurrency 42 \
--num-prompts 152 \
--random-input-len 30 \
--random-output-len 256 \
--num-prompts 120 \
--random-range-ratio 1
--random-range-ratio 1 \
--image-resolution 1920x1080 \
--image-count 1 \
--seed 1
```
### Qwen3.6-27B 1P IN64K OUT1K PREFIX90 50ms
@@ -272,6 +279,7 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -284,15 +292,12 @@ python3 -m sglang.launch_server \
--device npu \
--chunked-prefill-size 32768 \
--max-prefill-tokens 32768 \
--mamba-radix-cache-strategy extra_buffer \
--mamba-scheduler-strategy extra_buffer \
--trust-remote-code \
--max-running-requests 20 \
--max-mamba-cache-size 160 \
--mem-fraction-static 0.82 \
--cuda-graph-bs 1 2 5 10 15 17 19 20 \
--enable-prefill-delayer \
--prefill-delayer-queue-min-ratio 0.7 \
--prefill-delayer-max-delay-ms 20000 \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
@@ -306,8 +311,8 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6400` = `int(64000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 57600` = `round(64000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6400` = `round(64000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -317,12 +322,12 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 80 \
--gsp-prompts-per-group 20 \
--gsp-system-prompt-len 57600 \
--gsp-question-len 6400 \
--gsp-output-len 1000 \
--max-concurrency 20 \
--num-prompts 80 \
--num-prompts 20 \
--request-rate inf
```
@@ -417,10 +422,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 54 \
--num-prompts 216 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 216 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-27B W8A8 1P IN64K OUT1K 50ms
@@ -508,10 +514,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 6 \
--num-prompts 12 \
--random-input-len 64000 \
--random-output-len 1000 \
--num-prompts 12 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-27B W8A8 2P IN128K OUT1K 50ms
@@ -601,10 +608,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 4 \
--num-prompts 16 \
--random-input-len 128000 \
--random-output-len 1000 \
--num-prompts 16 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-27B W8A8 2P IN16K OUT1K 50ms
@@ -654,7 +662,8 @@ export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -670,17 +679,17 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 58000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 29 \
--max-mamba-cache-size 58 \
--mem-fraction-static 0.68 \
--cuda-graph-bs 1 2 8 12 16 20 24 26 28 29 \
--max-running-requests 37 \
--max-mamba-cache-size 74 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 1 2 3 4 6 8 10 12 14 16 18 20 21 23 24 25 26 27 28 29 30 31 33 35 37 \
--quantization modelslim \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-num-draft-tokens 5 \
--reasoning-parser qwen3 \
--tool-call-parser qwen3_coder
```
@@ -695,9 +704,107 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 29 \
--max-concurrency 37 \
--warmup-requests 4 \
--num-prompts 37 \
--random-input-len 16000 \
--random-output-len 1000 \
--num-prompts 116 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-27B W8A8 2P IN64K OUT1K 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 64k+1k
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 4 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 50000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 28 \
--max-mamba-cache-size 50 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 2 4 6 \
--enable-multimodal \
--quantization modelslim \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 6 \
--num-prompts 24 \
--random-input-len 64000 \
--random-output-len 1000 \
--random-range-ratio 1 \
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx
+167 -50
View File
@@ -8,8 +8,6 @@ metatags:
This page focuses on optimal configuration and benchmark results for Qwen3.6-35B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-35B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b).
On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details.
Use image **SGLang `>= v0.5.16`** for these NEXTN configurations. Without `--dataset-path`, `bench_serving --dataset-name random` downloads ShareGPT from Hugging Face; in offline environments, pass a local dataset path (for example a ShareGPT JSON file).
</Note>
### Low Latency
@@ -25,6 +23,7 @@ Use image **SGLang `>= v0.5.16`** for these NEXTN configurations. Without `--dat
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) |
@@ -83,8 +82,9 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
@@ -95,14 +95,15 @@ python3 -m sglang.launch_server \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 16384 \
--max-prefill-tokens 9999999 \
--max-total-tokens 365000 \
--prefill-max-requests 30 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--max-running-requests 120 \
--max-mamba-cache-size 240 \
--mem-fraction-static 0.78 \
--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120 \
--max-mamba-cache-size 120 \
--mem-fraction-static 0.85 \
--cuda-graph-bs 4 16 32 48 64 110 165 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
@@ -117,20 +118,23 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `RANDOM` dataset.
We tested it based on the `IMAGE` dataset with 1024x1024 resolution.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--dataset-name image \
--backend sglang-oai-chat \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 120 \
--num-prompts 120 \
--warmup-requests 120 \
--random-input-len 30 \
--random-output-len 1024 \
--num-prompts 480 \
--random-range-ratio 1 \
--image-resolution 1024x1024 \
--image-count 1 \
--seed 1 \
--request-rate inf
```
@@ -181,8 +185,9 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
@@ -193,14 +198,14 @@ python3 -m sglang.launch_server \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 16384 \
--max-prefill-tokens 150000 \
--max-total-tokens 200000 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--max-running-requests 50 \
--max-mamba-cache-size 55 \
--mem-fraction-static 0.8 \
--cuda-graph-bs 2 4 8 12 16 20 24 28 32 36 40 44 48 50 \
--max-running-requests 42 \
--max-mamba-cache-size 42 \
--mem-fraction-static 0.75 \
--cuda-graph-bs 4 8 16 24 48 64 80 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
@@ -215,20 +220,23 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `RANDOM` dataset.
We tested it based on the `IMAGE` dataset with 1920x1080 resolution.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--dataset-name image \
--backend sglang-oai-chat \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 50 \
--max-concurrency 42 \
--num-prompts 42 \
--warmup-requests 42 \
--random-input-len 30 \
--random-output-len 256 \
--num-prompts 200 \
--random-range-ratio 1 \
--image-resolution 1920x1080 \
--image-count 1 \
--seed 1 \
--request-rate inf
```
@@ -274,9 +282,9 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=1600
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1600
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
@@ -293,12 +301,12 @@ python3 -m sglang.launch_server \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-total-tokens 520960 \
--max-total-tokens 420000 \
--max-prefill-tokens 128000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 3 \
--max-mamba-cache-size 10 \
--max-mamba-cache-size 3 \
--mem-fraction-static 0.9 \
--cuda-graph-bs 1 2 3 \
--enable-multimodal \
@@ -320,15 +328,118 @@ We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 3 \
--warmup-requests 3 \
--num-prompts 3 \
--random-input-len 128000 \
--random-output-len 1000 \
--num-prompts 3 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-35B-A3B 1P IN128K OUT1K PREFIX90 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: BF16
**Dataset**: 128k+1k (90% prefix cache hit rate)
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size 16384 \
--max-prefill-tokens 65536 \
--trust-remote-code \
--enable-prefill-delayer \
--mamba-scheduler-strategy extra_buffer \
--max-running-requests 103 \
--max-mamba-cache-size 85 \
--mem-fraction-static 0.85 \
--cuda-graph-bs 2 4 8 16 32 48 64 80 96 103 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--reasoning-parser qwen3 \
--tool-call-parser qwen3_coder
```
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 115200` = `round(128000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 12800` = `round(128000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
python -m sglang.bench_serving \
--dataset-name generated-shared-prefix \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 412 \
--gsp-system-prompt-len 115200 \
--gsp-question-len 12800 \
--gsp-output-len 1000 \
--max-concurrency 103 \
--num-prompts 412 \
--request-rate inf
```
### Qwen3.6-35B-A3B 1P IN254K OUT1K
@@ -378,6 +489,7 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -415,15 +527,16 @@ We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--warmup-requests 1 \
--num-prompts 1 \
--random-input-len 254000 \
--random-output-len 1000 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-35B-A3B 1P IN3K5 OUT1K5 50ms
@@ -469,7 +582,7 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1
export HCCL_BUFFSIZE=100
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
@@ -513,15 +626,16 @@ We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 122 \
--warmup-requests 122 \
--num-prompts 122 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 122 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-35B-A3B 1P IN64K OUT1K 50ms
@@ -611,15 +725,16 @@ We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 10 \
--warmup-requests 10 \
--num-prompts 40 \
--random-input-len 64000 \
--random-output-len 1000 \
--num-prompts 40 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3.6-35B-A3B 1P IN64K OUT1K PREFIX90 50ms
@@ -664,9 +779,9 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=300
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=300
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
@@ -685,7 +800,7 @@ python3 -m sglang.launch_server \
--max-total-tokens 470784 \
--max-prefill-tokens 65536 \
--trust-remote-code \
--mamba-radix-cache-strategy extra_buffer \
--mamba-scheduler-strategy extra_buffer \
--max-running-requests 40 \
--max-mamba-cache-size 200 \
--mem-fraction-static 0.9 \
@@ -705,8 +820,8 @@ python3 -m sglang.launch_server \
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```bash Command
@@ -718,7 +833,7 @@ python -m sglang.bench_serving \
--gsp-num-groups 1 \
--gsp-prompts-per-group 40 \
--gsp-system-prompt-len 58982 \
--gsp-question-len 6553 \
--gsp-question-len 6554 \
--gsp-output-len 1024 \
--max-concurrency 40 \
--num-prompts 40 \
@@ -773,6 +888,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -811,13 +927,14 @@ We tested it based on the `RANDOM` dataset.
```bash Command
python -m sglang.bench_serving \
--dataset-name random \
--dataset-path /path/to/dataset \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--warmup-requests 1 \
--num-prompts 1 \
--random-input-len 984000 \
--random-output-len 1000 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx
+10 -5
View File
@@ -118,10 +118,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 64 \
--num-prompts 256 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 256 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-8B W8A8 1P IN3K5 OUT1K5 5ms
@@ -174,6 +175,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
@@ -212,11 +214,13 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-attempts 5 \
--max-concurrency 1 \
--num-prompts 4 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 4 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-8B W8A8 1P IN6K OUT1K5 BS16
@@ -307,9 +311,10 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-attempts 5 \
--max-concurrency 16 \
--num-prompts 16 \
--random-input-len 6144 \
--random-output-len 1500 \
--num-prompts 16 \
--random-range-ratio 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx
+14 -9
View File
@@ -72,16 +72,17 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=2000
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1
@@ -119,7 +120,7 @@ python3 -m sglang.launch_server \
--mamba-ssm-dtype bfloat16 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--reasoning-parser qwen3 \
--tool-call-parser qwen
--tool-call-parser qwen3_coder
```
#### Benchmark
@@ -133,10 +134,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--num-prompts 1 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 50ms
@@ -193,6 +195,7 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1
@@ -234,7 +237,7 @@ python3 -m sglang.launch_server \
--deepep-mode auto \
--cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 \
--reasoning-parser qwen3 \
--tool-call-parser qwen
--tool-call-parser qwen3_coder
```
#### Benchmark
@@ -248,10 +251,11 @@ python -m sglang.bench_serving \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 300 \
--num-prompts 300 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 300 \
--random-range-ratio 1
--random-range-ratio 1 \
--seed 1
```
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN6K OUT1K5 BS16
@@ -299,12 +303,12 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_HCCL_BUFFSIZE=2000
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
@@ -362,9 +366,10 @@ python -m sglang.bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-attempts 5 \
--max-concurrency 16 \
--num-prompts 16 \
--random-input-len 6144 \
--random-output-len 1500 \
--num-prompts 16 \
--random-range-ratio 1
```
docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx
+285
View File
@@ -0,0 +1,285 @@
---
title: "DeepSeek-V4-Flash"
metatags:
description: "Deploy DeepSeek-V4-Flash model with SGLang on Ascend NPUs, including multi-node PD disaggregation mode."
---
## Introduction
DeepSeek-V4-Flash is a Mixture-of-Experts (MoE) large language model developed by DeepSeek, featuring 284B total
parameters with 13B active parameters. It employs hybrid sparse attention (CSA + HCA) and DeepSeekMoE architecture with
a bundled multi-token prediction (MTP) draft head for speculative decoding, supporting a 1M-token context length.
This document demonstrates the deployment of DeepSeek-V4-Flash on Ascend NPUs using SGLang, including multi-node PD
disaggregation mode, feature configuration, and performance optimization.
This document is validated and written based on **SGLang v0.5.16**. The current model (DeepSeek-V4-Flash) is fully
supported in this version. To use the latest features (e.g., PD disaggregation, speculative decoding), it is
recommended to use v0.5.16 or a later version.
## Supported features
| Feature | Example usage |
|-------------------------------|-----------------------------------------------------------------------------------------------|
| Attention Backend | `--attention-backend dsv4` |
| Tensor Parallelism | `--tp-size 16` |
| Data Parallelism | `--dp-size 16` |
| Expert Parallelism | `--ep-size 16 \`<br/>`--moe-a2a-backend deepep \`<br/>`--deepep-mode auto` |
| PD Disaggregation | `--disaggregation-mode prefill \`<br/>`--disaggregation-transfer-backend ascend` |
| Quantization | `--quantization modelslim` |
| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;<br/>control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 4 8 16 24 36 40 48 56` |
| Speculative Decoding | `--speculative-algorithm EAGLE \`<br/>`--speculative-num-steps 2 \`<br/>`--speculative-eagle-topk 1 \`<br/>`--speculative-num-draft-tokens 3` |
| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` |
| DP LM Head | `--enable-dp-lm-head` |
| DP Attention | `--enable-dp-attention` |
<Note>
The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment
mode, and workload. For parameter details, see
[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for
recommended configurations for each deployment scenario, see [Best practices](#best-practices).
</Note>
For feature compatibility and conflict information between features,
see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility).
## Prerequisites
### Environment
Before following this tutorial, complete the environment setup in the documents below:
- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started.
It walks you through launching the official container image, starting the SGLang server, and sending a test request.
Recommended if you are new to SGLang on Ascend.
- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide.
It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a
Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize
the environment instead of using the official image.
### Model weights
Before downloading model weights, check the model size to reserve enough disk space.
For multi-node deployment, download the weights to a shared directory accessible to all nodes.
- [DeepSeek-V4-Flash-w8a8-mtp](https://www.modelscope.cn/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp) (W8A8 quantized version, 300GB)
<Info>
The W8A8 variant (300GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2
node or 8 dies on A3 (4 cards).
</Info>
This is the minimum recommended configuration. For optimized configurations,
see [Best practices](#best-practices), which may require additional cards or nodes.
For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3),
see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware).
## Installation
<Warning>
Ensure sufficient disk space before pulling images. The Docker image requires at least **30GB** of free space.
</Warning>
The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the
online platform. You can directly pull it.
Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag.
For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build).
<Tabs>
<Tab title="Atlas 800I A3">
```bash Command
docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16
docker run -itd --shm-size=16g --name ${NAME} \
--privileged=true --net=host \
-v /var/queue_schedule:/var/queue_schedule \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /usr/local/sbin:/usr/local/sbin \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \
--device=/dev/davinci0:/dev/davinci0 \
--device=/dev/davinci1:/dev/davinci1 \
--device=/dev/davinci2:/dev/davinci2 \
--device=/dev/davinci3:/dev/davinci3 \
--device=/dev/davinci4:/dev/davinci4 \
--device=/dev/davinci5:/dev/davinci5 \
--device=/dev/davinci6:/dev/davinci6 \
--device=/dev/davinci7:/dev/davinci7 \
--device=/dev/davinci8:/dev/davinci8 \
--device=/dev/davinci9:/dev/davinci9 \
--device=/dev/davinci10:/dev/davinci10 \
--device=/dev/davinci11:/dev/davinci11 \
--device=/dev/davinci12:/dev/davinci12 \
--device=/dev/davinci13:/dev/davinci13 \
--device=/dev/davinci14:/dev/davinci14 \
--device=/dev/davinci15:/dev/davinci15 \
--device=/dev/davinci_manager:/dev/davinci_manager \
--device=/dev/hisi_hdc:/dev/hisi_hdc \
--entrypoint=bash \
quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16
```
</Tab>
<Tab title="Atlas 800I A2">
```bash Command
docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16
docker run -itd --shm-size=16g --name ${NAME} \
--privileged=true --net=host \
-v /var/queue_schedule:/var/queue_schedule \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /usr/local/sbin:/usr/local/sbin \
-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \
--device=/dev/davinci0:/dev/davinci0 \
--device=/dev/davinci1:/dev/davinci1 \
--device=/dev/davinci2:/dev/davinci2 \
--device=/dev/davinci3:/dev/davinci3 \
--device=/dev/davinci4:/dev/davinci4 \
--device=/dev/davinci5:/dev/davinci5 \
--device=/dev/davinci6:/dev/davinci6 \
--device=/dev/davinci7:/dev/davinci7 \
--device=/dev/davinci_manager:/dev/davinci_manager \
--device=/dev/hisi_hdc:/dev/hisi_hdc \
--entrypoint=bash \
quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16
```
</Tab>
</Tabs>
<Tip>
- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the
container, for example: `-v /path/to/models:/models`.
- Replace `${NAME}` with your own container name or remove `--name` to use default name.
</Tip>
## Online service deployment
### Single-node PD mixed deployment
Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios
with limited hardware resources. In such scenarios, DeepSeek-V4-Flash can be launched with half of an Atlas 800I A3
node (4 cards, 8 dies) or a full Atlas 800I A2 node (8 cards). The configuration below was validated on Atlas 800I A3
(4 cards, 8 dies) with `--tp-size 8`.
```bash Command
# ============================================================
# Before running, update the following variable:
# MODEL_PATH: path to the model weights directory
# ============================================================
MODEL_PATH=/path/to/model-weights
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export HCCL_BUFFSIZE=8
export HCCL_OP_EXPANSION_MODE=AIV
export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DSV4_FP4_EXPERTS=False
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False
export SGLANG_OPT_FP8_WO_A_GEMM=0
export SGLANG_OPT_FUSE_WQA_WKV=0
export SGLANG_OPT_USE_FUSED_HASH_TOPK=False
export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False
export SGLANG_OPT_USE_TILELANG_MHC_POST=False
export SGLANG_OPT_USE_TILELANG_MHC_PRE=False
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:14699"
export SGLANG_ZBAL_LOCAL_MEM_SIZE=62084
export STREAMS_PER_DEVICE=32
export ZBAL_ENABLE_GRAPH=1
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path "${MODEL_PATH}" \
--page-size 128 \
--tp-size 8 \
--trust-remote-code \
--device npu \
--attention-backend dsv4 \
--watchdog-timeout 9000 \
--host 127.0.0.1 --port 6688 \
--mem-fraction-static 0.8 \
--prefill-max-requests 160 \
--max-prefill-tokens 80000 \
--chunked-prefill-size 131072 \
--max-running-requests 160 \
--dp-size 8 --enable-dp-attention \
--moe-a2a-backend deepep --deepep-mode auto \
--quantization modelslim --enable-dp-lm-head \
--kv-cache-dtype bfloat16
```
For the complete benchmark data and additional single-node configurations, see
[DeepSeek-V4-Flash Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#single-node-pd-mixed).
### Multi-node PD disaggregation deployment
PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving
throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized
deployment commands and benchmark data, see
[DeepSeek-V4-Flash Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#pd-disaggregation).
## Functional verification
After the service is started, you can invoke the model by sending a prompt:
```shell
# ============================================================
# Before running, update the following variables:
# HOST: the server host address (e.g., localhost)
# PORT: the server port number (e.g., 6688)
# ============================================================
curl http://${HOST}:${PORT}/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "deepseek-v4-flash",
"messages": [{"role": "user", "content": "What is the capital of France?"}]
}'
```
Expected result: an HTTP 200 response with the generated text containing "Paris".
Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more
testing examples (Health Check, Generate, Chat Completions, and port usage guidance),
see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service).
## Accuracy evaluation
For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation).
## Performance
For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing).
## Best practices
### Best practice configuration reference
For complete optimal configurations with deployment scripts and benchmark commands, see the
[DeepSeek-V4-Flash Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash) page.
## Performance tuning
For the full list of supported features, see [Supported features](#supported-features). For detailed optimization
guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning).
## FAQ
For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).