minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[NPU] Best Practice Docs Splitting (#27663)

Browse Source
This commit is contained in:
jianzhao-xu
2026-06-13 22:16:20 +08:00
committed by GitHub
parent 0e592395c7
commit 29128f31fd
14 changed files with 10812 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/docs.json
+18
View File
@@ -885,6 +885,24 @@
"docs/hardware-platforms/ascend-npus/ascend_contribution_guide", "docs/hardware-platforms/ascend-npus/ascend_contribution_guide",
"docs/hardware-platforms/ascend-npus/ascend_npu_support_new_models", "docs/hardware-platforms/ascend-npus/ascend_npu_support_new_models",
"docs/hardware-platforms/ascend-npus/ascend_npu_best_practice", "docs/hardware-platforms/ascend-npus/ascend_npu_best_practice",
{
"group": "Best Practice",
"pages": [
"docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1",
"docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2",
"docs/hardware-platforms/ascend-npus/best_practice/glm5_1",
"docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6",
"docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b",
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct"
]
},
"docs/hardware-platforms/ascend-npus/ascend_npu_optimization", "docs/hardware-platforms/ascend-npus/ascend_npu_optimization",
"docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation", "docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation",
"docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing", "docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing",
docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx
+1282
View File
File diff suppressed because it is too large Load Diff
docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx
+590
View File
@@ -0,0 +1,590 @@
---
title: "DeepSeek-V3.2"
metatags:
description: "Best Practice for DeepSeek-V3.2 on Ascend NPU"
---
This guide describes the best practice data for DeepSeek-V3.2 on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-26ms) |
| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs8) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 107ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs16) |
## Optimal Configuration
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K 26ms
**Model**: DeepSeek-V3.2
**Hardware**: Atlas 800I A3
**Cards**: 32
**Deploy Mode**: PD Disaggregation
**Quantization**: W8A8 INT8
**Dataset**: 128K+1K
**TPOT**: 26ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# P_IP: prefill node IP address
# D_IP: decode node IP address
# ASCEND_MF_STORE_URL: prefill node IP with port
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
P_IP=('<your prefill ip1>' '<your prefill ip2>')
D_IP=('<your decode ip1>' '<your decode ip2>')
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
MODEL_PATH=/path/to/model-weights
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode prefill \
--host ${P_IP[$i]} \
--port 8000 \
--dist-init-addr ${P_IP[0]}:5000 \
--disaggregation-bootstrap-port 8998 \
--node-rank $i \
--nnodes 2 \
--tp 32 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.73 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 68000 \
--max-running-requests 1 \
--moe-a2a-backend deepep \
--deepep-mode normal \
--quantization modelslim \
--disaggregation-transfer-backend ascend \
--disable-cuda-graph \
--moe-dense-tp-size 1 \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--attn-cp-size 32 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 2
NODE_RANK=$i
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode decode \
--host ${D_IP[$i]} \
--port 8001 \
--dist-init-addr ${D_IP[0]}:5000 \
--node-rank $i \
--nnodes 2 \
--tp 32 \
--dp 8 \
--ep 32 \
--moe-dense-tp-size 1 \
--enable-dp-attention \
--enable-dp-lm-head \
--watchdog-timeout 9000 \
--mem-fraction-static 0.79 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 68000 \
--max-running-requests 32 \
--cuda-graph-max-bs 4 \
--moe-a2a-backend deepep \
--deepep-mode low_latency \
--quantization modelslim \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--disaggregation-transfer-backend ascend
NODE_RANK=$i
break
fi
done
```
```shell Command
# ============================================================
# Before running, replace the following placeholders:
# <your prefill ip>: prefill node IP address
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
# ============================================================
python -m sglang_router.launch_router \
--pd-disaggregation \
--policy cache_aware \
--prefill http://<your prefill ip>:8000 8998 \
--decode http://<your decode ip1>:8001 \
--host 127.0.0.1 \
--port 6688 \
--mini-lb
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 8 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 8 \
--random-range-ratio 1
```
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K BS16
**Model**: DeepSeek-V3.2
**Hardware**: Atlas 800I A3
**Cards**: 32
**Deploy Mode**: PD Disaggregation
**Quantization**: W8A8 INT8
**Dataset**: 128K+1K
**TPOT**: 107ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# P_IP: prefill node IP address
# D_IP: decode node IP address
# ASCEND_MF_STORE_URL: prefill node IP with port
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
P_IP=('<your prefill ip1>' '<your prefill ip2>')
D_IP=('<your decode ip1>' '<your decode ip2>')
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
MODEL_PATH=/path/to/model-weights
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode prefill \
--host ${P_IP[$i]} \
--port 8000 \
--dist-init-addr ${P_IP[0]}:5000 \
--disaggregation-bootstrap-port 8998 \
--node-rank $i \
--nnodes 2 \
--tp 32 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.73 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 68000 \
--max-running-requests 1 \
--moe-a2a-backend deepep \
--deepep-mode normal \
--quantization modelslim \
--disaggregation-transfer-backend ascend \
--disable-cuda-graph \
--moe-dense-tp-size 1 \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--attn-cp-size 32 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 2
NODE_RANK=$i
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode decode \
--host ${D_IP[$i]} \
--port 8001 \
--dist-init-addr ${D_IP[0]}:5000 \
--node-rank $i \
--nnodes 2 \
--tp 32 \
--dp 8 \
--ep 32 \
--moe-dense-tp-size 1 \
--enable-dp-attention \
--enable-dp-lm-head \
--watchdog-timeout 9000 \
--mem-fraction-static 0.79 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 68000 \
--max-running-requests 32 \
--cuda-graph-max-bs 4 \
--moe-a2a-backend deepep \
--deepep-mode low_latency \
--quantization modelslim \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--disaggregation-transfer-backend ascend
NODE_RANK=$i
break
fi
done
```
```shell Command
# ============================================================
# Before running, replace the following placeholders:
# <your prefill ip>: prefill node IP address
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
# ============================================================
python -m sglang_router.launch_router \
--pd-disaggregation \
--policy cache_aware \
--prefill http://<your prefill ip>:8000 8998 \
--decode http://<your decode ip1>:8001 \
--host 127.0.0.1 \
--port 6688 \
--mini-lb
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 16 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 16 \
--random-range-ratio 1
```
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K BS8
**Model**: DeepSeek-V3.2
**Hardware**: Atlas 800I A3
**Cards**: 32
**Deploy Mode**: PD Disaggregation
**Quantization**: W8A8 INT8
**Dataset**: 128K+1K
**TPOT**: 26ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# P_IP: prefill node IP address
# D_IP: decode node IP address
# ASCEND_MF_STORE_URL: prefill node IP with port
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
P_IP=('<your prefill ip1>' '<your prefill ip2>')
D_IP=('<your decode ip1>' '<your decode ip2>')
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
MODEL_PATH=/path/to/model-weights
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1200
export HCCL_SOCKET_IFNAME=<network-interface>
export TASK_QUEUE_ENABLE=2
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode prefill \
--host ${P_IP[$i]} \
--port 8000 \
--dist-init-addr ${P_IP[0]}:5000 \
--disaggregation-bootstrap-port 8998 \
--node-rank $i \
--nnodes 2 \
--tp 32 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.73 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 68000 \
--max-running-requests 1 \
--moe-a2a-backend deepep \
--deepep-mode normal \
--quantization modelslim \
--disaggregation-transfer-backend ascend \
--disable-cuda-graph \
--moe-dense-tp-size 1 \
--enable-nsa-prefill-context-parallel \
--nsa-prefill-cp-mode in-seq-split \
--attn-cp-size 32 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 2
NODE_RANK=$i
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
python3 -m sglang.launch_server \
--model-path ${MODEL_PATH} \
--disaggregation-mode decode \
--host ${D_IP[$i]} \
--port 8001 \
--dist-init-addr ${D_IP[0]}:5000 \
--node-rank $i \
--nnodes 2 \
--tp 32 \
--dp 8 \
--ep 32 \
--moe-dense-tp-size 1 \
--enable-dp-attention \
--enable-dp-lm-head \
--watchdog-timeout 9000 \
--mem-fraction-static 0.79 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 68000 \
--max-running-requests 32 \
--cuda-graph-max-bs 4 \
--moe-a2a-backend deepep \
--deepep-mode low_latency \
--quantization modelslim \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--disaggregation-transfer-backend ascend
NODE_RANK=$i
break
fi
done
```
```shell Command
# ============================================================
# Before running, replace the following placeholders:
# <your prefill ip>: prefill node IP address
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
# ============================================================
python -m sglang_router.launch_router \
--pd-disaggregation \
--policy cache_aware \
--prefill http://<your prefill ip>:8000 8998 \
--decode http://<your decode ip1>:8001 \
--host 127.0.0.1 \
--port 6688 \
--mini-lb
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 8 \
--random-input-len 131072 \
--random-output-len 1024 \
--num-prompts 8 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx
+1730
View File
File diff suppressed because it is too large Load Diff
docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx
+1715
View File
File diff suppressed because it is too large Load Diff
docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx
+1090
View File
File diff suppressed because it is too large Load Diff
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx
+307
View File
@@ -0,0 +1,307 @@
---
title: "Qwen3-8B"
metatags:
description: "Best Practice for Qwen3-8B on Ascend NPU"
---
This guide describes the best practice data for Qwen3-8B on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 5ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-5ms) |
| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 6K+1.5K | 11.79ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in6k-out1k5-bs16) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 37ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-37ms) |
## Optimal Configuration
### Qwen3-8B W8A8 1P IN3K5 OUT1K5 37ms
**Model**: Qwen3-8B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 37ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 70 \
--max-prefill-tokens 16384 \
--disable-radix-cache \
--chunked-prefill-size 16384 \
--tp-size 1 \
--mem-fraction-static 0.85 \
--cuda-graph-bs 8 12 24 36 48 51 55 60 63 64 66 68 70 \
--dtype bfloat16 \
--speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 64 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 256 \
--random-range-ratio 1
```
### Qwen3-8B W8A8 1P IN3K5 OUT1K5 5ms
**Model**: Qwen3-8B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 5ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 1 \
--max-prefill-tokens 16384 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--tp-size 2 \
--mem-fraction-static 0.894 \
--cuda-graph-bs 1 \
--dtype bfloat16 \
--speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 4 \
--random-range-ratio 1
```
### Qwen3-8B W8A8 1P IN6K OUT1K5 BS16
**Model**: Qwen3-8B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 6K+1.5K
**TPOT**: 11.79ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 16 \
--max-prefill-tokens 16384 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--tp-size 2 \
--mem-fraction-static 0.894 \
--cuda-graph-bs 1 5 15 16 \
--dtype bfloat16 \
--speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 16 \
--random-input-len 6144 \
--random-output-len 1500 \
--num-prompts 16 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx
+226
View File
@@ -0,0 +1,226 @@
---
title: "Qwen3-235B-A22B"
metatags:
description: "Best Practice for Qwen3-235B-A22B on Ascend NPU"
---
This guide describes the best practice data for Qwen3-235B-A22B on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 11K+1.5K | 8ms | BF16 | [Optimal Configuration](#qwen3-235b-a22b-bf16-8p-in11k-out1k5-8ms) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-w8a8-8p-in3k5-out1k5-50-1ms) |
## Optimal Configuration
### Qwen3-235B-A22B BF16 8P IN11K OUT1K5 8ms
**Model**: Qwen3-235B-A22B
**Hardware**: Atlas 800I A3
**Cards**: 8
**Deploy Mode**: PD Mixed
**Quantization**: BF16
**Dataset**: 11K+1.5K
**TPOT**: 8ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1600
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--max-running-requests 1 \
--dtype bfloat16 \
--chunked-prefill-size -1 \
--max-prefill-tokens 16384 \
--speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--disable-radix-cache \
--enable-dp-lm-head \
--tp 16 \
--mem-fraction-static 0.78 \
--cuda-graph-bs 1
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 11000 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1
```
### Qwen3-235B-A22B W8A8 8P IN3K5 OUT1K5 50.1ms
**Model**: Qwen3-235B-A22B
**Hardware**: Atlas 800I A3
**Cards**: 8
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50.1ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=570
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_NPU_PROFILING=0
export SGLANG_NPU_PROFILING_BS=27
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 432 \
--context-length 8192 \
--dtype bfloat16 \
--chunked-prefill-size 94208 \
--max-prefill-tokens 458880 \
--sampling-backend ascend \
--ep-dispatch-algorithm static \
--init-expert-location /root/.cache/modelscope/hub/models/hot_map/235B_3_5k_decode.pt \
--disable-radix-cache \
--moe-a2a-backend ascend_fuseep \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--tp 16 \
--dp-size 16 \
--enable-dp-attention \
--enable-dp-lm-head \
--mem-fraction-static 0.8 \
--cuda-graph-bs 1 2 4 8 16 20 24 26 27
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 432 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1728 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx
+419
View File
@@ -0,0 +1,419 @@
---
title: "Qwen3-30B-A3B"
metatags:
description: "Best Practice for Qwen3-30B-A3B on Ascend NPU"
---
This guide describes the best practice data for Qwen3-30B-A3B on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 10ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-10ms) |
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 6K+1.5K | 10.25ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in6k-out1k5-bs16) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1K+100 | 10000ms | BF16 | [Optimal Configuration](#qwen3-30b-a3b-bf16-1p-in1k-out100) |
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-50ms) |
## Optimal Configuration
### Qwen3-30B-A3B BF16 1P IN1K OUT100
**Model**: Qwen3-30B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: BF16
**Dataset**: 1K+100
**TPOT**: 10000ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_LAUNCH_BLOCKING=0
export DP_ROUND_ROBIN=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_ALGO=level0:NA;level1:ring
export HCCL_SOCKET_IFNAME=<network-interface>
export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_USE_MAX_DP_ATT=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--max-running-requests 168 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 8300 \
--speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 7 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 8 \
--tp-size 2 \
--enable-dp-attention \
--dp-size 2 \
--mem-fraction-static 0.85 \
--cuda-graph-bs 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 162 \
--random-input-len 1000 \
--random-output-len 100 \
--num-prompts 624 \
--random-range-ratio 1
```
### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 10ms
**Model**: Qwen3-30B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 10ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_LAUNCH_BLOCKING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 162 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tp-size 2 \
--mem-fraction-static 0.87 \
--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1
```
### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 50ms
**Model**: Qwen3-30B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_LAUNCH_BLOCKING=0
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 162 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tp-size 2 \
--mem-fraction-static 0.87 \
--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 160 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 640 \
--random-range-ratio 1
```
### Qwen3-30B-A3B W8A8 1P IN6K OUT1K5 BS16
**Model**: Qwen3-30B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 6K+1.5K
**TPOT**: 10.25ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=400
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export TRANSFORMERS_VERBOSITY=error
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 16 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--tp-size 2 \
--mem-fraction-static 0.6 \
--cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 16 \
--random-input-len 6144 \
--random-output-len 1500 \
--num-prompts 16 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx
+312
View File
@@ -0,0 +1,312 @@
---
title: "Qwen3-32B"
metatags:
description: "Best Practice for Qwen3-32B on Ascend NPU"
---
This guide describes the best practice data for Qwen3-32B on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-32B | Atlas 800I A3 | 8 | PD Mixed | 18K+4K | 6ms | BF16 | [Optimal Configuration](#qwen3-32b-bf16-8p-in18k-out4k-6ms) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms-a2) |
| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) |
## Optimal Configuration
### Qwen3-32B BF16 8P IN18K OUT4K 6ms
**Model**: Qwen3-32B
**Hardware**: Atlas 800I A3
**Cards**: 8
**Deploy Mode**: PD Mixed
**Quantization**: BF16
**Dataset**: 18K+4K
**TPOT**: 6ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--max-running-requests 1 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 65536 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--tp-size 16 \
--mem-fraction-static 0.72 \
--cuda-graph-bs 1 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 18000 \
--random-output-len 4000 \
--num-prompts 1 \
--random-range-ratio 1
```
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms A2
**Model**: Qwen3-32B
**Hardware**: Atlas 800I A2
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 101 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tp-size 4 \
--mem-fraction-static 0.845 \
--cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 100 \
--random-input-len 3584 \
--random-output-len 1536 \
--num-prompts 400 \
--random-range-ratio 1
```
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms
**Model**: Qwen3-32B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--nnodes 1 \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 101 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-prefill-tokens 35000 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--tp-size 4 \
--mem-fraction-static 0.845 \
--cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 100 \
--random-input-len 3584 \
--random-output-len 1536 \
--num-prompts 400 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx
+1155
View File
File diff suppressed because it is too large Load Diff
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx
+703
View File
@@ -0,0 +1,703 @@
---
title: "Qwen3.6-27B"
metatags:
description: "Best Practice for Qwen3.6-27B on Ascend NPU"
---
This guide describes the best practice data for Qwen3.6-27B on the Ascend NPU.
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-1p-in1024x1024-30-out1024-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-2p-in64k-out1k-prefix90-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in3k5-out1k5-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) |
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) |
## Optimal Configuration
### Qwen3.6-27B 1P IN1024X1024 30 OUT1024 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 1024x1024 (30)+1024
*Format: resolution (input tokens) + output tokens*
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_PROFILING=1
export SGLANG_NPU_PROFILING_STAGE=prefill
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 52000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 50 \
--max-mamba-cache-size 60 \
--mem-fraction-static 0.76 \
--cuda-graph-bs 2 4 8 16 24 32 40 42 45 50 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--mm-enable-dp-encoder
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 48 \
--random-input-len 30 \
--random-output-len 1024 \
--num-prompts 48 \
--random-range-ratio 1
```
### Qwen3.6-27B 1P IN1080P 30 OUT256 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 1080p_30+256
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_PROFILING=1
export SGLANG_NPU_PROFILING_STAGE=prefill
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 48000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 30 \
--max-mamba-cache-size 40 \
--mem-fraction-static 0.76 \
--cuda-graph-bs 2 4 8 16 24 28 30 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--mm-enable-dp-encoder
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 30 \
--random-input-len 30 \
--random-output-len 256 \
--num-prompts 120 \
--random-range-ratio 1
```
### Qwen3.6-27B 2P IN64K OUT1K PREFIX90 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 64K+1K (90% prefix cache hit rate)
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size 32768 \
--max-prefill-tokens 32768 \
--mamba-scheduler-strategy extra_buffer \
--trust-remote-code \
--max-running-requests 20 \
--max-mamba-cache-size 108 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 1 2 4 8 12 15 17 19 20 \
--enable-prefill-delayer \
--prefill-delayer-queue-min-ratio 0.8 \
--prefill-delayer-max-delay-ms 30000 \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6399` = `int(64000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```shell Command
python -m sglang.bench_serving \
--dataset-name generated-shared-prefix \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 80 \
--gsp-system-prompt-len 57600 \
--gsp-question-len 6399 \
--gsp-output-len 1000 \
--max-concurrency 20 \
--num-prompts 80 \
--request-rate inf
```
### Qwen3.6-27B W8A8 1P IN3K5 OUT1K5 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 60000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 64 \
--max-mamba-cache-size 74 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 2 8 16 32 40 45 50 54 \
--enable-multimodal \
--quantization modelslim \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 54 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 216 \
--random-range-ratio 1
```
### Qwen3.6-27B W8A8 2P IN128K OUT1K 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 128K+1K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 4 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 74000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 6 \
--max-mamba-cache-size 7 \
--mem-fraction-static 0.63 \
--cuda-graph-bs 1 2 4 5 6 \
--enable-multimodal \
--quantization modelslim \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 4 \
--random-input-len 128000 \
--random-output-len 1000 \
--num-prompts 16 \
--random-range-ratio 1
```
### Qwen3.6-27B W8A8 2P IN16K OUT1K 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 16K+1K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 4 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 50000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 28 \
--max-mamba-cache-size 50 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 2 8 12 16 20 24 26 28 \
--enable-multimodal \
--quantization modelslim \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 28 \
--random-input-len 16000 \
--random-output-len 1000 \
--num-prompts 112 \
--random-range-ratio 1
```
### Qwen3.6-27B W8A8 2P IN64K OUT1K 50ms
**Model**: Qwen3.6-27B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 64K+1K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 4 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 50000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 28 \
--max-mamba-cache-size 50 \
--mem-fraction-static 0.7 \
--cuda-graph-bs 2 4 6 \
--enable-multimodal \
--quantization modelslim \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 6 \
--random-input-len 64000 \
--random-output-len 1000 \
--num-prompts 24 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx
+903
View File
@@ -0,0 +1,903 @@
---
title: "Qwen3.6-35B-A3B"
metatags:
description: "Best Practice for Qwen3.6-35B-A3B on Ascend NPU"
---
This guide describes the best practice data for Qwen3.6-35B-A3B on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 254K+1K | 16.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in254k-out1k) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) |
| Qwen3.6-35B-A3B | Atlas 800I A3 | 2 | PD Mixed | 984K+1K | 40.91ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) |
## Optimal Configuration
### Qwen3.6-35B-A3B 1P IN1024X1024 30 OUT1024 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 1024x1024 (30)+1024
*Format: resolution (input tokens) + output tokens*
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 16384 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--max-running-requests 120 \
--max-mamba-cache-size 240 \
--mem-fraction-static 0.78 \
--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 120 \
--random-input-len 30 \
--random-output-len 1024 \
--num-prompts 480 \
--random-range-ratio 1 \
--request-rate inf
```
### Qwen3.6-35B-A3B 1P IN1080P 30 OUT256 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 1080p_30+256
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 16384 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--max-running-requests 50 \
--max-mamba-cache-size 55 \
--mem-fraction-static 0.8 \
--cuda-graph-bs 2 4 8 12 16 20 24 28 32 36 40 44 48 50 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 50 \
--random-input-len 30 \
--random-output-len 256 \
--num-prompts 200 \
--random-range-ratio 1 \
--request-rate inf
```
### Qwen3.6-35B-A3B 1P IN128K OUT1K 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 128K+1K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1600
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 128000 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--max-running-requests 3 \
--max-mamba-cache-size 10 \
--mem-fraction-static 0.63 \
--cuda-graph-bs 1 2 3 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 3 \
--random-input-len 128000 \
--random-output-len 1000 \
--num-prompts 12 \
--random-range-ratio 1
```
### Qwen3.6-35B-A3B 1P IN128K OUT1K PREFIX90 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 128K+1K (90% prefix cache hit rate)
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size 16384 \
--max-prefill-tokens 65536 \
--trust-remote-code \
--enable-prefill-delayer \
--mamba-scheduler-strategy extra_buffer \
--max-running-requests 103 \
--max-mamba-cache-size 85 \
--mem-fraction-static 0.85 \
--cuda-graph-bs 2 4 8 16 32 48 64 80 96 103 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6399` = `int(64000 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```shell Command
python -m sglang.bench_serving \
--dataset-name generated-shared-prefix \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 412 \
--gsp-system-prompt-len 57600 \
--gsp-question-len 6399 \
--gsp-output-len 1000 \
--max-concurrency 103 \
--num-prompts 412 \
--request-rate inf
```
### Qwen3.6-35B-A3B 1P IN254K OUT1K
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 254K+1K
**TPOT**: 16.1ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size 131072 \
--max-prefill-tokens 254000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 1 \
--max-mamba-cache-size 6 \
--mem-fraction-static 0.65 \
--cuda-graph-bs 1 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 254000 \
--random-output-len 1000 \
--num-prompts 1 \
--random-range-ratio 1
```
### Qwen3.6-35B-A3B 1P IN3K5 OUT1K5 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=1
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 43400 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--prefill-delayer-max-delay-passes 50 \
--max-running-requests 124 \
--max-mamba-cache-size 124 \
--mem-fraction-static 0.8 \
--cuda-graph-bs 4 16 32 64 96 112 116 120 124 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 124 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 496 \
--random-range-ratio 1
```
### Qwen3.6-35B-A3B 1P IN64K OUT1K 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 64K+1K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-total-tokens 600000 \
--max-prefill-tokens 65536 \
--disable-radix-cache \
--trust-remote-code \
--enable-prefill-delayer \
--max-running-requests 10 \
--max-mamba-cache-size 20 \
--mem-fraction-static 0.65 \
--cuda-graph-bs 2 4 8 12 14 16 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 10 \
--random-input-len 64000 \
--random-output-len 1000 \
--num-prompts 40 \
--random-range-ratio 1
```
### Qwen3.6-35B-A3B 1P IN64K OUT1K PREFIX90 50ms
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 1
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 64K+1K (90% prefix cache hit rate)
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GDN_ATTN_BACKEND_TRITON=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=300
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 2 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size -1 \
--max-prefill-tokens 65536 \
--trust-remote-code \
--enable-prefill-delayer \
--mamba-scheduler-strategy extra_buffer \
--max-running-requests 42 \
--max-mamba-cache-size 210 \
--mem-fraction-static 0.71 \
--cuda-graph-bs 2 8 16 24 32 36 40 42 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4
```
#### Benchmark
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion.
`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix.
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
```shell Command
python -m sglang.bench_serving \
--dataset-name generated-shared-prefix \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--gsp-num-groups 1 \
--gsp-prompts-per-group 42 \
--gsp-system-prompt-len 58982 \
--gsp-question-len 6553 \
--gsp-output-len 1024 \
--max-concurrency 42 \
--num-prompts 42 \
--request-rate inf
```
### Qwen3.6-35B-A3B 2P IN984K OUT1K
**Model**: Qwen3.6-35B-A3B
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 984K+1K
**TPOT**: 40.91ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--tp-size 4 \
--nnodes 1 \
--attention-backend ascend \
--device npu \
--chunked-prefill-size 131072 \
--max-prefill-tokens 984000 \
--disable-radix-cache \
--trust-remote-code \
--max-running-requests 1 \
--max-mamba-cache-size 6 \
--mem-fraction-static 0.68 \
--cuda-graph-bs 1 \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--context-length 1010000
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 984000 \
--random-output-len 1000 \
--num-prompts 1 \
--random-range-ratio 1
```
docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx
+362
View File
@@ -0,0 +1,362 @@
---
title: "Qwen3-Next-80B-A3B-Instruct"
metatags:
description: "Best Practice for Qwen3-Next-80B-A3B-Instruct on Ascend NPU"
---
This guide describes the best practice data for Qwen3-Next-80B-A3B-Instruct on the Ascend NPU.
### Low Latency
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 20ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-20ms) |
| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 6K+1.5K | 15.62ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in6k-out1k5-bs16) |
### High Throughput
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
| --- | --- | --- | --- | --- | --- | --- | --- |
| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-50ms) |
## Optimal Configuration
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 20ms
**Model**: Qwen3-Next-80B-A3B-Instruct
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 20ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_WARMUP_TIMEOUT=3600
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
export ZBCCL_ENABLE_GRAPH=1
export ZBCCL_LOCAL_MEM_SIZE=60416
export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--page-size 128 \
--tp-size 2 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.85 \
--disable-radix-cache \
--max-prefill-tokens 28672 \
--context-length 26384 \
--max-total-tokens 122304 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-running-requests 2 \
--cuda-graph-bs 2 \
--mamba-ssm-dtype bfloat16 \
--speculative-draft-model-path $DRAFT_MODEL_PATH
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 1 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 1 \
--random-range-ratio 1
```
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 50ms
**Model**: Qwen3-Next-80B-A3B-Instruct
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 3.5K+1.5K
**TPOT**: 50ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=64
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_WARMUP_TIMEOUT=3600
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648
export STREAMS_PER_DEVICE=32
export ZBAL_ENABLE_GRAPH=1
export ZBAL_HCCL_OP=allreduce,_allgather_base,allgather,broadcast,scatter,reduce_scatter,_reduce_scatter_base,alltoall_base
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--page-size 128 \
--tp-size 4 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.75 \
--disable-radix-cache \
--max-prefill-tokens 14080 \
--context-length 26384 \
--chunked-prefill-size -1 \
--max-running-requests 300 \
--mamba-ssm-dtype bfloat16 \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--dp-size 2 \
--enable-dp-attention \
--enable-dp-lm-head \
--moe-a2a-backend deepep \
--deepep-mode auto \
--cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 300 \
--random-input-len 3500 \
--random-output-len 1500 \
--num-prompts 300 \
--random-range-ratio 1
```
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN6K OUT1K5 BS16
**Model**: Qwen3-Next-80B-A3B-Instruct
**Hardware**: Atlas 800I A3
**Cards**: 2
**Deploy Mode**: PD Mixed
**Quantization**: W8A8 INT8
**Dataset**: 6K+1.5K
**TPOT**: 15.62ms
#### Model Deployment
```bash Command
# ============================================================
# Before running, update the following variables:
# MODEL_PATH: path to the model weights directory
# DRAFT_MODEL_PATH: path to the draft model weights directory
# HCCL_SOCKET_IFNAME: network interface name for HCCL
# GLOO_SOCKET_IFNAME: network interface name for Gloo
# ============================================================
MODEL_PATH=/path/to/model-weights
DRAFT_MODEL_PATH=/path/to/draft-model-weights
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export ASCEND_USE_FIA=1
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export FORCE_DRAFT_MODEL_NON_QUANT=1
export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_BUFFSIZE=2000
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_WARMUP_TIMEOUT=3600
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
export ZBCCL_ENABLE_GRAPH=1
export ZBCCL_LOCAL_MEM_SIZE=60416
export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 --port 6688 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--page-size 128 \
--tp-size 4 \
--watchdog-timeout 9000 \
--mem-fraction-static 0.85 \
--disable-radix-cache \
--max-prefill-tokens 28672 \
--context-length 81920 \
--max-total-tokens 122304 \
--dp-size 2 \
--enable-dp-attention \
--enable-dp-lm-head \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 \
--max-running-requests 16 \
--cuda-graph-bs 2 4 8 \
--mamba-ssm-dtype bfloat16 \
--speculative-draft-model-path $DRAFT_MODEL_PATH
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving \
--dataset-name random \
--backend sglang \
--host 127.0.0.1 \
--port 6688 \
--max-concurrency 16 \
--random-input-len 6144 \
--random-output-len 1500 \
--num-prompts 16 \
--random-range-ratio 1
```