[NPU] Best Practice Docs Splitting (#27663)
This commit is contained in:
jianzhao-xu
@@ -885,6 +885,24 @@
|
||||
"docs/hardware-platforms/ascend-npus/ascend_contribution_guide",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_support_new_models",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_best_practice",
|
||||
{
|
||||
"group": "Best Practice",
|
||||
"pages": [
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/glm5_1",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b",
|
||||
"docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct"
|
||||
]
|
||||
},
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_optimization",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation",
|
||||
"docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,590 @@
|
||||
---
|
||||
title: "DeepSeek-V3.2"
|
||||
metatags:
|
||||
description: "Best Practice for DeepSeek-V3.2 on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for DeepSeek-V3.2 on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-26ms) |
|
||||
| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 26ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs8) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 107ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs16) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K 26ms
|
||||
|
||||
**Model**: DeepSeek-V3.2
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 32
|
||||
|
||||
**Deploy Mode**: PD Disaggregation
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 128K+1K
|
||||
|
||||
**TPOT**: 26ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# P_IP: prefill node IP address
|
||||
# D_IP: decode node IP address
|
||||
# ASCEND_MF_STORE_URL: prefill node IP with port
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
P_IP=('<your prefill ip1>' '<your prefill ip2>')
|
||||
D_IP=('<your decode ip1>' '<your decode ip2>')
|
||||
|
||||
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
|
||||
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
|
||||
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
|
||||
echo "${LOCAL_HOST1}"
|
||||
echo "${LOCAL_HOST2}"
|
||||
# prefill
|
||||
for i in "${!P_IP[@]}";
|
||||
do
|
||||
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
|
||||
then
|
||||
echo "${P_IP[$i]}"
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=1200
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export TASK_QUEUE_ENABLE=2
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode prefill \
|
||||
--host ${P_IP[$i]} \
|
||||
--port 8000 \
|
||||
--dist-init-addr ${P_IP[0]}:5000 \
|
||||
--disaggregation-bootstrap-port 8998 \
|
||||
--node-rank $i \
|
||||
--nnodes 2 \
|
||||
--tp 32 \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.73 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 68000 \
|
||||
--max-running-requests 1 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode normal \
|
||||
--quantization modelslim \
|
||||
--disaggregation-transfer-backend ascend \
|
||||
--disable-cuda-graph \
|
||||
--moe-dense-tp-size 1 \
|
||||
--enable-nsa-prefill-context-parallel \
|
||||
--nsa-prefill-cp-mode in-seq-split \
|
||||
--attn-cp-size 32 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 1 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 2
|
||||
NODE_RANK=$i
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# decode
|
||||
for i in "${!D_IP[@]}";
|
||||
do
|
||||
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
|
||||
then
|
||||
echo "${D_IP[$i]}"
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=400
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode decode \
|
||||
--host ${D_IP[$i]} \
|
||||
--port 8001 \
|
||||
--dist-init-addr ${D_IP[0]}:5000 \
|
||||
--node-rank $i \
|
||||
--nnodes 2 \
|
||||
--tp 32 \
|
||||
--dp 8 \
|
||||
--ep 32 \
|
||||
--moe-dense-tp-size 1 \
|
||||
--enable-dp-attention \
|
||||
--enable-dp-lm-head \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.79 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 68000 \
|
||||
--max-running-requests 32 \
|
||||
--cuda-graph-max-bs 4 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode low_latency \
|
||||
--quantization modelslim \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--disaggregation-transfer-backend ascend
|
||||
NODE_RANK=$i
|
||||
break
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
```shell Command
|
||||
# ============================================================
|
||||
# Before running, replace the following placeholders:
|
||||
# <your prefill ip>: prefill node IP address
|
||||
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
|
||||
# ============================================================
|
||||
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
--prefill http://<your prefill ip>:8000 8998 \
|
||||
--decode http://<your decode ip1>:8001 \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--mini-lb
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 8 \
|
||||
--random-input-len 131072 \
|
||||
--random-output-len 1024 \
|
||||
--num-prompts 8 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K BS16
|
||||
|
||||
**Model**: DeepSeek-V3.2
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 32
|
||||
|
||||
**Deploy Mode**: PD Disaggregation
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 128K+1K
|
||||
|
||||
**TPOT**: 107ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# P_IP: prefill node IP address
|
||||
# D_IP: decode node IP address
|
||||
# ASCEND_MF_STORE_URL: prefill node IP with port
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
P_IP=('<your prefill ip1>' '<your prefill ip2>')
|
||||
D_IP=('<your decode ip1>' '<your decode ip2>')
|
||||
|
||||
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
|
||||
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
|
||||
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
|
||||
echo "${LOCAL_HOST1}"
|
||||
echo "${LOCAL_HOST2}"
|
||||
# prefill
|
||||
for i in "${!P_IP[@]}";
|
||||
do
|
||||
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
|
||||
then
|
||||
echo "${P_IP[$i]}"
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=1200
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export TASK_QUEUE_ENABLE=2
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode prefill \
|
||||
--host ${P_IP[$i]} \
|
||||
--port 8000 \
|
||||
--dist-init-addr ${P_IP[0]}:5000 \
|
||||
--disaggregation-bootstrap-port 8998 \
|
||||
--node-rank $i \
|
||||
--nnodes 2 \
|
||||
--tp 32 \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.73 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 68000 \
|
||||
--max-running-requests 1 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode normal \
|
||||
--quantization modelslim \
|
||||
--disaggregation-transfer-backend ascend \
|
||||
--disable-cuda-graph \
|
||||
--moe-dense-tp-size 1 \
|
||||
--enable-nsa-prefill-context-parallel \
|
||||
--nsa-prefill-cp-mode in-seq-split \
|
||||
--attn-cp-size 32 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 1 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 2
|
||||
NODE_RANK=$i
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# decode
|
||||
for i in "${!D_IP[@]}";
|
||||
do
|
||||
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
|
||||
then
|
||||
echo "${D_IP[$i]}"
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=400
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode decode \
|
||||
--host ${D_IP[$i]} \
|
||||
--port 8001 \
|
||||
--dist-init-addr ${D_IP[0]}:5000 \
|
||||
--node-rank $i \
|
||||
--nnodes 2 \
|
||||
--tp 32 \
|
||||
--dp 8 \
|
||||
--ep 32 \
|
||||
--moe-dense-tp-size 1 \
|
||||
--enable-dp-attention \
|
||||
--enable-dp-lm-head \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.79 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 68000 \
|
||||
--max-running-requests 32 \
|
||||
--cuda-graph-max-bs 4 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode low_latency \
|
||||
--quantization modelslim \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--disaggregation-transfer-backend ascend
|
||||
NODE_RANK=$i
|
||||
break
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
```shell Command
|
||||
# ============================================================
|
||||
# Before running, replace the following placeholders:
|
||||
# <your prefill ip>: prefill node IP address
|
||||
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
|
||||
# ============================================================
|
||||
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
--prefill http://<your prefill ip>:8000 8998 \
|
||||
--decode http://<your decode ip1>:8001 \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--mini-lb
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 16 \
|
||||
--random-input-len 131072 \
|
||||
--random-output-len 1024 \
|
||||
--num-prompts 16 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K BS8
|
||||
|
||||
**Model**: DeepSeek-V3.2
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 32
|
||||
|
||||
**Deploy Mode**: PD Disaggregation
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 128K+1K
|
||||
|
||||
**TPOT**: 26ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# P_IP: prefill node IP address
|
||||
# D_IP: decode node IP address
|
||||
# ASCEND_MF_STORE_URL: prefill node IP with port
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
P_IP=('<your prefill ip1>' '<your prefill ip2>')
|
||||
D_IP=('<your decode ip1>' '<your decode ip2>')
|
||||
|
||||
export ASCEND_MF_STORE_URL="tcp://<your prefill ip1>:24670"
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
|
||||
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
|
||||
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
|
||||
echo "${LOCAL_HOST1}"
|
||||
echo "${LOCAL_HOST2}"
|
||||
# prefill
|
||||
for i in "${!P_IP[@]}";
|
||||
do
|
||||
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
|
||||
then
|
||||
echo "${P_IP[$i]}"
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=1200
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export TASK_QUEUE_ENABLE=2
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode prefill \
|
||||
--host ${P_IP[$i]} \
|
||||
--port 8000 \
|
||||
--dist-init-addr ${P_IP[0]}:5000 \
|
||||
--disaggregation-bootstrap-port 8998 \
|
||||
--node-rank $i \
|
||||
--nnodes 2 \
|
||||
--tp 32 \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.73 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 68000 \
|
||||
--max-running-requests 1 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode normal \
|
||||
--quantization modelslim \
|
||||
--disaggregation-transfer-backend ascend \
|
||||
--disable-cuda-graph \
|
||||
--moe-dense-tp-size 1 \
|
||||
--enable-nsa-prefill-context-parallel \
|
||||
--nsa-prefill-cp-mode in-seq-split \
|
||||
--attn-cp-size 32 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 1 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 2
|
||||
NODE_RANK=$i
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# decode
|
||||
for i in "${!D_IP[@]}";
|
||||
do
|
||||
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
|
||||
then
|
||||
echo "${D_IP[$i]}"
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=400
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode decode \
|
||||
--host ${D_IP[$i]} \
|
||||
--port 8001 \
|
||||
--dist-init-addr ${D_IP[0]}:5000 \
|
||||
--node-rank $i \
|
||||
--nnodes 2 \
|
||||
--tp 32 \
|
||||
--dp 8 \
|
||||
--ep 32 \
|
||||
--moe-dense-tp-size 1 \
|
||||
--enable-dp-attention \
|
||||
--enable-dp-lm-head \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.79 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 68000 \
|
||||
--max-running-requests 32 \
|
||||
--cuda-graph-max-bs 4 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode low_latency \
|
||||
--quantization modelslim \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--disaggregation-transfer-backend ascend
|
||||
NODE_RANK=$i
|
||||
break
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
```shell Command
|
||||
# ============================================================
|
||||
# Before running, replace the following placeholders:
|
||||
# <your prefill ip>: prefill node IP address
|
||||
# <your decode ip1>: first decode node IP address (decode may have distributed nodes)
|
||||
# ============================================================
|
||||
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
--prefill http://<your prefill ip>:8000 8998 \
|
||||
--decode http://<your decode ip1>:8001 \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--mini-lb
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 8 \
|
||||
--random-input-len 131072 \
|
||||
--random-output-len 1024 \
|
||||
--num-prompts 8 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,307 @@
|
||||
---
|
||||
title: "Qwen3-8B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3-8B on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3-8B on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 5ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-5ms) |
|
||||
| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 6K+1.5K | 11.79ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in6k-out1k5-bs16) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 37ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-37ms) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3-8B W8A8 1P IN3K5 OUT1K5 37ms
|
||||
|
||||
**Model**: Qwen3-8B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 37ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 70 \
|
||||
--max-prefill-tokens 16384 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size 16384 \
|
||||
--tp-size 1 \
|
||||
--mem-fraction-static 0.85 \
|
||||
--cuda-graph-bs 8 12 24 36 48 51 55 60 63 64 66 68 70 \
|
||||
--dtype bfloat16 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 64 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 256 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-8B W8A8 1P IN3K5 OUT1K5 5ms
|
||||
|
||||
**Model**: Qwen3-8B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 5ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 1 \
|
||||
--max-prefill-tokens 16384 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--tp-size 2 \
|
||||
--mem-fraction-static 0.894 \
|
||||
--cuda-graph-bs 1 \
|
||||
--dtype bfloat16 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 4 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 5
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 4 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-8B W8A8 1P IN6K OUT1K5 BS16
|
||||
|
||||
**Model**: Qwen3-8B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 6K+1.5K
|
||||
|
||||
**TPOT**: 11.79ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 16 \
|
||||
--max-prefill-tokens 16384 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--tp-size 2 \
|
||||
--mem-fraction-static 0.894 \
|
||||
--cuda-graph-bs 1 5 15 16 \
|
||||
--dtype bfloat16 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 4 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 5
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 16 \
|
||||
--random-input-len 6144 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 16 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
@@ -0,0 +1,226 @@
|
||||
---
|
||||
title: "Qwen3-235B-A22B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3-235B-A22B on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3-235B-A22B on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 11K+1.5K | 8ms | BF16 | [Optimal Configuration](#qwen3-235b-a22b-bf16-8p-in11k-out1k5-8ms) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-w8a8-8p-in3k5-out1k5-50-1ms) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3-235B-A22B BF16 8P IN11K OUT1K5 8ms
|
||||
|
||||
**Model**: Qwen3-235B-A22B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 8
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: BF16
|
||||
|
||||
**Dataset**: 11K+1.5K
|
||||
|
||||
**TPOT**: 8ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=1600
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--max-running-requests 1 \
|
||||
--dtype bfloat16 \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 16384 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 4 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 5 \
|
||||
--disable-radix-cache \
|
||||
--enable-dp-lm-head \
|
||||
--tp 16 \
|
||||
--mem-fraction-static 0.78 \
|
||||
--cuda-graph-bs 1
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 11000 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 1 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-235B-A22B W8A8 8P IN3K5 OUT1K5 50.1ms
|
||||
|
||||
**Model**: Qwen3-235B-A22B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 8
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50.1ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=570
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_NPU_PROFILING=0
|
||||
export SGLANG_NPU_PROFILING_BS=27
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 432 \
|
||||
--context-length 8192 \
|
||||
--dtype bfloat16 \
|
||||
--chunked-prefill-size 94208 \
|
||||
--max-prefill-tokens 458880 \
|
||||
--sampling-backend ascend \
|
||||
--ep-dispatch-algorithm static \
|
||||
--init-expert-location /root/.cache/modelscope/hub/models/hot_map/235B_3_5k_decode.pt \
|
||||
--disable-radix-cache \
|
||||
--moe-a2a-backend ascend_fuseep \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--tp 16 \
|
||||
--dp-size 16 \
|
||||
--enable-dp-attention \
|
||||
--enable-dp-lm-head \
|
||||
--mem-fraction-static 0.8 \
|
||||
--cuda-graph-bs 1 2 4 8 16 20 24 26 27
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 432 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 1728 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
@@ -0,0 +1,419 @@
|
||||
---
|
||||
title: "Qwen3-30B-A3B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3-30B-A3B on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3-30B-A3B on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 10ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-10ms) |
|
||||
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 6K+1.5K | 10.25ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in6k-out1k5-bs16) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1K+100 | 10000ms | BF16 | [Optimal Configuration](#qwen3-30b-a3b-bf16-1p-in1k-out100) |
|
||||
| Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-50ms) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3-30B-A3B BF16 1P IN1K OUT100
|
||||
|
||||
**Model**: Qwen3-30B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: BF16
|
||||
|
||||
**Dataset**: 1K+100
|
||||
|
||||
**TPOT**: 10000ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_LAUNCH_BLOCKING=0
|
||||
export DP_ROUND_ROBIN=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_ALGO=level0:NA;level1:ring
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export INF_NAN_MODE_FORCE_DISABLE=1
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
|
||||
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_USE_MAX_DP_ATT=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--max-running-requests 168 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 8300 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 7 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 8 \
|
||||
--tp-size 2 \
|
||||
--enable-dp-attention \
|
||||
--dp-size 2 \
|
||||
--mem-fraction-static 0.85 \
|
||||
--cuda-graph-bs 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 162 \
|
||||
--random-input-len 1000 \
|
||||
--random-output-len 100 \
|
||||
--num-prompts 624 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 10ms
|
||||
|
||||
**Model**: Qwen3-30B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 10ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_LAUNCH_BLOCKING=0
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=400
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 162 \
|
||||
--disable-radix-cache \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 35000 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--tp-size 2 \
|
||||
--mem-fraction-static 0.87 \
|
||||
--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 1 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 50ms
|
||||
|
||||
**Model**: Qwen3-30B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_LAUNCH_BLOCKING=0
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=400
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 162 \
|
||||
--disable-radix-cache \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 35000 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--tp-size 2 \
|
||||
--mem-fraction-static 0.87 \
|
||||
--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 160 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 640 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-30B-A3B W8A8 1P IN6K OUT1K5 BS16
|
||||
|
||||
**Model**: Qwen3-30B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 6K+1.5K
|
||||
|
||||
**TPOT**: 10.25ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=400
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export TRANSFORMERS_VERBOSITY=error
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 16 \
|
||||
--disable-radix-cache \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 4 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 5 \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 35000 \
|
||||
--tp-size 2 \
|
||||
--mem-fraction-static 0.6 \
|
||||
--cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 16 \
|
||||
--random-input-len 6144 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 16 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
@@ -0,0 +1,312 @@
|
||||
---
|
||||
title: "Qwen3-32B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3-32B on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3-32B on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-32B | Atlas 800I A3 | 8 | PD Mixed | 18K+4K | 6ms | BF16 | [Optimal Configuration](#qwen3-32b-bf16-8p-in18k-out4k-6ms) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms-a2) |
|
||||
| Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3-32B BF16 8P IN18K OUT4K 6ms
|
||||
|
||||
**Model**: Qwen3-32B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 8
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: BF16
|
||||
|
||||
**Dataset**: 18K+4K
|
||||
|
||||
**TPOT**: 6ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--max-running-requests 1 \
|
||||
--disable-radix-cache \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 65536 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 4 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 5 \
|
||||
--tp-size 16 \
|
||||
--mem-fraction-static 0.72 \
|
||||
--cuda-graph-bs 1 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 18000 \
|
||||
--random-output-len 4000 \
|
||||
--num-prompts 1 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms A2
|
||||
|
||||
**Model**: Qwen3-32B
|
||||
|
||||
**Hardware**: Atlas 800I A2
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_DEEPGEMM=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 101 \
|
||||
--disable-radix-cache \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 35000 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--tp-size 4 \
|
||||
--mem-fraction-static 0.845 \
|
||||
--cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 100 \
|
||||
--random-input-len 3584 \
|
||||
--random-output-len 1536 \
|
||||
--num-prompts 400 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms
|
||||
|
||||
**Model**: Qwen3-32B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_DEEPGEMM=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 101 \
|
||||
--disable-radix-cache \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 35000 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--tp-size 4 \
|
||||
--mem-fraction-static 0.845 \
|
||||
--cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \
|
||||
--dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 100 \
|
||||
--random-input-len 3584 \
|
||||
--random-output-len 1536 \
|
||||
--num-prompts 400 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,703 @@
|
||||
---
|
||||
title: "Qwen3.6-27B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3.6-27B on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3.6-27B on the Ascend NPU.
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-1p-in1024x1024-30-out1024-50ms) |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-2p-in64k-out1k-prefix90-50ms) |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in3k5-out1k5-50ms) |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) |
|
||||
| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3.6-27B 1P IN1024X1024 30 OUT1024 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 1024x1024 (30)+1024
|
||||
|
||||
*Format: resolution (input tokens) + output tokens*
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_PROFILING=1
|
||||
export SGLANG_NPU_PROFILING_STAGE=prefill
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 52000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 50 \
|
||||
--max-mamba-cache-size 60 \
|
||||
--mem-fraction-static 0.76 \
|
||||
--cuda-graph-bs 2 4 8 16 24 32 40 42 45 50 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--mm-enable-dp-encoder
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 48 \
|
||||
--random-input-len 30 \
|
||||
--random-output-len 1024 \
|
||||
--num-prompts 48 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-27B 1P IN1080P 30 OUT256 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 1080p_30+256
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_PROFILING=1
|
||||
export SGLANG_NPU_PROFILING_STAGE=prefill
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_VIT_ENABLE_CUDA_GRAPH=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 48000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 30 \
|
||||
--max-mamba-cache-size 40 \
|
||||
--mem-fraction-static 0.76 \
|
||||
--cuda-graph-bs 2 4 8 16 24 28 30 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--mm-enable-dp-encoder
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 30 \
|
||||
--random-input-len 30 \
|
||||
--random-output-len 256 \
|
||||
--num-prompts 120 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-27B 2P IN64K OUT1K PREFIX90 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 64K+1K (90% prefix cache hit rate)
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GDN_ATTN_BACKEND_TRITON=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size 32768 \
|
||||
--max-prefill-tokens 32768 \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 20 \
|
||||
--max-mamba-cache-size 108 \
|
||||
--mem-fraction-static 0.7 \
|
||||
--cuda-graph-bs 1 2 4 8 12 15 17 19 20 \
|
||||
--enable-prefill-delayer \
|
||||
--prefill-delayer-queue-min-ratio 0.8 \
|
||||
--prefill-delayer-max-delay-ms 30000 \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
|
||||
`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion.
|
||||
`--gsp-question-len 6399` = `int(64000 * (1 - 0.9))` is the unique per-request suffix.
|
||||
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name generated-shared-prefix \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--gsp-num-groups 1 \
|
||||
--gsp-prompts-per-group 80 \
|
||||
--gsp-system-prompt-len 57600 \
|
||||
--gsp-question-len 6399 \
|
||||
--gsp-output-len 1000 \
|
||||
--max-concurrency 20 \
|
||||
--num-prompts 80 \
|
||||
--request-rate inf
|
||||
```
|
||||
|
||||
### Qwen3.6-27B W8A8 1P IN3K5 OUT1K5 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 60000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 64 \
|
||||
--max-mamba-cache-size 74 \
|
||||
--mem-fraction-static 0.7 \
|
||||
--cuda-graph-bs 2 8 16 32 40 45 50 54 \
|
||||
--enable-multimodal \
|
||||
--quantization modelslim \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 54 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 216 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-27B W8A8 2P IN128K OUT1K 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 128K+1K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 4 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 74000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 6 \
|
||||
--max-mamba-cache-size 7 \
|
||||
--mem-fraction-static 0.63 \
|
||||
--cuda-graph-bs 1 2 4 5 6 \
|
||||
--enable-multimodal \
|
||||
--quantization modelslim \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 4 \
|
||||
--random-input-len 128000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 16 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-27B W8A8 2P IN16K OUT1K 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 16K+1K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 4 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 50000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 28 \
|
||||
--max-mamba-cache-size 50 \
|
||||
--mem-fraction-static 0.7 \
|
||||
--cuda-graph-bs 2 8 12 16 20 24 26 28 \
|
||||
--enable-multimodal \
|
||||
--quantization modelslim \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 28 \
|
||||
--random-input-len 16000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 112 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-27B W8A8 2P IN64K OUT1K 50ms
|
||||
|
||||
**Model**: Qwen3.6-27B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 64K+1K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 4 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 50000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 28 \
|
||||
--max-mamba-cache-size 50 \
|
||||
--mem-fraction-static 0.7 \
|
||||
--cuda-graph-bs 2 4 6 \
|
||||
--enable-multimodal \
|
||||
--quantization modelslim \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 6 \
|
||||
--random-input-len 64000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 24 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
@@ -0,0 +1,903 @@
|
||||
---
|
||||
title: "Qwen3.6-35B-A3B"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3.6-35B-A3B on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3.6-35B-A3B on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 254K+1K | 16.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in254k-out1k) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) |
|
||||
| Qwen3.6-35B-A3B | Atlas 800I A3 | 2 | PD Mixed | 984K+1K | 40.91ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN1024X1024 30 OUT1024 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 1024x1024 (30)+1024
|
||||
|
||||
*Format: resolution (input tokens) + output tokens*
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 16384 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--max-running-requests 120 \
|
||||
--max-mamba-cache-size 240 \
|
||||
--mem-fraction-static 0.78 \
|
||||
--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 120 \
|
||||
--random-input-len 30 \
|
||||
--random-output-len 1024 \
|
||||
--num-prompts 480 \
|
||||
--random-range-ratio 1 \
|
||||
--request-rate inf
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN1080P 30 OUT256 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 1080p_30+256
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 16384 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--max-running-requests 50 \
|
||||
--max-mamba-cache-size 55 \
|
||||
--mem-fraction-static 0.8 \
|
||||
--cuda-graph-bs 2 4 8 12 16 20 24 28 32 36 40 44 48 50 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 50 \
|
||||
--random-input-len 30 \
|
||||
--random-output-len 256 \
|
||||
--num-prompts 200 \
|
||||
--random-range-ratio 1 \
|
||||
--request-rate inf
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN128K OUT1K 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 128K+1K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GDN_ATTN_BACKEND_TRITON=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=1600
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 128000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--max-running-requests 3 \
|
||||
--max-mamba-cache-size 10 \
|
||||
--mem-fraction-static 0.63 \
|
||||
--cuda-graph-bs 1 2 3 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 3 \
|
||||
--random-input-len 128000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 12 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN128K OUT1K PREFIX90 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 128K+1K (90% prefix cache hit rate)
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size 16384 \
|
||||
--max-prefill-tokens 65536 \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--max-running-requests 103 \
|
||||
--max-mamba-cache-size 85 \
|
||||
--mem-fraction-static 0.85 \
|
||||
--cuda-graph-bs 2 4 8 16 32 48 64 80 96 103 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
|
||||
`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion.
|
||||
`--gsp-question-len 6399` = `int(64000 * (1 - 0.9))` is the unique per-request suffix.
|
||||
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name generated-shared-prefix \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--gsp-num-groups 1 \
|
||||
--gsp-prompts-per-group 412 \
|
||||
--gsp-system-prompt-len 57600 \
|
||||
--gsp-question-len 6399 \
|
||||
--gsp-output-len 1000 \
|
||||
--max-concurrency 103 \
|
||||
--num-prompts 412 \
|
||||
--request-rate inf
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN254K OUT1K
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 254K+1K
|
||||
|
||||
**TPOT**: 16.1ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size 131072 \
|
||||
--max-prefill-tokens 254000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 1 \
|
||||
--max-mamba-cache-size 6 \
|
||||
--mem-fraction-static 0.65 \
|
||||
--cuda-graph-bs 1 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 254000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 1 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN3K5 OUT1K5 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=1
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 43400 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--prefill-delayer-max-delay-passes 50 \
|
||||
--max-running-requests 124 \
|
||||
--max-mamba-cache-size 124 \
|
||||
--mem-fraction-static 0.8 \
|
||||
--cuda-graph-bs 4 16 32 64 96 112 116 120 124 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 124 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 496 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN64K OUT1K 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 64K+1K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-total-tokens 600000 \
|
||||
--max-prefill-tokens 65536 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--max-running-requests 10 \
|
||||
--max-mamba-cache-size 20 \
|
||||
--mem-fraction-static 0.65 \
|
||||
--cuda-graph-bs 2 4 8 12 14 16 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 10 \
|
||||
--random-input-len 64000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 40 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 1P IN64K OUT1K PREFIX90 50ms
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 1
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 64K+1K (90% prefix cache hit rate)
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GDN_ATTN_BACKEND_TRITON=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=300
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 2 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 65536 \
|
||||
--trust-remote-code \
|
||||
--enable-prefill-delayer \
|
||||
--mamba-scheduler-strategy extra_buffer \
|
||||
--max-running-requests 42 \
|
||||
--max-mamba-cache-size 210 \
|
||||
--mem-fraction-static 0.71 \
|
||||
--cuda-graph-bs 2 8 16 24 32 36 40 42 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`):
|
||||
`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion.
|
||||
`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix.
|
||||
`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name generated-shared-prefix \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--gsp-num-groups 1 \
|
||||
--gsp-prompts-per-group 42 \
|
||||
--gsp-system-prompt-len 58982 \
|
||||
--gsp-question-len 6553 \
|
||||
--gsp-output-len 1024 \
|
||||
--max-concurrency 42 \
|
||||
--num-prompts 42 \
|
||||
--request-rate inf
|
||||
```
|
||||
|
||||
### Qwen3.6-35B-A3B 2P IN984K OUT1K
|
||||
|
||||
**Model**: Qwen3.6-35B-A3B
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 984K+1K
|
||||
|
||||
**TPOT**: 40.91ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--tp-size 4 \
|
||||
--nnodes 1 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--chunked-prefill-size 131072 \
|
||||
--max-prefill-tokens 984000 \
|
||||
--disable-radix-cache \
|
||||
--trust-remote-code \
|
||||
--max-running-requests 1 \
|
||||
--max-mamba-cache-size 6 \
|
||||
--mem-fraction-static 0.68 \
|
||||
--cuda-graph-bs 1 \
|
||||
--enable-multimodal \
|
||||
--mm-attention-backend ascend_attn \
|
||||
--dtype bfloat16 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--context-length 1010000
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 984000 \
|
||||
--random-output-len 1000 \
|
||||
--num-prompts 1 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
+362
@@ -0,0 +1,362 @@
|
||||
---
|
||||
title: "Qwen3-Next-80B-A3B-Instruct"
|
||||
metatags:
|
||||
description: "Best Practice for Qwen3-Next-80B-A3B-Instruct on Ascend NPU"
|
||||
---
|
||||
|
||||
This guide describes the best practice data for Qwen3-Next-80B-A3B-Instruct on the Ascend NPU.
|
||||
|
||||
### Low Latency
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 20ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-20ms) |
|
||||
| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 6K+1.5K | 15.62ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in6k-out1k5-bs16) |
|
||||
|
||||
### High Throughput
|
||||
|
||||
| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-50ms) |
|
||||
|
||||
|
||||
## Optimal Configuration
|
||||
|
||||
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 20ms
|
||||
|
||||
**Model**: Qwen3-Next-80B-A3B-Instruct
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 20ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
|
||||
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
|
||||
export FORCE_DRAFT_MODEL_NON_QUANT=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=2000
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_WARMUP_TIMEOUT=3600
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
export ZBCCL_ENABLE_GRAPH=1
|
||||
export ZBCCL_LOCAL_MEM_SIZE=60416
|
||||
export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--page-size 128 \
|
||||
--tp-size 2 \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.85 \
|
||||
--disable-radix-cache \
|
||||
--max-prefill-tokens 28672 \
|
||||
--context-length 26384 \
|
||||
--max-total-tokens 122304 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-running-requests 2 \
|
||||
--cuda-graph-bs 2 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 1 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 1 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 50ms
|
||||
|
||||
**Model**: Qwen3-Next-80B-A3B-Instruct
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 3.5K+1.5K
|
||||
|
||||
**TPOT**: 50ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export FORCE_DRAFT_MODEL_NON_QUANT=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=64
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_WARMUP_TIMEOUT=3600
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export ZBAL_ENABLE_GRAPH=1
|
||||
export ZBAL_HCCL_OP=allreduce,_allgather_base,allgather,broadcast,scatter,reduce_scatter,_reduce_scatter_base,alltoall_base
|
||||
export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--page-size 128 \
|
||||
--tp-size 4 \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--disable-radix-cache \
|
||||
--max-prefill-tokens 14080 \
|
||||
--context-length 26384 \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-running-requests 300 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--dp-size 2 \
|
||||
--enable-dp-attention \
|
||||
--enable-dp-lm-head \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode auto \
|
||||
--cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 300 \
|
||||
--random-input-len 3500 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 300 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
|
||||
### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN6K OUT1K5 BS16
|
||||
|
||||
**Model**: Qwen3-Next-80B-A3B-Instruct
|
||||
|
||||
**Hardware**: Atlas 800I A3
|
||||
|
||||
**Cards**: 2
|
||||
|
||||
**Deploy Mode**: PD Mixed
|
||||
|
||||
**Quantization**: W8A8 INT8
|
||||
|
||||
**Dataset**: 6K+1.5K
|
||||
|
||||
**TPOT**: 15.62ms
|
||||
|
||||
#### Model Deployment
|
||||
|
||||
```bash Command
|
||||
# ============================================================
|
||||
# Before running, update the following variables:
|
||||
# MODEL_PATH: path to the model weights directory
|
||||
# DRAFT_MODEL_PATH: path to the draft model weights directory
|
||||
# HCCL_SOCKET_IFNAME: network interface name for HCCL
|
||||
# GLOO_SOCKET_IFNAME: network interface name for Gloo
|
||||
# ============================================================
|
||||
|
||||
MODEL_PATH=/path/to/model-weights
|
||||
DRAFT_MODEL_PATH=/path/to/draft-model-weights
|
||||
|
||||
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
sysctl -w vm.swappiness=0
|
||||
sysctl -w kernel.numa_balancing=0
|
||||
sysctl -w kernel.sched_migration_cost_ns=50000
|
||||
|
||||
unset https_proxy
|
||||
unset http_proxy
|
||||
unset HTTPS_PROXY
|
||||
unset HTTP_PROXY
|
||||
unset ASCEND_LAUNCH_BLOCKING
|
||||
|
||||
source /usr/local/Ascend/ascend-toolkit/set_env.sh
|
||||
source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export ASCEND_USE_FIA=1
|
||||
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
|
||||
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export FORCE_DRAFT_MODEL_NON_QUANT=1
|
||||
export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_BUFFSIZE=2000
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_WARMUP_TIMEOUT=3600
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
export ZBCCL_ENABLE_GRAPH=1
|
||||
export ZBCCL_LOCAL_MEM_SIZE=60416
|
||||
export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
--host 127.0.0.1 --port 6688 \
|
||||
--trust-remote-code \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--page-size 128 \
|
||||
--tp-size 4 \
|
||||
--watchdog-timeout 9000 \
|
||||
--mem-fraction-static 0.85 \
|
||||
--disable-radix-cache \
|
||||
--max-prefill-tokens 28672 \
|
||||
--context-length 81920 \
|
||||
--max-total-tokens 122304 \
|
||||
--dp-size 2 \
|
||||
--enable-dp-attention \
|
||||
--enable-dp-lm-head \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 3 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 4 \
|
||||
--speculative-draft-model-quantization unquant \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-running-requests 16 \
|
||||
--cuda-graph-bs 2 4 8 \
|
||||
--mamba-ssm-dtype bfloat16 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH
|
||||
```
|
||||
|
||||
#### Benchmark
|
||||
|
||||
We tested it based on the `RANDOM` dataset.
|
||||
|
||||
```shell Command
|
||||
python -m sglang.bench_serving \
|
||||
--dataset-name random \
|
||||
--backend sglang \
|
||||
--host 127.0.0.1 \
|
||||
--port 6688 \
|
||||
--max-concurrency 16 \
|
||||
--random-input-len 6144 \
|
||||
--random-output-len 1500 \
|
||||
--num-prompts 16 \
|
||||
--random-range-ratio 1
|
||||
```
|
||||
Reference in New Issue
Block a user