[NPU] update ascend docs (#21807)
This commit is contained in:
@@ -158,7 +158,6 @@ done
|
||||
```
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
@@ -275,7 +274,6 @@ done
|
||||
```
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
@@ -559,7 +557,6 @@ done
|
||||
```
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
@@ -760,7 +757,6 @@ done
|
||||
```
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
@@ -909,7 +905,6 @@ do
|
||||
--disaggregation-transfer-backend ascend \
|
||||
--disaggregation-mode decode \
|
||||
--nnodes 2 --node-rank $i \
|
||||
--prefill-round-robin-balance \
|
||||
--dist-init-addr ${D_IP[0]}:10000
|
||||
break
|
||||
fi
|
||||
@@ -918,7 +913,6 @@ done
|
||||
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
@@ -978,7 +972,6 @@ D_IP=('your decode ip1' 'your decode ip2')
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
|
||||
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
|
||||
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
|
||||
@@ -1054,7 +1047,6 @@ done
|
||||
```
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
@@ -1973,7 +1965,6 @@ done
|
||||
```
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
|
||||
@@ -262,7 +262,6 @@ done
|
||||
2. SGLang Model Gateway (former Router):
|
||||
|
||||
```shell
|
||||
export SGLANG_DP_ROUND_ROBIN=1
|
||||
python -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
|
||||
@@ -94,6 +94,97 @@ python -m sglang.launch_server \
|
||||
--mem-fraction-static 0.8
|
||||
```
|
||||
|
||||
#### Running Qwen3-235B-A22B-Instruct-2507 with 256K long sequence on 2 x Atlas 800I A3 without CP
|
||||
|
||||
This example uses **PD disaggregation** for long-sequence inference and keeps **context parallel disabled**.
|
||||
|
||||
Set the shared environment variables on both nodes first:
|
||||
|
||||
```shell
|
||||
export ASCEND_USE_FIA=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export ASCEND_MF_STORE_URL="tcp://<PREFILL_HOST_IP>:12345"
|
||||
export HCCL_SOCKET_IFNAME=<NETWORK_IFACE>
|
||||
export GLOO_SOCKET_IFNAME=<NETWORK_IFACE>
|
||||
|
||||
MODEL_PATH=/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-2507-w8a8
|
||||
```
|
||||
|
||||
**Prefill node:**
|
||||
|
||||
```shell
|
||||
export ASCEND_LAUNCH_BLOCKING=1
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export HCCL_BUFFSIZE=1500
|
||||
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
|
||||
export DEEPEP_NORMAL_LONG_SEQ_ROUND=128
|
||||
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend ascend \
|
||||
--disaggregation-bootstrap-port 8995 \
|
||||
--attention-backend ascend \
|
||||
--disable-radix-cache \
|
||||
--quantization modelslim \
|
||||
--chunked-prefill-size -1 \
|
||||
--skip-server-warmup \
|
||||
--device npu \
|
||||
--tp-size 16 \
|
||||
--mem-fraction-static 0.45 \
|
||||
--max-running-requests 1 \
|
||||
--host <PREFILL_HOST_IP> \
|
||||
--port 8000 \
|
||||
--dist-init-addr <PREFILL_HOST_IP>:5000 \
|
||||
--nnodes 1 \
|
||||
--node-rank 0 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode normal
|
||||
```
|
||||
|
||||
**Decode node:**
|
||||
|
||||
```shell
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export HCCL_BUFFSIZE=4000
|
||||
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096
|
||||
export DEEPEP_NORMAL_LONG_SEQ_ROUND=16
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path ${MODEL_PATH} \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend ascend \
|
||||
--attention-backend ascend \
|
||||
--mem-fraction-static 0.8 \
|
||||
--disable-cuda-graph \
|
||||
--device npu \
|
||||
--disable-radix-cache \
|
||||
--quantization modelslim \
|
||||
--chunked-prefill-size 8192 \
|
||||
--skip-server-warmup \
|
||||
--tp-size 16 \
|
||||
--max-running-requests 1 \
|
||||
--host <DECODE_HOST_IP> \
|
||||
--port 8232 \
|
||||
--moe-a2a-backend deepep \
|
||||
--deepep-mode low_latency \
|
||||
--disable-overlap-schedule
|
||||
```
|
||||
|
||||
**Router:**
|
||||
|
||||
```shell
|
||||
python3 -m sglang_router.launch_router \
|
||||
--pd-disaggregation \
|
||||
--policy cache_aware \
|
||||
--prefill http://<PREFILL_HOST_IP>:8000 8995 \
|
||||
--decode http://<DECODE_HOST_IP>:8232 \
|
||||
--host <ROUTER_HOST_IP> \
|
||||
--port 6689 \
|
||||
--prometheus-port 29010
|
||||
```
|
||||
|
||||
#### Running Qwen3-VL-8B-Instruct on 1 x Atlas 800I A3.
|
||||
|
||||
Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)
|
||||
|
||||
Reference in New Issue
Block a user