diff --git a/docs/platforms/ascend/ascend_npu_best_practice.md b/docs/platforms/ascend/ascend_npu_best_practice.md index bcda94a2e..a66d677da 100644 --- a/docs/platforms/ascend/ascend_npu_best_practice.md +++ b/docs/platforms/ascend/ascend_npu_best_practice.md @@ -158,7 +158,6 @@ done ``` ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -275,7 +274,6 @@ done ``` ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -559,7 +557,6 @@ done ``` ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -760,7 +757,6 @@ done ``` ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -909,7 +905,6 @@ do --disaggregation-transfer-backend ascend \ --disaggregation-mode decode \ --nnodes 2 --node-rank $i \ - --prefill-round-robin-balance \ --dist-init-addr ${D_IP[0]}:10000 break fi @@ -918,7 +913,6 @@ done ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -978,7 +972,6 @@ D_IP=('your decode ip1' 'your decode ip2') export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_DP_ROUND_ROBIN=1 LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -1054,7 +1047,6 @@ done ``` ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ @@ -1973,7 +1965,6 @@ done ``` ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ diff --git a/docs/platforms/ascend/ascend_npu_deepseek_example.md b/docs/platforms/ascend/ascend_npu_deepseek_example.md index cdecb544c..abda404d5 100644 --- a/docs/platforms/ascend/ascend_npu_deepseek_example.md +++ b/docs/platforms/ascend/ascend_npu_deepseek_example.md @@ -262,7 +262,6 @@ done 2. SGLang Model Gateway (former Router): ```shell -export SGLANG_DP_ROUND_ROBIN=1 python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ diff --git a/docs/platforms/ascend/ascend_npu_qwen3_examples.md b/docs/platforms/ascend/ascend_npu_qwen3_examples.md index 5278a22a1..7ceedd351 100644 --- a/docs/platforms/ascend/ascend_npu_qwen3_examples.md +++ b/docs/platforms/ascend/ascend_npu_qwen3_examples.md @@ -94,6 +94,97 @@ python -m sglang.launch_server \ --mem-fraction-static 0.8 ``` +#### Running Qwen3-235B-A22B-Instruct-2507 with 256K long sequence on 2 x Atlas 800I A3 without CP + +This example uses **PD disaggregation** for long-sequence inference and keeps **context parallel disabled**. + +Set the shared environment variables on both nodes first: + +```shell +export ASCEND_USE_FIA=1 +export SGLANG_SET_CPU_AFFINITY=1 +export ASCEND_MF_STORE_URL="tcp://:12345" +export HCCL_SOCKET_IFNAME= +export GLOO_SOCKET_IFNAME= + +MODEL_PATH=/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-2507-w8a8 +``` + +**Prefill node:** + +```shell +export ASCEND_LAUNCH_BLOCKING=1 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_BUFFSIZE=1500 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=128 +export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8995 \ + --attention-backend ascend \ + --disable-radix-cache \ + --quantization modelslim \ + --chunked-prefill-size -1 \ + --skip-server-warmup \ + --device npu \ + --tp-size 16 \ + --mem-fraction-static 0.45 \ + --max-running-requests 1 \ + --host \ + --port 8000 \ + --dist-init-addr :5000 \ + --nnodes 1 \ + --node-rank 0 \ + --moe-a2a-backend deepep \ + --deepep-mode normal +``` + +**Decode node:** + +```shell +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export HCCL_BUFFSIZE=4000 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --disable-cuda-graph \ + --device npu \ + --disable-radix-cache \ + --quantization modelslim \ + --chunked-prefill-size 8192 \ + --skip-server-warmup \ + --tp-size 16 \ + --max-running-requests 1 \ + --host \ + --port 8232 \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency \ + --disable-overlap-schedule +``` + +**Router:** + +```shell +python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8995 \ + --decode http://:8232 \ + --host \ + --port 6689 \ + --prometheus-port 29010 +``` + #### Running Qwen3-VL-8B-Instruct on 1 x Atlas 800I A3. Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)