diff --git a/docs_new/docs.json b/docs_new/docs.json index f5e049a66..2780ce2c5 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -877,6 +877,24 @@ { "group": "Ascend NPUs", "pages": [ + { + "group": "Model Tutorials", + "pages": [ + "docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1", + "docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2", + "docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1", + "docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6", + "docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b", + "docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct" + ] + }, "docs/hardware-platforms/ascend-npus/ascend_npu_quick_start", "docs/hardware-platforms/ascend-npus/ascend_npu", "docs/hardware-platforms/ascend-npus/ascend_npu_support_features", diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx deleted file mode 100644 index a29020b97..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx +++ /dev/null @@ -1,6927 +0,0 @@ ---- -title: "Best Practice on Ascend NPU" -metatags: - description: "Documentation for Best Practice on Ascend NPU" ---- -This section describes the best practice data of mainstream LLM models such as DeepSeek and Qwen on the Ascend NPU. If -you encounter issues or have any questions, please [open an issue](https://github.com/sgl-project/sglang/issues). - -## DeepSeek Series Models - -### Low Latency - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Deepseek-R1Atlas 800I A332PD Disaggregation6K+1.6K20msW8A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A332PD Disaggregation3.9K+1K19msW8A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A332PD Disaggregation3.5K+1.5K19msW8A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A332PD Disaggregation3.5K+1K19msW8A8 INT8Optimal Configuration
DeepSeek-V3.2Atlas 800I A332PD Disaggregation128K+1K26msW8A8 INT8Optimal Configuration
- -### High Throughput - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Deepseek-R1Atlas 800I A332PD Disaggregation3.5K+1.5K50msW8A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A324PD Disaggregation2K+2K50msW8A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A38PD Mixed2K+2K50msW4A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A316PD Disaggregation2K+2K50msW4A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A38PD Mixed3.5K+1.5K50msW4A8 INT8Optimal Configuration
Deepseek-R1Atlas 800I A316PD Disaggregation3.5K+1.5K50msW4A8 INT8Optimal Configuration
- -## Qwen Series Models - -### Low Latency - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Qwen3-235B-A22BAtlas 800I A38PD Mixed11K+1K10msBF16Optimal Configuration
Qwen3-32BAtlas 800I A34PD Mixed6K+1.5K18msBF16Optimal Configuration
Qwen3-32BAtlas 800I A34PD Mixed4K+1.5K11msBF16Optimal Configuration
Qwen3-32BAtlas 800I A38PD Mixed18K+4K6msBF16Optimal Configuration
Qwen3-32BAtlas 800I A28PD Mixed6K+1.5K18msW8A8 INT8Optimal Configuration
Qwen3-32BAtlas 800I A28PD Mixed4K+1.5K11msBF16Optimal Configuration
Qwen3-32BAtlas 800I A32PD Mixed1K+0.3K12msW8A8 INT8Optimal Configuration
Qwen3-32BAtlas 800I A32PD Mixed6K+1.5K17msW8A8 INT8Optimal Configuration
Qwen3-8BAtlas 800I A31PD Mixed1K+0.3K7msW8A8 INT8Optimal Configuration
Qwen3-8BAtlas 800I A31PD Mixed6K+1.5K12msW8A8 INT8Optimal Configuration
Qwen3-8BAtlas 800I A31PD Mixed3.5K+1.5K5msW8A8 INT8Optimal Configuration
Qwen3-30B-A3BAtlas 800I A31PD Mixed6K+1.5K10msW8A8 INT8Optimal Configuration
Qwen3-30B-A3BAtlas 800I A31PD Mixed1K+0.3K7msW8A8 INT8Optimal Configuration
Qwen3-Next-A3B-InstructAtlas 800I A32PD Mixed1K+0.3K14.21msW8A8 INT8Optimal Configuration
Qwen3-Next-A3B-InstructAtlas 800I A32PD Mixed6K+1.5K15.62msW8A8 INT8Optimal Configuration
Qwen3-Next-A3B-InstructAtlas 800I A31PD Mixed3.5K+1.5K20msW8A8 INT8Optimal Configuration
Qwen3-14BAtlas 800I A31PD Mixed3.5K+1.5K9msW8A8 INT8Optimal Configuration
Qwen3.5-27BAtlas 800I A32PD Mixed3.5K+1.5K20msW8A8 INT8Optimal Configuration
Qwen3.5-27BAtlas 800I A31PD Mixed16K+1K20msW8A8 INT8Optimal Configuration
Qwen3.5-27BAtlas 800I A31PD Mixed64K+1K20msW8A8 INT8Optimal Configuration
Qwen3.5-397B-A17BAtlas 800I A38PD Mixed3.5K+1.5K22msW4A8Optimal Configuration
- -### High Throughput - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Qwen3-235B-A22BAtlas 800I A324PD Disaggregation3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-235B-A22BAtlas 800I A38PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-235B-A22BAtlas 800I A38PD Mixed2K+2K50msW8A8 INT8Optimal Configuration
Qwen3-235B-A22BAtlas 800I A316PD Mixed2K+2K50msW8A8 INT8Optimal Configuration
Qwen3-32BAtlas 800I A32PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-32BAtlas 800I A32PD Mixed2K+2K50msW8A8 INT8Optimal Configuration
Qwen3-30B-A3BAtlas 800I A31PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-Coder-480B-A35B-InstructAtlas 800I A324PD Disaggregation3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-Coder-480B-A35B-InstructAtlas 800I A316PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-Coder-480B-A35B-InstructAtlas 800I A38PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-Next-80B-A3B-InstructAtlas 800I A32PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-32BAtlas 800I A28PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-32BAtlas 800I A28PD Mixed2K+2K50msW8A8 INT8Optimal Configuration
Qwen3-14BAtlas 800I A31PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3-8BAtlas 800I A31PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3.5-27BAtlas 800I A31PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
Qwen3.5-27BAtlas 800I A32PD Mixed16K+1K50msW8A8 INT8Optimal Configuration
Qwen3.5-27BAtlas 800I A32PD Mixed64K+1K50msW8A8 INT8Optimal Configuration
Qwen3.5-397B-A17BAtlas 800I A38PD Mixed3.5K+1.5K50msW4A8Optimal Configuration
- -## MiniMax Series Models - -### Low Latency - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
MiniMax-M2.5Atlas 800I A38PD Mixed3.5K+1.5K20msW8A8 INT8Optimal Configuration
MiniMax-M2.5Atlas 800I A38PD Mixed128K+1K20msW8A8 INT8Optimal Configuration
- -### High Throughput - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
MiniMax-M2.5Atlas 800I A38PD Mixed3.5K+1.5K50msW8A8 INT8Optimal Configuration
MiniMax-M2.5Atlas 800I A38PD Mixed64K+1K50msW8A8 INT8Optimal Configuration
MiniMax-M2.5Atlas 800I A38PD Mixed128K+1K50msW8A8 INT8Optimal Configuration
MiniMax-M2.5Atlas 800I A34PD Mixed64K+1K50msW8A8 INT8Optimal Configuration
MiniMax-M2.5Atlas 800I A316PD Disaggregation64K+1K50msW8A8 INT8Optimal Configuration
MiniMax-M2.5Atlas 800I A316PD Disaggregation128K+1K50msW8A8 INT8Optimal Configuration
- -## MiMo-V2-Flash Series Models - -### Low Latency - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTTFTQuantizationConfiguration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation16K+1974msW8A8 INT8Optimal Configuration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation32K+12,147msW8A8 INT8Optimal Configuration
- -### High Throughput - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation16K+1K16msW8A8 INT8Optimal Configuration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation32K+1K12msW8A8 INT8Optimal Configuration
- -## Kimi Series Models - -### Low Latency - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Kimi-K2.5-w4a8Atlas 800I A38PD Mixed3.5K+1.5K20msW4A8 INT8Optimal Configuration
- -### High Throughput - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Kimi-K2.5-w4a8Atlas 800I A38PD Mixed3.5K+1.5K50msW4A8 INT8Optimal Configuration
- -## GLM Series Models - -### High Throughput - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
GLM-5.1Atlas 800I A316PD Mixed3.5K+1.5K41msW4A8Optimal Configuration
GLM-5.1Atlas 800I A332PD Disaggregation16K+1K23msW4A8Optimal Configuration
GLM-5.1Atlas 800I A348PD Disaggregation64K+1K+90% cache hit45msW4A8Optimal Configuration
GLM-5.1Atlas 800I A348PD Disaggregation128K+1K+90% cache hit32msW4A8Optimal Configuration
- -## Optimal Configuration - -### DeepSeek-R1 3_5K-1_5K 50ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') - -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export SGLANG_USE_AG_AFTER_QLORA=1 - export HCCL_BUFFSIZE=800 - export TASK_QUEUE_ENABLE=2 - export SGLANG_NPU_FUSED_MOE_MODE=2 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.778 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 16 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 60000 --moe-a2a-backend ascend_fuseep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 4 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=600 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 - export TASK_QUEUE_ENABLE=1 - export SGLANG_NPU_FUSED_MOE_MODE=1 - export SGLANG_LM_HEAD_TP=8 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.82 --max-running-requests 1024 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend ascend_fuseep --enable-dp-attention --deepep-mode low_latency --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 1024 --random-input-len 3584 --random-output-len 1536 --num-prompts 7168 --random-range-ratio 1 --request-rate 40 -``` - -### DeepSeek-R1 2K-2K 50ms on A3 24 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 24Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1600 - export TASK_QUEUE_ENABLE=2 - export SGLANG_USE_AG_AFTER_QLORA=1 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.8 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 20 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 4 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=800 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=102 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export SGLANG_NPU_FUSED_MOE_MODE=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.81 --max-running-requests 1088 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend ascend_fuseep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang \ ---host 127.0.0.1 \ ---port 6688 \ ---max-concurrency 1088 \ ---random-input-len 2048 \ ---random-output-len 2048 \ ---num-prompts 12800 \ ---random-range-ratio 1 \ ---request-rate 24 -``` - -### DeepSeek-R1 6K-1_6K 20ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 6K+1.6K - -TPOT: 20ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') - -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 4 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 8 \ - --mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 6000 \ - --random-output-len 1600 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.9K+1K - -TPOT: 19ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 4 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr DIP1:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 16 \ - --mem-fraction-static 0.75 --max-running-requests 32 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done -``` - -```bash Command -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 3900 \ - --random-output-len 1024 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 3_5K-1_5K 19ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 19ms - -#### Model Deployment - -Please Turn to [DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 3500 \ - --random-output-len 1500 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 3_5K-1K 19ms on A3 32 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1K - -TPOT: 19ms - -#### Model Deployment - -Please Turn to [DeepSeek-R1 3_9K-1K 19ms on A3 32 Cards Disaggregation Mode](#deepseek-r1-3_9k-1k-19ms-on-a3-32-cards-disaggregation-mode) - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 32 \ - --random-input-len 3500 \ - --random-output-len 1024 \ - --num-prompts 32 \ - --random-range-ratio 1 \ - --request-rate 16 -``` - -### DeepSeek-R1 2K-2K 50ms on A3 8 Cards Mixed Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 -export HCCL_BUFFSIZE=1600 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_USE_FIA_NZ=1 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ ---tp 16 \ ---trust-remote-code \ ---attention-backend ascend \ ---device npu \ ---quantization modelslim \ ---watchdog-timeout 9000 \ ---host 127.0.0.1 --port 6699 \ ---cuda-graph-bs 4 8 20 21 22 \ ---mem-fraction-static 0.78 \ ---max-running-requests 352 \ ---disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 1500 \ ---moe-a2a-backend deepep --deepep-mode auto \ ---enable-dp-attention --dp-size 16 --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ ---dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 352 --random-input-len 2048 --random-output-len 2048 --num-prompts 1408 --random-range-ratio 1 -``` - -### DeepSeek-R1 2K-2K 50ms on A3 16 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" - -P_IP=('your prefill ip1') - -D_IP=('your decode ip1') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=2600 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.7 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 10240 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=900 - export SGLANG_DP_ROUND_ROBIN=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 - export TASK_QUEUE_ENABLE=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ - --mem-fraction-static 0.8 --max-running-requests 448 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 448 --random-input-len 2048 --random-output-len 2048 --num-prompts 1792 --random-range-ratio 1 --request-rate 32 -``` - -### DeepSeek-R1 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56 -export HCCL_BUFFSIZE=1200 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_USE_FIA_NZ=1 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ ---tp 16 \ ---trust-remote-code \ ---attention-backend ascend \ ---device npu \ ---quantization modelslim \ ---watchdog-timeout 9000 \ ---host 127.0.0.1 --port 6699 \ ---cuda-graph-bs 4 8 12 14 \ ---mem-fraction-static 0.77 \ ---max-running-requests 224 \ ---context-length 8188 --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 3000 \ ---moe-a2a-backend deepep --deepep-mode auto \ ---enable-dp-attention --dp-size 16 --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 224 --random-input-len 3500 --random-output-len 1500 --num-prompts 896 --random-range-ratio 1 -``` - -### DeepSeek-R1 3_5K-1_5K 50ms on A3 16 Cards Disaggregation Mode - -Model: Deepseek R1 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" - -P_IP=('your prefill ip1') - -D_IP=('your decode ip1') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export ENABLE_MOE_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=3500 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.62 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 32 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 20480 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 8 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=800 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - export TASK_QUEUE_ENABLE=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ - --mem-fraction-static 0.805 --max-running-requests 416 --attention-backend ascend --device npu --quantization modelslim \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 416 --random-input-len 3500 --random-output-len 1500 --num-prompts 1664 --random-range-ratio 1 -``` - -### DeepSeek-V3.2 128K-1K 26ms on A3 32 Cards Disaggregation Mode - -Model: DeepSeek-V3.2-W8A8 - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 128K+1K - -TPOT: 26ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24670" - -P_IP=('your prefill ip1' 'your prefill ip2') -D_IP=('your decode ip1' 'your decode ip2') -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1200 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --tp 32 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host ${P_IP[$i]} --port 8000 \ - --mem-fraction-static 0.73 \ - --disable-radix-cache --chunked-prefill-size -1 --max-prefill-tokens 68000 \ - --max-running-requests 1 \ - --moe-a2a-backend deepep --deepep-mode normal \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --disaggregation-mode prefill \ - --disable-cuda-graph \ - --nnodes 2 --node-rank $i \ - --disaggregation-bootstrap-port 8995 \ - --moe-dense-tp-size 1 \ - --enable-dsa-prefill-context-parallel \ - --dsa-prefill-cp-mode in-seq-split \ - --attn-cp-size 32 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dist-init-addr ${P_IP[0]}:10000 - break - fi -done - - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - - export TASK_QUEUE_ENABLE=0 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - DP=8 - export HCCL_BUFFSIZE=400 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 - - python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --tp 32 \ - --dp ${DP} \ - --ep 32 \ - --moe-dense-tp-size 1 \ - --enable-dp-attention \ - --enable-dp-lm-head \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host ${D_IP[$i]} --port 8001 \ - --mem-fraction-static 0.79 \ - --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 68000 \ - --max-running-requests 32 \ - --cuda-graph-max-bs 4 \ - --moe-a2a-backend deepep \ - --deepep-mode low_latency \ - --quantization modelslim \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disaggregation-transfer-backend ascend \ - --disaggregation-mode decode \ - --nnodes 2 --node-rank $i \ - --dist-init-addr ${D_IP[0]}:10000 - break - fi -done -``` - - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP1:8000 8995 \ - --decode http://D_IP1:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --max-concurrency 8 --random-input-len 131076 --random-output-len 1024 --num-prompts 8 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 3_5K-1_5K 50ms on A3 24 Cards Disaggregation Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 24Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_DP_ROUND_ROBIN=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -MODEL_PATH=xxx -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24667" -P_IP=('your prefill ip1') -D_IP=('your decode ip1' 'your decode ip2') - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - - -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 - export HCCL_BUFFSIZE=4300 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - export STREAMS_PER_DEVICE=32 - - # Prefill - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \ - --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \ - --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --mem-fraction-static 0.6 \ - --disable-radix-cache \ - --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --max-running-requests 128 --chunked-prefill-size 94208 --max-prefill-tokens 262144 \ - --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --dtype bfloat16 - NODE_RANK=$i - break - fi -done - - -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export DP_ROUND_ROBIN=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=65536 - export HCCL_BUFFSIZE=800 - export HCCL_SOCKET_IFNAME=data0.3001 - export GLOO_SOCKET_IFNAME=data0.3001 - export STREAMS_PER_DEVICE=32 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode \ - --host ${D_IP[$i]} --port 8001 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --mem-fraction-static 0.83 --max-running-requests 768 \ - --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --cuda-graph-bs 6 8 12 15 18 20 22 24 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-draft-model-quantization unquant \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --dist-init-addr xxx:5000 \ - --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 \ - --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```shell Command -export SGLANG_DP_ROUND_ROBIN=1 -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://PIP:8000 8995 \ - --decode http://DIP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang-oai --host 127.0.0.1 --port 7239 --max-concurrency 860 --random-input-len 3500 --random-output-len 1500 --num-prompts 3440 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=570 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 432 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 94208 --max-prefill-tokens 458880 --sampling-backend ascend \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend ascend_fuseep --speculative-draft-model-quantization unquant \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.8 --cuda-graph-bs 1 2 4 8 16 20 24 26 27 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 272 --random-input-len 3500 --random-output-len 1500 --num-prompts 1088 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 2K-2K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=450 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=147456 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 624 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 73728 --max-prefill-tokens 458880 --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disable-radix-cache --moe-a2a-backend ascend_fuseep \ - --tp 16 --dp-size 16 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.83 --cuda-graph-bs 4 8 16 24 28 29 30 32 34 36 37 38 39 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 480 --random-input-len 2048 --random-output-len 2048 --num-prompts 480 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 2K-2K 50ms on A3 16 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1600 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" - -MIX_IP=('IP1' 'IP2') - -for i in "${!MIX_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${MIX_IP[$i]}" || "$LOCAL_HOST2" == "${MIX_IP[$i]}" ]]; - then - echo "${MIX_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - - python -m sglang.launch_server --model-path ${MODEL_PATH} \ - --host 127.0.0.1 --port 7439 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --mem-fraction-static 0.8 --max-running-requests 768 \ - --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend deepep --deepep-mode auto --cuda-graph-bs 6 8 10 12 18 24 \ - --dist-init-addr ${MIX_IP[0]}:5000 --chunked-prefill-size 131072 --max-prefill-tokens 458880 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx --speculative-draft-model-quantization= unquant \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --context-length 8192 --disable-radix-cache \ - --enable-dp-lm-head --dtype bfloat16 - NODE_RANK=$i - break - fi -done - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 768 --random-input-len 2000 --random-output-len 2000 --num-prompts 768 --random-range-ratio 1 -``` - -### Qwen3-235B-A22B 11K-1K 10ms on A3 8 Cards Mixed Mode - -Model: Qwen3-235B-A22B-W8A8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 11K+1K - -TPOT: 10ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1600 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 1 --dtype bfloat16 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --disable-radix-cache --enable-dp-lm-head \ - --tp 16 --mem-fraction-static 0.78 --cuda-graph-bs 1 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 1 --random-input-len 11000 --random-output-len 1000 --num-prompts 1 --random-range-ratio 1 -``` - -### Qwen3-32B 6K-1_5K 18ms on A3 4 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 4Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 18ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 32 \ - --disable-radix-cache \ - --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 8 16 24 32 --dtype bfloat16 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 32 --random-output-len 1500 --random-input-len 6000 --num-prompts 32 --random-range-ratio 1 -``` - -### Qwen3-32B 4K-1_5K 11ms on A3 4 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 4Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 4K+1.5K - -TPOT: 11ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 1 \ - --disable-radix-cache \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 1 --dtype bfloat16 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 4096 --num-prompts 4 -``` - -### Qwen3-32B 18K-4K 6ms on A3 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 18K+4K - -TPOT: 6ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 1 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 65536 \ - --tp-size 16 --mem-fraction-static 0.72 --cuda-graph-bs 1 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 1 --random-output-len 18000 --random-input-len 4000 --num-prompts 1 -``` - -### Qwen3-32B 3_5K-1_5K 50ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 78 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-prefill-tokens 49152 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tp-size 4 --mem-fraction-static 0.72 --cuda-graph-bs 16 32 64 68 72 78 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 78 --random-output-len 1500 --random-input-len 3500 --num-prompts 312 --random-range-ratio 1 -``` - -### Qwen3-32B 2K-2K 50ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 120 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 49152 \ - --tp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 54 60 66 72 78 84 90 108 114 120 --dtype bfloat16 - -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 120 --random-output-len 2000 --random-input-len 2000 --num-prompts 480 --random-range-ratio 1 -``` - -### Qwen3-30B-A3B 3_5K-1_5K 50ms on A3 1 Card Mixed Mode - -Model: Qwen3-30B-A3B-Instruct-2507 - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export ASCEND_LAUNCH_BLOCKING=0 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 162 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 35000 \ - --tp-size 2 --mem-fraction-static 0.87 --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ - --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 156 --random-input-len 3500 --random-output-len 1500 --num-prompts 624 --random-range-ratio 1 -``` - -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 24 Cards Disaggregation Mode - -Model: Qwen3-Coder-480B-A35B-Instruct - -Hardware: Atlas 800I A3 24Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_NPU_FUSED_MOE_MODE=2 - -MODEL_PATH=xxx -export ASCEND_MF_STORE_URL="tcp://PIP:24667" -P_IP=('PIP') -D_IP=('DIP1' 'DIP2') -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - - -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=327680 - export HCCL_BUFFSIZE=1550 - export TASK_QUEUE_ENABLE=2 - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \ - --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \ - --nnodes 1 --node-rank $i --tp-size 16 --dp-size 2 --mem-fraction-static 0.7 \ - --disable-radix-cache \ - --attention-backend ascend --device npu --quantization modelslim --disaggregation-transfer-backend ascend \ - --max-running-requests 16 --chunked-prefill-size 20480 --max-prefill-tokens 20480 \ - --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --dtype bfloat16 \ - --disable-overlap-schedule - NODE_RANK=$i - break - fi -done - -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - source /usr/local/Ascend/ascend-toolkit/set_env.sh - source /usr/local/Ascend/nnal/atb/set_env.sh - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=65536 - export HCCL_BUFFSIZE=600 - export SGLANG_NPU_FUSED_MOE_MODE=2 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode \ - --host ${D_IP[$i]} --port 8001 --trust-remote-code \ - --nnodes 2 --node-rank $i --tp-size 32 --dp-size 4 --mem-fraction-static 0.75 --max-running-requests 544 \ - --attention-backend ascend --device npu --quantization modelslim --enable-dp-attention \ - --moe-a2a-backend ascend_fuseep --cuda-graph-bs 16 32 56 72 80 88 96 104 112 120 128 136 \ - --dist-init-addr DIP1:5000 \ - --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --enable-dp-lm-head --dtype bfloat16 --tokenizer-worker-num 4 --load-balance-method round_robin - NODE_RANK=$i - break - fi -done - -``` - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://PIP:8000 8995 \ - --decode http://DIP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 410 --random-input-len 3500 --random-output-len 1500 --num-prompts 1640 --random-range-ratio 1 --request-rate 8 -``` - -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 16 Cards Mixed Mode - -Model: Qwen3-Coder-480B-A35B-Instruct - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=1800 -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx -export HCCL_OP_EXPANSION_MODE="AIV" - -MIX_IP=('IP1' 'IP2') - -for i in "${!MIX_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${MIX_IP[$i]}" || "$LOCAL_HOST2" == "${MIX_IP[$i]}" ]]; - then - echo "${MIX_IP[$i]}" - - python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 2 --node-rank $i \ - --dist-init-addr 141.61.133.128:5000 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 288 --context-length 8192 --dtype bfloat16 \ - --chunked-prefill-size 114688 --max-prefill-tokens 458880 \ - --disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto \ - --tp 32 --dp-size 4 --enable-dp-attention --enable-dp-lm-head --mem-fraction-static 0.7 --cuda-graph-bs 56 64 72 - NODE_RANK=$i - break - fi -done -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 288 --random-input-len 3500 --random-output-len 1500 --num-prompts 1152 --random-range-ratio 1 --request-rate 20 -``` - -### Qwen3-Coder-480B-A35B-Instruct 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3-Coder-480B-A35B-Instruct - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=2100 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" - -python -m sglang.launch_server --model-path $MODEL_PATH \ ---host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ ---attention-backend ascend --device npu --quantization modelslim \ ---max-running-requests 80 --context-length 8192 --dtype bfloat16 \ ---chunked-prefill-size 28672 --max-prefill-tokens 458880 \ ---disable-radix-cache --moe-a2a-backend deepep --deepep-mode auto --enable-dp-attention --enable-dp-lm-head \ ---tp 16 --dp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 16 20 24 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 80 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 -``` - -### Qwen3-Next-80B-A3B-Instruct 3_5K-1_5K 50ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell Command -export cann_path=/usr/local/Ascend/ascend-toolkit/latest -source /usr/local/Ascend/driver/bin/setenv.bash -source ${cann_path}/../set_env.sh -source ${cann_path}/../../nnal/atb/set_env.sh -source ${cann_path}/opp/vendors/customize/bin/set_env.bash -export ASCEND_HOME_PATH=${cann_path} -source /usr/local/Ascend/8.5.0/bisheng_toolkit/set_env.sh - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -export LD_LIBRARY_PATH=/usr/local/Ascend/cann-9.0.0/opp/vendors/custom_transformer/op_api/lib:${LD_LIBRARY_PATH} - -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_ALGO="level0:NA;level1:ring" - -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=0 -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 - -ZBAL_HCCL_OP="allreduce,_allgather_base,allgather,broadcast,scatter,reduce_scatter,_reduce_scatter_base,alltoall_base" -export HCCL_BUFFSIZE=64 -export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669" - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBAL_ENABLE_GRAPH=1 -MODEL_PATH=/home/weights/Qwen3-Next-80B-A3B-Instruct-W8A8 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 4 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.75 \ - --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ - --chunked-prefill-size -1 --max-running-requests 300 \ - --mamba-ssm-dtype bfloat16 \ - --quantization modelslim \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ - --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ - --moe-a2a-backend deepep --deepep-mode auto \ - --cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --max-concurrency 300 --random-output-len 1536 --random-input-len 3584 --num-prompts 300 --random-range-ratio 1 -``` - -### Qwen3-32B 6K-1_5K 18ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 18ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 32 \ - --disable-radix-cache \ - --chunked-prefill-size 24576 --max-prefill-tokens 65536 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 8 16 24 32 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7439 --max-concurrency 32 --random-output-len 1500 --random-input-len 6000 --num-prompts 32 --random-range-ratio 1 -``` - -### Qwen3-32B 4K-1_5K 11ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 4K+1.5K - -TPOT: 11ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 32 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 65536 \ - --tp-size 8 --mem-fraction-static 0.72 --cuda-graph-bs 1 4 6 12 18 24 30 32 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 4096 --num-prompts 4 -``` - -### Qwen3-32B 1K-0_3K 12ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 12ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 4 --mem-fraction-static 0.843 --cuda-graph-bs 1 4 8 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 -``` - -### Qwen3-32B 6K-1_5K 17ms on A3 2 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 17ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 4 --mem-fraction-static 0.843 --cuda-graph-bs 1 4 10 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-8B 1K-0_3K 7ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 7ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 2 --mem-fraction-static 0.894 --cuda-graph-bs 1 2 4 6 9 10 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 -``` - -### Qwen3-8B 6K-1_5K 12ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 12ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu --quantization modelslim \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 16384 \ - --tp-size 2 --mem-fraction-static 0.894 --cuda-graph-bs 1 5 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-32B 3_5K-1_5K 50ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 78 \ - --disable-radix-cache --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-prefill-tokens 65536 \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --tp-size 4 --mem-fraction-static 0.72 --cuda-graph-bs 1 4 8 16 32 64 68 72 78 --dtype bfloat16 --base-gpu-id 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 78 --random-output-len 1500 --random-input-len 3500 --num-prompts 312 --random-range-ratio 1 -``` - -### Qwen3-32B 2K-2K 50ms on A2 8 Cards Mixed Mode - -Model: Qwen3-32B - -Hardware: Atlas 800I A2 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 2K+2K - -TPOT: 50ms - -#### Model Deployment - -```shell Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 120 \ - --disable-radix-cache \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-prefill-tokens 49152 --base-gpu-id 4 \ - --tp-size 4 --mem-fraction-static 0.7 --cuda-graph-bs 54 60 66 72 78 84 90 108 114 120 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 120 --random-output-len 2000 --random-input-len 2000 --num-prompts 120 --random-range-ratio 1 -``` - -### Qwen3-30B-A3B 6K-1_5K 10ms on A3 1 Cards Mixed Mode - -Model: Qwen3-30B-A3B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 10ms - -#### Model Deployment - -```bash Command -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 16 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 35000 \ - --tp-size 2 --mem-fraction-static 0.6 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-30B-A3B 1K-0_3K 7ms on A3 1 Cards Mixed Mode - -Model: Qwen3-30B-A3B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 7ms - -#### Model Deployment - -```bash Command -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=400 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --max-running-requests 8 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \ - --chunked-prefill-size -1 --max-prefill-tokens 35000 \ - --tp-size 2 --mem-fraction-static 0.7 --cuda-graph-bs 1 2 3 4 5 6 7 8 --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7339 --random-range-ratio 1 --max-concurrency 8 --random-output-len 300 --random-input-len 1024 --num-prompts 8 -``` - -### Qwen3-Next 1K-0_3K 14_21ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 1K+0.3K - -TPOT: 14.21ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=2000 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 4 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.75 \ - --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-running-requests 312 \ - --cuda-graph-bs 2 4 16 32 48 64 80 96 128 140 156 \ - --mamba-ssm-dtype bfloat16 \ - --base-gpu-id 0 \ - --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ - --moe-a2a-backend deepep --deepep-mode auto \ -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 16 --random-output-len 300 --random-input-len 1024 --num-prompts 16 -``` - -### Qwen3-Next 6K-1_5K 15_62ms on A3 2 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 6K+1.5K - -TPOT: 15.62ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=5 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 - -MODEL_PATH=xxx - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export HCCL_BUFFSIZE=2000 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 4 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.75 \ - --disable-radix-cache --max-prefill-tokens 14080 --context-length 26384 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-running-requests 312 \ - --cuda-graph-bs 2 4 16 32 48 64 80 96 128 140 156 \ - --mamba-ssm-dtype bfloat16 \ - --base-gpu-id 0 \ - --speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \ - --moe-a2a-backend deepep --deepep-mode auto \ -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 16 --random-output-len 1500 --random-input-len 6144 --num-prompts 16 -``` - -### Qwen3-14B 3_5K-1_5K 9ms on A3 1 Cards Mixed Mode - -Model: Qwen3-14B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 9ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export ASCEND_USE_FIA=0 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --disable-radix-cache --mem-fraction-static 0.8 \ - --tp-size 1 --dp-size 1 \ - --sampling-backend ascend --max-running-requests 8 \ - --served-model-name Qwen3-14B \ - --chunked-prefill-size -1 \ - --cuda-graph-bs 8 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --schedule-conservativeness 0.01 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 8 --random-range-ratio 1 -``` - -### Qwen3-14B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode - -Model: Qwen3-14B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export ASCEND_USE_FIA=0 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --disable-radix-cache --mem-fraction-static 0.89 \ - --tp-size 1 --dp-size 2 \ - --sampling-backend ascend --max-running-requests 144 \ - --max-prefill-tokens 12288 \ - --served-model-name Qwen3-14B \ - --chunked-prefill-size -1 \ - --cuda-graph-bs 8 16 32 44 48 50 52 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --schedule-conservativeness 0.01 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 144 --random-output-len 1500 --random-input-len 3500 --num-prompts 576 --random-range-ratio 1 -``` - -### Qwen3-8B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --disable-radix-cache --mem-fraction-static 0.9 \ - --tp-size 1 \ - --max-running-requests 70 \ - --max-prefill-tokens 16384 \ - --served-model-name Qwen3-8B \ - --chunked-prefill-size 16384 \ - --cuda-graph-bs 8 12 24 36 48 51 55 60 63 64 66 68 70 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 64 --random-output-len 1500 --random-input-len 3500 --num-prompts 256 --random-range-ratio 1 -``` - -### Qwen3-8B 3_5K-1_5K 5ms on A3 1 Cards Mixed Mode - -Model: Qwen3-8B - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 5ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -MODEL_PATH=xxx - -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_OP_EXPANSION_MODE="AIV" -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python -m sglang.launch_server --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ - --attention-backend ascend --device npu \ - --disable-radix-cache --mem-fraction-static 0.894 \ - --tp-size 2 \ - --max-running-requests 1 \ - --max-prefill-tokens 16384 \ - --served-model-name Qwen3-8B \ - --chunked-prefill-size -1 \ - --cuda-graph-bs 1 \ - --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \ - --speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 7239 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 4 --random-range-ratio 1 -``` - -### Qwen3-Next 3_5K-1_5K 20ms on A3 1 Cards Mixed Mode - -Model: Qwen3-Next-80B-A3B-Instruct - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 20ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 -export HCCL_OP_EXPANSION_MODE="AIV" -export TASK_QUEUE_ENABLE=1 -export ASCEND_USE_FIA=1 -export SGLANG_NPU_USE_MULTI_STREAM=0 -export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export FORCE_DRAFT_MODEL_NON_QUANT=1 -export HCCL_BUFFSIZE=2000 -export ZBCCL_LOCAL_MEM_SIZE=60416 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - -export ZBCCL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 -export ZBCCL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBCCL_ENABLE_GRAPH=1 - -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -MODEL_PATH=xxx - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --page-size 128 \ - --tp-size 2 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --host 127.0.0.1 --port 6699 \ - --mem-fraction-static 0.85 \ - --disable-radix-cache --max-prefill-tokens 28672 --context-length 26384 --max-total-tokens 122304 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 --max-running-requests 2 \ - --cuda-graph-bs 2 \ - --mamba-ssm-dtype bfloat16 \ - --speculative-draft-model-path /path/to/Qwen3-Next-80B-A3B-Instruct -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 1 -``` - -### Qwen3.5-27B 3_5K-1_5K 20ms on A3 2 Cards Mixed Mode - -Model: Eco-Tech/Qwen3.5-27B-w8a8-mtp - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 20ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY - -# on-demand set device -export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 - -export ASCEND_LAUNCH_BLOCKING=1 -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=3000 -export HCCL_OP_EXPANSION_MODE="AIV" -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_NPU_PROFILING=0 -export SGLANG_DISAGGEGATION_WAITING_TIMEOUT=3600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - -MODEL_PATH=xxx - -python -m sglang.launch_server --model-path ${MODEL_PATH} \ - --attention-backend ascend \ - --host 127.0.0.1 --port 6699 \ - --device npu \ - --tp-size 4\ - --trust-remote-code \ - --watchdog-timeout 9000 \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 186000 \ - --enable-prefill-delayer \ - --prefill-delayer-max-delay-passes 200 \ - --disable-radix-cache \ - --mem-fraction-static 0.94 \ - --max-total-tokens 700000 \ - --max-running-requests 38 \ - --max-mamba-cache-size 200 \ - --quantization modelslim \ - --dtype bfloat16 \ - --mamba-ssm-dtype bfloat16 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --cuda-graph-bs 1 2 4 8 12 18 24 32 34 36 38 \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 6699 --dataset-name random --max-concurrency 38 --num-prompts 152 --random-range-ratio 1 --random-output-len 1500 --random-input-len 3500 -``` - -### Qwen3.5-27B 16K-1K 20ms on A3 1 Cards Mixed Mode - -Model: Eco-Tech/Qwen3.5-27B-w8a8-mtp - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 16K+1K - -TPOT: 20ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 - -# on-demand set device -export ASCEND_RT_VISIBLE_DEVICES=8,9 - -MODEL_PATH=xxx - -sglang serve --model-path ${MODEL_PATH} \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size -1 --max-prefill-tokens 65000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 --max-running-requests 32 --max-mamba-cache-size 32 \ - --mem-fraction-static 0.85 \ - --port 8001 \ - --cuda-graph-bs 2 3 4 5 6 \ - --enable-multimodal \ - --quantization modelslim \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 310000 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 8001 --dataset-name random --max-concurrency 32 --num-prompts 128 --random-range-ratio 1 --random-output-len 1000 --random-input-len 16000 -``` - -### Qwen3.5-27B 64K-1K 20ms on A3 1 Cards Mixed Mode - -Model: Eco-Tech/Qwen3.5-27B-w8a8-mtp - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 64K+1K - -TPOT: 20ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY - -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_NPU_PROFILING=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 -# on-demand set device -export ASCEND_RT_VISIBLE_DEVICES=4,5 - -MODEL_PATH=xxx - -python -m sglang.launch_server --model-path ${MODEL_PATH} \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size -1 --max-prefill-tokens 130000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 --max-running-requests 32 --max-mamba-cache-size 18 \ - --mem-fraction-static 0.5 \ - --port 8004 \ - --cuda-graph-bs 2 3 4 \ - --enable-multimodal \ - --quantization modelslim \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 280000 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 8004 --dataset-name random --max-concurrency 9 --num-prompts 36 --random-range-ratio 1 --random-output-len 1000 --random-input-len 64000 -``` - -### Qwen3.5-27B 3_5K-1_5K 50ms on A3 1 Cards Mixed Mode - -Model: Eco-Tech/Qwen3.5-27B-w8a8-mtp - -Hardware: Atlas 800I A3 1Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY - -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 - -MODEL_PATH=xxx - -python -m sglang.launch_server --model-path ${MODEL_PATH} \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size -1 --max-prefill-tokens 60000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 --max-running-requests 48 --max-mamba-cache-size 60 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 2 8 16 32 48 \ - --enable-multimodal \ - --quantization modelslim \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 --mamba-ssm-dtype bfloat16 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 8000 --dataset-name random --max-concurrency 48 --num-prompts 192 --random-range-ratio 1 --random-output-len 1500 --random-input-len 3500 -``` - -### Qwen3.5-27B 16K-1K 50ms on A3 2 Cards Mixed Mode - -Model: Eco-Tech/Qwen3.5-27B-w8a8-mtp - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 16K+1K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY - -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 -# on-demand set device -export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --attention-backend ascend \ - --device npu \ - --tp-size 4 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size -1 --max-prefill-tokens 50000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 --max-running-requests 28 --max-mamba-cache-size 50 \ - --mem-fraction-static 0.7 \ - --port 8001 \ - --cuda-graph-bs 2 8 12 16 20 24 28\ - --enable-multimodal \ - --quantization modelslim \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 --mamba-ssm-dtype bfloat16 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 8001 --dataset-name random --max-concurrency 28 --num-prompts 152 --random-range-ratio 1 --random-output-len 1000 --random-input-len 16000 -``` - -### Qwen3.5-27B 64K-1K 50ms on A3 2 Cards Mixed Mode - -Model: Eco-Tech/Qwen3.5-27B-w8a8-mtp - -Hardware: Atlas 800I A3 2Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 64K+1K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY - -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 -# on-demand set device -export ASCEND_RT_VISIBLE_DEVICES=4,5,6,7 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ - --attention-backend ascend \ - --device npu \ - --tp-size 4 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size -1 --max-prefill-tokens 200000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 --max-running-requests 32 --max-mamba-cache-size 22 \ - --mem-fraction-static 0.5 \ - --port 9000 \ - --cuda-graph-bs 2 4 8 11 12 13 \ - --enable-multimodal \ - --quantization modelslim \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 850000 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 9000 --dataset-name random --max-concurrency 9 --num-prompts 36 --random-range-ratio 1 --random-output-len 1000 --random-input-len 64000 -``` - -### Qwen3.5-397B-A17B 3_5K-1_5K 22ms on A3 8 Cards Mixed Mode - -Model: Qwen3.5-397B-A17B - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 22ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export ASCEND_USE_FIA=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 -export HCCL_BUFFSIZE=3000 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_ZBAL_LOCAL_MEM_SIZE=58624 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669" -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBAL_ENABLE_GRAPH=1 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server \ ---model-path $MODEL_PATH \ ---attention-backend ascend \ ---device npu \ ---tp-size 16 \ ---chunked-prefill-size -1 --max-prefill-tokens 35000 \ ---disable-radix-cache \ ---trust-remote-code \ ---host 127.0.0.1 --max-running-requests 160 \ ---mem-fraction-static 0.8 \ ---port 6699 \ ---cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 \ ---quantization modelslim \ ---enable-multimodal --moe-a2a-backend deepep --deepep-mode auto \ ---mm-attention-backend ascend_attn \ ---dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 128000 \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---speculative-draft-model-quantization unquant \ ---dp-size 8 --enable-dp-attention --enable-dp-lm-head \ ---enable-prefill-delayer --prefill-delayer-max-delay-passes 100 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 120 --random-output-len 1500 --random-input-len 3500 --num-prompts 480 -``` - -### Qwen3.5-397B-A17B 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Qwen3.5-397B-A17B - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export ASCEND_USE_FIA=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 -export HCCL_BUFFSIZE=3000 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669" -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True -export ZBAL_ENABLE_GRAPH=1 - -MODEL_PATH=xxx - -python3 -m sglang.launch_server \ ---model-path $MODEL_PATH \ ---attention-backend ascend \ ---device npu \ ---tp-size 16 \ ---chunked-prefill-size -1 --max-prefill-tokens 17500 \ ---disable-radix-cache \ ---trust-remote-code \ ---host 127.0.0.1 --max-running-requests 432 \ ---mem-fraction-static 0.75 \ ---port 6699 \ ---cuda-graph-bs 2 4 6 8 12 16 20 24 28 32 36 40 44 48 52 56 \ ---quantization modelslim \ ---enable-multimodal --moe-a2a-backend deepep --deepep-mode auto \ ---mm-attention-backend ascend_attn \ ---dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 280000 \ ---dp-size 8 --enable-dp-attention --enable-dp-lm-head \ ---speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ ---speculative-draft-model-quantization unquant \ ---enable-prefill-delayer --prefill-delayer-max-delay-passes 200 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 352 --random-output-len 1500 --random-input-len 3500 --num-prompts 1408 -``` - -### MiniMax-M2.5 3_5K-1_5K Low Latency on A3 8 Cards Mixed Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export HCCL_OP_EXPANSION_MODE=AIV -export TASK_QUEUE_ENABLE=1 - -export HCCL_BUFFSIZE=1500 -export ASCEND_USE_FIA=1 -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=224000 - -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 \ - --port 32001 \ - --tp-size 16 \ - --dp-size 16 \ - --enable-dp-attention \ - --mem-fraction-static 0.75 \ - --max-running-requests 128 \ - --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-token 8192 \ - --cuda-graph-bs 2 4 6 8 \ - --moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --dtype bfloat16 \ - --tokenizer-worker-num 2 \ - --prefill-delayer-max-delay-passes 500 \ - --enable-prefill-delayer -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32001 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 --max-concurrency 80 -``` -### MiniMax-M2.5 128K-1K Low Latency on A3 8 Cards Mixed Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 128K+1K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export TASK_QUEUE_ENABLE=1 - -export ASCEND_USE_FIA=1 -export HCCL_BUFFSIZE=1600 -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1 -export SGLANG_NPU_FUSEEP_DECODE_ONLY=1 - -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 \ - --port 32000 \ - --tp-size 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --prefill-delayer-max-delay-passes 100 \ - --enable-prefill-delayer \ - --mem-fraction-static 0.65 \ - --max-running-requests 8 \ - --chunked-prefill-size -1 --max-prefill-token 130000 \ - --cuda-graph-bs 1 2 4 \ - --moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --dtype bfloat16 \ - --trust-remote-code \ - --tokenizer-worker-num 8 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 131072 --random-output-len 1024 --num-prompts 8 --random-range-ratio 1 --max-concurrency 2 -``` -### MiniMax-M2.5 3_5K-1_5K High Throughput on A3 8 Cards Mixed Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export HCCL_OP_EXPANSION_MODE=AIV -export TASK_QUEUE_ENABLE=1 - -export HCCL_BUFFSIZE=800 -export ASCEND_USE_FIA=1 -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 - -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 \ - --port 32001 \ - --tp-size 16 \ - --enable-dp-attention \ - --dp-size 16 \ - --mem-fraction-static 0.75 \ - --max-running-requests 480 \ - --disable-radix-cache \ - --prefill-delayer-max-delay-passes 500 \ - --enable-prefill-delayer \ - --chunked-prefill-size -1 --max-prefill-token 8192 \ - --cuda-graph-bs 8 16 24 32 48 64 80 \ - --moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32001 --random-input-len 3500 --random-output-len 1500 --num-prompts 1280 --random-range-ratio 1 --max-concurrency 320 -``` - -### MiniMax-M2.5 64K-1K High Throughput on A3 8 Cards Mixed Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 64K+1K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -export TASK_QUEUE_ENABLE=1 - -export ASCEND_USE_FIA=1 -export HCCL_BUFFSIZE=1600 -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1 -export SGLANG_NPU_FUSEEP_DECODE_ONLY=1 - -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 \ - --port 32000 \ - --tp-size 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --prefill-delayer-max-delay-passes 100 \ - --enable-prefill-delayer \ - --mem-fraction-static 0.65 \ - --max-running-requests 72 \ - --chunked-prefill-size -1 --max-prefill-token 180000 \ - --cuda-graph-bs 8 16 24 32 40 \ - --moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --dtype bfloat16 \ - --trust-remote-code \ - --tokenizer-worker-num 8 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 65536 --random-output-len 1024 --num-prompts 144 --random-range-ratio 1 --max-concurrency 36 -``` -### MiniMax-M2.5 128K-1K High Throughput on A3 8 Cards Mixed Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 128K+1K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export TASK_QUEUE_ENABLE=1 - -export ASCEND_USE_FIA=1 -export HCCL_BUFFSIZE=1600 -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1 -export SGLANG_NPU_FUSEEP_DECODE_ONLY=1 - -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 \ - --port 32000 \ - --tp-size 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --prefill-delayer-max-delay-passes 100 \ - --enable-prefill-delayer \ - --mem-fraction-static 0.65 \ - --max-running-requests 36 \ - --chunked-prefill-size -1 --max-prefill-token 130000 \ - --cuda-graph-bs 8 16 24 \ - --moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --dtype bfloat16 \ - --trust-remote-code \ - --tokenizer-worker-num 8 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 131072 --random-output-len 1024 --num-prompts 128 --random-range-ratio 1 --max-concurrency 32 -``` -### MiniMax-M2.5 64K-1K High Throughput on A3 4 Cards Mixed Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 4Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 64K+1K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export TASK_QUEUE_ENABLE=1 - -export ASCEND_USE_FIA=0 -export HCCL_BUFFSIZE=1600 -export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1 -export SGLANG_NPU_FUSEEP_DECODE_ONLY=1 - -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 \ - --port 32000 \ - --tp-size 8 \ - --enable-dp-attention \ - --prefill-delayer-max-delay-passes 500 \ - --enable-prefill-delayer \ - --mem-fraction-static 0.65 \ - --max-running-requests 36 \ - --chunked-prefill-size -1 --max-prefill-token 150000 \ - --cuda-graph-bs 8 16 24 32 40 \ - --moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --dtype bfloat16 \ - --trust-remote-code \ - --tokenizer-worker-num 8 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 65536 --random-output-len 1024 --num-prompts 144 --random-range-ratio 1 --max-concurrency 36 -``` -### MiniMax-M2.5 64K-1K High Throughput on A3 16 Cards Disaggregation Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 64K+1K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your_prefill_ip:24667" - -P_IP=('your_prefill_ip') -D_IP=('your_decode_ip') -D_MASTER="${D_IP[0]}:8001" -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot - -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - export ASCEND_USE_FIA=1 - export HCCL_BUFFSIZE=2500 - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 - export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 32000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.43 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 128 \ - --chunked-prefill-size -1 --max-prefill-tokens 58000 --moe-a2a-backend deepep --deepep-mode normal \ - --tokenizer-worker-num 16 \ - --dp-size 2 --enable-dp-attention --dtype bfloat16 --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant --skip-server-warmup - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export HCCL_BUFFSIZE=1600 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_NPU_FUSED_MOE_MODE=2 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} --disaggregation-decode-extra-slots 96 \ - --cuda-graph-bs 8 16 24 32 40 \ - --port 33000 --trust-remote-code \ - --tp-size 16 --mem-fraction-static 0.76 --attention-backend ascend --device npu --quantization modelslim \ - --nnodes 1 --node-rank $i --dist-init-addr $D_MASTER \ - --disaggregation-transfer-backend ascend --max-running-requests 80 \ - --chunked-prefill-size -1 --moe-a2a-backend ascend_fuseep --deepep-mode low_latency \ - --tokenizer-worker-num 16 \ - --dp-size 2 --enable-dp-attention --dtype bfloat16 \ - --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant - - NODE_RANK=$i - break - fi -done -``` - -```shell Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy round_robin \ - --prefill http://your_prefill_ip:32000 8998 \ - --decode http://your_decode_ip:33000 \ - --host 127.0.0.1 \ - --mini-lb \ - --port 6688 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --random-input-len 65536 --random-output-len 1024 --num-prompts 640 --random-range-ratio 1 --max-concurrency 160 -``` -### MiniMax-M2.5 128K-1K High Throughput on A3 16 Cards Disaggregation Mode - -Model: MiniMax-M2.5 - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 128K+1K - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -export SGLANG_SET_CPU_AFFINITY=1 -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your_prefill_ip:24667" - -P_IP=('your_prefill_ip') -D_IP=('your_decode_ip') -D_MASTER="${D_IP[0]}:8001" -MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot - -EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model -export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - export ASCEND_USE_FIA=1 - export HCCL_BUFFSIZE=2500 - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 - export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 32000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.43 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 128 \ - --chunked-prefill-size -1 --max-prefill-tokens 130000 --moe-a2a-backend deepep --deepep-mode normal \ - --tokenizer-worker-num 16 \ - --dp-size 2 --enable-dp-attention --dtype bfloat16 --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 2 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 3 \ - --speculative-draft-model-quantization unquant --skip-server-warmup - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export HCCL_BUFFSIZE=1600 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_NPU_FUSED_MOE_MODE=2 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} --disaggregation-decode-extra-slots 96 \ - --cuda-graph-bs 2 4 8 \ - --port 33000 --trust-remote-code \ - --tp-size 16 --mem-fraction-static 0.76 --attention-backend ascend --device npu --quantization modelslim \ - --nnodes 1 --node-rank $i --dist-init-addr $D_MASTER \ - --disaggregation-transfer-backend ascend --max-running-requests 80 \ - --chunked-prefill-size -1 --moe-a2a-backend ascend_fuseep --deepep-mode low_latency \ - --tokenizer-worker-num 8 \ - --dp-size 2 --enable-dp-attention --dtype bfloat16 \ - --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $EAGLE_MODEL_PATH \ - --speculative-num-steps 2 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 3 \ - --speculative-draft-model-quantization unquant - - NODE_RANK=$i - break - fi -done -``` - -```shell Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy round_robin \ - --prefill http://your_prefill_ip:32000 8998 \ - --decode http://your_decode_ip:33000 \ - --host 127.0.0.1 \ - --mini-lb \ - --port 6688 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --random-input-len 131072 --random-output-len 1024 --num-prompts 192 --random-range-ratio 1 --max-concurrency 48 -``` - - -### Kimi K2.5 w4a8 3_5K-1_5K 20ms on A3 8 Cards Mixed Mode - -Model: Kimi-K2.5-w4a8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 20ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export STREAMS_PER_DEVICE=32 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48 -export HCCL_BUFFSIZE=1200 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -MODEL_PATH=xxx -DRAFT_PATH=xxx - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH --quantization modelslim --dtype bfloat16 \ - --model-loader-extra-config '{"enable_multithread_load": true}' \ - --host 0.0.0.0 --port 6699 \ - --trust-remote-code --device npu --attention-backend ascend \ - --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.78 --max-running-requests 64 \ - --chunked-prefill-size 32768 --context-length 8192 --max-prefill-tokens 16384 \ - --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ - --enable-dp-attention --dp-size 16 \ - --moe-a2a-backend deepep --deepep-mode auto \ - --cuda-graph-bs 1 2 3 4 --disable-radix-cache \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_PATH \ - --speculative-num-steps 4 --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 \ - --speculative-draft-model-quantization unquant -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 64 --random-output-len 1500 --random-input-len 3500 --num-prompts 64 -``` - - -### Kimi K2.5 w4a8 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode - -Model: Kimi-K2.5-w4a8 - -Hardware: Atlas 800I A3 8Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 50ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash - -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export STREAMS_PER_DEVICE=32 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 -export HCCL_BUFFSIZE=1200 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 - -MODEL_PATH=xxx -DRAFT_PATH=xxx - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH --quantization modelslim --dtype bfloat16 \ - --model-loader-extra-config '{"enable_multithread_load": true}' \ - --host 0.0.0.0 --port 6699 \ - --trust-remote-code --device npu --attention-backend ascend \ - --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.7 --max-running-requests 120 \ - --chunked-prefill-size 32768 --context-length 8192 --max-prefill-tokens 16384 \ - --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ - --enable-dp-attention --dp-size 16 \ - --moe-a2a-backend deepep --deepep-mode auto \ - --cuda-graph-bs 1 2 4 8 12 16 24 32 48 64 96 120 --disable-radix-cache \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_PATH \ - --speculative-num-steps 4 --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 \ - --speculative-draft-model-quantization unquant -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 120 --random-output-len 1500 --random-input-len 3500 --num-prompts 120 -``` - -### GLM-5.1 3_5K-1_5K 41ms on A3 16 Cards Mixed Mode - -Model: [GLM-5.1](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) - -The model is quantized, with MTP layers excluded from quantization. - -Hardware: Atlas 800I A3 16Card - -DeployMode: PD Mixed - -Dataset: random - -Input Output Length: 3.5K+1.5K - -TPOT: 41ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export PYTHONPATH=/path/to/sglang/python:$PYTHONPATH - -export STREAMS_PER_DEVICE=32 - -export HCCL_SOCKET_IFNAME=your_nic -export GLOO_SOCKET_IFNAME=your_nic - -MODEL_PATH=/path/to/GLM-5.1-w4a8 - -P_IP=('your ip1' 'your ip2') -P_MASTER="${P_IP[0]}:4567" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` - -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=2500 - python -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --dist-init-addr ${P_IP[0]}:5000 \ - --tp-size 32 --nnodes 2 --node-rank $i \ - --dp-size 16 --enable-dp-attention \ - --chunked-prefill-size 131072 --max-prefill-tokens 280000 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.65 \ - --port 8001 \ - --served-model-name glm-5 \ - --cuda-graph-max-bs 8 \ - --max-running-requests 128 \ - --quantization modelslim \ - --speculative-draft-model-quantization unquant \ - --moe-a2a-backend deepep --deepep-mode auto \ - --load-balance-method round_robin \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 - NODE_RANK=$i - break - fi -done -``` - - -**Quantization Configuration:** - -- `--quantization modelslim` is only applicable for quantized models. -- `--speculative-draft-model-quantization unquant` should be configured based on model specs, turned on for non-quantized MTP layers. - - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 8001 --random-range-ratio 1 --random-output-len 1500 --random-input-len 3500 --num-prompts 320 -``` - -### GLM-5.1 16K-1K 23ms on A3 32 Cards Disaggregation Mode - -Model: [GLM-5.1](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) - -Hardware: Atlas 800I A3 32Card - -DeployMode: PD Disaggregation - -Dataset: random - -Input Output Length: 16K+1K - -TPOT: 23ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTHONPATH=/path/to/sglang/python:$PYTHONPATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://${P_IP[0]}:24707" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -P_IP=('your prefill ip1' 'your prefill ip2') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=/path/to/GLM-5.1-w4a8 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - export ENABLE_PROFILING=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export HCCL_BUFFSIZE=8 - unset PYTORCH_NPU_ALLOC_CONF - export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 - export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True - export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://${P_IP[0]}:24672" - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port 8998 --dist-init-addr ${P_IP[0]}:5000 --trust-remote-code --nnodes 2 --node-rank $i \ - --tp-size 32 --mem-fraction-static 0.75 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 64 \ - --served-model-name glm-5 --chunked-prefill-size 524288 --max-prefill-tokens 180000 --moe-a2a-backend deepep --deepep-mode normal \ - --disable-shared-experts-fusion --disable-cuda-graph --dtype bfloat16 \ - --dp-size 4 --enable-dp-attention \ - --load-balance-method round_robin \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ - --attn-cp-size 8 \ - --enable-dp-lm-head --moe-dense-tp 1 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 - export TASK_QUEUE_ENABLE=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export SGLANG_NPU_USE_MULTI_STREAM=1 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8003 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --ep-size 32 \ - --mem-fraction-static 0.87 --max-running-requests 128 --attention-backend ascend --device npu --quantization modelslim \ - --served-model-name glm-5 --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 - NODE_RANK=$i - break - fi -done -``` - -```shell Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy round_robin \ - --prefill http://your_prefill_ip1:8000 8998 \ - --decode http://your_decode_ip1:8003 \ - --host 127.0.0.1 \ - --port 6688 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 8003 --random-range-ratio 1 --random-output-len 1000 --random-input-len 16000 --num-prompts 192 -``` - -### GLM-5.1 64K-1K-90%_cache_hit 45ms on A3 48 Cards Disaggregation Mode - -Model: [GLM-5.1](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) - -Hardware: Atlas 800I A3 48Card - -DeployMode: PD Disaggregation - -Dataset: random (90% cache hit) - -Input Output Length: 64K+1K - -TPOT: 45ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTHONPATH=/path/to/sglang/python:$PYTHONPATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://${P_IP[0]}:24709" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=1200 -export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=1200 - -P_IP=('your prefill ip1' 'your prefill ip2' 'your prefill ip3' 'your prefill ip4') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=/path/to/GLM-5.1-w4a8 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - export ENABLE_PROFILING=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export ZBAL_HCCL_OP="send,recv" - export HCCL_BUFFSIZE=128 - unset PYTORCH_NPU_ALLOC_CONF - export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 - export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True - export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://${P_IP[$i]}:24691" - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998 + i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 4 --mem-fraction-static 0.72 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 16 \ - --served-model-name glm-5 --chunked-prefill-size 16384 --max-prefill-tokens 180000 --moe-a2a-backend deepep --deepep-mode normal \ - --disable-shared-experts-fusion --disable-cuda-graph --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ - --attn-cp-size 4 \ - --enable-dp-lm-head --moe-dense-tp 1 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --pp-size 4 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=300 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40 - export TASK_QUEUE_ENABLE=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export SGLANG_NPU_USE_MULTI_STREAM=1 - export SGLANG_LM_HEAD_TP=4 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8003 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --enable-dp-attention --ep-size 32 \ - --mem-fraction-static 0.85 --max-running-requests 320 --attention-backend ascend --device npu --quantization modelslim \ - --served-model-name glm-5 --moe-a2a-backend deepep --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disaggregation-enable-decode-radix-cache - NODE_RANK=$i - break - fi -done -``` - -```shell Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy round_robin \ - --prefill http://your_prefill_ip1:8000 8998 \ - --prefill http://your_prefill_ip2:8000 8999 \ - --prefill http://your_prefill_ip3:8000 9000 \ - --prefill http://your_prefill_ip4:8000 9001 \ - --decode http://your_decode_ip1:8003 \ - --host 127.0.0.1 \ - --port 6688 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset (90% cache hit), this dataset is generated through [this tool](https://github.com/rayn-zzz/aisbench_auto_tools_prefix/tree/main). - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 8003 --random-range-ratio 1 --random-output-len 1000 --random-input-len 64000 --num-prompts 192 -``` - -### GLM-5.1 128K-1K-90%_cache_hit 32ms on A3 48 Cards Disaggregation Mode - -Model: [GLM-5.1](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) - -Hardware: Atlas 800I A3 48Card - -DeployMode: PD Disaggregation - -Dataset: random (90% cache hit) - -Input Output Length: 128K+1K - -TPOT: 32ms - -#### Model Deployment - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/op_api/lib/:${LD_LIBRARY_PATH} -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTHONPATH=/path/to/sglang/python:$PYTHONPATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://${P_IP[0]}:24709" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=1200 -export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=1200 - -P_IP=('your prefill ip1' 'your prefill ip2') -P1_IP=('your prefill ip3' 'your prefill ip4') -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=/path/to/GLM-5.1-w4a8 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" - -# prefill group 1 -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - export ENABLE_PROFILING=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export ZBAL_HCCL_OP="send,recv" - export HCCL_BUFFSIZE=128 - unset PYTORCH_NPU_ALLOC_CONF - export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 - export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True - export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://${P_IP[0]}:24691" - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port 8998 --trust-remote-code --nnodes 2 --node-rank $i --dist-init-addr ${P_IP[0]}:5000 \ - --tp-size 4 --mem-fraction-static 0.72 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 32 \ - --served-model-name glm-5 --chunked-prefill-size 16384 --max-prefill-tokens 180000 --moe-a2a-backend deepep --deepep-mode normal \ - --disable-shared-experts-fusion --disable-cuda-graph --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ - --attn-cp-size 4 \ - --enable-dp-lm-head --moe-dense-tp 1 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --pp-size 8 - NODE_RANK=$i - break - fi -done - -# prefill group 2 -for i in "${!P1_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P1_IP[$i]}" || "$LOCAL_HOST2" == "${P1_IP[$i]}" ]]; - then - echo "${P1_IP[$i]}" - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export TASK_QUEUE_ENABLE=2 - export ENABLE_PROFILING=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export ZBAL_HCCL_OP="send,recv" - export HCCL_BUFFSIZE=128 - unset PYTORCH_NPU_ALLOC_CONF - export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 - export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True - export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True - export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://${P1_IP[0]}:24691" - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P1_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port 8999 --trust-remote-code --nnodes 2 --node-rank $i --dist-init-addr ${P1_IP[0]}:5000 \ - --tp-size 4 --mem-fraction-static 0.72 --attention-backend ascend --device npu --quantization modelslim \ - --disaggregation-transfer-backend ascend --max-running-requests 32 \ - --served-model-name glm-5 --chunked-prefill-size 16384 --max-prefill-tokens 180000 --moe-a2a-backend deepep --deepep-mode normal \ - --disable-shared-experts-fusion --disable-cuda-graph --dtype bfloat16 \ - --speculative-draft-model-quantization unquant \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ - --attn-cp-size 4 \ - --enable-dp-lm-head --moe-dense-tp 1 \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --pp-size 8 - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=200 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 - export TASK_QUEUE_ENABLE=0 - export HCCL_SOCKET_IFNAME=your_nic - export GLOO_SOCKET_IFNAME=your_nic - - export SGLANG_NPU_USE_MULTI_STREAM=1 - - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8003 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 --enable-dp-attention --ep-size 32 \ - --mem-fraction-static 0.865 --max-running-requests 96 --attention-backend ascend --device npu --quantization modelslim \ - --served-model-name glm-5 --moe-a2a-backend deepep --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 4 5 6 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 \ - --tokenizer-worker-num 32 --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --disaggregation-decode-enable-radix-cache - NODE_RANK=$i - break - fi -done -``` - -```shell Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy round_robin \ - --prefill http://your_prefill_ip1:8000 8998 \ - --prefill http://your_prefill_ip3:8000 8999 \ - --decode http://your_decode_ip1:8003 \ - --host 127.0.0.1 \ - --port 6688 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset (90% cache hit), this dataset is generated through [this tool](https://github.com/rayn-zzz/aisbench_auto_tools_prefix/tree/main). - -```bash Command -python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 8003 --random-range-ratio 1 --random-output-len 1000 --random-input-len 131072 --num-prompts 192 -``` - -### MiMo-V2-Flash 16K+1 974ms on A3 24 Cards Disaggregation Mode - -**Model:** MiMo-V2-Flash - -**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) - -**Deploy Mode:** PD Disaggregation - -**Dataset:** random - -**Input/Output Length:** 16K+1 - -**TTFT:** 974ms (P99: 2,068ms) - -**Quantization:** W8A8 INT8 - -#### Model Deployment - -**Prefill Node:** - -```bash -# High performance CPU configuration -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -# Bind CPU -export SGLANG_SET_CPU_AFFINITY=1 - -# Proxy settings (unset if needed) -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -# Performance environment variables -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 -export HCCL_BUFFSIZE=1024 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_NPU_PROFILING=0 -export SGLANG_NPU_PROFILING_STAGE="prefill" -export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 -export ASCEND_MF_STORE_URL="tcp://:24669" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 -export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_BF16_DISPATCH=0 -export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_CONNECT_TIMEOUT=1800 -export ASCEND_USE_FIA=1 - -# Launch prefill server -python3 -m sglang.launch_server \ - --model-path /path/to/MiMo-V2-Flash-w8a8-all-0512 \ - --attention-backend ascend \ - --device npu \ - --tp-size 8 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 8192 \ - --trust-remote-code --port 10000 \ - --host --max-running-requests 64 \ - --mem-fraction-static 0.8 \ - --swa-full-tokens-ratio 0.3 \ - --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8996 \ - --disable-radix-cache \ - --disable-cuda-graph \ - --disable-piecewise-cuda-graph \ - --dp-size 2 -``` - -**Decode Node:** - -```bash -# High performance CPU configuration -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -# Bind CPU -export SGLANG_SET_CPU_AFFINITY=1 - -# Proxy settings (unset if needed) -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -# Performance environment variables -export STREAMS_PER_DEVICE=32 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 -export HCCL_BUFFSIZE=800 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo -export SGLANG_NPU_PROFILING=0 -export SGLANG_NPU_PROFILING_STAGE="prefill" -export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 -export ASCEND_MF_STORE_URL="tcp://:24669" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 -export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_BF16_DISPATCH=0 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export HCCL_CONNECT_TIMEOUT=1800 -export SGLANG_PROFILE_WITH_STACK=True -export ASCEND_USE_FIA=1 - -# Launch decode server -python3 -m sglang.launch_server \ - --model-path /path/to/MiMo-V2-Flash-w8a8-all-0512/ \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 --nnodes 1 --node-rank 0 \ - --trust-remote-code --port 10001 \ - --host --max-running-requests 64 \ - --mem-fraction-static 0.8 \ - --swa-full-tokens-ratio 0.3 \ - --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ - --disaggregation-mode decode --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8996 \ - --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --enable-multi-layer-eagle \ - --disable-radix-cache \ - --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ - --moe-a2a-backend deepep --deepep-mode low_latency -``` - -**Router:** - -```bash -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://:10000 \ - --decode http://:10001 \ - --host \ - --port 9903 \ - --health-check-interval-secs 3600 \ - --mini-lb -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset with throughput focus (0.4 request rate). - -```bash -python3 -m sglang.bench_serving --backend sglang --host --port 9903 --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --tokenize-prompt --random-input-len 16000 --random-output-len 1 --request-rate 0.4 --random-range-ratio 1 --num-prompts 128 --max-concurrency 64 -``` - -### MiMo-V2-Flash 32K+1 2,147ms on A3 24 Cards Disaggregation Mode - -**Model:** MiMo-V2-Flash - -**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) - -**Deploy Mode:** PD Disaggregation - -**Dataset:** random - -**Input/Output Length:** 32K+1 - -**TTFT:** 2,147ms (P99: 4,546ms) - -**Quantization:** W8A8 INT8 - -#### Model Deployment - -Same deployment configuration as [MiMo-V2-Flash 16K+1 974ms](#mimo-v2-flash-16k-1-974ms-on-a3-24-cards-disaggregation-mode). - -### MiMo-V2-Flash 16K+1K 16ms on A3 24 Cards Disaggregation Mode - -**Model:** MiMo-V2-Flash - -**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) - -**Deploy Mode:** PD Disaggregation - -**Dataset:** random - -**Input/Output Length:** 16K+1K - -**TTFT:** 11,326ms (P99: 28,160ms) - -**TPOT:** 16ms (P99: 23ms) - -**Quantization:** W8A8 INT8 - -#### Model Deployment - -Same deployment configuration as [MiMo-V2-Flash 16K+1 974ms](#mimo-v2-flash-16k-1-974ms-on-a3-24-cards-disaggregation-mode). - -#### Benchmark - -We tested it based on the `RANDOM` dataset with low latency focus (infinite request rate). - -```bash -python3 -m sglang.bench_serving --backend sglang --host --port 9903 --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --tokenize-prompt --random-input-len 16000 --random-output-len 1000 --request-rate inf --random-range-ratio 1 --num-prompts 128 --max-concurrency 64 -``` - -### MiMo-V2-Flash 32K+1K 12ms on A3 24 Cards Disaggregation Mode - -**Model:** MiMo-V2-Flash - -**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) - -**Deploy Mode:** PD Disaggregation - -**Dataset:** random - -**Input/Output Length:** 32K+1K - -**TTFT:** 35,144ms (P99: 55,004ms) - -**TPOT:** 12ms (P99: 17ms) - -**Quantization:** W8A8 INT8 - -#### Model Deployment - -Same deployment configuration as [MiMo-V2-Flash 16K+1 974ms](#mimo-v2-flash-16k-1-974ms-on-a3-24-cards-disaggregation-mode). diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx deleted file mode 100644 index 823e2a046..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx +++ /dev/null @@ -1,300 +0,0 @@ ---- -title: "DeepSeek Examples" -metatags: - description: "Examples for running DeepSeek models on Ascend NPUs, including PD mixed mode, PD disaggregation, and SGLang Model Gateway." ---- - -## Running DeepSeek-V3 - - -If you need to download model weights, check the model size at [ModelScope](https://www.modelscope.cn/models) to reserve enough space. - - -### Running DeepSeek in PD mixed mode on 1 x Atlas 800I A3. - -W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - -```shell Launch Server -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#Deepep communication settings -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 -export HCCL_BUFFSIZE=1600 - -#spec overlap -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -#npu acceleration operator -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --tp 16 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --cuda-graph-bs 8 16 24 28 32 \ - --mem-fraction-static 0.68 \ - --max-running-requests 128 \ - --context-length 8188 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 16384 \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --enable-dp-attention \ - --dp-size 4 \ - --enable-dp-lm-head \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --dtype bfloat16 -``` - -### Running DeepSeek with PD disaggregation mode on 2 x Atlas 800I A3. - -W4A8 Model weights could be found [here](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8). - -1. Prefill: - -```bash Command -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#memfabric config store -export ASCEND_MF_STORE_URL="tcp://:" - -#Deepep communication settings -export HCCL_BUFFSIZE=1536 - -#npu acceleration operator -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 -export TASK_QUEUE_ENABLE=2 - -python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --host $PREFILL_HOST_IP \ - --port 8000 \ - --disaggregation-mode prefill \ - --disaggregation-bootstrap-port 8996 \ - --disaggregation-transfer-backend ascend \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --mem-fraction-static 0.6 \ - --attention-backend ascend \ - --device npu \ - --load-balance-method round_robin \ - --max-running-requests 8 \ - --context-length 8192 \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 28680 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --dp-size 2 \ - --enable-dp-attention \ - --disable-shared-experts-fusion \ - --dtype bfloat16 -``` - -2. Decode: - -```bash Command -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -#memfabric config store -export ASCEND_MF_STORE_URL="tcp://:" - -#Deepep communication settings -export HCCL_BUFFSIZE=720 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 - -#spec overlap -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -#npu acceleration operator -unset TASK_QUEUE_ENABLE -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -# suggest max-running-requests <= max-cuda-graph-bs * dp_size, Because when this value is exceeded, performance will significantly degrade. -python -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host $DECODE_HOST_IP \ - --port 8001 \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --tp-size 16 \ - --dp-size 16 \ - --mem-fraction-static 0.8 \ - --max-running-requests 352 \ - --attention-backend ascend \ - --device npu \ - --moe-a2a-backend deepep \ - --enable-dp-attention \ - --deepep-mode low_latency \ - --enable-dp-lm-head \ - --cuda-graph-bs 8 10 12 14 16 18 20 22 \ - --disaggregation-transfer-backend ascend \ - --watchdog-timeout 9000 \ - --context-length 8192 \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --disable-shared-experts-fusion \ - --dtype bfloat16 \ - --tokenizer-worker-num 4 -``` - -3. SGLang Model Gateway (former Router) - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://:8000 8996 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 -``` - -### Running DeepSeek with PD disaggregation on 4 x Atlas 800I A3. - -W8A8 Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). - -1. Prefill & Decode: - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -export SGLANG_SET_CPU_AFFINITY=1 -unset ASCEND_LAUNCH_BLOCKING -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh -export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 - -export ASCEND_MF_STORE_URL="tcp://your prefill ip1:24669" - -P_IP=('your prefill ip1' 'your prefill ip2') - -D_IP=('your decode ip1' 'your decode ip2') - -MODEL_PATH=xxx - -export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_USE_FIA_NZ=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export HCCL_BUFFSIZE=1536 - export TASK_QUEUE_ENABLE=2 - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \ - --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \ - --tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu \ - --disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \ - --chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \ - --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \ - --dp-size 2 --enable-dp-attention --disable-shared-experts-fusion --dtype bfloat16 --enable-attn-tp-input-scattered - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export HCCL_BUFFSIZE=650 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 - export TASK_QUEUE_ENABLE=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \ - --port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \ - --mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu \ - --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \ - --cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \ - --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 \ - --load-balance-method decode_round_robin - NODE_RANK=$i - break - fi -done -``` - -2. SGLang Model Gateway (former Router): - -```bash Command -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://P_IP:8000 8998 \ - --prefill http://P_IP:8000 8999 \ - --decode http://D_IP:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --mini-lb -``` - -## Testing the Service - -Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For testing examples (Health Check, Generate, Chat Completions, and port usage guidance), see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). - -### Test GSM8K - -```python Test GSM8K -from types import SimpleNamespace -from sglang.test.few_shot_gsm8k import run_eval - -def gsm8k(): - args = SimpleNamespace( - num_shots=5, - data_path=None, - num_questions=200, - max_new_tokens=512, - parallel=32, - host=f"http://127.0.0.1", - port=6688, - ) - metrics = run_eval(args) - print(f"{metrics=}") - print(f"{metrics['accuracy']=}") -if __name__ == "__main__": - gsm8k() -``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx index b16d1cefb..d60cf4f09 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_faq.mdx @@ -288,3 +288,14 @@ FP8 model is not supported ### Solution Replace model weights, e.g., switch Qwen/Qwen3.5-27B-FP8 to Eco-Tech/Qwen3.5-27B-w8a8-mtp. + +## 8. Docker image versions: stable release vs. daily build + +Docker images for Ascend NPU are available in two types: + +- **Stable release** — validated version with a specific tag, e.g., `quay.io/ascend/sglang:v0.5.10-npu.rc1-a3`. + Recommended for production deployments. +- **Daily build** — includes the latest development changes, e.g., `quay.io/ascend/sglang:main-cann8.5.0-a3`. Use this + if you need the latest features or bug fixes that have not yet been included in a stable release. + +If you encounter issues with a stable release, try switching to a daily build to see if the issue has been resolved in the latest development version. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx deleted file mode 100644 index cf22b3748..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx +++ /dev/null @@ -1,240 +0,0 @@ ---- -title: "GLM-5 examples" -metatags: - description: "Documentation for GLM-5 examples" ---- -## Introduction - -The GLM (General Language Model) series is an open-source bilingual large language model family jointly developed by the KEG Laboratory of Tsinghua University and Zhipu AI. This series of models has performed outstandingly in the field of Chinese NLP with its unique unified pre-training framework and bilingual capabilities. [GLM-5](https://huggingface.co/zai-org/GLM-5) adopts the DeepSeek-V3/V3.2 architecture, including the sparse attention (DSA) and multi-token prediction (MTP). Ascend supports GLM-5 with 0Day based on the SGLang inference framework, achieving low-code seamless enablement and compatibility with the mainstream distributed parallel capabilities within the current SGLang framework. We welcome developers to download and experience it. - -## Environment Preparation - -### Model Weight - - -If you need to download model weights, check the model size at [ModelScope](https://www.modelscope.cn/models) to reserve enough space. - - -- `GLM-5.0`(BF16 version): [Download model weight](https://www.modelscope.cn/models/ZhipuAI/GLM-5). -- `GLM-5.0-w4a8`(Quantized version without mtp): [Download model weight](https://modelers.cn/models/Eco-Tech/GLM-5-w4a8). -- You can use [msmodelslim](https://gitcode.com/Ascend/msmodelslim) to quantify the model naively. - - -### Installation - - -Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. - - -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. - - - - -```bash Command -docker pull quay.io/ascend/sglang:v0.5.10-npu.rc1-a3 - -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci8:/dev/davinci8 \ ---device=/dev/davinci9:/dev/davinci9 \ ---device=/dev/davinci10:/dev/davinci10 \ ---device=/dev/davinci11:/dev/davinci11 \ ---device=/dev/davinci12:/dev/davinci12 \ ---device=/dev/davinci13:/dev/davinci13 \ ---device=/dev/davinci14:/dev/davinci14 \ ---device=/dev/davinci15:/dev/davinci15 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -quay.io/ascend/sglang:v0.5.10-npu.rc1-a3 -``` - - - - -```bash Command -docker pull quay.io/ascend/sglang:v0.5.10-npu.rc1-910b - -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -quay.io/ascend/sglang:v0.5.10-npu.rc1-910b -``` - - - - -### Best Practices -Note: Using this image for **best practices**, you need to update transformers to version 5.3.0 -``` -# reinstall transformers - -# Install transformers version 5.3.0 from PyPI -pip install transformers==5.3.0 - -# Install from GitHub v5.3.0 tag from GitHub -pip install git+https://github.com/huggingface/transformers.git@v5.3.0 -``` - -## Deployment - -### Single-node Deployment - -- Quantized model `glm5_w4a8` can be deployed on 1 Atlas 800 A3 (64G × 16) . - -Run the following script to execute online inference. - -```shell Launch Server -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 16384 --max-prefill-tokens 280000 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --served-model-name glm-5 \ - --cuda-graph-bs 16 \ - --moe-a2a-backend deepep --deepep-mode auto -``` - -### Multi-node Deployment - -- `GLM-5-bf16`: require at least 2 Atlas 800 A3 (64G × 16). - -**A3 series** - -Modify the IP of 2 nodes, then run the same scripts on two nodes. - -**node 0/1** - -```shell Launch Multi-node Server -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV - -# Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - - -P_IP=('your ip1' 'your ip2') -P_MASTER="${P_IP[0]}:your port" -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 - -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 32 --nnodes 2 --node-rank $i --dist-init-addr $P_MASTER \ - --chunked-prefill-size 16384 --max-prefill-tokens 131072 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.8\ - --port 8000 \ - --served-model-name glm-5 \ - --cuda-graph-max-bs 16 \ - --disable-radix-cache - NODE_RANK=$i - break - fi -done - -``` - -### Prefill-Decode Disaggregation - -Not tested yet. - -## Testing the Service - -Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For testing examples (Health Check, Chat Completions, and port usage guidance), see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). - -### Using Benchmark - -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling) for details. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx deleted file mode 100644 index 926eab477..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx +++ /dev/null @@ -1,311 +0,0 @@ ---- -title: "Kimi K2.5 examples" -metatags: - description: "Documentation for Kimi K2.5 examples" ---- -## Introduction - -Kimi K2.5 is an open-source, native multimodal agentic model built through continual pretraining on approximately 15 trillion mixed visual and text tokens atop Kimi-K2-Base. It seamlessly integrates vision and language understanding with advanced agentic capabilities, instant and thinking modes, as well as conversational and agentic paradigms. - -## Environment Preparation - -### Model Weight - -- `Kimi-K2.5`: Download Kimi-K2.5 from [modelers](https://modelers.cn/models/moonshotai/Kimi-K2.5), [modelscope](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.5-w4a8) or [huggingface](https://huggingface.co/moonshotai/Kimi-K2.5). -- `Kimi-K2.5-w4a8`: Download Kimi-K2.5-w4a8 from [modelers](https://modelers.cn/models/Eco-Tech/Kimi-K2.5-w4a8), [modelscope](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.5-w4a8). -- You can use [msmodelslim](https://gitcode.com/Ascend/msmodelslim) to quantize `Kimi-K2.5-w4a8` from `Kimi-K2.5`. - - -### Installation - -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. - -```bash Command -# Atlas 800 A3 -docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-openEuler-20260427 - -# Start container -docker run -itd --shm-size=16g --privileged=true --name ${NAME} --net=host \ - -v /var/queue_schedule:/var/queue_schedule \ - -v /etc/ascend_install.info:/etc/ascend_install.info \ - -v /usr/local/sbin:/usr/local/sbin \ - -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ - -v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ - --device=/dev/davinci0:/dev/davinci0 \ - --device=/dev/davinci1:/dev/davinci1 \ - --device=/dev/davinci2:/dev/davinci2 \ - --device=/dev/davinci3:/dev/davinci3 \ - --device=/dev/davinci4:/dev/davinci4 \ - --device=/dev/davinci5:/dev/davinci5 \ - --device=/dev/davinci6:/dev/davinci6 \ - --device=/dev/davinci7:/dev/davinci7 \ - --device=/dev/davinci8:/dev/davinci8 \ - --device=/dev/davinci9:/dev/davinci9 \ - --device=/dev/davinci10:/dev/davinci10 \ - --device=/dev/davinci11:/dev/davinci11 \ - --device=/dev/davinci12:/dev/davinci12 \ - --device=/dev/davinci13:/dev/davinci13 \ - --device=/dev/davinci14:/dev/davinci14 \ - --device=/dev/davinci15:/dev/davinci15 \ - --device=/dev/davinci_manager:/dev/davinci_manager \ - --device=/dev/hisi_hdc:/dev/hisi_hdc \ - --entrypoint=bash \ - swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-openEuler-20260427 -``` - -### Best Practices -Note: Using this image for **best practices**, switch to the more stable version 4.57.1 of transformers. -``` -# reinstall transformers - -# Install transformers version 4.57.1 from PyPI -pip install transformers==4.57.1 - -# Install from GitHub v4.57.1 tag from GitHub -pip install git+https://github.com/huggingface/transformers.git@v4.57.1 -``` - -## Deployment - -### Single-node Deployment - -- Quantized model `Kimi-K2.5-w4a8` can be deployed on 1 Atlas 800 A3 (64G x 16) . - -Run the following script to execute online inference. - -```shell Launch Server -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 -export HCCL_BUFFSIZE=1500 -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python -m sglang.launch_server \ - --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ - --model-loader-extra-config '{"enable_multithread_load": true}' \ - --host 0.0.0.0 --port 8000 \ - --trust-remote-code --device npu --attention-backend ascend \ - --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.77 --max-running-requests 256 \ - --chunked-prefill-size 32768 --context-length 8192 --max-prefill-tokens 16384 \ - --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ - --enable-dp-attention --dp-size 16 \ - --moe-a2a-backend deepep --deepep-mode auto \ - --cuda-graph-bs 1 2 4 8 9 10 12 16 --disable-radix-cache \ - --enable-prefill-delayer --prefill-delayer-max-delay-passes 200 \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path ${DRAFT_MODEL_PATH} \ - --speculative-num-steps 4 --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 \ - --speculative-draft-model-quantization unquant -``` - -### Multi-node Deployment - -- Quantized model `Kimi-K2.5-w4a8` also can be deployed on 2 Atlas 800 A3 (64G x 16) . - -**A3 series** - -Modify the IP of 2 nodes, then run the same scripts on two nodes. - -**node 0/1** - -```shell Launch Multi-node Server -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 -export HCCL_BUFFSIZE=2400 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 -export SGLANG_NPU_USE_MLAPO=1 - -# Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here -export HCCL_SOCKET_IFNAME=xxx -export GLOO_SOCKET_IFNAME=xxx - -NODES_IP=('your ip1' 'your ip2') - -LOCAL_IPS=$(hostname -I) -for i in "${!NODES_IP[@]}"; -do - if [[ " ${LOCAL_IPS} " =~ " ${NODES_IP[$i]} " ]]; - then - echo "${NODES_IP[$i]}" - python -m sglang.launch_server \ - --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ - --model-loader-extra-config '{"enable_multithread_load": true}' \ - --host 0.0.0.0 --port 8880 --dist-init-addr ${NODES_IP[0]}:5000 --nnodes 2 --node-rank $i \ - --trust-remote-code --device npu --attention-backend ascend \ - --tp-size 32 --base-gpu-id 0 --mem-fraction-static 0.62 --max-running-requests 386 \ - --chunked-prefill-size 65536 --context-length 8192 --max-prefill-tokens 16384 \ - --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ - --enable-dp-attention --dp-size 32 \ - --moe-a2a-backend deepep --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 10 12 --disable-radix-cache \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path ${DRAFT_MODEL_PATH} \ - --speculative-num-steps 4 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 \ - --speculative-draft-model-quantization unquant - NODE_RANK=$i - break - fi -done - -``` - -### Prefill-Decode Disaggregation - -Here is an example of a large scale EP scenario (3P1D) for `Kimi-K2.5-w4a8` on 5 Atlas 800 A3 (64G x 16) . - -Modify the IP of 5 nodes, then run the same scripts on 5 nodes. Then start an additional Router on the first P node. - -```shell -# System Settings -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=10 -sysctl -w kernel.numa_balancing=0 - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -P_IP=('your prefill ip1' 'your prefill ip2' 'your prefill ip3') -D_IP=('your decode ip1' 'your decode ip2') -LOCAL_IPS=$(hostname -I) - -export ASCEND_MF_STORE_URL="tcp://${P_IP[0]}:24669" - -for i in "${!P_IP[@]}"; -do - if [[ " ${LOCAL_IPS} " =~ " ${P_IP[$i]} " ]]; - then - echo "Prefill -> ${P_IP[$i]}" - - export HCCL_SOCKET_IFNAME=lo - export GLOO_SOCKET_IFNAME=lo - - export HCCL_BUFFSIZE=3600 - - export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 - - python -m sglang.launch_server \ - --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ - --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ - --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --nnodes 1 --node-rank 0 \ - --trust-remote-code --device npu --attention-backend ascend \ - --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.68 --max-running-requests 8 \ - --chunked-prefill-size 32768 --context-length 81920 \ - --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ - --enable-dp-attention --dp-size 4 \ - --disable-radix-cache \ - --moe-a2a-backend deepep --deepep-mode auto - NODE_RANK=$i - exit 0 - fi -done - - -for i in "${!D_IP[@]}"; -do - if [[ " ${LOCAL_IPS} " =~ " ${D_IP[$i]} " ]]; - then - echo "Decode -> ${D_IP[$i]}" - - # Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here - export HCCL_SOCKET_IFNAME=xxx - export GLOO_SOCKET_IFNAME=xxx - - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 - export HCCL_BUFFSIZE=2400 - - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - - export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 - - python -m sglang.launch_server \ - --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ - --disaggregation-mode decode --disaggregation-transfer-backend ascend \ - --host ${D_IP[$i]} --port 8001 --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i \ - --trust-remote-code --device npu --attention-backend ascend \ - --tp-size 32 --base-gpu-id 0 --mem-fraction-static 0.74 --max-running-requests 128 \ - --chunked-prefill-size 65536 --context-length 81920 \ - --disable-radix-cache \ - --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ - --enable-dp-attention --dp-size 32 \ - --moe-a2a-backend deepep --deepep-mode auto \ - --cuda-graph-bs 4 - NODE_RANK=$i - exit 0 - fi -done -``` - -SGLang Model Gateway (former Router) - -```shell -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://:8000 8998 \ - --prefill http://:8000 8999 \ - --prefill http://:8000 9000 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 \ -``` - - -### Using Benchmark - -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling) for details. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx deleted file mode 100644 index de4d0d8bd..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx +++ /dev/null @@ -1,354 +0,0 @@ ---- -title: "Qwen3.5 examples" -metatags: - description: "Documentation for Qwen3.5 examples" ---- -## Environment Preparation - -### Installation - - -Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. If you need to download model weights, check the model size at [ModelScope](https://www.modelscope.cn/models) to reserve enough space. - - -The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. - - - - -```bash Command -docker pull quay.io/ascend/sglang:v0.5.10-npu.rc1-a3 - -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci8:/dev/davinci8 \ ---device=/dev/davinci9:/dev/davinci9 \ ---device=/dev/davinci10:/dev/davinci10 \ ---device=/dev/davinci11:/dev/davinci11 \ ---device=/dev/davinci12:/dev/davinci12 \ ---device=/dev/davinci13:/dev/davinci13 \ ---device=/dev/davinci14:/dev/davinci14 \ ---device=/dev/davinci15:/dev/davinci15 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -quay.io/ascend/sglang:v0.5.10-npu.rc1-a3 -``` - - - - -```bash Command -docker pull quay.io/ascend/sglang:v0.5.10-npu.rc1-910b - -docker run -itd --shm-size=16g --privileged=true --name ${NAME} \ ---privileged=true --net=host \ --v /var/queue_schedule:/var/queue_schedule \ --v /etc/ascend_install.info:/etc/ascend_install.info \ --v /usr/local/sbin:/usr/local/sbin \ --v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ --v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ ---device=/dev/davinci0:/dev/davinci0 \ ---device=/dev/davinci1:/dev/davinci1 \ ---device=/dev/davinci2:/dev/davinci2 \ ---device=/dev/davinci3:/dev/davinci3 \ ---device=/dev/davinci4:/dev/davinci4 \ ---device=/dev/davinci5:/dev/davinci5 \ ---device=/dev/davinci6:/dev/davinci6 \ ---device=/dev/davinci7:/dev/davinci7 \ ---device=/dev/davinci_manager:/dev/davinci_manager \ ---device=/dev/hisi_hdc:/dev/hisi_hdc \ ---entrypoint=bash \ -quay.io/ascend/sglang:v0.5.10-npu.rc1-910b -``` - - - - -## Deployment - -### Single-node Deployment - -Run the following script to execute online inference. - -#### Qwen3.5 397B - - -Recommended model: [`Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp`](https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp) - - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - -#### Qwen3.5 122B - - -Recommended model: [`Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp`](https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-122B-A10B-w8a8-mtp) - - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 8 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - -#### Qwen3.5 35B - - -Recommended model: [`Eco-Tech/Qwen3.5-35B-A3B-w8a8-mtp`](https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-35B-A3B-w8a8-mtp) - - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 --nnodes 1 --node-rank 0 \ - --chunked-prefill-size 4096 --max-prefill-tokens 280000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.7 \ - --port 8000 \ - --cuda-graph-bs 16 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 -``` - -#### Qwen3.5 27B - - -Recommended model: [`Eco-Tech/Qwen3.5-27B-w8a8-mtp`](https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-27B-w8a8-mtp) - - -```bash Command -# high performance cpu -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1000 -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 2 \ - --chunked-prefill-size -1 --max-prefill-tokens 120000 \ - --disable-radix-cache \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.8 \ - --port 8000 \ - --cuda-graph-bs 32 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn -``` - -### Multi-node Deployment - - -Recommended model: [`Qwen/Qwen3.5-35B-A3B`](https://www.modelscope.cn/models/Qwen/Qwen3.5-35B-A3B) - -Other Qwen3.5 series models can also be deployed in multi-node configurations following this workflow. Simply change `--model-path` to the corresponding model, and adjust parameters like `--tp-size`, `--nnodes`, and `--mem-fraction-static` according to the model size and available resources. - - -**A2 series** - -Modify the IP of 2 nodes, then run the same scripts on two nodes. - -**node 0/1** - -```bash Command -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 -# bind cpu -export SGLANG_SET_CPU_AFFINITY=1 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING -# cann -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export STREAMS_PER_DEVICE=32 -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_MULTI_STREAM=1 -export HCCL_BUFFSIZE=1000 - -# Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here -export HCCL_SOCKET_IFNAME=lo -export GLOO_SOCKET_IFNAME=lo - - -P_IP=('your ip1' 'your ip2') -P_MASTER="${P_IP[0]}:your port" - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --attention-backend ascend \ - --device npu \ - --tp-size 8 --nnodes 2 --node-rank $i --dist-init-addr $P_MASTER \ - --chunked-prefill-size 16384 --max-prefill-tokens 131072 \ - --trust-remote-code \ - --host 127.0.0.1 \ - --mem-fraction-static 0.8\ - --port 8000 \ - --served-model-name qwen3.5 \ - --cuda-graph-max-bs 16 \ - --disable-radix-cache - NODE_RANK=$i - break - fi -done -``` -### Prefill-Decode Disaggregation - -Not tested yet. - -## Testing the Service - -Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For testing examples (Health Check, Generate, Chat Completions, Multimodal Chat Completions, and port usage guidance), see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). - -### Using Benchmark - -Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling) for details. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx deleted file mode 100644 index 24f35022e..000000000 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx +++ /dev/null @@ -1,293 +0,0 @@ ---- -title: "Qwen3 Examples" -metatags: - description: "Documentation for Qwen3 Examples" ---- -## Qwen3 examples - - -If you need to download model weights, check the model size at [ModelScope](https://www.modelscope.cn/models) to reserve enough space. - - -### Running Qwen3 - -#### Running Qwen3-32B on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-32B) - -```shell Launch Server -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export HCCL_OP_EXPANSION_MODE=AIV - -python -m sglang.launch_server \ - --device npu \ - --attention-backend ascend \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-32B \ - --mem-fraction-static 0.8 -``` - -#### Running Qwen3-32B on 1 x Atlas 800I A3 with Qwen3-32B-Eagle3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-32B) - -Speculative model weights could be found [here](https://huggingface.co/Zhihu-ai/Zhi-Create-Qwen3-32B-Eagle3) - -```shell Launch Server with Eagle3 -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - -python -m sglang.launch_server \ - --device npu \ - --attention-backend ascend \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-32B \ - --mem-fraction-static 0.8 \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path Qwen/Qwen3-32B-Eagle3 \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 -``` - -#### Running Qwen3-30B-A3B MOE on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-30B-A3B) - -```shell Launch Server -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export HCCL_OP_EXPANSION_MODE=AIV -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 - -python -m sglang.launch_server \ - --device npu \ - --attention-backend ascend \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-30B-A3B \ - --mem-fraction-static 0.8 -``` - -#### Running Qwen3-235B-A22B-Instruct-2507 MOE on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507) - -```shell Launch Server -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 - -python -m sglang.launch_server \ - --model-path Qwen/Qwen3-235B-A22B-Instruct-2507 \ - --tp-size 16 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --watchdog-timeout 9000 \ - --mem-fraction-static 0.8 -``` - -#### Running Qwen3-235B-A22B-Instruct-2507 with 256K long sequence on 2 x Atlas 800I A3 without CP - -This example uses **PD disaggregation** for long-sequence inference and keeps **context parallel disabled**. - -Set the shared environment variables on both nodes first: - -```bash Command -export ASCEND_USE_FIA=1 -export SGLANG_SET_CPU_AFFINITY=1 -export ASCEND_MF_STORE_URL="tcp://:12345" -export HCCL_SOCKET_IFNAME= -export GLOO_SOCKET_IFNAME= - -MODEL_PATH=/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-2507-w8a8 -``` - -**Prefill node:** - -```bash Command -export ASCEND_LAUNCH_BLOCKING=1 -export HCCL_BUFFSIZE=1500 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=128 -export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8995 \ - --attention-backend ascend \ - --disable-radix-cache \ - --chunked-prefill-size -1 \ - --skip-server-warmup \ - --device npu \ - --tp-size 16 \ - --mem-fraction-static 0.45 \ - --max-running-requests 1 \ - --host \ - --port 8000 \ - --dist-init-addr :5000 \ - --nnodes 1 \ - --node-rank 0 \ - --moe-a2a-backend deepep \ - --deepep-mode normal -``` - -**Decode node:** - -```bash Command -export HCCL_BUFFSIZE=4000 -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 -export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 - -python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --attention-backend ascend \ - --mem-fraction-static 0.8 \ - --disable-cuda-graph \ - --device npu \ - --disable-radix-cache \ - --chunked-prefill-size 8192 \ - --skip-server-warmup \ - --tp-size 16 \ - --max-running-requests 1 \ - --host \ - --port 8232 \ - --moe-a2a-backend deepep \ - --deepep-mode low_latency \ - --disable-overlap-schedule -``` - -**Router:** - -```bash Command -python3 -m sglang_router.launch_router \ - --pd-disaggregation \ - --policy cache_aware \ - --prefill http://:8000 8995 \ - --decode http://:8232 \ - --host \ - --port 6689 \ - --prometheus-port 29010 -``` - -#### Running Qwen3-235B-A22B-Instruct-2507-W8A8 with Prefill Context Parallel (CP) on 2 x Atlas 800I A3 - -This example enables **Prefill Context Parallel** (`--enable-prefill-context-parallel`) to split the context across CP ranks during prefill, reducing per-device memory pressure and improving TTFT for long sequences. PD disaggregation is required. - -> **Constraints** -> - Prefill side must set `--max-running-requests 1` (PCP only supports batch_size=1) -> - `--attn-cp-size` must evenly divide `--tp-size`; each CP rank occupies `tp_size / cp_size` NPUs - -**Prefill node ``:** - -```shell Launch Server -export SGLANG_SET_CPU_AFFINITY=1 -export ASCEND_MF_STORE_URL="tcp://:23456" -export ASCEND_USE_FIA=True - -python3 -m sglang.launch_server \ - --model-path /mnt/share/weights/Qwen3-235B-A22B-Instruct-2507-W8A8 \ - --trust-remote-code \ - --disaggregation-mode prefill \ - --disaggregation-transfer-backend ascend \ - --disaggregation-bootstrap-port 8995 \ - --quantization modelslim \ - --attention-backend ascend \ - --skip-server-warmup \ - --mem-fraction-static 0.7 \ - --chunked-prefill-size 32768 \ - --device npu \ - --base-gpu-id 0 \ - --tp-size 16 \ - --enable-prefill-context-parallel \ - --attn-cp-size 2 \ - --moe-dp-size 2 \ - --max-running-requests 1 \ - --host \ - --port 8000 \ - --nnodes 1 \ - --node-rank 0 \ - --dist-init-addr :6688 -``` - -Key parameters for PCP: - -| Parameter | Value | Description | -|-----------|-------|-------------| -| `--enable-prefill-context-parallel` | flag | Enable PCP feature | -| `--attn-cp-size` | 2 | Split context across 2 CP ranks (each rank handles half the sequence) | -| `--moe-dp-size` | 2 | MoE DP size, should match `--attn-cp-size` | -| `--max-running-requests` | 1 | Required by PCP (batch_size=1 constraint) | - -**Decode node (``):** - -```shell Launch Server -export ASCEND_MF_STORE_URL="tcp://141.61.39.231:23456" -export ASCEND_USE_FIA=True - -python3 -m sglang.launch_server \ - --model-path /mnt/share/weights/Qwen3-235B-A22B-Instruct-2507-W8A8 \ - --trust-remote-code \ - --disaggregation-mode decode \ - --disaggregation-transfer-backend ascend \ - --quantization modelslim \ - --attention-backend ascend \ - --disable-radix-cache \ - --disable-cuda-graph \ - --mem-fraction-static 0.7 \ - --chunked-prefill-size 32768 \ - --skip-server-warmup \ - --device npu \ - --base-gpu-id 0 \ - --tp-size 8 \ - --max-running-requests 32 \ - --host \ - --port 8001 \ - --nnodes 1 \ - --node-rank 0 \ - --dist-init-addr :6688 -``` - -> **Note:** `ASCEND_MF_STORE_URL` on both nodes must point to the same KV store (typically the Prefill node IP). `ASCEND_USE_FIA=True` enables fast interconnect aggregation for KV transfer. PCP is a Prefill-only feature; the Decode side needs no CP-related flags. - -#### Running Qwen3-VL-8B-Instruct on 1 x Atlas 800I A3. - -Model weights could be found [here](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) - -```shell Launch Server -export SGLANG_SET_CPU_AFFINITY=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export STREAMS_PER_DEVICE=32 -export HCCL_BUFFSIZE=1536 -export HCCL_OP_EXPANSION_MODE=AIV - -python -m sglang.launch_server \ - --enable-multimodal \ - --attention-backend ascend \ - --mm-attention-backend ascend_attn \ - --trust-remote-code \ - --tp-size 4 \ - --model-path Qwen/Qwen3-VL-8B-Instruct \ - --mem-fraction-static 0.8 -``` - -## Testing the Service - -Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For testing examples (Health Check, Generate, Chat Completions, and port usage guidance), see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx index ec5aa42d2..2b722d294 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx @@ -23,9 +23,10 @@ This guide describes the best practice data for DeepSeek-R1 on the Ascend NPU. | DeepSeek-R1 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50.36ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-8p-in3k5-out1k5-50-36ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-50ms) | - ## Optimal Configuration + + ### DeepSeek-R1 W4A8 1P1D 16P IN3K5 OUT1K5 41ms **Model**: DeepSeek-R1 @@ -212,6 +213,8 @@ python -m sglang.bench_serving \ --request-rate 24 ``` + + ### DeepSeek-R1 W4A8 8P IN3K5 OUT1K5 50.36ms **Model**: DeepSeek-R1 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx index 7e2a310ce..125a18178 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx @@ -19,9 +19,10 @@ This guide describes the best practice data for DeepSeek-V3.2 on the Ascend NPU. | --- | --- | --- | --- | --- | --- | --- | --- | | DeepSeek-V3.2 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 107ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v3-2-w8a8-1p1d-32p-in128k-out1k-bs16) | - ## Optimal Configuration + + ### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K 26ms **Model**: DeepSeek-V3.2 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx index 1019bd7b8..97e940d73 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx @@ -25,9 +25,10 @@ This guide describes the best practice data for GLM-5.1 on the Ascend NPU. | GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) | | GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 64K+1K (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) | - ## Optimal Configuration + + ### GLM-5.1 W4A8 16P IN3K5 OUT1K5 50ms **Model**: GLM-5.1 @@ -124,6 +125,8 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` + + ### GLM-5.1 W4A8 1P1D 32P IN128K OUT1K **Model**: GLM-5.1 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx index eacfd0597..b6786733b 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx @@ -27,9 +27,10 @@ This guide describes the best practice data for Kimi-K2.6 on the Ascend NPU. | Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1080p_30+256 | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1080p-30-out256-50ms) | | Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-50ms) | - ## Optimal Configuration + + ### Kimi-K2.6 W4A8 16P IN64K OUT1K 100ms **Model**: Kimi-K2.6 @@ -133,6 +134,8 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` + + ### Kimi-K2.6 W4A8 1P1D 16P IN128K OUT1K 100ms **Model**: Kimi-K2.6 @@ -1605,6 +1608,8 @@ python -m sglang.bench_serving \ --warmup-requests 0 ``` + + ### Kimi-K2.6 W4A8 8P IN3K5 OUT1K5 50ms **Model**: Kimi-K2.6 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx index 574c6b00b..8aa732e7d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx @@ -24,9 +24,10 @@ This guide describes the best practice data for MiniMax-M2.5 on the Ascend NPU. | MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in128k-out1k-prefix90-50ms) | | MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-50ms) | - ## Optimal Configuration + + ### MiniMax-M2.5 W8A8 1P1D 16P IN128K OUT1K PREFIX90 50ms **Model**: MiniMax-M2.5 @@ -441,6 +442,8 @@ python -m sglang.bench_serving \ --request-rate inf ``` + + ### MiniMax-M2.5 W8A8 4P IN32K OUT1K 50ms **Model**: MiniMax-M2.5 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx index 6070d8ab1..ba72fddaf 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx @@ -19,9 +19,10 @@ This guide describes the best practice data for Qwen3-8B on the Ascend NPU. | --- | --- | --- | --- | --- | --- | --- | --- | | Qwen3-8B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 37ms | W8A8 INT8 | [Optimal Configuration](#qwen3-8b-w8a8-1p-in3k5-out1k5-37ms) | - ## Optimal Configuration + + ### Qwen3-8B W8A8 1P IN3K5 OUT1K5 37ms **Model**: Qwen3-8B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx index 8d9cf860e..a24de2350 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx @@ -18,9 +18,10 @@ This guide describes the best practice data for Qwen3-235B-A22B on the Ascend NP | --- | --- | --- | --- | --- | --- | --- | --- | | Qwen3-235B-A22B | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-235b-a22b-w8a8-8p-in3k5-out1k5-50-1ms) | - ## Optimal Configuration + + ### Qwen3-235B-A22B BF16 8P IN11K OUT1K5 8ms **Model**: Qwen3-235B-A22B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx index 00ca721fb..180cf4b32 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx @@ -20,7 +20,6 @@ This guide describes the best practice data for Qwen3-30B-A3B on the Ascend NPU. | Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1K+100 | 10000ms | BF16 | [Optimal Configuration](#qwen3-30b-a3b-bf16-1p-in1k-out100) | | Qwen3-30B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-30b-a3b-w8a8-1p-in3k5-out1k5-50ms) | - ## Optimal Configuration ### Qwen3-30B-A3B BF16 1P IN1K OUT100 @@ -223,6 +222,8 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` + + ### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 50ms **Model**: Qwen3-30B-A3B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx index 7b8d3439f..1db3b9978 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx @@ -19,7 +19,6 @@ This guide describes the best practice data for Qwen3-32B on the Ascend NPU. | Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms-a2) | | Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) | - ## Optimal Configuration ### Qwen3-32B BF16 8P IN18K OUT4K 6ms @@ -214,6 +213,8 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` + + ### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms **Model**: Qwen3-32B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx index 8af7a7a89..e3df0d0fc 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx @@ -26,7 +26,6 @@ This guide describes the best practice data for Qwen3.5-397B on the Ascend NPU. | Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64K+1K | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-50ms) | | Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-prefix90-50ms) | - ## Optimal Configuration ### Qwen3.5-397B W4A8 8P IN128K OUT1K 20ms @@ -699,6 +698,8 @@ python -m sglang.bench_serving \ --warmup-requests 64 ``` + + ### Qwen3.5-397B W4A8 8P IN3K5 OUT1K5 50ms **Model**: Qwen3.5-397B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx index 9ba09c583..68abd0bf9 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx @@ -18,7 +18,6 @@ This guide describes the best practice data for Qwen3.6-27B on the Ascend NPU. | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) | - ## Optimal Configuration ### Qwen3.6-27B 1P IN1024X1024 30 OUT1024 50ms @@ -322,6 +321,8 @@ python -m sglang.bench_serving \ --request-rate inf ``` + + ### Qwen3.6-27B W8A8 1P IN3K5 OUT1K5 50ms **Model**: Qwen3.6-27B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx index ad3b6b0b9..18b9e50c2 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx @@ -25,7 +25,6 @@ This guide describes the best practice data for Qwen3.6-35B-A3B on the Ascend NP | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) | | Qwen3.6-35B-A3B | Atlas 800I A3 | 2 | PD Mixed | 984K+1K | 40.91ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) | - ## Optimal Configuration ### Qwen3.6-35B-A3B 1P IN1024X1024 30 OUT1024 50ms @@ -513,6 +512,8 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` + + ### Qwen3.6-35B-A3B 1P IN3K5 OUT1K5 50ms **Model**: Qwen3.6-35B-A3B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx index 0f4456948..9a10ae29d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx @@ -19,7 +19,6 @@ This guide describes the best practice data for Qwen3-Next-80B-A3B-Instruct on t | --- | --- | --- | --- | --- | --- | --- | --- | | Qwen3-Next-80B-A3B-Instruct | Atlas 800I A3 | 2 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-next-80b-a3b-instruct-w8a8-2p-in3k5-out1k5-50ms) | - ## Optimal Configuration ### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 20ms @@ -133,6 +132,8 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` + + ### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 50ms **Model**: Qwen3-Next-80B-A3B-Instruct diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx new file mode 100644 index 000000000..9e04d522a --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx @@ -0,0 +1,214 @@ +--- +title: "DeepSeek-R1" +metatags: + description: "Deploy DeepSeek-R1 model with SGLang on Ascend NPUs, including single-node and multi-node PD disaggregation modes." +--- + +## Introduction + +DeepSeek-R1 is a Mixture-of-Experts (MoE) large language model developed by DeepSeek, featuring 671B total parameters +with 37B active parameters. It employs Multi-head Latent Attention (MLA) and DeepSeekMoE architecture, with built-in +multi-token prediction (MTP) for speculative decoding. The model excels at reasoning, math, and code tasks through +reinforcement learning-based training. + +This document demonstrates the deployment of DeepSeek-R1 on Ascend NPUs using SGLang, including single-node PD mixed +mode, multi-node PD disaggregation mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (DeepSeek-R1) is fully supported in +this version. To use the latest features (e.g., PD disaggregation, speculative decoding), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 16` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 4 8 20 21 22` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | +| MLAPO | `export SGLANG_NPU_USE_MLAPO=1` | +| Multistream MoE | `export SGLANG_NPU_USE_MULTI_STREAM=1` | +| NZ Weight Format | `export SGLANG_USE_FIA_NZ=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +Before downloading model weights, check the model size to reserve enough disk space. + + +- [DeepSeek-R1-0528-W4A8](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w4a8) (Quantized version) +- [DeepSeek-R1-0528-W8A8](https://modelers.cn/models/Modelers_Park/DeepSeek-R1-0528-w8a8) (Quantized version) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +The Docker image requires at least **30 GB** of free space. Ensure sufficient disk space before pulling images. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[DeepSeek-R1 Best Practice — W4A8 8P PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1#w4a8-8p-pd-mixed). + +### Multi-node PD disaggregation deployment + +PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving +throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[DeepSeek-R1 Best Practice — W4A8 16P PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1#w4a8-16p-pd-disaggregation). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6689) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[DeepSeek-R1 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx new file mode 100644 index 000000000..63676919f --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx @@ -0,0 +1,205 @@ +--- +title: "DeepSeek-V3.2" +metatags: + description: "Deploy DeepSeek-V3.2 model with SGLang on Ascend NPUs, including multi-node PD disaggregation mode." +--- + +## Introduction + +DeepSeek-V3.2 is a Mixture-of-Experts (MoE) large language model developed by DeepSeek, featuring 685B total parameters +with 37B active parameters. It employs Multi-head Latent Attention (MLA) and DeepSeekMoE architecture, with built-in +multi-token prediction (MTP) for speculative decoding. DeepSeek-V3.2 introduces the DeepSeek Sparse Attention (DSA) +mechanism, a fine-grained sparse attention mechanism powered by a lightning indexer, enabling significant efficiency +improvements in long-context scenarios. + +This document demonstrates the deployment of DeepSeek-V3.2 on Ascend NPUs using SGLang, including multi-node PD +disaggregation mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (DeepSeek-V3.2) is fully supported in +this version. To use the latest features (e.g., PD disaggregation, speculative decoding, DSA context parallel), it is +recommended to use v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 32` | +| Data Parallelism | `--dp-size 8` | +| Expert Parallelism | `--ep-size 32 \`
`--moe-a2a-backend deepep \`
`--deepep-mode low_latency` | +| Context Parallelism | `--enable-nsa-prefill-context-parallel \`
`--nsa-prefill-cp-mode in-seq-split \`
`--attn-cp-size 32` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 3 4 5 6 7 8` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [DeepSeek-V3.2-W8A8](https://www.modelscope.cn/models/sgl-npu/DeepSeek-V3.2-W8A8) (Quantized version) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Multi-node PD disaggregation deployment + +PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving +throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[DeepSeek-V3.2 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2#pd-disaggregation). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6689) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[DeepSeek-V3.2 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx new file mode 100644 index 000000000..ae8807096 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx @@ -0,0 +1,217 @@ +--- +title: "GLM-5.1" +metatags: + description: "Deploy GLM-5.1 model with SGLang on Ascend NPUs, including single-node and multi-node deployment modes." +--- + +## Introduction + +GLM-5.1 is a Mixture-of-Experts (MoE) large language model developed by Z.ai, featuring 744B total parameters with 40B +active parameters. It uses 256 routed experts (top-8) plus one shared expert, with Multi-head Latent Attention (MLA) +and DeepSeek Sparse Attention (DSA), and a built-in multi-token prediction (MTP) head for speculative decoding. +The model features built-in bilingual (Chinese-English) capabilities with a unified pre-training framework, excelling at +reasoning, math, code, and tool calling tasks. GLM-5.1 supports both Thinking mode (step-by-step reasoning) and Instruct +mode (direct response), with a native context window of approximately 200K tokens. + +This document demonstrates the deployment of GLM-5.1 on Ascend NPUs using SGLang, including single-node and multi-node +deployment, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (GLM-5.1) is fully supported in +this version. To use the latest features (e.g., speculative decoding, multi-node deployment), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 16` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | +| Context Parallelism | `--enable-nsa-prefill-context-parallel \`
`--nsa-prefill-cp-mode in-seq-split \`
`--attn-cp-size 4` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 16384` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 3 4 5 6` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [GLM-5.1](https://www.modelscope.cn/models/ZhipuAI/GLM-5.1) (BF16) +- [GLM-5.1-w4a8](https://www.modelscope.cn/models/Eco-Tech/GLM-5.1-w4a8) (Quantized version) +- You can use [msmodelslim](https://gitcode.com/Ascend/msmodelslim) to quantize the model naively. + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Multi-node PD mixed deployment + +Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes using tensor parallelism while keeping +prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single +node can provide. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[GLM-5.1 Best Practice — Multi-node PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/glm5_1#multi-node-pd-mixed). + +### Multi-node PD disaggregation deployment + +PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving +throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[GLM-5.1 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/glm5_1#pd-disaggregation). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6689) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[GLM-5.1 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/glm5_1) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx new file mode 100644 index 000000000..039308a12 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx @@ -0,0 +1,226 @@ +--- +title: "Kimi-K2.6" +metatags: + description: "Deploy Kimi-K2.6 model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Kimi-K2.6 is an open-source, native multimodal agentic model developed by Moonshot AI, built through continual +pretraining on approximately 15 trillion mixed visual and text tokens atop Kimi-K2-Base. It is a Mixture-of-Experts +(MoE) model featuring Multi-head Latent Attention (MLA) and MoE architecture, with 1T total parameters and 32B +active parameters. The model seamlessly integrates vision and language understanding with advanced agentic +capabilities, supporting both instant and thinking modes as well as conversational and agentic paradigms. + +This document demonstrates the deployment of Kimi-K2.6 on Ascend NPUs using SGLang, including single-node PD mixed +mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Kimi-K2.6) is fully supported in +this version. To use the latest features (e.g., speculative decoding, multimodal), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 16` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 32768` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 4 8 12 16 24 32 48 64 96 120` | +| Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 4 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 5 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | +| MLAPO | `export SGLANG_NPU_USE_MLAPO=1` | +| Multistream MoE | `export SGLANG_NPU_USE_MULTI_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Kimi-K2.6](https://www.modelscope.cn/models/moonshotai/Kimi-K2.6) (BF16) +- [Kimi-K2.6-w4a8](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.6-w4a8) (W4A8 quantized version) +- [kimi-k2.6-eagle3](https://www.modelscope.cn/models/lightseekorg/kimi-k2.6-eagle3) (EAGLE3 draft model for speculative decoding) + +You can use [msmodelslim](https://gitcode.com/Ascend/msmodelslim) to quantize `Kimi-K2.6-w4a8` from `Kimi-K2.6`. + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Kimi K2.6 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6#single-node). + +### Multi-node online deployment + +Multi-node deployment distributes the model across multiple Atlas 800I A3 nodes using tensor parallelism while keeping +prefill and decode on the same nodes (PD mixed mode), suitable for scenarios that need more device memory than a single +node can provide. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Kimi-K2.6 Best Practice — Multi-node On A3](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6#multi-node). + +### Multi-node PD disaggregation deployment + +PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving +throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Kimi-K2.6 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6#pd-disaggregation). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6689) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Kimi-K2.6 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx new file mode 100644 index 000000000..57841612f --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx @@ -0,0 +1,212 @@ +--- +title: "MiniMax-M2.5" +metatags: + description: "Deploy MiniMax-M2.5 model with SGLang on Ascend NPUs, including single-node and PD disaggregation modes." +--- + +## Introduction + +MiniMax-M2.5 is a Mixture-of-Experts (MoE) large language model developed by MiniMax, featuring a sparse MoE +architecture with approximately 230B total parameters and 10B active parameters. It supports native long-context +processing up to 200K tokens. The model supports EAGLE3 speculative decoding with a +custom eagle model for accelerated inference, and excels at general language understanding, reasoning, and +long-context tasks. + +This document demonstrates the deployment of MiniMax-M2.5 on Ascend NPUs using SGLang, including single-node PD mixed +mode, multi-node PD disaggregation mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (MiniMax-M2.5) is fully supported in +this version. To use the latest features (e.g., PD disaggregation, speculative decoding), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 16` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend ascend_fuseep \`
`--deepep-mode auto` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 8192` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 8 16 24 32 48 64 80` | +| Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [MiniMax-M2.5-w8a8-QuaRot](https://www.modelscope.cn/models/Eco-Tech/MiniMax-M2.5-w8a8-QuaRot) (W8A8 quantized version) +- [MiniMax-M2.5-eagle-model](https://www.modelscope.cn/models/sgl-npu/MiniMax-M2.5-eagel-model-0318) (EAGLE3 draft model for speculative decoding) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[MiniMax-M2.5 Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5#single-node). + +### Multi-node PD disaggregation deployment + +PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving +throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[MiniMax-M2.5 Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5#pd-disaggregation). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6689) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[MiniMax-M2.5 Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx new file mode 100644 index 000000000..e22e6554c --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx @@ -0,0 +1,203 @@ +--- +title: "Qwen3-8B" +metatags: + description: "Deploy Qwen3-8B model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3-8B is a compact dense model in the Qwen3 series developed by Alibaba, featuring 8B parameters with +Grouped-Query Attention (GQA) and up to 128K context length. It delivers significant improvements in instruction +following, logical reasoning, text comprehension, mathematics, science, coding, and tool usage. The model supports +EAGLE3 speculative decoding for accelerated inference and is available in both standard and thinking/reasoning-enhanced +editions. + +This document demonstrates the deployment of Qwen3-8B on Ascend NPUs using SGLang, including single-node PD mixed mode, +feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3-8B) is fully supported in +this version. To use the latest features (e.g., speculative decoding), it is recommended to use v0.5.13 or a later +version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 2` | +| Quantization | `--quantization modelslim` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 8192` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 4 6 9 10 15 16` | +| Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Qwen3-8B](https://www.modelscope.cn/models/Qwen/Qwen3-8B) (BF16) +- [Qwen3-8B-W8A8](https://www.modelscope.cn/models/vllm-ascend/Qwen3-8B-w8a8) (W8A8 quantized version) +- [Eagle3-Qwen3-8B-zh](https://www.modelscope.cn/models/Zjcxy-SmartAI/Eagle3-Qwen3-8B-zh) (EAGLE3 draft model for speculative decoding) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3-8B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3-8B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx new file mode 100644 index 000000000..2a3100813 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx @@ -0,0 +1,424 @@ +--- +title: "Qwen3-235B-A22B" +metatags: + description: "Deploy Qwen3-235B-A22B model with SGLang on Ascend NPUs, including single-node, multi-node, and PD disaggregation modes." +--- + +## Introduction + +Qwen3-235B-A22B is a Mixture-of-Experts (MoE) large language model developed by Alibaba, featuring 235B total +parameters with 22B active parameters. It employs Grouped-Query Attention (GQA) and Qwen3MoE architecture, with +support for EAGLE3 speculative decoding for accelerated inference. The model excels at instruction following, +logical reasoning, text comprehension, mathematics, science, coding, and tool usage, available in both standard +and thinking/reasoning-enhanced editions. + +This document demonstrates the deployment of Qwen3-235B-A22B on Ascend NPUs using SGLang, including single-node PD mixed +mode, multi-node PD mixed mode, multi-node PD disaggregation mode, 256K long-sequence inference, Prefill Context Parallel, +feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3-235B-A22B) is fully supported in +this version. To use the latest features (e.g., PD disaggregation, speculative decoding), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 16` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend ascend_fuseep` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 94208` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 4 8 16 20 24 26 27` | +| Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | +| Context Parallelism | `--enable-prefill-context-parallel \`
`--attn-cp-size 2 \`
`--moe-dp-size 2` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +Before downloading model weights, check the model size to reserve enough disk space. + + +- [Qwen3-235B-A22B-Instruct-2507](https://www.modelscope.cn/models/Qwen/Qwen3-235B-A22B-Instruct-2507) (BF16) +- [Qwen3-235B-A22B-W8A8](https://www.modelscope.cn/models/vllm-ascend/Qwen3-235B-A22B-W8A8) (W8A8 quantized version) +- [Qwen3-235B-A22B-Eagle3](https://www.modelscope.cn/models/nv-community/Qwen3-235B-A22B-Eagle3) (EAGLE3 draft model for speculative decoding) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +The Docker image requires at least **30 GB** of free space. Ensure sufficient disk space before pulling images. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3-235B-A22B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b#single-node). + +### Multi-node PD disaggregation deployment + +#### 256K long-sequence PD disaggregation on 2 x Atlas 800I A3 (without CP) + +This configuration uses PD disaggregation for **256K long-sequence inference** on 2 x Atlas 800I A3 with context parallel disabled. +The following command is based on the **W8A8** quantized model. + +1. Set the shared environment variables on both prefill and decode nodes: + +```bash Shared environment +#============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# PREFILL_HOST_IP: prefill node IP address +# NETWORK_IFACE: network interface name (use ifconfig to find) +#============================================================ + +export ASCEND_USE_FIA=1 +export SGLANG_SET_CPU_AFFINITY=1 +export ASCEND_MF_STORE_URL="tcp://:12345" +export HCCL_SOCKET_IFNAME= +export GLOO_SOCKET_IFNAME= + +MODEL_PATH=/path/to/model-weights +``` + +2. Run on the **prefill node**: + +```bash Prefill node +#============================================================ +# Before running, update the following variable: +# PREFILL_HOST_IP: prefill node IP address +#============================================================ + +export ASCEND_LAUNCH_BLOCKING=1 +export HCCL_BUFFSIZE=1500 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=128 +export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8995 \ + --attention-backend ascend \ + --disable-radix-cache \ + --chunked-prefill-size -1 \ + --skip-server-warmup \ + --device npu \ + --tp-size 16 \ + --mem-fraction-static 0.45 \ + --max-running-requests 1 \ + --host \ + --port 8000 \ + --dist-init-addr :5000 \ + --nnodes 1 \ + --node-rank 0 \ + --moe-a2a-backend deepep \ + --deepep-mode normal +``` + +3. Run on the **decode node**: + +```bash Decode node +#============================================================ +# Before running, update the following variable: +# DECODE_HOST_IP: decode node IP address +#============================================================ + +export HCCL_BUFFSIZE=4000 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=16 + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --disable-cuda-graph \ + --device npu \ + --disable-radix-cache \ + --chunked-prefill-size 8192 \ + --skip-server-warmup \ + --tp-size 16 \ + --max-running-requests 1 \ + --host \ + --port 8232 \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency \ + --disable-overlap-schedule +``` + +4. Launch the SGLang Router (on any reachable node): + +```bash Router +#============================================================ +# Before running, update the following variables: +# PREFILL_HOST_IP: prefill node IP address +# DECODE_HOST_IP: decode node IP address +# ROUTER_HOST_IP: router node IP address +#============================================================ + +python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8995 \ + --decode http://:8232 \ + --host \ + --port 6689 \ + --prometheus-port 29010 +``` + +#### Prefill Context Parallel (PCP) on 2 x Atlas 800I A3 + +This configuration enables **Prefill Context Parallel** (`--enable-prefill-context-parallel`) to split the context +across CP ranks during prefill, reducing per-device memory pressure and improving TTFT for long sequences. +PD disaggregation is required. The following command is based on the **W8A8** quantized model. + + +**Constraints:** +- Prefill side must set `--max-running-requests 1` (PCP only supports batch_size=1) +- `--attn-cp-size` must evenly divide `--tp-size`; each CP rank occupies `tp_size / cp_size` NPUs + + +1. Run on the **prefill node**: + +```bash Prefill node +#============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# PREFILL_HOST_IP: prefill node IP address +#============================================================ + +export SGLANG_SET_CPU_AFFINITY=1 +export ASCEND_MF_STORE_URL="tcp://:23456" +export ASCEND_USE_FIA=True + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --trust-remote-code \ + --disaggregation-mode prefill \ + --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8995 \ + --quantization modelslim \ + --attention-backend ascend \ + --skip-server-warmup \ + --mem-fraction-static 0.7 \ + --chunked-prefill-size 32768 \ + --device npu \ + --base-gpu-id 0 \ + --tp-size 16 \ + --enable-prefill-context-parallel \ + --attn-cp-size 2 \ + --moe-dp-size 2 \ + --max-running-requests 1 \ + --host \ + --port 8000 \ + --nnodes 1 \ + --node-rank 0 \ + --dist-init-addr :6688 +``` + +Key parameters for PCP: + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `--enable-prefill-context-parallel` | flag | Enable PCP feature | +| `--attn-cp-size` | 2 | Split context across 2 CP ranks (each rank handles half the sequence) | +| `--moe-dp-size` | 2 | MoE DP size, should match `--attn-cp-size` | +| `--max-running-requests` | 1 | Required by PCP (batch_size=1 constraint) | + +2. Run on the **decode node**: + +```bash Decode node +#============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# DECODE_HOST_IP: decode node IP address +# PREFILL_HOST_IP: prefill node IP address (for ASCEND_MF_STORE_URL) +#============================================================ + +export ASCEND_MF_STORE_URL="tcp://:23456" +export ASCEND_USE_FIA=True + +python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --trust-remote-code \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --quantization modelslim \ + --attention-backend ascend \ + --disable-radix-cache \ + --disable-cuda-graph \ + --mem-fraction-static 0.7 \ + --chunked-prefill-size 32768 \ + --skip-server-warmup \ + --device npu \ + --base-gpu-id 0 \ + --tp-size 8 \ + --max-running-requests 32 \ + --host \ + --port 8001 \ + --nnodes 1 \ + --node-rank 0 \ + --dist-init-addr :6688 +``` + + +`ASCEND_MF_STORE_URL` on both nodes must point to the same KV store (typically the prefill node IP). +`ASCEND_USE_FIA=True` enables fast interconnect aggregation for KV transfer. +PCP is a prefill-only feature; the decode side needs no CP-related flags. + + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6689) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3-235B-A22B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx new file mode 100644 index 000000000..bbf252816 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx @@ -0,0 +1,203 @@ +--- +title: "Qwen3-30B-A3B" +metatags: + description: "Deploy Qwen3-30B-A3B model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3-30B-A3B is a Mixture-of-Experts (MoE) large language model developed by Alibaba, featuring 30B total +parameters with 3B active parameters. It employs Grouped-Query Attention (GQA) and Qwen3MoE architecture, with +support for EAGLE3 speculative decoding for accelerated inference. The model excels at instruction following, +logical reasoning, text comprehension, mathematics, science, coding, and tool usage, available in both standard +and thinking/reasoning-enhanced editions. + +This document demonstrates the deployment of Qwen3-30B-A3B on Ascend NPUs using SGLang, including single-node PD mixed +mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3-30B-A3B) is fully supported in +this version. To use the latest features (e.g., speculative decoding), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 2` | +| Data Parallelism | `--dp-size 2` | +| Quantization | `--quantization modelslim` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162` | +| Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +Before downloading model weights, check the model size to reserve enough disk space. + + +- [Qwen3-30B-A3B-Instruct-2507](https://www.modelscope.cn/models/Qwen/Qwen3-30B-A3B-Instruct-2507) (BF16, recommended) +- [Qwen3-30B-A3B-w8a8](https://www.modelscope.cn/models/Eco-Tech/Qwen3-30B-A3B-w8a8) (W8A8 quantized version) +- [Qwen3-a3B_eagle3](https://www.modelscope.cn/models/vllm-ascend/Qwen3-a3B_eagle3) (EAGLE3 draft model for speculative decoding) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +The Docker image requires at least **30 GB** of free space. Ensure sufficient disk space before pulling images. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3-30B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3-30B-A3B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx new file mode 100644 index 000000000..dc85ce4b2 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx @@ -0,0 +1,201 @@ +--- +title: "Qwen3-32B" +metatags: + description: "Deploy Qwen3-32B model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3-32B is a dense model in the Qwen3 series developed by Alibaba, featuring 32B parameters with Grouped-Query +Attention (GQA) and up to 128K (131K with YaRN) context length. It delivers significant improvements in instruction following, logical +reasoning, text comprehension, mathematics, science, coding, and tool usage. The model supports EAGLE3 speculative +decoding for accelerated inference and supports both standard and thinking/reasoning modes. + +This document demonstrates the deployment of Qwen3-32B on Ascend NPUs using SGLang, including single-node PD mixed mode, +feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3-32B) is fully supported in +this version. To use the latest features (e.g., speculative decoding), it is recommended to use v0.5.13 or a later +version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 4` | +| Quantization | `--quantization modelslim` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 4 8 16` | +| Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Qwen3-32B](https://www.modelscope.cn/models/Qwen/Qwen3-32B) (BF16) +- [Qwen3-32B-W8A8](https://www.modelscope.cn/models/vllm-ascend/Qwen3-32B-W8A8) (W8A8 quantized version) +- [Eagle3-Qwen3-32B-zh](https://www.modelscope.cn/models/Zjcxy-SmartAI/Eagle3-Qwen3-32B-zh) (EAGLE3 draft model for speculative decoding) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3-32B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3-32B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx new file mode 100644 index 000000000..70b6a7d23 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx @@ -0,0 +1,204 @@ +--- +title: "Qwen3.5-397B-A17B" +metatags: + description: "Deploy Qwen3.5-397B-A17B model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3.5-397B-A17B is the latest flagship model in the Qwen series developed by Alibaba, featuring a Gated Delta +Networks combined with sparse Mixture-of-Experts architecture (397B total parameters, 17B activated). It employs hybrid +attention with Gated Delta Networks (linear, O(n) complexity) combined with full attention every 4th layer, and MoE +routing with Top-10 active out of 512 routed experts plus a dedicated shared expert. The model supports multimodal +inputs (text, image, video) with native context lengths of up to 262,144 tokens, and includes built-in multi-token +prediction (MTP) for speculative decoding. + +This document demonstrates the deployment of Qwen3.5-397B-A17B on Ascend NPUs using SGLang, including single-node PD +mixed mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3.5-397B-A17B) is fully +supported in this version. To use the latest features (e.g., speculative decoding, multimodal), it is recommended to +use v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 8` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | +| Quantization | `--quantization modelslim` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 2 4 6 8 10 12 14 16 18 20` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp](https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp) (W4A8 quantized version with MTP) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3.5-397B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3.5-397B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx new file mode 100644 index 000000000..e4b01fb7c --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx @@ -0,0 +1,201 @@ +--- +title: "Qwen3.6-27B" +metatags: + description: "Deploy Qwen3.6-27B model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3.6-27B is a dense model in the Qwen3.6 series developed by Alibaba, featuring 27B parameters with a hybrid +architecture combining Gated Delta Networks (linear, O(n) complexity) with full attention every 4th layer. It supports +multimodal inputs (text, image, video) with native context lengths of up to 262,144 tokens, and includes built-in +multi-token prediction (MTP) for speculative decoding. The model delivers strong performance in instruction following, +reasoning, text comprehension, and tool usage. + +This document demonstrates the deployment of Qwen3.6-27B on Ascend NPUs using SGLang, including single-node PD mixed +mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3.6-27B) is fully supported +in this version. To use the latest features (e.g., speculative decoding, multimodal), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 2` | +| Quantization | `--quantization modelslim` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 32768` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 2 8 16 32 48` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Eco-Tech/Qwen3.6-27B-w8a8](https://www.modelscope.cn/models/Eco-Tech/Qwen3.6-27B-w8a8) (W8A8 quantized version) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3.6-27B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3.6-27B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx new file mode 100644 index 000000000..84bb2d9a5 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx @@ -0,0 +1,201 @@ +--- +title: "Qwen3.6-35B-A3B" +metatags: + description: "Deploy Qwen3.6-35B-A3B model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3.6-35B-A3B is a Mixture-of-Experts model in the Qwen3.6 series developed by Alibaba, featuring 35B total +parameters with 3B active parameters per token. It uses a hybrid architecture that combines Gated DeltaNet (linear, +O(n) complexity) with full attention applied every Nth layer, enabling efficient long-context modeling. The model +supports multimodal inputs (text, image, video) with long context lengths, and includes built-in multi-token prediction +(NEXTN) for speculative decoding. It delivers strong performance in instruction following, reasoning, text +comprehension, and tool usage. + +This document demonstrates the deployment of Qwen3.6-35B-A3B on Ascend NPUs using SGLang, including single-node PD mixed +mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3.6-35B-A3B) is fully supported +in this version. To use the latest features (e.g., speculative decoding, multimodal), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 2` | +| Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 16384` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Eco-Tech/Qwen3.6-35B-A3B-w8a8](https://www.modelscope.cn/models/Eco-Tech/Qwen3.6-35B-A3B-w8a8) (W8A8 quantized version) + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3.6-35B-A3B Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3.6-35B-A3B Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues. diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx new file mode 100644 index 000000000..eaf324817 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx @@ -0,0 +1,208 @@ +--- +title: "Qwen3-Next-80B-A3B-Instruct" +metatags: + description: "Deploy Qwen3-Next-80B-A3B-Instruct model with SGLang on Ascend NPUs, including single-node deployment modes." +--- + +## Introduction + +Qwen3-Next-80B-A3B-Instruct is a hybrid Gated DeltaNet-Transformer Mixture-of-Experts (MoE) model developed by Alibaba, +featuring 80B total parameters with 3B active parameters. It combines Gated DeltaNet SSM layers with attention layers in a +sparse MoE architecture, enabling efficient long-sequence modeling with linear complexity in the SSM layers. The model +supports multi-token prediction (NEXTN) for speculative decoding, delivering strong performance in instruction +following, reasoning, and text generation tasks. + +This document demonstrates the deployment of Qwen3-Next-80B-A3B-Instruct on Ascend NPUs using SGLang, including +single-node PD mixed mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.13**. The current model (Qwen3-Next-80B-A3B-Instruct) is +fully supported in this version. To use the latest features (e.g., speculative decoding), it is recommended to use +v0.5.13 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Tensor Parallelism | `--tp-size 4` | +| Data Parallelism | `--dp-size 2` | +| Expert Parallelism | `--ep-size 4 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | +| Quantization | `--quantization modelslim` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 2 4 8` | +| Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant \`
`--speculative-draft-model-path /path/to/draft-model-weights` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization#feature-compatibility). + +## Prerequisites + +### Model weights + + +If you need to download model weights, check the model size before downloading to reserve enough space. + + +- [Qwen3-Next-80B-A3B-Instruct](https://www.modelscope.cn/models/Qwen/Qwen3-Next-80B-A3B-Instruct) (BF16) +- [Qwen3-Next-80B-A3B-Instruct-W8A8](https://www.modelscope.cn/models/vllm-ascend/Qwen3-Next-80B-A3B-Instruct-W8A8) (W8A8 quantized version) + + +For EAGLE3 speculative decoding, use the BF16 weights as the draft model (set `--speculative-draft-model-path` to the BF16 weight path). + + +Ensure the available device memory exceeds the model weight size before deployment. For optimal throughput and latency, +refer to the [best practice configurations](#best-practices) which may require additional nodes or cards. + +It is recommended to download the model weights to a shared directory across multiple nodes. + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30 GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/ascend_npu_faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-a3 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:v0.5.13.post1-cann9.0.0-910b +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node online deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[Qwen3-Next-80B-A3B-Instruct Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct#single-node). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "What is the capital of France?", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/ascend_npu#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[Qwen3-Next-80B-A3B-Instruct Best Practice](/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/ascend_npu_optimization). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/ascend_npu_faq). +This section only covers model-specific issues.