From 51c18d9aa82e7446fe0c661411e26cef063f5a5f Mon Sep 17 00:00:00 2001 From: amote-i <49533125+amote-i@users.noreply.github.com> Date: Sat, 29 Aug 2026 09:38:52 +0800 Subject: [PATCH] [NPU] [DOC] update npu best practice (#36476) --- docs/docs.json | 3 + .../best-practices/deepseek_r1.mdx | 112 ++-- .../best-practices/deepseek_v3_2.mdx | 42 +- .../best-practices/deepseek_v4_flash.mdx | 486 ++++++++++++++++++ .../best-practices/glm_5_1.mdx | 350 +++---------- .../best-practices/glm_5_2.mdx | 226 ++++++++ .../best-practices/kimi_k2_6.mdx | 102 ++-- .../best-practices/mimo_v2_flash.mdx | 50 +- .../best-practices/minimax_m2_5.mdx | 66 ++- .../best-practices/qwen3_235b_a22b.mdx | 18 +- .../best-practices/qwen3_30b_a3b.mdx | 28 +- .../best-practices/qwen3_32b.mdx | 207 ++++---- .../best-practices/qwen3_5_397b.mdx | 171 +++--- .../best-practices/qwen3_6_27b.mdx | 205 ++++++-- .../best-practices/qwen3_6_35b_a3b.mdx | 217 ++++++-- .../best-practices/qwen3_8b.mdx | 15 +- .../qwen3_next_80b_a3b_instruct.mdx | 23 +- .../tutorials/deepseek_v4_flash.mdx | 285 ++++++++++ 18 files changed, 1862 insertions(+), 744 deletions(-) create mode 100644 docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx create mode 100644 docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx create mode 100644 docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx diff --git a/docs/docs.json b/docs/docs.json index 9dc1a9c1d..ab55b0d51 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1111,6 +1111,7 @@ "pages": [ "docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1", "docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2", + "docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash", "docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1", "docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2", "docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3", @@ -1133,7 +1134,9 @@ "pages": [ "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2", + "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1", + "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5", "docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash", diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx index 58a1945a0..46898890a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx @@ -14,7 +14,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1.5k | 16ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-16ms) | +| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1.5k | 20ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-20ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k-19-0ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.9k+1k | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k9-out1k-19-0ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 6k+1.6k | 20.5ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in6k-out1k6-20-5ms) | @@ -97,9 +97,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=4000 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=3500 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -112,12 +112,12 @@ do --node-rank 0 \ --nnodes 1 \ --tp-size 16 \ - --mem-fraction-static 0.62 \ + --mem-fraction-static 0.8 \ --quantization modelslim \ --max-running-requests 32 \ --context-length 8192 \ --disable-radix-cache \ - --chunked-prefill-size -1 \ + --chunked-prefill-size 20480 \ --max-prefill-tokens 20480 \ --moe-a2a-backend deepep \ --deepep-mode normal \ @@ -131,11 +131,11 @@ do --dtype bfloat16 \ --reasoning-parser deepseek-r1 \ --tool-call-parser deepseekv3 \ + --max-total-tokens 80000 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -146,11 +146,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export TASK_QUEUE_ENABLE=1 python3 -m sglang.launch_server \ @@ -161,7 +162,7 @@ do --nnodes 1 \ --tp-size 16 \ --dp-size 16 \ - --mem-fraction-static 0.805 \ + --mem-fraction-static 0.9 \ --max-running-requests 416 \ --quantization modelslim \ --moe-a2a-backend deepep \ @@ -182,11 +183,11 @@ do --load-balance-method round_robin \ --reasoning-parser deepseek-r1 \ --tool-call-parser deepseekv3 \ + --max-total-tokens 145792 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -218,12 +219,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --request-rate 24 \ --max-concurrency 416 \ + --num-prompts 1664 \ --random-input-len 3584 \ --random-output-len 1536 \ - --num-prompts 1664 \ --random-range-ratio 1 \ - --request-rate 24 + --seed 1 ``` @@ -269,15 +271,16 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh -export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 +export DEEPEP_HCCL_BUFFSIZE=2000 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 @@ -294,11 +297,11 @@ python3 -m sglang.launch_server \ --quantization modelslim \ --watchdog-timeout 9000 \ --cuda-graph-bs 4 8 12 14 \ - --mem-fraction-static 0.77 \ + --mem-fraction-static 0.9 \ --max-running-requests 224 \ --context-length 8188 \ --disable-radix-cache \ - --chunked-prefill-size -1 \ + --chunked-prefill-size 65536 \ --max-prefill-tokens 3000 \ --moe-a2a-backend deepep \ --deepep-mode auto \ @@ -311,7 +314,8 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --dtype bfloat16 \ --reasoning-parser deepseek-r1 \ - --tool-call-parser deepseekv3 + --tool-call-parser deepseekv3 \ + --max-total-tokens 100000 ``` #### Benchmark @@ -325,15 +329,16 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 224 \ + --num-prompts 896 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 896 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` -### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 16ms +### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 20ms **Model**: DeepSeek-R1 @@ -347,7 +352,7 @@ python -m sglang.bench_serving \ **Dataset**: 3.5k+1.5k -**TPOT**: 16ms +**TPOT**: 20ms #### Model Deployment @@ -400,9 +405,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1536 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1536 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -439,7 +444,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -450,11 +454,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=650 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=1 @@ -494,7 +499,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -529,12 +533,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --request-rate 16 \ --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 32 \ --random-range-ratio 1 \ - --request-rate 16 + --seed 42 ``` ### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 50ms @@ -606,11 +611,12 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=800 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072 + export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_USE_AG_AFTER_QLORA=1 export TASK_QUEUE_ENABLE=2 @@ -624,12 +630,13 @@ do --nnodes 1 \ --tp-size 16 \ --mem-fraction-static 0.778 \ + --quantization modelslim \ --max-running-requests 16 \ --disable-radix-cache \ --chunked-prefill-size -1 \ --max-prefill-tokens 60000 \ --moe-a2a-backend ascend_fuseep \ - --fuseep-mode 2 \ + --deepep-mode normal \ --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ @@ -645,7 +652,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -656,12 +662,14 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=600 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=600 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_LM_HEAD_TP=8 + export SGLANG_NPU_FUSED_MOE_MODE=1 export TASK_QUEUE_ENABLE=1 python3 -m sglang.launch_server \ @@ -676,8 +684,8 @@ do --dp-size 32 \ --mem-fraction-static 0.82 \ --max-running-requests 1024 \ + --quantization modelslim \ --moe-a2a-backend ascend_fuseep \ - --fuseep-mode 1 \ --enable-dp-attention \ --deepep-mode low_latency \ --moe-dense-tp 1 \ @@ -699,7 +707,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -734,12 +741,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --request-rate 40 \ --max-concurrency 1024 \ + --num-prompts 7168 \ --random-input-len 3584 \ --random-output-len 1536 \ - --num-prompts 7168 \ --random-range-ratio 1 \ - --request-rate 40 + --seed 1 ``` ### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K 19.0ms @@ -809,9 +817,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1536 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1536 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -848,7 +856,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -859,11 +866,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=650 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=1 @@ -903,7 +911,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -938,12 +945,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --request-rate 16 \ --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 3500 \ --random-output-len 1024 \ - --num-prompts 32 \ --random-range-ratio 1 \ - --request-rate 16 + --seed 1 ``` ### DeepSeek-R1 W8A8 2P1D 32P IN3K9 OUT1K 19.0ms @@ -1013,9 +1021,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1536 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1536 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -1052,7 +1060,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -1063,11 +1070,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=650 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=1 @@ -1107,7 +1115,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -1142,12 +1149,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --request-rate 16 \ --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 3900 \ --random-output-len 1024 \ - --num-prompts 32 \ --random-range-ratio 1 \ - --request-rate 16 + --seed 1 ``` ### DeepSeek-R1 W8A8 2P1D 32P IN6K OUT1K6 20.5ms @@ -1217,9 +1225,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1536 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1536 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -1255,7 +1263,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -1266,11 +1273,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=650 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=1 @@ -1309,7 +1317,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -1344,10 +1351,11 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --request-rate 16 \ --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 6000 \ --random-output-len 1600 \ - --num-prompts 32 \ --random-range-ratio 1 \ - --request-rate 16 + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx index 5ba3e2da8..bf08bdec5 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v3_2.mdx @@ -92,9 +92,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -120,8 +120,8 @@ do --disaggregation-transfer-backend ascend \ --disable-cuda-graph \ --moe-dense-tp-size 1 \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 32 \ --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ @@ -132,7 +132,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -143,11 +142,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=400 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=400 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=0 @@ -185,7 +185,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -219,10 +218,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 8 \ + --num-prompts 8 \ --random-input-len 131072 \ --random-output-len 1024 \ - --num-prompts 8 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### DeepSeek-V3.2 W8A8 1P1D 32P IN128K OUT1K BS16 @@ -290,9 +290,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -318,8 +318,8 @@ do --disaggregation-transfer-backend ascend \ --disable-cuda-graph \ --moe-dense-tp-size 1 \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 32 \ --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ @@ -330,7 +330,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -341,11 +340,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=400 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=400 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=0 @@ -383,7 +383,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -417,9 +416,9 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 16 \ + --num-prompts 16 \ --random-input-len 131072 \ --random-output-len 1024 \ - --num-prompts 16 \ --random-range-ratio 1 ``` @@ -488,9 +487,9 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -516,8 +515,8 @@ do --disaggregation-transfer-backend ascend \ --disable-cuda-graph \ --moe-dense-tp-size 1 \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 32 \ --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ @@ -528,7 +527,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -539,11 +537,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=400 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=400 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export TASK_QUEUE_ENABLE=0 @@ -581,7 +580,6 @@ do --trust-remote-code \ --attention-backend ascend \ --device npu - NODE_RANK=$i break fi done @@ -615,8 +613,8 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 8 \ + --num-prompts 8 \ --random-input-len 131072 \ --random-output-len 1024 \ - --num-prompts 8 \ --random-range-ratio 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx new file mode 100644 index 000000000..63a3b6628 --- /dev/null +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx @@ -0,0 +1,486 @@ +--- +title: "DeepSeek-V4-Flash" +metatags: + description: "Best Practice for DeepSeek-V4-Flash on Ascend NPU" +--- + + +This page focuses on optimal configuration and benchmark results for DeepSeek-V4-Flash on the Ascend NPU. + +On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. + + +### High Throughput + +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | +| DeepSeek-V4-Flash | Atlas 800I A3 | 16 | PD Disaggregation | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-1p1d-16p-in8k-out1k-50ms) | +| DeepSeek-V4-Flash | Atlas 800I A3 | 8 | PD Mixed | 32k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in32k-out1k-50ms) | +| DeepSeek-V4-Flash | Atlas 800I A3 | 8 | PD Mixed | 8k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-v4-flash-w8a8-8p-in8k-out1k-50ms) | + +## Optimal Configuration + + + +### DeepSeek-V4-Flash W8A8 1P1D 16P IN8K OUT1K 50ms + +**Model**: DeepSeek-V4-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 16 + +**Deploy Mode**: PD Disaggregation + +**Quantization**: W8A8 INT8 + +**Dataset**: 8k+1k + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# P_IP: prefill node IP address +# D_IP: decode node IP address +# ASCEND_MF_STORE_URL: prefill node IP with port +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash + +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 +export HCCL_OP_EXPANSION_MODE=AIV +export INF_NAN_MODE_FORCE_DISABLE=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DSV4_FP4_EXPERTS=False +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch +export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False +export SGLANG_OPT_FP8_WO_A_GEMM=0 +export SGLANG_OPT_FUSE_WQA_WKV=0 +export SGLANG_OPT_USE_FUSED_HASH_TOPK=False +export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False +export SGLANG_OPT_USE_TILELANG_MHC_POST=False +export SGLANG_OPT_USE_TILELANG_MHC_PRE=False +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +P_IP=('') +D_IP=('') + +export ASCEND_MF_STORE_URL="tcp://:24670" + +MODEL_PATH=/path/to/model-weights + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=8 + export HCCL_SOCKET_IFNAME= + export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 + export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 + export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 + export SGLANG_ZBAL_LOCAL_MEM_SIZE=62084 + export ZBAL_ENABLE_GRAPH=1 + export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --host ${P_IP[$i]} \ + --port 8000 \ + --disaggregation-bootstrap-port 8998 \ + --page-size 128 \ + --tp-size 16 \ + --trust-remote-code \ + --device npu \ + --attention-backend dsv4 \ + --watchdog-timeout 9000 \ + --disaggregation-transfer-backend ascend \ + --mem-fraction-static 0.62 \ + --prefill-max-requests 6 \ + --max-prefill-tokens 70000 \ + --chunked-prefill-size -1 \ + --max-running-requests 112 \ + --dp-size 16 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --deepep-mode normal \ + --quantization modelslim \ + --enable-dp-lm-head \ + --kv-cache-dtype bfloat16 \ + --disable-cuda-graph \ + --disable-radix-cache \ + --load-balance-method round_robin \ + --ep-dispatch-algorithm static + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 + export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 + export DEEPEP_NORMAL_LONG_SEQ_ROUND=8 + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=1200 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host ${D_IP[$i]} \ + --port 8001 \ + --page-size 128 \ + --tp-size 16 \ + --trust-remote-code \ + --device npu \ + --attention-backend dsv4 \ + --watchdog-timeout 9000 \ + --mem-fraction-static 0.75 \ + --prefill-max-requests 1 \ + --disable-radix-cache \ + --chunked-prefill-size 32768 \ + --disaggregation-transfer-backend ascend \ + --max-running-requests 896 \ + --dp-size 16 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --quantization modelslim \ + --enable-dp-lm-head \ + --kv-cache-dtype bfloat16 \ + --cuda-graph-bs 1 2 4 8 16 24 36 40 48 56 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 2 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 3 + break + fi +done +``` + +```bash Command +# ============================================================ +# Before running, replace the following placeholders: +# : prefill node IP address +# : decode node IP address +# ============================================================ + +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://:8000 8998 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --policy cache_aware +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --random-input-len 8000 \ + --random-output-len 1000 \ + --num-prompts 1600 \ + --max-concurrency 800 \ + --random-range-ratio 1 \ + --warmup-requests 0 \ + --request-rate inf \ + --seed 1 +``` + + + +### DeepSeek-V4-Flash W8A8 8P IN32K OUT1K 50ms + +**Model**: DeepSeek-V4-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 8 + +**Deploy Mode**: PD Mixed + +**Quantization**: W8A8 INT8 + +**Dataset**: 32k+1k + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + +MODEL_PATH=/path/to/model-weights +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash + +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 +export GLOO_SOCKET_IFNAME= +export HCCL_BUFFSIZE=8 +export HCCL_SOCKET_IFNAME= +export INF_NAN_MODE_FORCE_DISABLE=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 +export SGLANG_DISABLE_DRAFT_EXTEND_GRAPH=1 +export SGLANG_DSV4_FP4_EXPERTS=False +export SGLANG_DSV4_NPU_FUSED_COMPRESSOR=1 +export SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL=0 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 +export SGLANG_NPU_USE_MULTI_STREAM=1 +export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch +export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False +export SGLANG_OPT_FP8_WO_A_GEMM=0 +export SGLANG_OPT_FUSE_WQA_WKV=0 +export SGLANG_OPT_USE_FUSED_HASH_TOPK=False +export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False +export SGLANG_OPT_USE_TILELANG_MHC_POST=False +export SGLANG_OPT_USE_TILELANG_MHC_PRE=False +export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 +export SGLANG_ZBAL_LOCAL_MEM_SIZE=61000 +export STREAMS_PER_DEVICE=32 +export USE_FUSED_HC_PRE_ASCENDC=1 +export USE_NPU_MOE_GATING_TOP_K=1 +export ZBAL_ENABLE_GRAPH=1 +export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 6688 \ + --page-size 128 \ + --tp-size 16 \ + --trust-remote-code \ + --device npu \ + --prefill-max-requests 32 \ + --attention-backend dsv4 \ + --watchdog-timeout 9000 \ + --mem-fraction-static 0.7 \ + --chunked-prefill-size 131072 \ + --max-running-requests 64 \ + --dp-size 16 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --quantization modelslim \ + --enable-dp-lm-head \ + --kv-cache-dtype auto \ + --skip-server-warmup \ + --cuda-graph-bs 1 2 4 8 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 2 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 3 \ + --ep-size 16 \ + --disable-radix-cache +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --random-input-len 32000 \ + --random-output-len 1000 \ + --num-prompts 64 \ + --max-concurrency 64 \ + --random-range-ratio 1 \ + --warmup-requests 0 \ + --request-rate inf \ + --seed 1 \ + --max-attempts 3 +``` + +### DeepSeek-V4-Flash W8A8 8P IN8K OUT1K 50ms + +**Model**: DeepSeek-V4-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 8 + +**Deploy Mode**: PD Mixed + +**Quantization**: W8A8 INT8 + +**Dataset**: 8k+1k + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + +MODEL_PATH=/path/to/model-weights +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash + +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 +export GLOO_SOCKET_IFNAME= +export HCCL_BUFFSIZE=8 +export HCCL_SOCKET_IFNAME= +export INF_NAN_MODE_FORCE_DISABLE=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 +export SGLANG_DISABLE_DRAFT_EXTEND_GRAPH=1 +export SGLANG_DSV4_FP4_EXPERTS=False +export SGLANG_DSV4_NPU_FUSED_COMPRESSOR=1 +export SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 +export SGLANG_NPU_USE_MULTI_STREAM=1 +export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch +export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False +export SGLANG_OPT_FP8_WO_A_GEMM=0 +export SGLANG_OPT_FUSE_WQA_WKV=0 +export SGLANG_OPT_USE_FUSED_HASH_TOPK=False +export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False +export SGLANG_OPT_USE_TILELANG_MHC_POST=False +export SGLANG_OPT_USE_TILELANG_MHC_PRE=False +export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 +export SGLANG_ZBAL_LOCAL_MEM_SIZE=61000 +export STREAMS_PER_DEVICE=32 +export USE_FUSED_HC_PRE_ASCENDC=1 +export USE_NPU_MOE_GATING_TOP_K=1 +export ZBAL_ENABLE_GRAPH=1 +export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 6688 \ + --page-size 128 \ + --tp-size 16 \ + --trust-remote-code \ + --device npu \ + --prefill-max-requests 160 \ + --attention-backend dsv4 \ + --watchdog-timeout 9000 \ + --mem-fraction-static 0.7 \ + --chunked-prefill-size 131072 \ + --max-running-requests 160 \ + --dp-size 16 \ + --enable-dp-attention \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --quantization modelslim \ + --enable-dp-lm-head \ + --kv-cache-dtype auto \ + --skip-server-warmup \ + --cuda-graph-bs 1 2 4 8 10 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 2 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 3 \ + --ep-size 16 \ + --disable-radix-cache +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --random-input-len 8000 \ + --random-output-len 1000 \ + --num-prompts 320 \ + --max-concurrency 160 \ + --random-range-ratio 1 \ + --warmup-requests 0 \ + --request-rate inf \ + --seed 1 \ + --max-attempts 3 +``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx index 21df2cf68..04472d42f 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx @@ -25,7 +25,6 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 16k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) | | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64k+1k | 55.2ms | 7.58s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-55-2ms) | | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64k+1k | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) | -| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 65k+1.5k (100% prefix cache hit rate) | 33ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-48p-in65k-out1k5-prefix100-33ms) | | GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 128k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) | | GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 64k+1k (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) | @@ -77,13 +76,14 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=2500 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=2500 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -142,10 +142,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 128 \ + --num-prompts 128 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 128 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` @@ -219,15 +220,18 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=72 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= + export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 + export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 + export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 export TASK_QUEUE_ENABLE=2 + export ZBAL_ENABLE_GRAPH=1 python3 -m sglang.launch_server \ --model-path ${MODEL_PATH} \ @@ -246,7 +250,7 @@ do --disaggregation-transfer-backend ascend \ --max-running-requests 16 \ --served-model-name glm-5 \ - --chunked-prefill-size 8192 \ + --chunked-prefill-size 32768 \ --max-prefill-tokens 180000 \ --moe-a2a-backend deepep \ --deepep-mode normal \ @@ -257,13 +261,11 @@ do --enable-nsa-prefill-context-parallel \ --nsa-prefill-cp-mode in-seq-split \ --attn-cp-size 4 \ + --disable-radix-cache \ --enable-dp-lm-head \ --moe-dense-tp 1 \ --pp-size 8 \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -279,6 +281,8 @@ do export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export TASK_QUEUE_ENABLE=0 @@ -289,6 +293,7 @@ do --port 8001 \ --dist-init-addr ${D_IP[0]}:5000 \ --node-rank $i \ + --trust-remote-code \ --tp-size 32 \ --nnodes 2 \ --dp-size 32 \ @@ -312,10 +317,10 @@ do --dtype bfloat16 \ --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ - --trust-remote-code - NODE_RANK=$i + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 break fi done @@ -347,11 +352,12 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 1 \ + --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 131072 \ --random-output-len 1024 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### GLM-5.1 W4A8 1P1D 32P IN16K OUT1K 50ms @@ -420,11 +426,10 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -455,15 +460,14 @@ do --dp-size 4 \ --enable-dp-attention \ --load-balance-method round_robin \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 8 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -474,11 +478,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=650 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export TASK_QUEUE_ENABLE=0 @@ -519,7 +524,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -552,10 +556,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 128 \ + --num-prompts 512 \ --random-input-len 16384 \ --random-output-len 1024 \ - --num-prompts 512 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### GLM-5.1 W4A8 1P1D 32P IN64K OUT1K 55.2ms @@ -627,15 +632,19 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=72 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= + export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 + export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 + export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 export TASK_QUEUE_ENABLE=2 + export ZBAL_ENABLE_GRAPH=1 + export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ --model-path ${MODEL_PATH} \ @@ -645,6 +654,7 @@ do --dist-init-addr ${P_IP[0]}:5000 \ --disaggregation-bootstrap-port 8998 \ --node-rank $i \ + --trust-remote-code \ --tp-size 4 \ --nnodes 2 \ --mem-fraction-static 0.72 \ @@ -654,7 +664,7 @@ do --disaggregation-transfer-backend ascend \ --max-running-requests 16 \ --served-model-name glm-5 \ - --chunked-prefill-size 8192 \ + --chunked-prefill-size 32768 \ --max-prefill-tokens 180000 \ --moe-a2a-backend deepep \ --deepep-mode normal \ @@ -665,13 +675,10 @@ do --enable-nsa-prefill-context-parallel \ --nsa-prefill-cp-mode in-seq-split \ --attn-cp-size 4 \ + --disable-radix-cache \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --pp-size 8 \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ - --trust-remote-code - NODE_RANK=$i + --pp-size 8 break fi done @@ -687,6 +694,8 @@ do export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export TASK_QUEUE_ENABLE=0 @@ -720,10 +729,11 @@ do --dtype bfloat16 \ --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -755,11 +765,12 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 1 \ + --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 65536 \ --random-output-len 1024 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### GLM-5.1 W4A8 1P1D 32P IN64K OUT1K 50ms @@ -829,13 +840,12 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=72 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -873,7 +883,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -925,7 +934,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -958,10 +966,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ --random-input-len 65536 \ --random-output-len 1024 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### GLM-5.1 W4A8 1P1D 32P IN65K OUT1K5 PREFIX90 25ms @@ -1030,11 +1039,10 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -1055,7 +1063,7 @@ do --disaggregation-transfer-backend ascend \ --max-running-requests 64 \ --served-model-name glm-5 \ - --chunked-prefill-size 524288 \ + --chunked-prefill-size 53248 \ --max-prefill-tokens 180000 \ --moe-a2a-backend deepep \ --deepep-mode normal \ @@ -1065,15 +1073,14 @@ do --dp-size 4 \ --enable-dp-attention \ --load-balance-method round_robin \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 8 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -1084,11 +1091,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=650 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export TASK_QUEUE_ENABLE=0 @@ -1129,7 +1137,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -1154,8 +1161,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 59904` = `int(66560 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6656` = `int(66560 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 59904` = `round(66560 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6656` = `round(66560 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -1174,203 +1181,6 @@ python -m sglang.bench_serving \ --request-rate inf ``` -### GLM-5.1 W4A8 1P1D 48P IN65K OUT1K5 PREFIX100 33ms - -**Model**: GLM-5.1 - -**Hardware**: Atlas 800I A3 - -**Cards**: 48 - -**Deploy Mode**: PD Disaggregation - -**Quantization**: W4A8 INT8 - -**Dataset**: 65k+1.5k (100% prefix cache hit rate) - -**TPOT**: 33ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# P_IP: prefill node IP address -# D_IP: decode node IP address -# ASCEND_MF_STORE_URL: prefill node IP with port -# MODEL_PATH: path to the model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -P_IP=('' '') -D_IP=('' '' '' '') - -export ASCEND_MF_STORE_URL="tcp://:24670" - -MODEL_PATH=/path/to/model-weights - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 - export HCCL_SOCKET_IFNAME= - export TASK_QUEUE_ENABLE=2 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --host ${P_IP[$i]} \ - --port 8000 \ - --dist-init-addr ${P_IP[0]}:5000 \ - --disaggregation-bootstrap-port 8998 \ - --node-rank $i \ - --tp-size 32 \ - --nnodes 2 \ - --mem-fraction-static 0.72 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --max-running-requests 192 \ - --served-model-name glm-5 \ - --chunked-prefill-size 16384 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --disable-shared-experts-fusion \ - --disable-cuda-graph \ - --dtype bfloat16 \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ - --trust-remote-code - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=650 - export HCCL_SOCKET_IFNAME= - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 - export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 - export TASK_QUEUE_ENABLE=0 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host ${D_IP[$i]} \ - --port 8001 \ - --dist-init-addr ${D_IP[0]}:5000 \ - --node-rank $i \ - --tp-size 64 \ - --nnodes 4 \ - --dp-size 64 \ - --ep-size 64 \ - --enable-dp-attention \ - --mem-fraction-static 0.84 \ - --max-running-requests 192 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --served-model-name glm-5 \ - --moe-a2a-backend deepep \ - --deepep-mode low_latency \ - --enable-dp-lm-head \ - --moe-dense-tp 1 \ - --cuda-graph-bs 1 2 3 \ - --disaggregation-transfer-backend ascend \ - --watchdog-timeout 9000 \ - --context-length 180000 \ - --speculative-draft-model-quantization unquant \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ - --disable-shared-experts-fusion \ - --dtype bfloat16 \ - --load-balance-method round_robin \ - --reasoning-parser glm45 \ - --tool-call-parser glm47 \ - --trust-remote-code - NODE_RANK=$i - break - fi -done -``` - -```bash Command -# ============================================================ -# Before running, replace the following placeholders: -# : prefill node IP address -# : first decode node IP address (decode may have distributed nodes) -# ============================================================ - -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://:8000 8998 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --policy round_robin -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving \ - --dataset-name random \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 128 \ - --random-input-len 66560 \ - --random-output-len 1536 \ - --num-prompts 512 \ - --random-range-ratio 1 -``` - ### GLM-5.1 W4A8 2P1D 48P IN128K OUT1K PREFIX90 50ms **Model**: GLM-5.1 @@ -1438,13 +1248,12 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=72 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -1473,8 +1282,8 @@ do --disable-cuda-graph \ --dtype bfloat16 \ --speculative-draft-model-quantization unquant \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1482,7 +1291,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -1498,6 +1306,7 @@ do export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export TASK_QUEUE_ENABLE=0 @@ -1538,7 +1347,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -1564,8 +1372,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 117965` = `round(131072 * 0.9)` is the shared prefix portion. +`--gsp-question-len 13107` = `round(131072 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -1576,7 +1384,7 @@ python -m sglang.bench_serving \ --port 6688 \ --gsp-num-groups 1 \ --gsp-prompts-per-group 576 \ - --gsp-system-prompt-len 117964 \ + --gsp-system-prompt-len 117965 \ --gsp-question-len 13107 \ --gsp-output-len 1024 \ --max-concurrency 144 \ @@ -1651,13 +1459,12 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024 export DEEPEP_NORMAL_LONG_SEQ_ROUND=72 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export TASK_QUEUE_ENABLE=2 @@ -1685,8 +1492,8 @@ do --disable-cuda-graph \ --dtype bfloat16 \ --speculative-draft-model-quantization unquant \ - --enable-nsa-prefill-context-parallel \ - --nsa-prefill-cp-mode in-seq-split \ + --enable-prefill-cp \ + --cp-strategy zigzag \ --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1694,7 +1501,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -1705,11 +1511,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=300 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=300 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export TASK_QUEUE_ENABLE=0 @@ -1750,7 +1557,6 @@ do --reasoning-parser glm45 \ --tool-call-parser glm47 \ --trust-remote-code - NODE_RANK=$i break fi done @@ -1778,8 +1584,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -1791,7 +1597,7 @@ python -m sglang.bench_serving \ --gsp-num-groups 1 \ --gsp-prompts-per-group 1280 \ --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ + --gsp-question-len 6554 \ --gsp-output-len 1024 \ --max-concurrency 320 \ --num-prompts 1280 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx new file mode 100644 index 000000000..50334f60d --- /dev/null +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx @@ -0,0 +1,226 @@ +--- +title: "GLM-5.2" +metatags: + description: "Best Practice for GLM-5.2 on Ascend NPU" +--- + + +This page focuses on optimal configuration and benchmark results for GLM-5.2 on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [GLM-5.2 Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2). + +On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. + + +### High Throughput + +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | +| GLM-5.2 | Atlas 800I A3 | 32 | PD Disaggregation | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-2-w4a8-3p1d-32p-in16k-out1k-50ms) | + +## Optimal Configuration + + + +### GLM-5.2 W4A8 3P1D 32P IN16K OUT1K 50ms + +**Model**: GLM-5.2 + +**Hardware**: Atlas 800I A3 + +**Cards**: 32 + +**Deploy Mode**: PD Disaggregation + +**Quantization**: W4A8 INT8 + +**Dataset**: 16k+1k + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# P_IP: prefill node IP address +# D_IP: decode node IP address +# ASCEND_MF_STORE_URL: prefill node IP with port +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=1200 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=1200 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 +export TRANSFORMERS_VERBOSITY=error + +P_IP=('' '' '') +D_IP=('') + +export ASCEND_MF_STORE_URL="tcp://:24670" + +MODEL_PATH=/path/to/model-weights + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export DEEP_USE_ALLTOALL_MODE=1 + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=128 + export HCCL_SOCKET_IFNAME= + export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 + export SGLANG_PP_LAYER_PARTITION=18,20,24,16 + export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 + export TASK_QUEUE_ENABLE=2 + export ZBAL_HCCL_OP=send,recv + export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --host ${P_IP[$i]} \ + --port 8000 \ + --disaggregation-bootstrap-port $((8998 + $i)) \ + --trust-remote-code \ + --tp-size 4 \ + --mem-fraction-static 0.72 \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --disaggregation-transfer-backend ascend \ + --max-running-requests 16 \ + --served-model-name glm-5 \ + --chunked-prefill-size 4096 \ + --max-prefill-tokens 180000 \ + --moe-a2a-backend deepep \ + --deepep-mode normal \ + --disable-shared-experts-fusion \ + --disable-cuda-graph \ + --dtype bfloat16 \ + --speculative-draft-model-quantization unquant \ + --enable-nsa-prefill-context-parallel \ + --nsa-prefill-cp-mode in-seq-split \ + --attn-cp-size 4 \ + --enable-dp-lm-head \ + --moe-dense-tp 1 \ + --pp-size 4 \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 1 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 2 + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=300 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40 + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 + export SGLANG_NPU_USE_MULTI_STREAM=1 + export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 + export TASK_QUEUE_ENABLE=0 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host ${D_IP[$i]} \ + --port 8001 \ + --trust-remote-code \ + --tp-size 16 \ + --dp-size 16 \ + --enable-dp-attention \ + --ep-size 16 \ + --mem-fraction-static 0.895 \ + --max-running-requests 128 \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --served-model-name glm-5 \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency \ + --cuda-graph-max-bs 8 \ + --disaggregation-transfer-backend ascend \ + --watchdog-timeout 9000 \ + --context-length 180000 \ + --tokenizer-worker-num 8 \ + --prefill-round-robin-balance \ + --disable-shared-experts-fusion \ + --dtype bfloat16 \ + --load-balance-method round_robin \ + --speculative-draft-model-quantization unquant \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 + break + fi +done +``` + +```bash Command +# ============================================================ +# Before running, replace the following placeholders: +# , , : prefill node IP addresses +# : decode node IP address +# ============================================================ + +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://:8000 8998 \ + --prefill http://:8000 8999 \ + --prefill http://:8000 9000 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --policy round_robin +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 64 \ + --num-prompts 200 \ + --random-input-len 16000 \ + --random-output-len 1000 \ + --random-range-ratio 1 +``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx index 6dcdf5d68..56bee38e7 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx @@ -80,9 +80,9 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=4400 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=4400 export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 @@ -149,10 +149,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 32 \ + --num-prompts 32 \ --random-input-len 64000 \ --random-output-len 1000 \ - --num-prompts 32 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` @@ -261,7 +262,6 @@ do --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend - NODE_RANK=$i break fi done @@ -272,11 +272,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -313,7 +314,6 @@ do --speculative-draft-model-quantization unquant \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 - NODE_RANK=$i break fi done @@ -346,11 +346,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ + --request-rate inf \ --random-input-len 128000 \ --random-output-len 1000 \ - --num-prompts 1 \ --random-range-ratio 1 \ - --request-rate inf + --seed 1 ``` ### Kimi-K2.6 W4A8 1P1D 16P IN128K OUT1K PREFIX90 100ms @@ -460,7 +461,6 @@ do --sampling-backend ascend \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 - NODE_RANK=$i break fi done @@ -471,11 +471,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -512,7 +513,6 @@ do --speculative-draft-model-quantization unquant \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 - NODE_RANK=$i break fi done @@ -537,8 +537,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 115200` = `int(128000 * 0.9)` is the shared prefix portion. -`--gsp-question-len 12800` = `int(128000 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 115200` = `round(128000 * 0.9)` is the shared prefix portion. +`--gsp-question-len 12800` = `round(128000 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -626,8 +626,8 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1800 export HCCL_SOCKET_IFNAME= python3 -m sglang.launch_server \ @@ -659,7 +659,6 @@ do --sampling-backend ascend \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 - NODE_RANK=$i break fi done @@ -670,11 +669,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -711,7 +711,6 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 5 \ --speculative-draft-model-quantization unquant - NODE_RANK=$i break fi done @@ -744,11 +743,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ + --request-rate inf \ --random-input-len 64000 \ --random-output-len 1500 \ - --num-prompts 1 \ --random-range-ratio 1 \ - --request-rate inf + --seed 1 ``` ### Kimi-K2.6 W4A8 1P1D 16P IN64K OUT1K5 PREFIX90 100ms @@ -822,8 +822,8 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1800 export HCCL_SOCKET_IFNAME= python3 -m sglang.launch_server \ @@ -853,7 +853,6 @@ do --sampling-backend ascend \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 - NODE_RANK=$i break fi done @@ -864,11 +863,12 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1200 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -905,7 +905,6 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 5 \ --speculative-draft-model-quantization unquant - NODE_RANK=$i break fi done @@ -930,8 +929,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6400` = `int(64000 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 57600` = `round(64000 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6400` = `round(64000 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -977,8 +976,6 @@ python -m sglang.bench_serving \ # Before running, update the following variables: # MODEL_PATH: path to the model weights directory # DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ MODEL_PATH=/path/to/model-weights @@ -998,12 +995,13 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=1280 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export HCCL_BUFFSIZE=1500 export HCCL_OP_EXPANSION_MODE=AIV export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -1018,7 +1016,7 @@ python3 -m sglang.launch_server \ --device npu \ --attention-backend ascend \ --tp-size 16 \ - --mem-fraction-static 0.872 \ + --mem-fraction-static 0.865 \ --max-running-requests 176 \ --chunked-prefill-size 32768 \ --context-length 8192 \ @@ -1054,15 +1052,16 @@ python -m sglang.bench_serving \ --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ + --image-resolution 1024x1024 \ + --image-count 1 \ --max-concurrency 160 \ + --num-prompts 640 \ + --request-rate inf \ --random-input-len 30 \ --random-output-len 1024 \ - --num-prompts 640 \ --random-range-ratio 1 \ - --request-rate inf \ --warmup-requests 16 \ - --image-count 1 \ - --image-resolution 1024x1024 + --seed 1 ``` ### Kimi-K2.6 W4A8 8P IN1080P 30 OUT256 50ms @@ -1088,8 +1087,6 @@ python -m sglang.bench_serving \ # Before running, update the following variables: # MODEL_PATH: path to the model weights directory # DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ MODEL_PATH=/path/to/model-weights @@ -1109,12 +1106,13 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=1024 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export HCCL_BUFFSIZE=2400 export HCCL_OP_EXPANSION_MODE=AIV export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -1129,7 +1127,6 @@ python3 -m sglang.launch_server \ --device npu \ --attention-backend ascend \ --tp-size 16 \ - --base-gpu-id 0 \ --mem-fraction-static 0.852 \ --max-running-requests 64 \ --chunked-prefill-size 16384 \ @@ -1166,15 +1163,16 @@ python -m sglang.bench_serving \ --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ + --image-resolution 1920x1080 \ + --image-count 1 \ --max-concurrency 48 \ + --num-prompts 196 \ + --request-rate inf \ --random-input-len 30 \ --random-output-len 256 \ - --num-prompts 196 \ --random-range-ratio 1 \ - --request-rate inf \ --warmup-requests 16 \ - --image-count 1 \ - --image-resolution 1920x1080 + --seed 1 ``` ### Kimi-K2.6 W4A8 8P IN3K5 OUT1K5 20ms @@ -1221,15 +1219,16 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=1200 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1200 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_SET_CPU_AFFINITY=1 @@ -1281,12 +1280,14 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --max-attempts 5 \ --max-concurrency 64 \ + --num-prompts 256 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 256 \ --random-range-ratio 1 \ - --warmup-requests 0 + --warmup-requests 0 \ + --seed 1 ``` ### Kimi-K2.6 W4A8 8P IN3K5 OUT1K5 50ms @@ -1333,15 +1334,17 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=1200 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1200 +export HCCL_BUFFSIZE=200 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -1366,7 +1369,7 @@ python3 -m sglang.launch_server \ --dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs-decode 1 2 4 8 12 13 \ + --cuda-graph-bs-decode 1 2 4 8 11 12 13 \ --disable-radix-cache \ --model-loader-extra-config '{"enable_multithread_load": true}' \ --speculative-algorithm EAGLE3 \ @@ -1375,7 +1378,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 5 \ --speculative-draft-model-quantization unquant \ - --prefill-delayer-max-delay-passes 200 \ + --prefill-delayer-max-delay-passes 50 \ --enable-prefill-delayer \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 @@ -1392,9 +1395,10 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 192 \ + --num-prompts 768 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 768 \ --random-range-ratio 1 \ - --warmup-requests 0 + --warmup-requests 0 \ + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx index 9c01ca590..1a5d4f6cb 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx @@ -83,6 +83,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -103,8 +104,8 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1024 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 @@ -130,7 +131,6 @@ do --disable-cuda-graph \ --disable-piecewise-cuda-graph \ --dp-size 2 - NODE_RANK=$i break fi done @@ -141,8 +141,8 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 @@ -172,7 +172,6 @@ do --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency - NODE_RANK=$i break fi done @@ -206,11 +205,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 64 \ + --num-prompts 128 \ + --request-rate 0.4 \ --random-input-len 16000 \ --random-output-len 1 \ - --num-prompts 128 \ --random-range-ratio 1 \ - --request-rate 0.4 + --seed 1 ``` ### MiMo-V2-Flash 1P1D 12P IN16K OUT1K TPOT 20ms @@ -268,6 +268,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -288,8 +289,8 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1024 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 @@ -315,7 +316,6 @@ do --disable-cuda-graph \ --disable-piecewise-cuda-graph \ --dp-size 2 - NODE_RANK=$i break fi done @@ -326,8 +326,8 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 @@ -357,7 +357,6 @@ do --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency - NODE_RANK=$i break fi done @@ -391,11 +390,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 32 \ + --num-prompts 128 \ + --request-rate inf \ --random-input-len 16000 \ --random-output-len 1000 \ - --num-prompts 128 \ --random-range-ratio 1 \ - --request-rate inf + --seed 1 ``` ### MiMo-V2-Flash 1P1D 12P IN32K OUT1 TTFT 5s @@ -453,6 +453,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -473,8 +474,8 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1024 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 @@ -500,7 +501,6 @@ do --disable-cuda-graph \ --disable-piecewise-cuda-graph \ --dp-size 2 - NODE_RANK=$i break fi done @@ -511,8 +511,8 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 @@ -542,7 +542,6 @@ do --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency - NODE_RANK=$i break fi done @@ -576,11 +575,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 64 \ + --num-prompts 128 \ + --request-rate 0.4 \ --random-input-len 32000 \ --random-output-len 1 \ - --num-prompts 128 \ --random-range-ratio 1 \ - --request-rate 0.4 + --seed 1 ``` ### MiMo-V2-Flash 1P1D 12P IN32K OUT1K TPOT 20ms @@ -638,6 +638,7 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -658,8 +659,8 @@ do if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; then echo "${P_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=1024 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 @@ -685,7 +686,6 @@ do --disable-cuda-graph \ --disable-piecewise-cuda-graph \ --dp-size 2 - NODE_RANK=$i break fi done @@ -696,8 +696,8 @@ do if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; then echo "${D_IP[$i]}" + export DEEPEP_HCCL_BUFFSIZE=800 export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=800 export HCCL_SOCKET_IFNAME= export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 @@ -727,7 +727,6 @@ do --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode low_latency - NODE_RANK=$i break fi done @@ -760,10 +759,11 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 64 \ + --max-concurrency 48 \ + --num-prompts 192 \ + --request-rate inf \ --random-input-len 32000 \ --random-output-len 1000 \ - --num-prompts 128 \ --random-range-ratio 1 \ - --request-rate inf + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx index b9e5aa0c7..e842e3cd3 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx @@ -85,6 +85,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 export SGLANG_SET_CPU_AFFINITY=1 @@ -118,7 +119,8 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --tokenizer-worker-num 4 \ --reasoning-parser minimax-append-think \ - --tool-call-parser minimax-m2 + --tool-call-parser minimax-m2 \ + --device npu ``` #### Benchmark @@ -132,10 +134,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 18 \ + --num-prompts 72 \ --random-input-len 32768 \ --random-output-len 1024 \ - --num-prompts 72 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### MiniMax-M2.5 W8A8 4P IN64K OUT1K PREFIX90 50ms @@ -184,14 +187,15 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=1024 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 +export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 export TASK_QUEUE_ENABLE=1 @@ -210,7 +214,8 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 67000 \ --cuda-graph-bs 2 4 8 12 16 18 20 22 24 26 \ --moe-a2a-backend ascend_fuseep \ - --fuseep-mode 2 \ + --deepep-mode auto \ + --quantization modelslim \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 3 \ @@ -218,14 +223,15 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ --dtype bfloat16 \ - --trust-remote-code + --trust-remote-code \ + --device npu ``` #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -237,7 +243,7 @@ python -m sglang.bench_serving \ --gsp-num-groups 1 \ --gsp-prompts-per-group 104 \ --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ + --gsp-question-len 6554 \ --gsp-output-len 1024 \ --max-concurrency 26 \ --num-prompts 104 \ @@ -290,14 +296,16 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=1024 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 +export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 export TASK_QUEUE_ENABLE=1 @@ -318,7 +326,8 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 80000 \ --cuda-graph-bs 2 4 6 8 \ --moe-a2a-backend ascend_fuseep \ - --fuseep-mode 2 \ + --deepep-mode auto \ + --quantization modelslim \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 3 \ @@ -326,14 +335,15 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ --tokenizer-worker-num 4 \ - --dtype bfloat16 + --dtype bfloat16 \ + --device npu ``` #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 117965` = `round(131072 * 0.9)` is the shared prefix portion. +`--gsp-question-len 13107` = `round(131072 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -344,7 +354,7 @@ python -m sglang.bench_serving \ --port 6688 \ --gsp-num-groups 1 \ --gsp-prompts-per-group 16 \ - --gsp-system-prompt-len 117964 \ + --gsp-system-prompt-len 117965 \ --gsp-question-len 13107 \ --gsp-output-len 1024 \ --max-concurrency 4 \ @@ -398,13 +408,15 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=2048 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=2048 export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 +export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 export TASK_QUEUE_ENABLE=1 @@ -427,7 +439,6 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 8192 \ --cuda-graph-bs 1 2 3 4 5 6 \ --moe-a2a-backend ascend_fuseep \ - --fuseep-mode 2 \ --deepep-mode auto \ --quantization modelslim \ --speculative-algorithm EAGLE3 \ @@ -436,7 +447,8 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ - --dtype bfloat16 + --dtype bfloat16 \ + --device npu ``` #### Benchmark @@ -450,10 +462,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 112 \ + --num-prompts 448 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 448 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### MiniMax-M2.5 W8A8 8P IN3K5 OUT1K5 50ms @@ -502,8 +515,8 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=1024 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1024 export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 @@ -526,12 +539,11 @@ python3 -m sglang.launch_server \ --tool-call-parser minimax-m2 \ --prefill-delayer-max-delay-passes 500 \ --enable-prefill-delayer \ - --chunked-prefill-size -1 \ + --chunked-prefill-size 196608 \ --max-prefill-tokens 8192 \ --cuda-graph-bs 1 2 4 8 12 16 20 \ --moe-a2a-backend ascend_fuseep \ --fuseep-mode 2 \ - --deepep-mode auto \ --quantization modelslim \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ @@ -539,7 +551,8 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --speculative-draft-model-quantization unquant \ - --dtype bfloat16 + --dtype bfloat16 \ + --device npu ``` #### Benchmark @@ -553,8 +566,9 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 320 \ + --num-prompts 1280 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 1280 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx index 886683484..b73f67f46 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx @@ -70,13 +70,14 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=1600 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1600 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ @@ -116,10 +117,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ --random-input-len 11000 \ --random-output-len 1500 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-235B-A22B W8A8 8P IN3K5 OUT1K5 50.1ms @@ -166,8 +168,8 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=570 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=570 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -175,6 +177,7 @@ export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 @@ -186,6 +189,7 @@ python3 -m sglang.launch_server \ --node-rank 0 \ --attention-backend ascend \ --device npu \ + --quantization modelslim \ --max-running-requests 432 \ --context-length 8192 \ --dtype bfloat16 \ @@ -195,7 +199,6 @@ python3 -m sglang.launch_server \ --ep-dispatch-algorithm static \ --disable-radix-cache \ --moe-a2a-backend ascend_fuseep \ - --fuseep-mode 2 \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 3 \ @@ -223,8 +226,9 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 432 \ + --num-prompts 1728 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 1728 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx index e76a016b6..9324eaa3c 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx @@ -81,6 +81,7 @@ export INF_NAN_MODE_FORCE_DISABLE=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_USE_MAX_DP_ATT=1 @@ -125,10 +126,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 162 \ + --num-prompts 624 \ --random-input-len 1000 \ --random-output-len 100 \ - --num-prompts 624 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 \ + --max-attempts 4 ``` ### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 10ms @@ -176,13 +179,14 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_LAUNCH_BLOCKING=0 +export DEEPEP_HCCL_BUFFSIZE=400 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=400 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 @@ -224,10 +228,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 50ms @@ -275,8 +280,8 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_LAUNCH_BLOCKING=0 +export DEEPEP_HCCL_BUFFSIZE=400 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=400 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -323,10 +328,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 160 \ + --num-prompts 640 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 640 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-30B-A3B W8A8 1P IN6K OUT1K5 BS16 @@ -373,13 +379,14 @@ unset ASCEND_LAUNCH_BLOCKING source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh +export DEEPEP_HCCL_BUFFSIZE=400 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=400 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export TRANSFORMERS_VERBOSITY=error @@ -420,9 +427,10 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --max-attempts 5 \ --max-concurrency 16 \ + --num-prompts 16 \ --random-input-len 6144 \ --random-output-len 1500 \ - --num-prompts 16 \ --random-range-ratio 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx index 9bb5b5ed7..62d332d09 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx @@ -20,8 +20,8 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms-a2) | | Qwen3-32B | Atlas 800I A3 | 2 | PD Mixed | 3.5k+1.5k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-50ms) | +| Qwen3-32B | Atlas 800I A2 | 2 | PD Mixed | 3.5k+1.5k | 55ms | W8A8 INT8 | [Optimal Configuration](#qwen3-32b-w8a8-2p-in3k5-out1k5-55ms-a2) | ## Optimal Configuration @@ -117,108 +117,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ --random-input-len 18000 \ --random-output-len 4000 \ - --num-prompts 1 \ - --random-range-ratio 1 -``` - -### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms A2 - -**Model**: Qwen3-32B - -**Hardware**: Atlas 800I A2 - -**Cards**: 2 - -**Deploy Mode**: PD Mixed - -**Quantization**: W8A8 INT8 - -**Dataset**: 3.5k+1.5k - -**TPOT**: 50ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# MODEL_PATH: path to the model weights directory -# DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - -MODEL_PATH=/path/to/model-weights -DRAFT_MODEL_PATH=/path/to/draft-model-weights - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export GLOO_SOCKET_IFNAME= -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME= -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_NPU_USE_DEEPGEMM=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 6688 \ - --trust-remote-code \ - --nnodes 1 \ - --node-rank 0 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --max-running-requests 101 \ - --disable-radix-cache \ - --speculative-draft-model-quantization unquant \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 35000 \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --tp-size 4 \ - --mem-fraction-static 0.845 \ - --cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ - --dtype bfloat16 \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```bash Command -python -m sglang.bench_serving \ - --dataset-name random \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 100 \ - --random-input-len 3584 \ - --random-output-len 1536 \ - --num-prompts 400 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms @@ -313,8 +216,108 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 100 \ + --num-prompts 400 \ --random-input-len 3584 \ --random-output-len 1536 \ + --random-range-ratio 1 \ + --seed 1 +``` + +### Qwen3-32B W8A8 2P IN3K5 OUT1K5 55ms A2 + +**Model**: Qwen3-32B + +**Hardware**: Atlas 800I A2 + +**Cards**: 2 + +**Deploy Mode**: PD Mixed + +**Quantization**: W8A8 INT8 + +**Dataset**: 3.5k+1.5k + +**TPOT**: 55ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# DRAFT_MODEL_PATH: path to the draft model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + +MODEL_PATH=/path/to/model-weights +DRAFT_MODEL_PATH=/path/to/draft-model-weights + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export GLOO_SOCKET_IFNAME= +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME= +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_NPU_USE_DEEPGEMM=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 6688 \ + --trust-remote-code \ + --nnodes 1 \ + --node-rank 0 \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --max-running-requests 101 \ + --disable-radix-cache \ + --speculative-draft-model-quantization unquant \ + --chunked-prefill-size -1 \ + --max-prefill-tokens 35000 \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --tp-size 4 \ + --mem-fraction-static 0.845 \ + --cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 100 \ --num-prompts 400 \ - --random-range-ratio 1 + --random-input-len 3584 \ + --random-output-len 1536 \ + --random-range-ratio 1 \ + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx index 8fc099c5b..75724ee1a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx @@ -1,11 +1,13 @@ --- title: "Qwen3.5-397B-A17B" metatags: - description: "Best Practice for Qwen3.5-397B on Ascend NPU" + description: "Best Practice for Qwen3.5-397B-A17B on Ascend NPU" --- -This page focuses on optimal configuration and benchmark results for Qwen3.5-397B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.5-397B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b). +This page focuses on optimal configuration and benchmark results for Qwen3.5-397B-A17B on the Ascend NPU. For environment +setup, model weight download, feature configuration, and deployment instructions, etc., see +the [Qwen3.5-397B-A17B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b). On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. @@ -14,29 +16,29 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in128k-out1k-20ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in16k-out1k-20ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in3k5-out1k5-20ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-20ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-20ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-20ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 22.2ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-22-2ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 20ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-20ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in128k-out1k-50ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in128k-out1k-prefix90-50ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in16k-out1k-50ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in3k5-out1k5-50ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-50ms) | -| Qwen3.5-397B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-w4a8-8p-in64k-out1k-prefix90-50ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-50ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in128k-out1k-prefix90-50ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 16k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in16k-out1k-50ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 3.5k+1.5k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in3k5-out1k5-50ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-50ms) | +| Qwen3.5-397B-A17B | Atlas 800I A3 | 8 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#qwen3-5-397b-a17b-w4a8-8p-in64k-out1k-prefix90-50ms) | ## Optimal Configuration -### Qwen3.5-397B W4A8 8P IN128K OUT1K 20ms +### Qwen3.5-397B-A17B W4A8 8P IN128K OUT1K 20ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -87,6 +89,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -135,18 +138,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 2 \ --max-concurrency 3 \ + --num-prompts 3 \ --random-input-len 131072 \ --random-output-len 1024 \ - --num-prompts 3 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 2 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN128K OUT1K 50ms +### Qwen3.5-397B-A17B W4A8 8P IN128K OUT1K 50ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -197,6 +203,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -218,7 +225,6 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --max-running-requests 16 \ --mem-fraction-static 0.6 \ - --cuda-graph-bs 2 4 6 8 12 14 16 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -245,18 +251,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 8 \ --max-concurrency 10 \ + --num-prompts 10 \ --random-input-len 131072 \ --random-output-len 1024 \ - --num-prompts 10 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 8 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN128K OUT1K PREFIX90 50ms +### Qwen3.5-397B-A17B W4A8 8P IN128K OUT1K PREFIX90 50ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -296,17 +305,18 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=2200 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=2200 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -320,7 +330,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 131072 \ --max-mamba-cache-size 320 \ --prefill-max-requests 10 \ - --mamba-radix-cache-strategy extra_buffer \ + --mamba-scheduler-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 64 \ --mem-fraction-static 0.6 \ @@ -343,8 +353,8 @@ python3 -m sglang.launch_server \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 117965` = `round(131072 * 0.9)` is the shared prefix portion. +`--gsp-question-len 13107` = `round(131072 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -355,17 +365,19 @@ python -m sglang.bench_serving \ --port 6688 \ --gsp-num-groups 1 \ --gsp-prompts-per-group 40 \ - --gsp-system-prompt-len 117964 \ + --gsp-system-prompt-len 117965 \ --gsp-question-len 13107 \ --gsp-output-len 1024 \ --max-concurrency 40 \ --num-prompts 40 \ - --request-rate inf + --request-rate inf \ + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN16K OUT1K 20ms +### Qwen3.5-397B-A17B W4A8 8P IN16K OUT1K 20ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -416,6 +428,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -468,18 +481,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 32 \ --max-concurrency 40 \ + --num-prompts 40 \ --random-input-len 16384 \ --random-output-len 1024 \ - --num-prompts 40 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 32 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN16K OUT1K 50ms +### Qwen3.5-397B-A17B W4A8 8P IN16K OUT1K 50ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -530,6 +546,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -582,18 +599,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 8 \ --max-concurrency 132 \ + --num-prompts 132 \ --random-input-len 16384 \ --random-output-len 1024 \ - --num-prompts 132 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 8 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN3K5 OUT1K5 20ms +### Qwen3.5-397B-A17B W4A8 8P IN3K5 OUT1K5 22.2ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -605,7 +625,7 @@ python -m sglang.bench_serving \ **Dataset**: 3.5k+1.5k -**TPOT**: 20ms +**TPOT**: 22.2ms #### Model Deployment @@ -644,6 +664,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -695,18 +716,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 64 \ --max-concurrency 160 \ + --num-prompts 160 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 160 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 64 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN3K5 OUT1K5 50ms +### Qwen3.5-397B-A17B W4A8 8P IN3K5 OUT1K5 50ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -757,6 +781,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -808,18 +833,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 16 \ --max-concurrency 432 \ + --num-prompts 432 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 432 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 16 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN64K OUT1K 20ms +### Qwen3.5-397B-A17B W4A8 8P IN64K OUT1K 20ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -870,6 +898,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -922,18 +951,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --warmup-requests 6 \ --max-concurrency 6 \ + --num-prompts 6 \ --random-input-len 65536 \ --random-output-len 1024 \ - --num-prompts 6 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 6 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN64K OUT1K 50ms +### Qwen3.5-397B-A17B W4A8 8P IN64K OUT1K 50ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -984,6 +1016,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 @@ -1036,18 +1069,21 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 28 \ + --warmup-requests 8 \ + --max-concurrency 24 \ + --num-prompts 24 \ --random-input-len 65536 \ --random-output-len 1024 \ - --num-prompts 28 \ --random-range-ratio 1 \ + --seed 1 \ --request-rate inf \ - --warmup-requests 8 + --temperature 0.6 \ + --top-p 0.95 ``` -### Qwen3.5-397B W4A8 8P IN64K OUT1K PREFIX90 50ms +### Qwen3.5-397B-A17B W4A8 8P IN64K OUT1K PREFIX90 50ms -**Model**: Qwen3.5-397B +**Model**: Qwen3.5-397B-A17B **Hardware**: Atlas 800I A3 @@ -1087,17 +1123,18 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=2200 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096 export DEEPEP_NORMAL_LONG_SEQ_ROUND=20 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=2200 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -1110,7 +1147,7 @@ python3 -m sglang.launch_server \ --chunked-prefill-size -1 \ --max-prefill-tokens 65536 \ --max-mamba-cache-size 640 \ - --mamba-radix-cache-strategy extra_buffer \ + --mamba-scheduler-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 128 \ --mem-fraction-static 0.6 \ @@ -1137,8 +1174,8 @@ python3 -m sglang.launch_server \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -1148,11 +1185,13 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --gsp-num-groups 1 \ - --gsp-prompts-per-group 112 \ + --gsp-prompts-per-group 96 \ --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ + --gsp-question-len 6554 \ --gsp-output-len 1024 \ - --max-concurrency 112 \ - --num-prompts 112 \ - --request-rate inf + --max-concurrency 96 \ + --num-prompts 96 \ + --request-rate inf \ + --temperature 0.6 \ + --top-p 0.95 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx index 8663c47f4..5c52399c7 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx @@ -21,6 +21,7 @@ On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend | Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in64k-out1k-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) | +| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64k+1k | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) | ## Optimal Configuration @@ -75,7 +76,7 @@ export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=300 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_VIT_ENABLE_CUDA_GRAPH=1 @@ -92,10 +93,10 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 52000 \ --disable-radix-cache \ --trust-remote-code \ - --max-running-requests 50 \ + --max-running-requests 60 \ --max-mamba-cache-size 60 \ - --mem-fraction-static 0.76 \ - --cuda-graph-bs 2 4 8 16 24 32 40 42 45 50 \ + --mem-fraction-static 0.74 \ + --cuda-graph-bs 2 4 8 14 16 24 26 32 36 37 40 42 44 45 46 50 52 60 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -111,19 +112,22 @@ python3 -m sglang.launch_server \ #### Benchmark -We tested it based on the `RANDOM` dataset. +We tested it based on the `IMAGE` dataset with 1024x1024 resolution. ```bash Command python -m sglang.bench_serving \ - --dataset-name random \ - --backend sglang \ + --dataset-name image \ + --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 48 \ + --max-concurrency 60 \ + --num-prompts 240 \ --random-input-len 30 \ --random-output-len 1024 \ - --num-prompts 48 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --image-resolution 1024x1024 \ + --image-count 1 \ + --seed 1 ``` ### Qwen3.6-27B 1P IN1080P 30 OUT256 50ms @@ -171,10 +175,7 @@ export ASCEND_USE_FIA=1 export GLOO_SOCKET_IFNAME= export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_VIT_ENABLE_CUDA_GRAPH=1 export STREAMS_PER_DEVICE=32 @@ -187,13 +188,16 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ - --max-prefill-tokens 48000 \ + --max-prefill-tokens 82688 \ --disable-radix-cache \ --trust-remote-code \ - --max-running-requests 30 \ - --max-mamba-cache-size 40 \ - --mem-fraction-static 0.76 \ - --cuda-graph-bs 2 4 8 16 24 28 30 \ + --max-running-requests 38 \ + --max-mamba-cache-size 38 \ + --mem-fraction-static 0.7 \ + --cuda-graph-bs 1 2 4 8 10 12 16 20 24 28 30 32 35 38 \ + --enable-prefill-delayer \ + --prefill-delayer-queue-min-ratio 0.45 \ + --prefill-delayer-max-delay-ms 5500 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -202,26 +206,29 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --mm-enable-dp-encoder \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder ``` #### Benchmark -We tested it based on the `RANDOM` dataset. +We tested it based on the `IMAGE` dataset with 1920x1080 resolution. ```bash Command python -m sglang.bench_serving \ - --dataset-name random \ - --backend sglang \ + --dataset-name image \ + --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 30 \ + --warmup-requests 38 \ + --max-concurrency 42 \ + --num-prompts 152 \ --random-input-len 30 \ --random-output-len 256 \ - --num-prompts 120 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --image-resolution 1920x1080 \ + --image-count 1 \ + --seed 1 ``` ### Qwen3.6-27B 1P IN64K OUT1K PREFIX90 50ms @@ -272,6 +279,7 @@ export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -284,15 +292,12 @@ python3 -m sglang.launch_server \ --device npu \ --chunked-prefill-size 32768 \ --max-prefill-tokens 32768 \ - --mamba-radix-cache-strategy extra_buffer \ + --mamba-scheduler-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 20 \ --max-mamba-cache-size 160 \ --mem-fraction-static 0.82 \ --cuda-graph-bs 1 2 5 10 15 17 19 20 \ - --enable-prefill-delayer \ - --prefill-delayer-queue-min-ratio 0.7 \ - --prefill-delayer-max-delay-ms 20000 \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ --speculative-algorithm NEXTN \ @@ -306,8 +311,8 @@ python3 -m sglang.launch_server \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6400` = `int(64000 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 57600` = `round(64000 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6400` = `round(64000 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -317,12 +322,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --gsp-num-groups 1 \ - --gsp-prompts-per-group 80 \ + --gsp-prompts-per-group 20 \ --gsp-system-prompt-len 57600 \ --gsp-question-len 6400 \ --gsp-output-len 1000 \ --max-concurrency 20 \ - --num-prompts 80 \ + --num-prompts 20 \ --request-rate inf ``` @@ -417,10 +422,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 54 \ + --num-prompts 216 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 216 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3.6-27B W8A8 1P IN64K OUT1K 50ms @@ -508,10 +514,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 6 \ + --num-prompts 12 \ --random-input-len 64000 \ --random-output-len 1000 \ - --num-prompts 12 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3.6-27B W8A8 2P IN128K OUT1K 50ms @@ -601,10 +608,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 4 \ + --num-prompts 16 \ --random-input-len 128000 \ --random-output-len 1000 \ - --num-prompts 16 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3.6-27B W8A8 2P IN16K OUT1K 50ms @@ -654,7 +662,8 @@ export GLOO_SOCKET_IFNAME= export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -670,17 +679,17 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 58000 \ --disable-radix-cache \ --trust-remote-code \ - --max-running-requests 29 \ - --max-mamba-cache-size 58 \ - --mem-fraction-static 0.68 \ - --cuda-graph-bs 1 2 8 12 16 20 24 26 28 29 \ + --max-running-requests 37 \ + --max-mamba-cache-size 74 \ + --mem-fraction-static 0.7 \ + --cuda-graph-bs 1 2 3 4 6 8 10 12 14 16 18 20 21 23 24 25 26 27 28 29 30 31 33 35 37 \ --quantization modelslim \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ + --speculative-num-steps 4 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ + --speculative-num-draft-tokens 5 \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder ``` @@ -695,9 +704,107 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 29 \ + --max-concurrency 37 \ + --warmup-requests 4 \ + --num-prompts 37 \ --random-input-len 16000 \ --random-output-len 1000 \ - --num-prompts 116 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 +``` + +### Qwen3.6-27B W8A8 2P IN64K OUT1K 50ms + +**Model**: Qwen3.6-27B + +**Hardware**: Atlas 800I A3 + +**Cards**: 2 + +**Deploy Mode**: PD Mixed + +**Quantization**: W8A8 INT8 + +**Dataset**: 64k+1k + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + +MODEL_PATH=/path/to/model-weights +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_USE_FIA=1 +export GLOO_SOCKET_IFNAME= +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME= +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 6688 \ + --tp-size 4 \ + --nnodes 1 \ + --attention-backend ascend \ + --device npu \ + --chunked-prefill-size -1 \ + --max-prefill-tokens 50000 \ + --disable-radix-cache \ + --trust-remote-code \ + --max-running-requests 28 \ + --max-mamba-cache-size 50 \ + --mem-fraction-static 0.7 \ + --cuda-graph-bs 2 4 6 \ + --enable-multimodal \ + --quantization modelslim \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 \ + --mamba-ssm-dtype bfloat16 \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 6 \ + --num-prompts 24 \ + --random-input-len 64000 \ + --random-output-len 1000 \ + --random-range-ratio 1 \ + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx index 4b44614dc..6bf66c156 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx @@ -8,8 +8,6 @@ metatags: This page focuses on optimal configuration and benchmark results for Qwen3.6-35B-A3B on the Ascend NPU. For environment setup, model weight download, feature configuration, and deployment instructions, etc., see the [Qwen3.6-35B-A3B Model Tutorial](/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b). On A3 each card has 2 dies, so `--tp-size` is twice the card count; see [Ascend NPU Reference](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware) for details. - -Use image **SGLang `>= v0.5.16`** for these NEXTN configurations. Without `--dataset-path`, `bench_serving --dataset-name random` downloads ShareGPT from Hugging Face; in offline environments, pass a local dataset path (for example a ShareGPT JSON file). ### Low Latency @@ -25,6 +23,7 @@ Use image **SGLang `>= v0.5.16`** for these NEXTN configurations. Without `--dat | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) | | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) | | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) | | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5k+1.5k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) | | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) | | Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64k+1k (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) | @@ -83,8 +82,9 @@ export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_VIT_ENABLE_CUDA_GRAPH=1 export STREAMS_PER_DEVICE=32 python3 -m sglang.launch_server \ @@ -95,14 +95,15 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ - --max-prefill-tokens 16384 \ + --max-prefill-tokens 9999999 \ + --max-total-tokens 365000 \ + --prefill-max-requests 30 \ --disable-radix-cache \ --trust-remote-code \ - --enable-prefill-delayer \ --max-running-requests 120 \ - --max-mamba-cache-size 240 \ - --mem-fraction-static 0.78 \ - --cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120 \ + --max-mamba-cache-size 120 \ + --mem-fraction-static 0.85 \ + --cuda-graph-bs 4 16 32 48 64 110 165 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -117,20 +118,23 @@ python3 -m sglang.launch_server \ #### Benchmark -We tested it based on the `RANDOM` dataset. +We tested it based on the `IMAGE` dataset with 1024x1024 resolution. ```bash Command python -m sglang.bench_serving \ - --dataset-name random \ - --dataset-path /path/to/dataset \ - --backend sglang \ + --dataset-name image \ + --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 120 \ + --num-prompts 120 \ + --warmup-requests 120 \ --random-input-len 30 \ --random-output-len 1024 \ - --num-prompts 480 \ --random-range-ratio 1 \ + --image-resolution 1024x1024 \ + --image-count 1 \ + --seed 1 \ --request-rate inf ``` @@ -181,8 +185,9 @@ export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_VIT_ENABLE_CUDA_GRAPH=1 export STREAMS_PER_DEVICE=32 python3 -m sglang.launch_server \ @@ -193,14 +198,14 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ - --max-prefill-tokens 16384 \ + --max-prefill-tokens 150000 \ + --max-total-tokens 200000 \ --disable-radix-cache \ --trust-remote-code \ - --enable-prefill-delayer \ - --max-running-requests 50 \ - --max-mamba-cache-size 55 \ - --mem-fraction-static 0.8 \ - --cuda-graph-bs 2 4 8 12 16 20 24 28 32 36 40 44 48 50 \ + --max-running-requests 42 \ + --max-mamba-cache-size 42 \ + --mem-fraction-static 0.75 \ + --cuda-graph-bs 4 8 16 24 48 64 80 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -215,20 +220,23 @@ python3 -m sglang.launch_server \ #### Benchmark -We tested it based on the `RANDOM` dataset. +We tested it based on the `IMAGE` dataset with 1920x1080 resolution. ```bash Command python -m sglang.bench_serving \ - --dataset-name random \ - --dataset-path /path/to/dataset \ - --backend sglang \ + --dataset-name image \ + --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 50 \ + --max-concurrency 42 \ + --num-prompts 42 \ + --warmup-requests 42 \ --random-input-len 30 \ --random-output-len 256 \ - --num-prompts 200 \ --random-range-ratio 1 \ + --image-resolution 1920x1080 \ + --image-count 1 \ + --seed 1 \ --request-rate inf ``` @@ -274,9 +282,9 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=1600 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1600 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -293,12 +301,12 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ - --max-total-tokens 520960 \ + --max-total-tokens 420000 \ --max-prefill-tokens 128000 \ --disable-radix-cache \ --trust-remote-code \ --max-running-requests 3 \ - --max-mamba-cache-size 10 \ + --max-mamba-cache-size 3 \ --mem-fraction-static 0.9 \ --cuda-graph-bs 1 2 3 \ --enable-multimodal \ @@ -320,15 +328,118 @@ We tested it based on the `RANDOM` dataset. ```bash Command python -m sglang.bench_serving \ --dataset-name random \ - --dataset-path /path/to/dataset \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 3 \ + --warmup-requests 3 \ + --num-prompts 3 \ --random-input-len 128000 \ --random-output-len 1000 \ - --num-prompts 3 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 +``` + +### Qwen3.6-35B-A3B 1P IN128K OUT1K PREFIX90 50ms + +**Model**: Qwen3.6-35B-A3B + +**Hardware**: Atlas 800I A3 + +**Cards**: 1 + +**Deploy Mode**: PD Mixed + +**Quantization**: BF16 + +**Dataset**: 128k+1k (90% prefix cache hit rate) + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + +MODEL_PATH=/path/to/model-weights +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_USE_FIA=1 +export GLOO_SOCKET_IFNAME= +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME= +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 6688 \ + --tp-size 2 \ + --nnodes 1 \ + --attention-backend ascend \ + --device npu \ + --chunked-prefill-size 16384 \ + --max-prefill-tokens 65536 \ + --trust-remote-code \ + --enable-prefill-delayer \ + --mamba-scheduler-strategy extra_buffer \ + --max-running-requests 103 \ + --max-mamba-cache-size 85 \ + --mem-fraction-static 0.85 \ + --cuda-graph-bs 2 4 8 16 32 48 64 80 96 103 \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --dtype bfloat16 \ + --mamba-ssm-dtype bfloat16 \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder +``` + +#### Benchmark + +We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): +`--gsp-system-prompt-len 115200` = `round(128000 * 0.9)` is the shared prefix portion. +`--gsp-question-len 12800` = `round(128000 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. + +```bash Command +python -m sglang.bench_serving \ + --dataset-name generated-shared-prefix \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --gsp-num-groups 1 \ + --gsp-prompts-per-group 412 \ + --gsp-system-prompt-len 115200 \ + --gsp-question-len 12800 \ + --gsp-output-len 1000 \ + --max-concurrency 103 \ + --num-prompts 412 \ + --request-rate inf ``` ### Qwen3.6-35B-A3B 1P IN254K OUT1K @@ -378,6 +489,7 @@ export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -415,15 +527,16 @@ We tested it based on the `RANDOM` dataset. ```bash Command python -m sglang.bench_serving \ --dataset-name random \ - --dataset-path /path/to/dataset \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --warmup-requests 1 \ + --num-prompts 1 \ --random-input-len 254000 \ --random-output-len 1000 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3.6-35B-A3B 1P IN3K5 OUT1K5 50ms @@ -469,7 +582,7 @@ source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1 +export HCCL_BUFFSIZE=100 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -513,15 +626,16 @@ We tested it based on the `RANDOM` dataset. ```bash Command python -m sglang.bench_serving \ --dataset-name random \ - --dataset-path /path/to/dataset \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 122 \ + --warmup-requests 122 \ + --num-prompts 122 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 122 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3.6-35B-A3B 1P IN64K OUT1K 50ms @@ -611,15 +725,16 @@ We tested it based on the `RANDOM` dataset. ```bash Command python -m sglang.bench_serving \ --dataset-name random \ - --dataset-path /path/to/dataset \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 10 \ + --warmup-requests 10 \ + --num-prompts 40 \ --random-input-len 64000 \ --random-output-len 1000 \ - --num-prompts 40 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3.6-35B-A3B 1P IN64K OUT1K PREFIX90 50ms @@ -664,9 +779,9 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=300 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=300 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -685,7 +800,7 @@ python3 -m sglang.launch_server \ --max-total-tokens 470784 \ --max-prefill-tokens 65536 \ --trust-remote-code \ - --mamba-radix-cache-strategy extra_buffer \ + --mamba-scheduler-strategy extra_buffer \ --max-running-requests 40 \ --max-mamba-cache-size 200 \ --mem-fraction-static 0.9 \ @@ -705,8 +820,8 @@ python3 -m sglang.launch_server \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 58982` = `round(65536 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6554` = `round(65536 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```bash Command @@ -718,7 +833,7 @@ python -m sglang.bench_serving \ --gsp-num-groups 1 \ --gsp-prompts-per-group 40 \ --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ + --gsp-question-len 6554 \ --gsp-output-len 1024 \ --max-concurrency 40 \ --num-prompts 40 \ @@ -773,6 +888,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -811,13 +927,14 @@ We tested it based on the `RANDOM` dataset. ```bash Command python -m sglang.bench_serving \ --dataset-name random \ - --dataset-path /path/to/dataset \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --warmup-requests 1 \ + --num-prompts 1 \ --random-input-len 984000 \ --random-output-len 1000 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx index e9fcc72f6..82696fb69 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx @@ -118,10 +118,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 64 \ + --num-prompts 256 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 256 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-8B W8A8 1P IN3K5 OUT1K5 5ms @@ -174,6 +175,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ @@ -212,11 +214,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --max-attempts 5 \ --max-concurrency 1 \ + --num-prompts 4 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 4 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-8B W8A8 1P IN6K OUT1K5 BS16 @@ -307,9 +311,10 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --max-attempts 5 \ --max-concurrency 16 \ + --num-prompts 16 \ --random-input-len 6144 \ --random-output-len 1500 \ - --num-prompts 16 \ --random-range-ratio 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx index db1cd24c3..15bd5d67e 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx @@ -72,16 +72,17 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=2000 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 export FORCE_DRAFT_MODEL_NON_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=2000 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_SET_CPU_AFFINITY=1 @@ -119,7 +120,7 @@ python3 -m sglang.launch_server \ --mamba-ssm-dtype bfloat16 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --reasoning-parser qwen3 \ - --tool-call-parser qwen + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -133,10 +134,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ + --num-prompts 1 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 1 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 50ms @@ -193,6 +195,7 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_SET_CPU_AFFINITY=1 @@ -234,7 +237,7 @@ python3 -m sglang.launch_server \ --deepep-mode auto \ --cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 \ --reasoning-parser qwen3 \ - --tool-call-parser qwen + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -248,10 +251,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 300 \ + --num-prompts 300 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 300 \ - --random-range-ratio 1 + --random-range-ratio 1 \ + --seed 1 ``` ### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN6K OUT1K5 BS16 @@ -299,12 +303,12 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export DEEPEP_HCCL_BUFFSIZE=2000 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export FORCE_DRAFT_MODEL_NON_QUANT=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=2000 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -362,9 +366,10 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ + --max-attempts 5 \ --max-concurrency 16 \ + --num-prompts 16 \ --random-input-len 6144 \ --random-output-len 1500 \ - --num-prompts 16 \ --random-range-ratio 1 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx new file mode 100644 index 000000000..545bfbd18 --- /dev/null +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx @@ -0,0 +1,285 @@ +--- +title: "DeepSeek-V4-Flash" +metatags: + description: "Deploy DeepSeek-V4-Flash model with SGLang on Ascend NPUs, including multi-node PD disaggregation mode." +--- + +## Introduction + +DeepSeek-V4-Flash is a Mixture-of-Experts (MoE) large language model developed by DeepSeek, featuring 284B total +parameters with 13B active parameters. It employs hybrid sparse attention (CSA + HCA) and DeepSeekMoE architecture with +a bundled multi-token prediction (MTP) draft head for speculative decoding, supporting a 1M-token context length. + +This document demonstrates the deployment of DeepSeek-V4-Flash on Ascend NPUs using SGLang, including multi-node PD +disaggregation mode, feature configuration, and performance optimization. + +This document is validated and written based on **SGLang v0.5.16**. The current model (DeepSeek-V4-Flash) is fully +supported in this version. To use the latest features (e.g., PD disaggregation, speculative decoding), it is +recommended to use v0.5.16 or a later version. + +## Supported features + +| Feature | Example usage | +|-------------------------------|-----------------------------------------------------------------------------------------------| +| Attention Backend | `--attention-backend dsv4` | +| Tensor Parallelism | `--tp-size 16` | +| Data Parallelism | `--dp-size 16` | +| Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | +| PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | +| Quantization | `--quantization modelslim` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 4 8 16 24 36 40 48 56` | +| Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | +| Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | +| DP LM Head | `--enable-dp-lm-head` | +| DP Attention | `--enable-dp-attention` | + + +The values in the **Example usage** column are for illustration only. Adjust them according to your hardware, deployment +mode, and workload. For parameter details, see +[Feature descriptions](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-descriptions); for +recommended configurations for each deployment scenario, see [Best practices](#best-practices). + + +For feature compatibility and conflict information between features, +see [Feature Compatibility](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning#feature-compatibility). + +## Prerequisites + +### Environment + +Before following this tutorial, complete the environment setup in the documents below: + +- [Ascend NPU Quickstart](/docs/hardware-platforms/ascend-npus/getting-started/quick_start) — the fastest way to get started. + It walks you through launching the official container image, starting the SGLang server, and sending a test request. + Recommended if you are new to SGLang on Ascend. +- [SGLang Installation with NPU Support](/docs/hardware-platforms/ascend-npus/getting-started/installation) — the full installation guide. + It covers the component version mapping (CANN, TorchNPU, Triton, kernels, etc.), building from source or from a + Dockerfile, and recommended system settings (CPU power scheme, NUMA, swap). Use it when you need to install or customize + the environment instead of using the official image. + +### Model weights + +Before downloading model weights, check the model size to reserve enough disk space. +For multi-node deployment, download the weights to a shared directory accessible to all nodes. + +- [DeepSeek-V4-Flash-w8a8-mtp](https://www.modelscope.cn/models/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp) (W8A8 quantized version, 300GB) + + +The W8A8 variant (300GB) can be deployed on 8 × 64GB of device memory (`--tp-size 8`), which corresponds to one full A2 +node or 8 dies on A3 (4 cards). + + +This is the minimum recommended configuration. For optimized configurations, +see [Best practices](#best-practices), which may require additional cards or nodes. + +For the hardware specifications (memory per die, dies per card, and the difference between A2 and A3), +see [Ascend NPU Reference — Hardware](/docs/hardware-platforms/ascend-npus/reference/glossary#hardware). + +## Installation + + +Ensure sufficient disk space before pulling images. The Docker image requires at least **30GB** of free space. + + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the +online platform. You can directly pull it. + +Both **stable releases** and **daily builds** are available. The following command is based on the stable release tag. +For details, see [Docker image versions](/docs/hardware-platforms/ascend-npus/faq#8-docker-image-versions-stable-release-vs-daily-build). + + + + +```bash Command +docker pull quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci8:/dev/davinci8 \ +--device=/dev/davinci9:/dev/davinci9 \ +--device=/dev/davinci10:/dev/davinci10 \ +--device=/dev/davinci11:/dev/davinci11 \ +--device=/dev/davinci12:/dev/davinci12 \ +--device=/dev/davinci13:/dev/davinci13 \ +--device=/dev/davinci14:/dev/davinci14 \ +--device=/dev/davinci15:/dev/davinci15 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:cann9.0.0-a3-v0.5.16 +``` + + + + +```bash Command +docker pull quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 + +docker run -itd --shm-size=16g --name ${NAME} \ +--privileged=true --net=host \ +-v /var/queue_schedule:/var/queue_schedule \ +-v /etc/ascend_install.info:/etc/ascend_install.info \ +-v /usr/local/sbin:/usr/local/sbin \ +-v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ +-v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ +--device=/dev/davinci0:/dev/davinci0 \ +--device=/dev/davinci1:/dev/davinci1 \ +--device=/dev/davinci2:/dev/davinci2 \ +--device=/dev/davinci3:/dev/davinci3 \ +--device=/dev/davinci4:/dev/davinci4 \ +--device=/dev/davinci5:/dev/davinci5 \ +--device=/dev/davinci6:/dev/davinci6 \ +--device=/dev/davinci7:/dev/davinci7 \ +--device=/dev/davinci_manager:/dev/davinci_manager \ +--device=/dev/hisi_hdc:/dev/hisi_hdc \ +--entrypoint=bash \ +quay.io/ascend/sglang:cann9.0.0-910b-v0.5.16 +``` + + + + + +- If the model weights have already been downloaded to a shared directory, use `-v` to mount the model path into the + container, for example: `-v /path/to/models:/models`. +- Replace `${NAME}` with your own container name or remove `--name` to use default name. + + +## Online service deployment + +### Single-node PD mixed deployment + +Single-node deployment completes both prefill and decode within the same node (PD mixed mode), suitable for scenarios +with limited hardware resources. In such scenarios, DeepSeek-V4-Flash can be launched with half of an Atlas 800I A3 +node (4 cards, 8 dies) or a full Atlas 800I A2 node (8 cards). The configuration below was validated on Atlas 800I A3 +(4 cards, 8 dies) with `--tp-size 8`. + +```bash Command +# ============================================================ +# Before running, update the following variable: +# MODEL_PATH: path to the model weights directory +# ============================================================ + +MODEL_PATH=/path/to/model-weights + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash + +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export FORCE_DRAFT_MODEL_NON_QUANT=1 +export HCCL_BUFFSIZE=8 +export HCCL_OP_EXPANSION_MODE=AIV +export INF_NAN_MODE_FORCE_DISABLE=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DSV4_FP4_EXPERTS=False +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 +export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch +export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False +export SGLANG_OPT_FP8_WO_A_GEMM=0 +export SGLANG_OPT_FUSE_WQA_WKV=0 +export SGLANG_OPT_USE_FUSED_HASH_TOPK=False +export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False +export SGLANG_OPT_USE_TILELANG_MHC_POST=False +export SGLANG_OPT_USE_TILELANG_MHC_PRE=False +export SGLANG_SET_CPU_AFFINITY=1 +export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:14699" +export SGLANG_ZBAL_LOCAL_MEM_SIZE=62084 +export STREAMS_PER_DEVICE=32 +export ZBAL_ENABLE_GRAPH=1 +export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True + +python3 -m sglang.launch_server \ + --model-path "${MODEL_PATH}" \ + --page-size 128 \ + --tp-size 8 \ + --trust-remote-code \ + --device npu \ + --attention-backend dsv4 \ + --watchdog-timeout 9000 \ + --host 127.0.0.1 --port 6688 \ + --mem-fraction-static 0.8 \ + --prefill-max-requests 160 \ + --max-prefill-tokens 80000 \ + --chunked-prefill-size 131072 \ + --max-running-requests 160 \ + --dp-size 8 --enable-dp-attention \ + --moe-a2a-backend deepep --deepep-mode auto \ + --quantization modelslim --enable-dp-lm-head \ + --kv-cache-dtype bfloat16 +``` + +For the complete benchmark data and additional single-node configurations, see +[DeepSeek-V4-Flash Best Practice — PD Mixed On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#single-node-pd-mixed). + +### Multi-node PD disaggregation deployment + +PD disaggregation splits the prefill and decode stages onto separate nodes, reducing interference and improving +throughput for high-concurrency scenarios. This scenario is already covered in the best practice. For the complete, optimized +deployment commands and benchmark data, see +[DeepSeek-V4-Flash Best Practice — PD Disaggregation On A3](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash#pd-disaggregation). + +## Functional verification + +After the service is started, you can invoke the model by sending a prompt: + +```shell +# ============================================================ +# Before running, update the following variables: +# HOST: the server host address (e.g., localhost) +# PORT: the server port number (e.g., 6688) +# ============================================================ + +curl http://${HOST}:${PORT}/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "deepseek-v4-flash", + "messages": [{"role": "user", "content": "What is the capital of France?"}] + }' +``` + +Expected result: an HTTP 200 response with the generated text containing "Paris". + +Once the server prints `The server is fired up and ready to roll!` in the logs, it is ready to accept requests. For more +testing examples (Health Check, Generate, Chat Completions, and port usage guidance), +see [Testing the Service](/docs/hardware-platforms/ascend-npus/getting-started/installation#testing-the-service). + +## Accuracy evaluation + +For accuracy evaluation methods and datasets, see [Accuracy Evaluation on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/accuracy_evaluation). + +## Performance + +For performance data and benchmark commands, see [Performance Testing on Ascend NPU](/docs/hardware-platforms/ascend-npus/evaluation/performance_testing). + +## Best practices + +### Best practice configuration reference + +For complete optimal configurations with deployment scripts and benchmark commands, see the +[DeepSeek-V4-Flash Best Practice](/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash) page. + +## Performance tuning + +For the full list of supported features, see [Supported features](#supported-features). For detailed optimization +guidance, see [Optimization on Ascend NPU](/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning). + +## FAQ + +For common environment, installation, and general parameter issues, please refer to the [Ascend NPU FAQ](/docs/hardware-platforms/ascend-npus/faq).