From 4e1d25117bf3134a3a43d55f461ea8708853f0b3 Mon Sep 17 00:00:00 2001 From: jianzhao-xu <978716854@qq.com> Date: Mon, 22 Jun 2026 16:58:48 +0800 Subject: [PATCH] [NPU] update best practice docs from testcase (#28621) --- .../ascend_npu_accuracy_evaluation.mdx | 27 +- .../ascend-npus/best_practice/deepseek_r1.mdx | 76 ++- .../best_practice/deepseek_v3_2.mdx | 30 +- .../ascend-npus/best_practice/glm5_1.mdx | 124 ++-- .../ascend-npus/best_practice/kimi_k2_6.mdx | 586 ++++-------------- .../best_practice/minimax_m2_5.mdx | 538 +--------------- .../ascend-npus/best_practice/qwen3-8b.mdx | 2 +- .../best_practice/qwen3_235b_a22b.mdx | 2 +- .../best_practice/qwen3_30b_a3b.mdx | 4 +- .../ascend-npus/best_practice/qwen3_32b.mdx | 4 +- .../best_practice/qwen3_5_397b.mdx | 13 +- .../ascend-npus/best_practice/qwen3_6_27b.mdx | 36 +- .../best_practice/qwen3_6_35b_a3b.mdx | 67 +- .../qwen3_next_80b_a3b_instruct.mdx | 4 +- 14 files changed, 376 insertions(+), 1137 deletions(-) diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx index d367d7dc2..8a6d69d41 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_accuracy_evaluation.mdx @@ -87,17 +87,6 @@ cd evalscope/ pip install -e . ``` - -Before running any `evalscope` commands, make sure to unset proxy environment variables to prevent communication issues with the SGLang server: - -```shell -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -``` - - ### Online Text Model Testing This section covers online evaluation scenarios where the SGLang server is already running. @@ -254,6 +243,22 @@ requests.exceptions.SSLError: HTTPSConnectionPool(host='www.modelscope.cn', port ``` You can navigate to `/usr/local/python3.11.14/lib/python3.11/site-packages/requests/sessions.py`, find the `class Session` definition, and set `self.verify` to `False` to resolve this. +### EvalScope Request Retry Timeout +If EvalScope keeps retrying requests with errors like: +``` +2026-06-22 03:09:03 - evalscope - WARNING: Attempt 4 / 5 failed: ....... Retrying... +2026-06-22 03:09:14 - evalscope - INFO: Evaluating[ceval] 0%| 0/520 [Elapsed: 02:00 < Remaining: ?, ?it/s] +2026-06-22 03:09:19,557 - openai._base_client - INFO: Retrying request to /chat/completions in 0.447260 seconds +2026-06-22 03:09:26,088 - openai._base_client - INFO: Retrying request to /chat/completions in 0.992551 seconds +``` +This is usually caused by the HTTP proxy intercepting requests to the local SGLang server. Disable the proxy with: +```shell Command +unset http_proxy +unset https_proxy +unset HTTP_PROXY +unset HTTPS_PROXY +``` + ### Download Dataset Error For this error ``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx index 2b722d294..76d3c218d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx @@ -25,7 +25,7 @@ This guide describes the best practice data for DeepSeek-R1 on the Ascend NPU. ## Optimal Configuration - + ### DeepSeek-R1 W4A8 1P1D 16P IN3K5 OUT1K5 41ms @@ -126,7 +126,11 @@ do --dp-size 8 \ --enable-dp-attention \ --disable-shared-experts-fusion \ - --dtype bfloat16 + --dtype bfloat16 \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -172,7 +176,11 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --tokenizer-worker-num 4 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -213,7 +221,7 @@ python -m sglang.bench_serving \ --request-rate 24 ``` - + ### DeepSeek-R1 W4A8 8P IN3K5 OUT1K5 50.36ms @@ -416,7 +424,11 @@ do --enable-dp-attention \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --enable-attn-tp-input-scattered + --enable-attn-tp-input-scattered \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -466,7 +478,11 @@ do --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -613,7 +629,11 @@ do --enable-dp-attention \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --enable-attn-tp-input-scattered + --enable-attn-tp-input-scattered \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -663,7 +683,11 @@ do --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -806,7 +830,11 @@ do --enable-dp-attention \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --enable-attn-tp-input-scattered + --enable-attn-tp-input-scattered \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -856,7 +884,11 @@ do --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -999,7 +1031,11 @@ do --enable-dp-attention \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --enable-attn-tp-input-scattered + --enable-attn-tp-input-scattered \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -1049,7 +1085,11 @@ do --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -1191,7 +1231,11 @@ do --enable-dp-attention \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --enable-attn-tp-input-scattered + --enable-attn-tp-input-scattered \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -1240,7 +1284,11 @@ do --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx index 125a18178..dfa316644 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx @@ -122,7 +122,10 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 + --speculative-num-draft-tokens 2 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -171,7 +174,10 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --disaggregation-transfer-backend ascend + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -311,7 +317,10 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 + --speculative-num-draft-tokens 2 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -360,7 +369,10 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --disaggregation-transfer-backend ascend + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -500,7 +512,10 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 + --speculative-num-draft-tokens 2 \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi @@ -549,7 +564,10 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --disaggregation-transfer-backend ascend + --disaggregation-transfer-backend ascend \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu NODE_RANK=$i break fi diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx index 97e940d73..c7850757d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx @@ -10,6 +10,7 @@ This guide describes the best practice data for GLM-5.1 on the Ascend NPU. | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | +| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 20ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in128k-out1k-20ms) | | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64K+1K | 20ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-20ms) | | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 65K+1.5K (90% prefix cache hit rate) | 25ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in65k-out1k5-prefix90-25ms) | @@ -18,7 +19,6 @@ This guide describes the best practice data for GLM-5.1 on the Ascend NPU. | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | | GLM-5.1 | Atlas 800I A3 | 16 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-16p-in3k5-out1k5-50ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in128k-out1k) | | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 16K+1K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) | | GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64K+1K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) | | GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 65K+1.5K (100% prefix cache hit rate) | 33ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-48p-in65k-out1k5-prefix100-33ms) | @@ -51,11 +51,14 @@ This guide describes the best practice data for GLM-5.1 on the Ascend NPU. # ============================================================ # Before running, update the following variables: # MODEL_PATH: path to the model weights directory +# NODE_IPS: IP addresses of each node in the cluster # HCCL_SOCKET_IFNAME: network interface name for HCCL # GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ MODEL_PATH=/path/to/model-weights +NODE_IPS=('' '') + echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 @@ -81,31 +84,46 @@ export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 6688 \ - --attention-backend ascend \ - --device npu \ - --tp-size 32 \ - --nnodes 2 \ - --dp-size 16 \ - --enable-dp-attention \ - --chunked-prefill-size 65536 \ - --max-prefill-tokens 280000 \ - --trust-remote-code \ - --mem-fraction-static 0.65 \ - --served-model-name glm-5 \ - --cuda-graph-max-bs 8 \ - --max-running-requests 128 \ - --quantization modelslim \ - --speculative-draft-model-quantization unquant \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --load-balance-method round_robin \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +for i in "${!NODE_IPS[@]}"; +do + if [[ "$LOCAL_HOST1" == "${NODE_IPS[$i]}" || "$LOCAL_HOST2" == "${NODE_IPS[$i]}" ]]; + then + echo "${NODE_IPS[$i]}" + python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host ${NODE_IPS[$i]} --port 6688 \ + --nnodes 2 \ + --dist-init-addr ${NODE_IPS[0]}:5000 \ + --node-rank $i \ + --attention-backend ascend \ + --device npu \ + --tp-size 32 \ + --dp-size 16 \ + --enable-dp-attention \ + --chunked-prefill-size 65536 \ + --max-prefill-tokens 280000 \ + --trust-remote-code \ + --mem-fraction-static 0.65 \ + --served-model-name glm-5 \ + --cuda-graph-max-bs 16 \ + --max-running-requests 256 \ + --quantization modelslim \ + --speculative-draft-model-quantization unquant \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --load-balance-method round_robin \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 + break + fi +done ``` #### Benchmark @@ -121,13 +139,13 @@ python -m sglang.bench_serving \ --max-concurrency 128 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 512 \ + --num-prompts 128 \ --random-range-ratio 1 ``` -### GLM-5.1 W4A8 1P1D 32P IN128K OUT1K +### GLM-5.1 W4A8 1P1D 32P IN128K OUT1K 20ms **Model**: GLM-5.1 @@ -141,7 +159,7 @@ python -m sglang.bench_serving \ **Dataset**: 128K+1K -**TPOT**: 50ms +**TPOT**: 20ms #### Model Deployment @@ -234,7 +252,8 @@ do --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --pp-size 8 + --pp-size 8 \ + --trust-remote-code NODE_RANK=$i break fi @@ -284,7 +303,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --trust-remote-code NODE_RANK=$i break fi @@ -429,7 +449,8 @@ do --nsa-prefill-cp-mode in-seq-split \ --attn-cp-size 8 \ --enable-dp-lm-head \ - --moe-dense-tp 1 + --moe-dense-tp 1 \ + --trust-remote-code NODE_RANK=$i break fi @@ -483,7 +504,8 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --trust-remote-code NODE_RANK=$i break fi @@ -630,7 +652,8 @@ do --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --pp-size 8 + --pp-size 8 \ + --trust-remote-code NODE_RANK=$i break fi @@ -680,7 +703,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --trust-remote-code NODE_RANK=$i break fi @@ -827,7 +851,8 @@ do --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --pp-size 8 + --pp-size 8 \ + --trust-remote-code NODE_RANK=$i break fi @@ -877,7 +902,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --trust-remote-code NODE_RANK=$i break fi @@ -1022,7 +1048,8 @@ do --nsa-prefill-cp-mode in-seq-split \ --attn-cp-size 8 \ --enable-dp-lm-head \ - --moe-dense-tp 1 + --moe-dense-tp 1 \ + --trust-remote-code NODE_RANK=$i break fi @@ -1076,7 +1103,8 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --trust-remote-code NODE_RANK=$i break fi @@ -1217,7 +1245,8 @@ do --deepep-mode normal \ --disable-shared-experts-fusion \ --disable-cuda-graph \ - --dtype bfloat16 + --dtype bfloat16 \ + --trust-remote-code NODE_RANK=$i break fi @@ -1275,7 +1304,8 @@ do --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ - --load-balance-method round_robin + --load-balance-method round_robin \ + --trust-remote-code NODE_RANK=$i break fi @@ -1422,7 +1452,8 @@ do --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --pp-size 8 + --pp-size 8 \ + --trust-remote-code NODE_RANK=$i break fi @@ -1476,7 +1507,8 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --trust-remote-code NODE_RANK=$i break fi @@ -1629,7 +1661,8 @@ do --attn-cp-size 4 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --pp-size 4 + --pp-size 4 \ + --trust-remote-code NODE_RANK=$i break fi @@ -1683,7 +1716,8 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --trust-remote-code NODE_RANK=$i break fi diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx index b6786733b..037d169e5 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx @@ -21,15 +21,13 @@ This guide describes the best practice data for Kimi-K2.6 on the Ascend NPU. | Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-prefix90-100ms) | | Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1.5K | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-100ms) | | Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1.5K (90% prefix cache hit rate) | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-prefix90-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 24 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-24p-in128k-out1k-prefix90-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 24 | PD Disaggregation | 64K+1.5K (90% prefix cache hit rate) | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-24p-in64k-out1k5-prefix90-100ms) | | Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1024x1024-30-out1024-50ms) | | Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1080p_30+256 | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1080p-30-out256-50ms) | | Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-50ms) | ## Optimal Configuration - + ### Kimi-K2.6 W4A8 16P IN64K OUT1K 100ms @@ -54,6 +52,7 @@ This guide describes the best practice data for Kimi-K2.6 on the Ascend NPU. # Before running, update the following variables: # MODEL_PATH: path to the model weights directory # DRAFT_MODEL_PATH: path to the draft model weights directory +# NODE_IPS: IP addresses of each node in the cluster # HCCL_SOCKET_IFNAME: network interface name for HCCL # GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ @@ -61,6 +60,8 @@ This guide describes the best practice data for Kimi-K2.6 on the Ascend NPU. MODEL_PATH=/path/to/model-weights DRAFT_MODEL_PATH=/path/to/draft-model-weights +NODE_IPS=('' '') + echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor sysctl -w vm.swappiness=0 sysctl -w kernel.numa_balancing=0 @@ -86,35 +87,50 @@ export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 6688 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --dtype bfloat16 \ - --tp-size 32 \ - --nnodes 2 \ - --mem-fraction-static 0.55 \ - --max-running-requests 32 \ - --chunked-prefill-size 262144 \ - --context-length 75000 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 32 \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --cuda-graph-bs 1 \ - --disable-radix-cache \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" + +for i in "${!NODE_IPS[@]}"; +do + if [[ "$LOCAL_HOST1" == "${NODE_IPS[$i]}" || "$LOCAL_HOST2" == "${NODE_IPS[$i]}" ]]; + then + echo "${NODE_IPS[$i]}" + python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host ${NODE_IPS[$i]} --port 6688 \ + --nnodes 2 \ + --dist-init-addr ${NODE_IPS[0]}:5000 \ + --node-rank $i \ + --trust-remote-code \ + --attention-backend ascend \ + --device npu \ + --quantization modelslim \ + --dtype bfloat16 \ + --tp-size 32 \ + --mem-fraction-static 0.55 \ + --max-running-requests 32 \ + --chunked-prefill-size 262144 \ + --context-length 75000 \ + --enable-multimodal \ + --mm-attention-backend ascend_attn \ + --sampling-backend ascend \ + --enable-dp-attention \ + --dp-size 32 \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --cuda-graph-bs 1 \ + --disable-radix-cache \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --speculative-draft-model-quantization unquant + break + fi +done ``` #### Benchmark @@ -161,6 +177,7 @@ python -m sglang.bench_serving \ # D_IP: decode node IP address # ASCEND_MF_STORE_URL: prefill node IP with port # MODEL_PATH: path to the model weights directory +# DRAFT_MODEL_PATH: path to the draft model weights directory # HCCL_SOCKET_IFNAME: network interface name for HCCL # GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ @@ -192,6 +209,7 @@ D_IP=('') export ASCEND_MF_STORE_URL="tcp://:24670" MODEL_PATH=/path/to/model-weights +DRAFT_MODEL_PATH=/path/to/draft-model-weights LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -210,7 +228,6 @@ do export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 export ZBAL_ENABLE_GRAPH=1 - export ZBAL_HCCL_OP=send,recv export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ @@ -273,7 +290,7 @@ do --attention-backend ascend \ --device npu \ --tp-size 16 \ - --mem-fraction-static 0.82 \ + --mem-fraction-static 0.73 \ --max-running-requests 1 \ --enable-dp-attention \ --dp-size 1 \ @@ -284,7 +301,13 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 12 16 + --cuda-graph-bs 1 2 4 6 8 16 \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --speculative-draft-model-quantization unquant NODE_RANK=$i break fi @@ -318,8 +341,8 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ - --random-input-len 131072 \ - --random-output-len 1024 \ + --random-input-len 128000 \ + --random-output-len 1000 \ --num-prompts 1 \ --random-range-ratio 1 \ --request-rate inf @@ -350,6 +373,7 @@ python -m sglang.bench_serving \ # D_IP: decode node IP address # ASCEND_MF_STORE_URL: prefill node IP with port # MODEL_PATH: path to the model weights directory +# DRAFT_MODEL_PATH: path to the draft model weights directory # HCCL_SOCKET_IFNAME: network interface name for HCCL # GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ @@ -381,6 +405,7 @@ D_IP=('') export ASCEND_MF_STORE_URL="tcp://:24670" MODEL_PATH=/path/to/model-weights +DRAFT_MODEL_PATH=/path/to/draft-model-weights LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -399,7 +424,6 @@ do export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 export ZBAL_ENABLE_GRAPH=1 - export ZBAL_HCCL_OP=send,recv export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ @@ -460,10 +484,10 @@ do --attention-backend ascend \ --device npu \ --tp-size 16 \ - --mem-fraction-static 0.82 \ + --mem-fraction-static 0.73 \ --max-running-requests 2 \ --enable-dp-attention \ - --dp-size 2 \ + --dp-size 1 \ --enable-dp-lm-head \ --disable-radix-cache \ --enable-multimodal \ @@ -471,7 +495,13 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 12 + --cuda-graph-bs 1 2 4 6 8 16 \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --speculative-draft-model-quantization unquant NODE_RANK=$i break fi @@ -497,8 +527,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 115200` = `int(128000 * 0.9)` is the shared prefix portion. +`--gsp-question-len 12799` = `int(128000 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```shell Command @@ -508,12 +538,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --gsp-num-groups 1 \ - --gsp-prompts-per-group 8 \ - --gsp-system-prompt-len 117964 \ - --gsp-question-len 13107 \ - --gsp-output-len 1024 \ - --max-concurrency 2 \ - --num-prompts 8 \ + --gsp-prompts-per-group 4 \ + --gsp-system-prompt-len 115200 \ + --gsp-question-len 12799 \ + --gsp-output-len 1000 \ + --max-concurrency 1 \ + --num-prompts 4 \ --request-rate inf ``` @@ -542,6 +572,7 @@ python -m sglang.bench_serving \ # D_IP: decode node IP address # ASCEND_MF_STORE_URL: prefill node IP with port # MODEL_PATH: path to the model weights directory +# DRAFT_MODEL_PATH: path to the draft model weights directory # HCCL_SOCKET_IFNAME: network interface name for HCCL # GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ @@ -573,6 +604,7 @@ D_IP=('') export ASCEND_MF_STORE_URL="tcp://:24670" MODEL_PATH=/path/to/model-weights +DRAFT_MODEL_PATH=/path/to/draft-model-weights LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -591,7 +623,6 @@ do export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 export ZBAL_ENABLE_GRAPH=1 - export ZBAL_HCCL_OP=send,recv export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ @@ -655,7 +686,7 @@ do --device npu \ --tp-size 16 \ --mem-fraction-static 0.82 \ - --max-running-requests 16 \ + --max-running-requests 1 \ --enable-dp-attention \ --dp-size 1 \ --enable-dp-lm-head \ @@ -665,7 +696,13 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 16 + --cuda-graph-bs 16 \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --speculative-num-steps 4 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --speculative-draft-model-quantization unquant NODE_RANK=$i break fi @@ -699,8 +736,8 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --max-concurrency 1 \ - --random-input-len 65536 \ - --random-output-len 1536 \ + --random-input-len 64000 \ + --random-output-len 1500 \ --num-prompts 1 \ --random-range-ratio 1 \ --request-rate inf @@ -731,6 +768,7 @@ python -m sglang.bench_serving \ # D_IP: decode node IP address # ASCEND_MF_STORE_URL: prefill node IP with port # MODEL_PATH: path to the model weights directory +# DRAFT_MODEL_PATH: path to the draft model weights directory # HCCL_SOCKET_IFNAME: network interface name for HCCL # GLOO_SOCKET_IFNAME: network interface name for Gloo # ============================================================ @@ -762,6 +800,7 @@ D_IP=('') export ASCEND_MF_STORE_URL="tcp://:24670" MODEL_PATH=/path/to/model-weights +DRAFT_MODEL_PATH=/path/to/draft-model-weights LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` @@ -780,7 +819,6 @@ do export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 export ZBAL_ENABLE_GRAPH=1 - export ZBAL_HCCL_OP=send,recv export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ @@ -852,7 +890,13 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 12 + --cuda-graph-bs 1 2 4 6 8 \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --speculative-num-steps 4 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --speculative-draft-model-quantization unquant NODE_RANK=$i break fi @@ -878,199 +922,8 @@ python -m sglang_router.launch_router \ #### Benchmark We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. -`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. - -```shell Command -python -m sglang.bench_serving \ - --dataset-name generated-shared-prefix \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --gsp-num-groups 1 \ - --gsp-prompts-per-group 16 \ - --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ - --gsp-output-len 1536 \ - --max-concurrency 2 \ - --num-prompts 16 \ - --request-rate inf -``` - -### Kimi-K2.6 W4A8 1P1D 24P IN128K OUT1K PREFIX90 100ms - -**Model**: Kimi-K2.6 - -**Hardware**: Atlas 800I A3 - -**Cards**: 24 - -**Deploy Mode**: PD Disaggregation - -**Quantization**: W4A8 INT8 - -**Dataset**: 128K+1K (90% prefix cache hit rate) - -**TPOT**: 100ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# P_IP: prefill node IP address -# D_IP: decode node IP address -# ASCEND_MF_STORE_URL: prefill node IP with port -# MODEL_PATH: path to the model weights directory -# DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -P_IP=('') -D_IP=('' '') - -export ASCEND_MF_STORE_URL="tcp://:24670" - -MODEL_PATH=/path/to/model-weights -DRAFT_MODEL_PATH=/path/to/draft-model-weights - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1800 - export HCCL_SOCKET_IFNAME= - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --host ${P_IP[$i]} \ - --port 8000 \ - --disaggregation-bootstrap-port 8998 \ - --node-rank 0 \ - --quantization modelslim \ - --dtype bfloat16 \ - --nnodes 1 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 \ - --mem-fraction-static 0.78 \ - --max-running-requests 8 \ - --chunked-prefill-size 16384 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --sampling-backend ascend \ - --moe-a2a-backend deepep \ - --deepep-mode auto - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 - export HCCL_SOCKET_IFNAME= - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_NPU_USE_MLAPO=1 - export SGLANG_NPU_USE_MULTI_STREAM=1 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host ${D_IP[$i]} \ - --port 8001 \ - --dist-init-addr ${D_IP[0]}:5000 \ - --node-rank $i \ - --quantization modelslim \ - --dtype bfloat16 \ - --nnodes 2 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --tp-size 32 \ - --mem-fraction-static 0.82 \ - --max-running-requests 32 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 4 \ - --disable-radix-cache \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --cuda-graph-bs 8 \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 \ - --speculative-draft-model-quantization unquant - NODE_RANK=$i - break - fi -done -``` - -```shell Command -# ============================================================ -# Before running, replace the following placeholders: -# : prefill node IP address -# : first decode node IP address (decode may have distributed nodes) -# ============================================================ - -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://:8000 8998 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --policy cache_aware -``` - -#### Benchmark - -We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. +`--gsp-system-prompt-len 57600` = `int(64000 * 0.9)` is the shared prefix portion. +`--gsp-question-len 6399` = `int(64000 * (1 - 0.9))` is the unique per-request suffix. `--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. ```shell Command @@ -1081,204 +934,15 @@ python -m sglang.bench_serving \ --port 6688 \ --gsp-num-groups 1 \ --gsp-prompts-per-group 8 \ - --gsp-system-prompt-len 117964 \ - --gsp-question-len 13107 \ - --gsp-output-len 1024 \ - --max-concurrency 8 \ + --gsp-system-prompt-len 57600 \ + --gsp-question-len 6399 \ + --gsp-output-len 1500 \ + --max-concurrency 2 \ --num-prompts 8 \ --request-rate inf ``` -### Kimi-K2.6 W4A8 1P1D 24P IN64K OUT1K5 PREFIX90 100ms - -**Model**: Kimi-K2.6 - -**Hardware**: Atlas 800I A3 - -**Cards**: 24 - -**Deploy Mode**: PD Disaggregation - -**Quantization**: W4A8 INT8 - -**Dataset**: 64K+1.5K (90% prefix cache hit rate) - -**TPOT**: 100ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# P_IP: prefill node IP address -# D_IP: decode node IP address -# ASCEND_MF_STORE_URL: prefill node IP with port -# MODEL_PATH: path to the model weights directory -# DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -P_IP=('') -D_IP=('' '') - -export ASCEND_MF_STORE_URL="tcp://:24670" - -MODEL_PATH=/path/to/model-weights -DRAFT_MODEL_PATH=/path/to/draft-model-weights - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1800 - export HCCL_SOCKET_IFNAME= - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --host ${P_IP[$i]} \ - --port 8000 \ - --disaggregation-bootstrap-port 8998 \ - --node-rank 0 \ - --quantization modelslim \ - --dtype bfloat16 \ - --nnodes 1 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --tp-size 16 \ - --mem-fraction-static 0.78 \ - --max-running-requests 8 \ - --chunked-prefill-size 16384 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --sampling-backend ascend \ - --moe-a2a-backend deepep \ - --deepep-mode auto - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1200 - export HCCL_SOCKET_IFNAME= - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_NPU_USE_MLAPO=1 - export SGLANG_NPU_USE_MULTI_STREAM=1 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host ${D_IP[$i]} \ - --port 8001 \ - --dist-init-addr ${D_IP[0]}:5000 \ - --node-rank $i \ - --quantization modelslim \ - --dtype bfloat16 \ - --nnodes 2 \ - --trust-remote-code \ - --attention-backend ascend \ - --device npu \ - --tp-size 32 \ - --mem-fraction-static 0.82 \ - --max-running-requests 32 \ - --enable-multimodal \ - --mm-attention-backend ascend_attn \ - --sampling-backend ascend \ - --enable-dp-attention \ - --dp-size 4 \ - --disable-radix-cache \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ - --cuda-graph-bs 8 \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 1 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 2 \ - --speculative-draft-model-quantization unquant - NODE_RANK=$i - break - fi -done -``` - -```shell Command -# ============================================================ -# Before running, replace the following placeholders: -# : prefill node IP address -# : first decode node IP address (decode may have distributed nodes) -# ============================================================ - -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://:8000 8998 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --policy cache_aware -``` - -#### Benchmark - -We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. -`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. - -```shell Command -python -m sglang.bench_serving \ - --dataset-name generated-shared-prefix \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --gsp-num-groups 1 \ - --gsp-prompts-per-group 16 \ - --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ - --gsp-output-len 1536 \ - --max-concurrency 16 \ - --num-prompts 16 \ - --request-rate inf -``` + ### Kimi-K2.6 W4A8 8P IN1024X1024 30 OUT1024 50ms @@ -1437,11 +1101,13 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export HCCL_BUFFSIZE=1800 +export HCCL_BUFFSIZE=2400 +export HCCL_OP_EXPANSION_MODE=AIV export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -1455,26 +1121,29 @@ python3 -m sglang.launch_server \ --device npu \ --attention-backend ascend \ --tp-size 16 \ - --mem-fraction-static 0.7 \ - --max-running-requests 80 \ - --chunked-prefill-size -1 \ + --base-gpu-id 0 \ + --mem-fraction-static 0.74 \ + --max-running-requests 64 \ + --chunked-prefill-size 16384 \ --context-length 8192 \ - --prefill-max-requests 1 \ + --max-prefill-tokens 16384 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend \ - --moe-a2a-backend deepep \ - --deepep-mode auto \ --enable-dp-attention \ --dp-size 16 \ - --cuda-graph-bs 1 2 4 6 8 10 \ + --moe-a2a-backend deepep \ + --deepep-mode auto \ + --cuda-graph-bs-decode 1 2 3 4 \ --disable-radix-cache \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 4 \ + --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 \ - --speculative-draft-model-quantization unquant + --speculative-num-draft-tokens 3 \ + --speculative-draft-model-quantization unquant \ + --prefill-delayer-max-delay-passes 200 \ + --enable-prefill-delayer ``` #### Benchmark @@ -1487,12 +1156,13 @@ python -m sglang.bench_serving \ --backend sglang-oai-chat \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 20 \ + --max-concurrency 48 \ --random-input-len 30 \ --random-output-len 256 \ - --num-prompts 20 \ + --num-prompts 196 \ --random-range-ratio 1 \ --request-rate inf \ + --warmup-requests 16 \ --image-count 1 \ --image-resolution 1920x1080 ``` @@ -1608,8 +1278,6 @@ python -m sglang.bench_serving \ --warmup-requests 0 ``` - - ### Kimi-K2.6 W4A8 8P IN3K5 OUT1K5 50ms **Model**: Kimi-K2.6 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx index 8aa732e7d..e8a286484 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx @@ -10,439 +10,20 @@ This guide describes the best practice data for MiniMax-M2.5 on the Ascend NPU. | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 20ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in128k-out1k-prefix90-20ms) | +| MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 24.44ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in128k-out1k-prefix90-24-44ms) | | MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 20ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-20ms) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| MiniMax-M2.5 | Atlas 800I A3 | 16 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-1p1d-16p-in128k-out1k-prefix90-50ms) | -| MiniMax-M2.5 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-1p1d-16p-in64k-out1k-prefix90-50ms) | | MiniMax-M2.5 | Atlas 800I A3 | 4 | PD Mixed | 32K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-4p-in32k-out1k-50ms) | | MiniMax-M2.5 | Atlas 800I A3 | 4 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-4p-in64k-out1k-prefix90-50ms) | -| MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in128k-out1k-prefix90-50ms) | | MiniMax-M2.5 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#minimax-m2-5-w8a8-8p-in3k5-out1k5-50ms) | ## Optimal Configuration - - -### MiniMax-M2.5 W8A8 1P1D 16P IN128K OUT1K PREFIX90 50ms - -**Model**: MiniMax-M2.5 - -**Hardware**: Atlas 800I A3 - -**Cards**: 16 - -**Deploy Mode**: PD Disaggregation - -**Quantization**: W8A8 INT8 - -**Dataset**: 128K+1K (90% prefix cache hit rate) - -**TPOT**: 50ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# P_IP: prefill node IP address -# D_IP: decode node IP address -# ASCEND_MF_STORE_URL: prefill node IP with port -# MODEL_PATH: path to the model weights directory -# DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -P_IP=('') -D_IP=('') - -export ASCEND_MF_STORE_URL="tcp://:24670" - -MODEL_PATH=/path/to/model-weights -DRAFT_MODEL_PATH=/path/to/draft-model-weights -export PYTHONPATH=${DRAFT_MODEL_PATH}:$PYTHONPATH - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export ASCEND_USE_FIA=1 - export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=2500 - export HCCL_SOCKET_IFNAME= - export PROFILING_BS=8 - export PROFILING_STAGE=prefill - export PROFILING_step=30 - export TASK_QUEUE_ENABLE=2 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --host ${P_IP[$i]} \ - --port 8000 \ - --disaggregation-bootstrap-port 8998 \ - --trust-remote-code \ - --tp-size 16 \ - --mem-fraction-static 0.43 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --max-running-requests 128 \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 130000 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --tokenizer-worker-num 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --dtype bfloat16 \ - --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 2 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 3 \ - --speculative-draft-model-quantization unquant \ - --skip-server-warmup - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1600 - export HCCL_SOCKET_IFNAME= - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_NPU_FUSED_MOE_MODE=2 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host ${D_IP[$i]} \ - --port 8001 \ - --trust-remote-code \ - --tp-size 16 \ - --mem-fraction-static 0.76 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --disaggregation-decode-extra-slots 96 \ - --max-running-requests 80 \ - --chunked-prefill-size -1 \ - --moe-a2a-backend ascend_fuseep \ - --deepep-mode low_latency \ - --tokenizer-worker-num 8 \ - --dp-size 2 \ - --enable-dp-attention \ - --dtype bfloat16 \ - --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 2 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 3 \ - --speculative-draft-model-quantization unquant \ - --skip-server-warmup \ - --cuda-graph-bs 2 4 8 - NODE_RANK=$i - break - fi -done -``` - -```shell Command -# ============================================================ -# Before running, replace the following placeholders: -# : prefill node IP address -# : decode node IP address -# ============================================================ - -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://:8000 8998 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --policy round_robin --mini-lb -``` - -#### Benchmark - -We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. -`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. - -```shell Command -python -m sglang.bench_serving \ - --dataset-name generated-shared-prefix \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --gsp-num-groups 1 \ - --gsp-prompts-per-group 32 \ - --gsp-system-prompt-len 117964 \ - --gsp-question-len 13107 \ - --gsp-output-len 1024 \ - --max-concurrency 8 \ - --num-prompts 32 \ - --request-rate inf -``` - -### MiniMax-M2.5 W8A8 1P1D 16P IN64K OUT1K PREFIX90 50ms - -**Model**: MiniMax-M2.5 - -**Hardware**: Atlas 800I A3 - -**Cards**: 16 - -**Deploy Mode**: PD Disaggregation - -**Quantization**: W8A8 INT8 - -**Dataset**: 64K+1K (90% prefix cache hit rate) - -**TPOT**: 50ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# P_IP: prefill node IP address -# D_IP: decode node IP address -# ASCEND_MF_STORE_URL: prefill node IP with port -# MODEL_PATH: path to the model weights directory -# DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -P_IP=('') -D_IP=('') - -export ASCEND_MF_STORE_URL="tcp://:24670" - -MODEL_PATH=/path/to/model-weights -DRAFT_MODEL_PATH=/path/to/draft-model-weights -export PYTHONPATH=${DRAFT_MODEL_PATH}:$PYTHONPATH - -LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` -LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` -echo "${LOCAL_HOST1}" -echo "${LOCAL_HOST2}" -# prefill -for i in "${!P_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; - then - echo "${P_IP[$i]}" - export ASCEND_USE_FIA=1 - export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1 - export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048 - export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 - export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 - export ENABLE_PROFILING=0 - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=2500 - export HCCL_SOCKET_IFNAME= - export PROFILING_BS=8 - export PROFILING_STAGE=prefill - export PROFILING_step=30 - export TASK_QUEUE_ENABLE=2 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode prefill \ - --host ${P_IP[$i]} \ - --port 8000 \ - --disaggregation-bootstrap-port 8998 \ - --trust-remote-code \ - --tp-size 16 \ - --mem-fraction-static 0.43 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --max-running-requests 128 \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 58000 \ - --moe-a2a-backend deepep \ - --deepep-mode normal \ - --tokenizer-worker-num 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --dtype bfloat16 \ - --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --skip-server-warmup - NODE_RANK=$i - break - fi -done - -# decode -for i in "${!D_IP[@]}"; -do - if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; - then - echo "${D_IP[$i]}" - export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=1600 - export HCCL_SOCKET_IFNAME= - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_NPU_FUSED_MOE_MODE=2 - - python3 -m sglang.launch_server \ - --model-path ${MODEL_PATH} \ - --disaggregation-mode decode \ - --host ${D_IP[$i]} \ - --port 8001 \ - --trust-remote-code \ - --tp-size 16 \ - --mem-fraction-static 0.76 \ - --attention-backend ascend \ - --device npu \ - --quantization modelslim \ - --disaggregation-transfer-backend ascend \ - --disaggregation-decode-extra-slots 96 \ - --max-running-requests 80 \ - --chunked-prefill-size -1 \ - --moe-a2a-backend ascend_fuseep \ - --deepep-mode low_latency \ - --tokenizer-worker-num 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --dtype bfloat16 \ - --load-balance-method round_robin \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --skip-server-warmup \ - --cuda-graph-bs 2 4 8 16 24 32 40 - NODE_RANK=$i - break - fi -done -``` - -```shell Command -# ============================================================ -# Before running, replace the following placeholders: -# : prefill node IP address -# : decode node IP address -# ============================================================ - -python -m sglang_router.launch_router \ - --pd-disaggregation \ - --prefill http://:8000 8998 \ - --decode http://:8001 \ - --host 127.0.0.1 \ - --port 6688 \ - --policy round_robin --mini-lb -``` - -#### Benchmark - -We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 58982` = `int(65536 * 0.9)` is the shared prefix portion. -`--gsp-question-len 6553` = `int(65536 * (1 - 0.9))` is the unique per-request suffix. -`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. - -```shell Command -python -m sglang.bench_serving \ - --dataset-name generated-shared-prefix \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --gsp-num-groups 1 \ - --gsp-prompts-per-group 640 \ - --gsp-system-prompt-len 58982 \ - --gsp-question-len 6553 \ - --gsp-output-len 1024 \ - --max-concurrency 160 \ - --num-prompts 640 \ - --request-rate inf -``` - - + ### MiniMax-M2.5 W8A8 4P IN32K OUT1K 50ms @@ -661,7 +242,7 @@ python -m sglang.bench_serving \ --request-rate inf ``` -### MiniMax-M2.5 W8A8 8P IN128K OUT1K PREFIX90 20ms +### MiniMax-M2.5 W8A8 8P IN128K OUT1K PREFIX90 24.44ms **Model**: MiniMax-M2.5 @@ -675,7 +256,7 @@ python -m sglang.bench_serving \ **Dataset**: 128K+1K (90% prefix cache hit rate) -**TPOT**: 20ms +**TPOT**: 24.44ms #### Model Deployment @@ -772,117 +353,6 @@ python -m sglang.bench_serving \ --request-rate inf ``` -### MiniMax-M2.5 W8A8 8P IN128K OUT1K PREFIX90 50ms - -**Model**: MiniMax-M2.5 - -**Hardware**: Atlas 800I A3 - -**Cards**: 8 - -**Deploy Mode**: PD Mixed - -**Quantization**: W8A8 INT8 - -**Dataset**: 128K+1K (90% prefix cache hit rate) - -**TPOT**: 50ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# MODEL_PATH: path to the model weights directory -# DRAFT_MODEL_PATH: path to the draft model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - -MODEL_PATH=/path/to/model-weights -DRAFT_MODEL_PATH=/path/to/draft-model-weights -export PYTHONPATH=${DRAFT_MODEL_PATH}:$PYTHONPATH - -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export ASCEND_USE_FIA=1 -export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 -export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=1024 -export HCCL_SOCKET_IFNAME= -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000 -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 -export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 -export TASK_QUEUE_ENABLE=1 - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 6688 \ - --tp-size 16 \ - --dp-size 2 \ - --enable-dp-attention \ - --mem-fraction-static 0.65 \ - --max-running-requests 20 \ - --reasoning-parser minimax-append-think \ - --tool-call-parser minimax-m2 \ - --enable-prefill-delayer \ - --prefill-max-requests 4 \ - --chunked-prefill-size 160000 \ - --max-prefill-tokens 80000 \ - --cuda-graph-bs 2 4 6 8 10 16 \ - --moe-a2a-backend ascend_fuseep \ - --deepep-mode auto \ - --quantization modelslim \ - --speculative-algorithm EAGLE3 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant \ - --tokenizer-worker-num 4 \ - --dtype bfloat16 -``` - -#### Benchmark - -We tested it based on the `generated-shared-prefix` dataset with 90% cache hit (`repeat_rate = 0.9`): -`--gsp-system-prompt-len 117964` = `int(131072 * 0.9)` is the shared prefix portion. -`--gsp-question-len 13107` = `int(131072 * (1 - 0.9))` is the unique per-request suffix. -`--gsp-num-groups 1` keeps all requests in one prefix group for maximum cache reuse. - -```shell Command -python -m sglang.bench_serving \ - --dataset-name generated-shared-prefix \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --gsp-num-groups 1 \ - --gsp-prompts-per-group 80 \ - --gsp-system-prompt-len 117964 \ - --gsp-question-len 13107 \ - --gsp-output-len 1024 \ - --max-concurrency 20 \ - --num-prompts 80 \ - --request-rate inf -``` - ### MiniMax-M2.5 W8A8 8P IN3K5 OUT1K5 20ms **Model**: MiniMax-M2.5 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx index ba72fddaf..a73d9685a 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx @@ -21,7 +21,7 @@ This guide describes the best practice data for Qwen3-8B on the Ascend NPU. ## Optimal Configuration - + ### Qwen3-8B W8A8 1P IN3K5 OUT1K5 37ms diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx index a24de2350..ea5340a4d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx @@ -20,7 +20,7 @@ This guide describes the best practice data for Qwen3-235B-A22B on the Ascend NP ## Optimal Configuration - + ### Qwen3-235B-A22B BF16 8P IN11K OUT1K5 8ms diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx index 180cf4b32..ff16db1d1 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx @@ -22,6 +22,8 @@ This guide describes the best practice data for Qwen3-30B-A3B on the Ascend NPU. ## Optimal Configuration + + ### Qwen3-30B-A3B BF16 1P IN1K OUT100 **Model**: Qwen3-30B-A3B @@ -222,8 +224,6 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` - - ### Qwen3-30B-A3B W8A8 1P IN3K5 OUT1K5 50ms **Model**: Qwen3-30B-A3B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx index 1db3b9978..f29f18bed 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx @@ -21,6 +21,8 @@ This guide describes the best practice data for Qwen3-32B on the Ascend NPU. ## Optimal Configuration + + ### Qwen3-32B BF16 8P IN18K OUT4K 6ms **Model**: Qwen3-32B @@ -213,8 +215,6 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` - - ### Qwen3-32B W8A8 2P IN3K5 OUT1K5 50ms **Model**: Qwen3-32B diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx index e3df0d0fc..35c1e2423 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx @@ -28,6 +28,8 @@ This guide describes the best practice data for Qwen3.5-397B on the Ascend NPU. ## Optimal Configuration + + ### Qwen3.5-397B W4A8 8P IN128K OUT1K 20ms **Model**: Qwen3.5-397B @@ -698,8 +700,6 @@ python -m sglang.bench_serving \ --warmup-requests 64 ``` - - ### Qwen3.5-397B W4A8 8P IN3K5 OUT1K5 50ms **Model**: Qwen3.5-397B @@ -1085,7 +1085,7 @@ export DEEPEP_NORMAL_LONG_SEQ_ROUND=20 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=0 +export HCCL_BUFFSIZE=64 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True @@ -1098,6 +1098,7 @@ export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_LOCAL_MEM_SIZE=58672 export STREAMS_PER_DEVICE=32 export ZBAL_ENABLE_GRAPH=1 +export ZBAL_HCCL_OP=allreduce,_allgather_base,allgather,broadcast,scatter,reduce_scatter,_reduce_scatter_base,alltoall_base export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ @@ -1146,11 +1147,11 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --gsp-num-groups 1 \ - --gsp-prompts-per-group 128 \ + --gsp-prompts-per-group 112 \ --gsp-system-prompt-len 58982 \ --gsp-question-len 6553 \ --gsp-output-len 1024 \ - --max-concurrency 128 \ - --num-prompts 128 \ + --max-concurrency 112 \ + --num-prompts 112 \ --request-rate inf ``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx index 68abd0bf9..0f4d84823 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx @@ -10,9 +10,9 @@ This guide describes the best practice data for Qwen3.6-27B on the Ascend NPU. | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-1p-in1024x1024-30-out1024-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-2p-in64k-out1k-prefix90-50ms) | +| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1024x1024-30-out1024-50ms) | +| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) | +| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-2p-in64k-out1k-prefix90-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in3k5-out1k5-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) | @@ -20,6 +20,8 @@ This guide describes the best practice data for Qwen3.6-27B on the Ascend NPU. ## Optimal Configuration + + ### Qwen3.6-27B 1P IN1024X1024 30 OUT1024 50ms **Model**: Qwen3.6-27B @@ -30,7 +32,7 @@ This guide describes the best practice data for Qwen3.6-27B on the Ascend NPU. **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 1024x1024 (30)+1024 @@ -131,7 +133,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 1080p_30+256 @@ -230,7 +232,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 64K+1K (90% prefix cache hit rate) @@ -321,8 +323,6 @@ python -m sglang.bench_serving \ --request-rate inf ``` - - ### Qwen3.6-27B W8A8 1P IN3K5 OUT1K5 50ms **Model**: Qwen3.6-27B @@ -553,13 +553,13 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh source /usr/local/Ascend/nnal/atb/set_env.sh export ASCEND_USE_FIA=1 +export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SET_CPU_AFFINITY=1 export STREAMS_PER_DEVICE=32 @@ -572,16 +572,14 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ - --max-prefill-tokens 50000 \ + --max-prefill-tokens 58000 \ --disable-radix-cache \ --trust-remote-code \ - --max-running-requests 28 \ - --max-mamba-cache-size 50 \ - --mem-fraction-static 0.7 \ - --cuda-graph-bs 2 8 12 16 20 24 26 28 \ - --enable-multimodal \ + --max-running-requests 29 \ + --max-mamba-cache-size 58 \ + --mem-fraction-static 0.68 \ + --cuda-graph-bs 1 2 8 12 16 20 24 26 28 29 \ --quantization modelslim \ - --mm-attention-backend ascend_attn \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ --speculative-algorithm NEXTN \ @@ -600,10 +598,10 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 28 \ + --max-concurrency 29 \ --random-input-len 16000 \ --random-output-len 1000 \ - --num-prompts 112 \ + --num-prompts 116 \ --random-range-ratio 1 ``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx index 18b9e50c2..e26cd951d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx @@ -10,23 +10,25 @@ This guide describes the best practice data for Qwen3.6-35B-A3B on the Ascend NP | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 254K+1K | 16.1ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in254k-out1k) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 254K+1K | 16.1ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in254k-out1k) | ### High Throughput | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) | -| Qwen3.6-35B-A3B | Atlas 800I A3 | 2 | PD Mixed | 984K+1K | 40.91ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1024x1024 (30)+1024 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1024x1024-30-out1024-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in1080p-30-out256-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 128K+1K (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in128k-out1k-prefix90-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in3k5-out1k5-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-1p-in64k-out1k-prefix90-50ms) | +| Qwen3.6-35B-A3B | Atlas 800I A3 | 2 | PD Mixed | 984K+1K | 40.91ms | BF16 | [Optimal Configuration](#qwen3-6-35b-a3b-2p-in984k-out1k) | ## Optimal Configuration + + ### Qwen3.6-35B-A3B 1P IN1024X1024 30 OUT1024 50ms **Model**: Qwen3.6-35B-A3B @@ -37,7 +39,7 @@ This guide describes the best practice data for Qwen3.6-35B-A3B on the Ascend NP **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 1024x1024 (30)+1024 @@ -135,7 +137,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 1080p_30+256 @@ -231,7 +233,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 128K+1K @@ -286,7 +288,6 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 128000 \ --disable-radix-cache \ --trust-remote-code \ - --enable-prefill-delayer \ --max-running-requests 3 \ --max-mamba-cache-size 10 \ --mem-fraction-static 0.63 \ @@ -314,7 +315,7 @@ python -m sglang.bench_serving \ --max-concurrency 3 \ --random-input-len 128000 \ --random-output-len 1000 \ - --num-prompts 12 \ + --num-prompts 3 \ --random-range-ratio 1 ``` @@ -328,7 +329,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 128K+1K (90% prefix cache hit rate) @@ -429,7 +430,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 254K+1K @@ -512,8 +513,6 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` - - ### Qwen3.6-35B-A3B 1P IN3K5 OUT1K5 50ms **Model**: Qwen3.6-35B-A3B @@ -524,7 +523,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 3.5K+1.5K @@ -577,12 +576,11 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 43400 \ --disable-radix-cache \ --trust-remote-code \ - --enable-prefill-delayer \ - --prefill-delayer-max-delay-passes 50 \ - --max-running-requests 124 \ - --max-mamba-cache-size 124 \ + --prefill-max-requests 12 \ + --max-running-requests 122 \ + --max-mamba-cache-size 122 \ --mem-fraction-static 0.8 \ - --cuda-graph-bs 4 16 32 64 96 112 116 120 124 \ + --cuda-graph-bs 4 16 32 64 96 116 120 122 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -603,10 +601,10 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 124 \ + --max-concurrency 122 \ --random-input-len 3500 \ --random-output-len 1500 \ - --num-prompts 496 \ + --num-prompts 122 \ --random-range-ratio 1 ``` @@ -620,7 +618,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 64K+1K @@ -716,7 +714,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 64K+1K (90% prefix cache hit rate) @@ -769,12 +767,11 @@ python3 -m sglang.launch_server \ --chunked-prefill-size -1 \ --max-prefill-tokens 65536 \ --trust-remote-code \ - --enable-prefill-delayer \ --mamba-scheduler-strategy extra_buffer \ - --max-running-requests 42 \ + --max-running-requests 40 \ --max-mamba-cache-size 210 \ --mem-fraction-static 0.71 \ - --cuda-graph-bs 2 8 16 24 32 36 40 42 \ + --cuda-graph-bs 2 8 16 24 32 36 40 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -799,12 +796,12 @@ python -m sglang.bench_serving \ --host 127.0.0.1 \ --port 6688 \ --gsp-num-groups 1 \ - --gsp-prompts-per-group 42 \ + --gsp-prompts-per-group 40 \ --gsp-system-prompt-len 58982 \ --gsp-question-len 6553 \ --gsp-output-len 1024 \ - --max-concurrency 42 \ - --num-prompts 42 \ + --max-concurrency 40 \ + --num-prompts 40 \ --request-rate inf ``` @@ -818,7 +815,7 @@ python -m sglang.bench_serving \ **Deploy Mode**: PD Mixed -**Quantization**: W8A8 INT8 +**Quantization**: BF16 **Dataset**: 984K+1K diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx index 9a10ae29d..f768cccc4 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx @@ -21,6 +21,8 @@ This guide describes the best practice data for Qwen3-Next-80B-A3B-Instruct on t ## Optimal Configuration + + ### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 20ms **Model**: Qwen3-Next-80B-A3B-Instruct @@ -132,8 +134,6 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` - - ### Qwen3-Next-80B-A3B-Instruct W8A8 2P IN3K5 OUT1K5 50ms **Model**: Qwen3-Next-80B-A3B-Instruct