From 2260e612f6092000d899dab42da92c549f015c4a Mon Sep 17 00:00:00 2001 From: jianzhao-xu <978716854@qq.com> Date: Mon, 29 Jun 2026 11:30:27 +0800 Subject: [PATCH] [NPU] update best practicce docs from testcase (#29492) --- docs_new/docs.json | 1 + .../ascend-npus/best_practice/deepseek_r1.mdx | 46 +- .../best_practice/deepseek_v3_2.mdx | 12 + .../ascend-npus/best_practice/glm5_1.mdx | 79 +- .../ascend-npus/best_practice/kimi_k2_6.mdx | 119 +-- .../best_practice/mimo_v2_flash.mdx | 769 ++++++++++++++++++ .../best_practice/minimax_m2_5.mdx | 12 +- .../ascend-npus/best_practice/qwen3-8b.mdx | 12 +- .../best_practice/qwen3_235b_a22b.mdx | 10 +- .../best_practice/qwen3_30b_a3b.mdx | 16 +- .../ascend-npus/best_practice/qwen3_32b.mdx | 12 +- .../best_practice/qwen3_5_397b.mdx | 67 +- .../ascend-npus/best_practice/qwen3_6_27b.mdx | 228 +++--- .../best_practice/qwen3_6_35b_a3b.mdx | 47 +- .../qwen3_next_80b_a3b_instruct.mdx | 12 +- 15 files changed, 1183 insertions(+), 259 deletions(-) create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash.mdx diff --git a/docs_new/docs.json b/docs_new/docs.json index ea3d4d377..24285957e 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -917,6 +917,7 @@ "docs/hardware-platforms/ascend-npus/best_practice/glm5_1", "docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6", "docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5", + "docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash", "docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b", "docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b", "docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b", diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx index 420f26fff..31658d2ed 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_r1.mdx @@ -12,7 +12,7 @@ This page focuses on optimal configuration and benchmark results for DeepSeek-R1 | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1.5K | 18.9ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-18-9ms) | +| DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1.5K | 16ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-16ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1K | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k-19-0ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.9K+1K | 19.0ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k9-out1k-19-0ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 6K+1.6K | 20.5ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in6k-out1k6-20-5ms) | @@ -21,13 +21,13 @@ This page focuses on optimal configuration and benchmark results for DeepSeek-R1 | Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | | --- | --- | --- | --- | --- | --- | --- | --- | -| DeepSeek-R1 | Atlas 800I A3 | 16 | PD Disaggregation | 3.5K+1.5K | 41ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-1p1d-16p-in3k5-out1k5-41ms) | -| DeepSeek-R1 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50.36ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-8p-in3k5-out1k5-50-36ms) | +| DeepSeek-R1 | Atlas 800I A3 | 16 | PD Disaggregation | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-1p1d-16p-in3k5-out1k5-50ms) | +| DeepSeek-R1 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#deepseek-r1-w4a8-8p-in3k5-out1k5-50ms) | | DeepSeek-R1 | Atlas 800I A3 | 32 | PD Disaggregation | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#deepseek-r1-w8a8-2p1d-32p-in3k5-out1k5-50ms) | ## Optimal Configuration -### DeepSeek-R1 W4A8 1P1D 16P IN3K5 OUT1K5 41ms +### DeepSeek-R1 W4A8 1P1D 16P IN3K5 OUT1K5 50ms **Model**: DeepSeek-R1 @@ -41,7 +41,7 @@ This page focuses on optimal configuration and benchmark results for DeepSeek-R1 **Dataset**: 3.5K+1.5K -**TPOT**: 41ms +**TPOT**: 50ms #### Model Deployment @@ -127,6 +127,8 @@ do --enable-dp-attention \ --disable-shared-experts-fusion \ --dtype bfloat16 \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -177,6 +179,8 @@ do --dtype bfloat16 \ --tokenizer-worker-num 4 \ --load-balance-method round_robin \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -223,7 +227,7 @@ python -m sglang.bench_serving \ -### DeepSeek-R1 W4A8 8P IN3K5 OUT1K5 50.36ms +### DeepSeek-R1 W4A8 8P IN3K5 OUT1K5 50ms **Model**: DeepSeek-R1 @@ -237,7 +241,7 @@ python -m sglang.bench_serving \ **Dataset**: 3.5K+1.5K -**TPOT**: 50.36ms +**TPOT**: 50ms #### Model Deployment @@ -305,7 +309,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 ``` #### Benchmark @@ -327,7 +333,7 @@ python -m sglang.bench_serving \ -### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 18.9ms +### DeepSeek-R1 W8A8 2P1D 32P IN3K5 OUT1K5 16ms **Model**: DeepSeek-R1 @@ -341,7 +347,7 @@ python -m sglang.bench_serving \ **Dataset**: 3.5K+1.5K -**TPOT**: 18.9ms +**TPOT**: 16ms #### Model Deployment @@ -427,6 +433,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -481,6 +489,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -632,6 +642,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -686,6 +698,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -833,6 +847,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -887,6 +903,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -1034,6 +1052,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -1088,6 +1108,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -1234,6 +1256,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --enable-attn-tp-input-scattered \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ @@ -1287,6 +1311,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ + --reasoning-parser deepseek-r1 \ + --tool-call-parser deepseekv3 \ --disaggregation-transfer-backend ascend \ --trust-remote-code \ --attention-backend ascend \ diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx index 02e324ea9..1abb8266e 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx @@ -125,6 +125,8 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ + --reasoning-parser deepseek-v3 \ + --tool-call-parser deepseekv32 \ --trust-remote-code \ --attention-backend ascend \ --device npu @@ -177,6 +179,8 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --disaggregation-transfer-backend ascend \ + --reasoning-parser deepseek-v3 \ + --tool-call-parser deepseekv32 \ --trust-remote-code \ --attention-backend ascend \ --device npu @@ -320,6 +324,8 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ + --reasoning-parser deepseek-v3 \ + --tool-call-parser deepseekv32 \ --trust-remote-code \ --attention-backend ascend \ --device npu @@ -372,6 +378,8 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --disaggregation-transfer-backend ascend \ + --reasoning-parser deepseek-v3 \ + --tool-call-parser deepseekv32 \ --trust-remote-code \ --attention-backend ascend \ --device npu @@ -515,6 +523,8 @@ do --speculative-num-steps 1 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ + --reasoning-parser deepseek-v3 \ + --tool-call-parser deepseekv32 \ --trust-remote-code \ --attention-backend ascend \ --device npu @@ -567,6 +577,8 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --disaggregation-transfer-backend ascend \ + --reasoning-parser deepseek-v3 \ + --tool-call-parser deepseekv32 \ --trust-remote-code \ --attention-backend ascend \ --device npu diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx index 52790e77f..27e6344ac 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx @@ -10,22 +10,22 @@ This page focuses on optimal configuration and benchmark results for GLM-5.1 on ### Low Latency -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -| --- | --- | --- | --- | --- | --- | --- | --- | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 20ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in128k-out1k-20ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64K+1K | 20ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-20ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 65K+1.5K (90% prefix cache hit rate) | 25ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in65k-out1k5-prefix90-25ms) | +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 65K+1.5K (90% prefix cache hit rate) | 25ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in65k-out1k5-prefix90-25ms) | ### High Throughput -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -| --- | --- | --- | --- | --- | --- | --- | --- | -| GLM-5.1 | Atlas 800I A3 | 16 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-16p-in3k5-out1k5-50ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 16K+1K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) | -| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64K+1K | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) | -| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 65K+1.5K (100% prefix cache hit rate) | 33ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-48p-in65k-out1k5-prefix100-33ms) | -| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) | -| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 64K+1K (90% prefix cache hit rate) | 50ms | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) | +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| GLM-5.1 | Atlas 800I A3 | 16 | PD Mixed | 3.5K+1.5K | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-16p-in3k5-out1k5-50ms) | +| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 128K+1K | 56.4ms | 13.1s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in128k-out1k-56-4ms) | +| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 16K+1K | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in16k-out1k-50ms) | +| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64K+1K | 55.2ms | 7.58s | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-55-2ms) | +| GLM-5.1 | Atlas 800I A3 | 32 | PD Disaggregation | 64K+1K | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-32p-in64k-out1k-50ms) | +| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 65K+1.5K (100% prefix cache hit rate) | 33ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-1p1d-48p-in65k-out1k5-prefix100-33ms) | +| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-2p1d-48p-in128k-out1k-prefix90-50ms) | +| GLM-5.1 | Atlas 800I A3 | 48 | PD Disaggregation | 64K+1K (90% prefix cache hit rate) | 50ms | - | W4A8 INT8 | [Optimal Configuration](#glm-5-1-w4a8-4p1d-48p-in64k-out1k-prefix90-50ms) | ## Optimal Configuration @@ -122,7 +122,9 @@ do --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 break fi done @@ -147,7 +149,7 @@ python -m sglang.bench_serving \ -### GLM-5.1 W4A8 1P1D 32P IN128K OUT1K 20ms +### GLM-5.1 W4A8 1P1D 32P IN128K OUT1K 56.4ms **Model**: GLM-5.1 @@ -161,7 +163,9 @@ python -m sglang.bench_serving \ **Dataset**: 128K+1K -**TPOT**: 20ms +**TPOT**: 56.4ms + +**TTFT**: 13.1s #### Model Deployment @@ -241,7 +245,7 @@ do --disaggregation-transfer-backend ascend \ --max-running-requests 16 \ --served-model-name glm-5 \ - --chunked-prefill-size 16384 \ + --chunked-prefill-size 8192 \ --max-prefill-tokens 180000 \ --moe-a2a-backend deepep \ --deepep-mode normal \ @@ -255,6 +259,8 @@ do --enable-dp-lm-head \ --moe-dense-tp 1 \ --pp-size 8 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -306,6 +312,8 @@ do --dtype bfloat16 \ --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -452,6 +460,8 @@ do --attn-cp-size 8 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -507,6 +517,8 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -547,7 +559,7 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` -### GLM-5.1 W4A8 1P1D 32P IN64K OUT1K 20ms +### GLM-5.1 W4A8 1P1D 32P IN64K OUT1K 55.2ms **Model**: GLM-5.1 @@ -561,7 +573,9 @@ python -m sglang.bench_serving \ **Dataset**: 64K+1K -**TPOT**: 20ms +**TPOT**: 55.2ms + +**TTFT**: 7.58s #### Model Deployment @@ -641,7 +655,7 @@ do --disaggregation-transfer-backend ascend \ --max-running-requests 16 \ --served-model-name glm-5 \ - --chunked-prefill-size 16384 \ + --chunked-prefill-size 8192 \ --max-prefill-tokens 180000 \ --moe-a2a-backend deepep \ --deepep-mode normal \ @@ -655,6 +669,8 @@ do --enable-dp-lm-head \ --moe-dense-tp 1 \ --pp-size 8 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -706,6 +722,8 @@ do --dtype bfloat16 \ --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -854,6 +872,8 @@ do --enable-dp-lm-head \ --moe-dense-tp 1 \ --pp-size 8 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -905,6 +925,8 @@ do --dtype bfloat16 \ --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1051,6 +1073,8 @@ do --attn-cp-size 8 \ --enable-dp-lm-head \ --moe-dense-tp 1 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1106,6 +1130,8 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1248,6 +1274,8 @@ do --disable-shared-experts-fusion \ --disable-cuda-graph \ --dtype bfloat16 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1289,7 +1317,6 @@ do --quantization modelslim \ --served-model-name glm-5 \ --moe-a2a-backend deepep \ - --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ @@ -1307,6 +1334,8 @@ do --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1455,6 +1484,8 @@ do --enable-dp-lm-head \ --moe-dense-tp 1 \ --pp-size 8 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1510,6 +1541,8 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1664,6 +1697,8 @@ do --enable-dp-lm-head \ --moe-dense-tp 1 \ --pp-size 4 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break @@ -1719,6 +1754,8 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ + --reasoning-parser glm45 \ + --tool-call-parser glm47 \ --trust-remote-code NODE_RANK=$i break diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx index fd7395133..403e6a796 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/kimi_k2_6.mdx @@ -10,22 +10,22 @@ This page focuses on optimal configuration and benchmark results for Kimi-K2.6 o ### Low Latency -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 20ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-20ms) | +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 20ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-20ms) | ### High Throughput -| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | Quantization | Configuration | -| --- | --- | --- | --- | --- | --- | --- | --- | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Mixed | 64K+1K | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-16p-in64k-out1k-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128K+1K | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-prefix90-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1.5K | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1.5K (90% prefix cache hit rate) | 100ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-prefix90-100ms) | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1024x1024 (30)+1024 | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1024x1024-30-out1024-50ms) | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1080p_30+256 | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1080p-30-out256-50ms) | -| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-50ms) | +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Mixed | 64K+1K | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-16p-in64k-out1k-100ms) | +| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128K+1K | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-100ms) | +| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 128K+1K (90% prefix cache hit rate) | 100ms | 5s | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in128k-out1k-prefix90-100ms) | +| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1.5K | 100ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-100ms) | +| Kimi-K2.6 | Atlas 800I A3 | 16 | PD Disaggregation | 64K+1.5K (90% prefix cache hit rate) | 100ms | 3s | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-1p1d-16p-in64k-out1k5-prefix90-100ms) | +| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1024x1024 (30)+1024 | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1024x1024-30-out1024-50ms) | +| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 1080p_30+256 | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in1080p-30-out256-50ms) | +| Kimi-K2.6 | Atlas 800I A3 | 8 | PD Mixed | 3.5K+1.5K | 50ms | - | W4A8 INT8 | [Optimal Configuration](#kimi-k2-6-w4a8-8p-in3k5-out1k5-50ms) | ## Optimal Configuration @@ -111,7 +111,7 @@ do --quantization modelslim \ --dtype bfloat16 \ --tp-size 32 \ - --mem-fraction-static 0.55 \ + --mem-fraction-static 0.662 \ --max-running-requests 32 \ --chunked-prefill-size 262144 \ --context-length 75000 \ @@ -129,7 +129,9 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 break fi done @@ -248,14 +250,13 @@ do --device npu \ --tp-size 16 \ --disable-radix-cache \ - --disable-cuda-graph \ --mem-fraction-static 0.78 \ - --max-running-requests 1 \ + --max-running-requests 2 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --chunked-prefill-size 16384 \ - --prefill-max-requests 1 \ - --max-prefill-tokens 131072 \ + --prefill-max-requests 2 \ + --max-prefill-tokens 65536 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --sampling-backend ascend @@ -292,8 +293,8 @@ do --attention-backend ascend \ --device npu \ --tp-size 16 \ - --mem-fraction-static 0.73 \ - --max-running-requests 1 \ + --mem-fraction-static 0.82 \ + --max-running-requests 2 \ --enable-dp-attention \ --dp-size 1 \ --enable-dp-lm-head \ @@ -309,7 +310,9 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 NODE_RANK=$i break fi @@ -366,6 +369,8 @@ python -m sglang.bench_serving \ **TPOT**: 100ms +**TTFT**: 5s + #### Model Deployment ```bash Command @@ -452,7 +457,9 @@ do --max-prefill-tokens 65536 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ - --sampling-backend ascend + --sampling-backend ascend \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 NODE_RANK=$i break fi @@ -486,7 +493,7 @@ do --attention-backend ascend \ --device npu \ --tp-size 16 \ - --mem-fraction-static 0.73 \ + --mem-fraction-static 0.82 \ --max-running-requests 2 \ --enable-dp-attention \ --dp-size 1 \ @@ -503,7 +510,9 @@ do --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 NODE_RANK=$i break fi @@ -619,13 +628,8 @@ do then echo "${P_IP[$i]}" export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=8 + export HCCL_BUFFSIZE=1800 export HCCL_SOCKET_IFNAME= - export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 - export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 - export ZBAL_ENABLE_GRAPH=1 - export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ --model-path ${MODEL_PATH} \ @@ -653,7 +657,9 @@ do --max-prefill-tokens 65536 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ - --sampling-backend ascend + --sampling-backend ascend \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 NODE_RANK=$i break fi @@ -688,7 +694,7 @@ do --device npu \ --tp-size 16 \ --mem-fraction-static 0.82 \ - --max-running-requests 1 \ + --max-running-requests 16 \ --enable-dp-attention \ --dp-size 1 \ --enable-dp-lm-head \ @@ -699,6 +705,8 @@ do --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs 16 \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 4 \ @@ -761,6 +769,8 @@ python -m sglang.bench_serving \ **TPOT**: 100ms +**TTFT**: 3s + #### Model Deployment ```bash Command @@ -815,13 +825,8 @@ do then echo "${P_IP[$i]}" export GLOO_SOCKET_IFNAME= - export HCCL_BUFFSIZE=8 + export HCCL_BUFFSIZE=1800 export HCCL_SOCKET_IFNAME= - export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 - export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24699 - export SGLANG_ZBAL_LOCAL_MEM_SIZE=61184 - export ZBAL_ENABLE_GRAPH=1 - export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ --model-path ${MODEL_PATH} \ @@ -847,7 +852,9 @@ do --max-prefill-tokens 65536 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ - --sampling-backend ascend + --sampling-backend ascend \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 NODE_RANK=$i break fi @@ -893,6 +900,8 @@ do --moe-a2a-backend deepep \ --deepep-mode auto \ --cuda-graph-bs 1 2 4 6 8 \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 4 \ @@ -1013,7 +1022,7 @@ python3 -m sglang.launch_server \ --device npu \ --attention-backend ascend \ --tp-size 16 \ - --mem-fraction-static 0.76 \ + --mem-fraction-static 0.872 \ --max-running-requests 176 \ --chunked-prefill-size 32768 \ --context-length 8192 \ @@ -1025,7 +1034,7 @@ python3 -m sglang.launch_server \ --dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 8 9 10 11 \ + --cuda-graph-bs-decode 1 2 4 8 9 10 11 \ --disable-radix-cache \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ @@ -1034,7 +1043,9 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 3 \ --speculative-draft-model-quantization unquant \ --prefill-delayer-max-delay-passes 200 \ - --enable-prefill-delayer + --enable-prefill-delayer \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 ``` #### Benchmark @@ -1124,7 +1135,7 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --tp-size 16 \ --base-gpu-id 0 \ - --mem-fraction-static 0.74 \ + --mem-fraction-static 0.852 \ --max-running-requests 64 \ --chunked-prefill-size 16384 \ --context-length 8192 \ @@ -1145,7 +1156,9 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 3 \ --speculative-draft-model-quantization unquant \ --prefill-delayer-max-delay-passes 200 \ - --enable-prefill-delayer + --enable-prefill-delayer \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 ``` #### Benchmark @@ -1237,7 +1250,7 @@ python3 -m sglang.launch_server \ --quantization modelslim \ --dtype bfloat16 \ --tp-size 16 \ - --mem-fraction-static 0.753 \ + --mem-fraction-static 0.865 \ --max-running-requests 80 \ --chunked-prefill-size 32768 \ --context-length 6144 \ @@ -1249,7 +1262,7 @@ python3 -m sglang.launch_server \ --dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 3 4 5 \ + --cuda-graph-bs-decode 1 2 3 4 5 \ --disable-radix-cache \ --model-loader-extra-config '{"enable_multithread_load": true}' \ --speculative-algorithm EAGLE3 \ @@ -1259,7 +1272,9 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 5 \ --speculative-draft-model-quantization unquant \ --prefill-delayer-max-delay-passes 200 \ - --enable-prefill-delayer + --enable-prefill-delayer \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 ``` #### Benchmark @@ -1346,7 +1361,7 @@ python3 -m sglang.launch_server \ --quantization modelslim \ --dtype bfloat16 \ --tp-size 16 \ - --mem-fraction-static 0.783 \ + --mem-fraction-static 0.895 \ --max-running-requests 208 \ --chunked-prefill-size 32768 \ --context-length 6144 \ @@ -1358,7 +1373,7 @@ python3 -m sglang.launch_server \ --dp-size 16 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 8 12 13 \ + --cuda-graph-bs-decode 1 2 4 8 12 13 \ --disable-radix-cache \ --model-loader-extra-config '{"enable_multithread_load": true}' \ --speculative-algorithm EAGLE3 \ @@ -1368,7 +1383,9 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 5 \ --speculative-draft-model-quantization unquant \ --prefill-delayer-max-delay-passes 200 \ - --enable-prefill-delayer + --enable-prefill-delayer \ + --reasoning-parser kimi_k2 \ + --tool-call-parser kimi_k2 ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash.mdx new file mode 100644 index 000000000..e16850355 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/mimo_v2_flash.mdx @@ -0,0 +1,769 @@ +--- +title: "MiMo-V2-Flash" +metatags: + description: "Best Practice for MiMo-V2-Flash on Ascend NPU" +--- + +This guide describes the best practice data for MiMo-V2-Flash on the Ascend NPU. + +### Low Latency + +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 16K+1K | 20ms | - | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in16k-out1k-tpot-20ms) | +| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 32K+1K | 20ms | - | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in32k-out1k-tpot-20ms) | + +### High Throughput + +| Model | Hardware | Cards | Deploy Mode | Dataset | TPOT | TTFT | Quantization | Configuration | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 16K+1 | - | 5s | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in16k-out1-ttft-5s) | +| MiMo-V2-Flash | Atlas 800I A3 | 12 | PD Disaggregation | 32K+1 | - | 5s | W8A8 INT8 | [Optimal Configuration](#mimo-v2-flash-1p1d-12p-in32k-out1-ttft-5s) | + +## Optimal Configuration + + + +### MiMo-V2-Flash 1P1D 12P IN16K OUT1 TTFT 5s + +**Model**: MiMo-V2-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 12 + +**Deploy Mode**: PD Disaggregation + +**Quantization**: W8A8 INT8 + +**Dataset**: 16K+1 + +**TTFT**: 5s + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# P_IP: prefill node IP address +# D_IP: decode node IP address +# ASCEND_MF_STORE_URL: prefill node IP with port +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_USE_FIA=1 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_CONNECT_TIMEOUT=1800 +export HCCL_OP_EXPANSION_MODE=AIV +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +P_IP=('') +D_IP=('') + +export ASCEND_MF_STORE_URL="tcp://:24670" + +MODEL_PATH=/path/to/model-weights + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=1024 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 + export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --host ${P_IP[$i]} \ + --port 8000 \ + --disaggregation-bootstrap-port 8998 \ + --node-rank 0 \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 \ + --nnodes 1 \ + --chunked-prefill-size 8192 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --disaggregation-transfer-backend ascend \ + --disable-radix-cache \ + --disable-cuda-graph \ + --disable-piecewise-cuda-graph \ + --dp-size 2 + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=800 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host ${D_IP[$i]} \ + --port 8001 \ + --attention-backend ascend \ + --device npu \ + --tp-size 16 \ + --nnodes 1 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --disaggregation-transfer-backend ascend \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --enable-multi-layer-eagle \ + --disable-radix-cache \ + --dp-size 2 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency + NODE_RANK=$i + break + fi +done +``` + +```shell Command +# ============================================================ +# Before running, replace the following placeholders: +# : prefill node IP address +# : decode node IP address +# ============================================================ + +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8998 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --health-check-interval-secs 3600 --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```shell Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 64 \ + --random-input-len 16000 \ + --random-output-len 1 \ + --num-prompts 128 \ + --random-range-ratio 1 \ + --request-rate 0.4 +``` + +### MiMo-V2-Flash 1P1D 12P IN16K OUT1K TPOT 20ms + +**Model**: MiMo-V2-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 12 + +**Deploy Mode**: PD Disaggregation + +**Quantization**: W8A8 INT8 + +**Dataset**: 16K+1K + +**TPOT**: 20ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# P_IP: prefill node IP address +# D_IP: decode node IP address +# ASCEND_MF_STORE_URL: prefill node IP with port +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_USE_FIA=1 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_CONNECT_TIMEOUT=1800 +export HCCL_OP_EXPANSION_MODE=AIV +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +P_IP=('') +D_IP=('') + +export ASCEND_MF_STORE_URL="tcp://:24670" + +MODEL_PATH=/path/to/model-weights + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=1024 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 + export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --host ${P_IP[$i]} \ + --port 8000 \ + --disaggregation-bootstrap-port 8998 \ + --node-rank 0 \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 \ + --nnodes 1 \ + --chunked-prefill-size 8192 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --disaggregation-transfer-backend ascend \ + --disable-radix-cache \ + --disable-cuda-graph \ + --disable-piecewise-cuda-graph \ + --dp-size 2 + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=800 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host ${D_IP[$i]} \ + --port 8001 \ + --attention-backend ascend \ + --device npu \ + --tp-size 16 \ + --nnodes 1 \ + --trust-remote-code \ + --max-running-requests 32 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --cuda-graph-bs 1 2 4 8 12 16 \ + --disaggregation-transfer-backend ascend \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --enable-multi-layer-eagle \ + --disable-radix-cache \ + --dp-size 2 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency + NODE_RANK=$i + break + fi +done +``` + +```shell Command +# ============================================================ +# Before running, replace the following placeholders: +# : prefill node IP address +# : decode node IP address +# ============================================================ + +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8998 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --health-check-interval-secs 3600 --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```shell Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 32 \ + --random-input-len 16000 \ + --random-output-len 1000 \ + --num-prompts 128 \ + --random-range-ratio 1 \ + --request-rate inf +``` + +### MiMo-V2-Flash 1P1D 12P IN32K OUT1 TTFT 5s + +**Model**: MiMo-V2-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 12 + +**Deploy Mode**: PD Disaggregation + +**Quantization**: W8A8 INT8 + +**Dataset**: 32K+1 + +**TTFT**: 5s + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# P_IP: prefill node IP address +# D_IP: decode node IP address +# ASCEND_MF_STORE_URL: prefill node IP with port +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_USE_FIA=1 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_CONNECT_TIMEOUT=1800 +export HCCL_OP_EXPANSION_MODE=AIV +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +P_IP=('') +D_IP=('') + +export ASCEND_MF_STORE_URL="tcp://:24670" + +MODEL_PATH=/path/to/model-weights + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=1024 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 + export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --host ${P_IP[$i]} \ + --port 8000 \ + --disaggregation-bootstrap-port 8998 \ + --node-rank 0 \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 \ + --nnodes 1 \ + --chunked-prefill-size 8192 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --disaggregation-transfer-backend ascend \ + --disable-radix-cache \ + --disable-cuda-graph \ + --disable-piecewise-cuda-graph \ + --dp-size 2 + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=800 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host ${D_IP[$i]} \ + --port 8001 \ + --attention-backend ascend \ + --device npu \ + --tp-size 16 \ + --nnodes 1 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --disaggregation-transfer-backend ascend \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --enable-multi-layer-eagle \ + --disable-radix-cache \ + --dp-size 2 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency + NODE_RANK=$i + break + fi +done +``` + +```shell Command +# ============================================================ +# Before running, replace the following placeholders: +# : prefill node IP address +# : decode node IP address +# ============================================================ + +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8998 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --health-check-interval-secs 3600 --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```shell Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 64 \ + --random-input-len 32000 \ + --random-output-len 1 \ + --num-prompts 128 \ + --random-range-ratio 1 \ + --request-rate 0.4 +``` + +### MiMo-V2-Flash 1P1D 12P IN32K OUT1K TPOT 20ms + +**Model**: MiMo-V2-Flash + +**Hardware**: Atlas 800I A3 + +**Cards**: 12 + +**Deploy Mode**: PD Disaggregation + +**Quantization**: W8A8 INT8 + +**Dataset**: 32K+1K + +**TPOT**: 20ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# P_IP: prefill node IP address +# D_IP: decode node IP address +# ASCEND_MF_STORE_URL: prefill node IP with port +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + + +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export ASCEND_USE_FIA=1 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 +export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export HCCL_CONNECT_TIMEOUT=1800 +export HCCL_OP_EXPANSION_MODE=AIV +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +P_IP=('') +D_IP=('') + +export ASCEND_MF_STORE_URL="tcp://:24670" + +MODEL_PATH=/path/to/model-weights + +LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` +LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'` +echo "${LOCAL_HOST1}" +echo "${LOCAL_HOST2}" +# prefill +for i in "${!P_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]]; + then + echo "${P_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=1024 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 + export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode prefill \ + --host ${P_IP[$i]} \ + --port 8000 \ + --disaggregation-bootstrap-port 8998 \ + --node-rank 0 \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 \ + --nnodes 1 \ + --chunked-prefill-size 8192 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --disaggregation-transfer-backend ascend \ + --disable-radix-cache \ + --disable-cuda-graph \ + --disable-piecewise-cuda-graph \ + --dp-size 2 + NODE_RANK=$i + break + fi +done + +# decode +for i in "${!D_IP[@]}"; +do + if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]]; + then + echo "${D_IP[$i]}" + export GLOO_SOCKET_IFNAME= + export HCCL_BUFFSIZE=800 + export HCCL_SOCKET_IFNAME= + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 + + python3 -m sglang.launch_server \ + --model-path ${MODEL_PATH} \ + --disaggregation-mode decode \ + --host ${D_IP[$i]} \ + --port 8001 \ + --attention-backend ascend \ + --device npu \ + --tp-size 16 \ + --nnodes 1 \ + --trust-remote-code \ + --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --disaggregation-transfer-backend ascend \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --enable-multi-layer-eagle \ + --disable-radix-cache \ + --dp-size 2 \ + --enable-dp-attention \ + --enable-dp-lm-head \ + --moe-a2a-backend deepep \ + --deepep-mode low_latency + NODE_RANK=$i + break + fi +done +``` + +```shell Command +# ============================================================ +# Before running, replace the following placeholders: +# : prefill node IP address +# : decode node IP address +# ============================================================ + +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://:8000 8998 \ + --decode http://:8001 \ + --host 127.0.0.1 \ + --port 6688 \ + --health-check-interval-secs 3600 --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```shell Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 64 \ + --random-input-len 32000 \ + --random-output-len 1000 \ + --num-prompts 128 \ + --random-range-ratio 1 \ + --request-rate inf +``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx index 6e9755ae5..1e00313e4 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/minimax_m2_5.mdx @@ -115,7 +115,9 @@ python3 -m sglang.launch_server \ --speculative-draft-model-quantization unquant \ --dtype bfloat16 \ --trust-remote-code \ - --tokenizer-worker-num 4 + --tokenizer-worker-num 4 \ + --reasoning-parser minimax-append-think \ + --tool-call-parser minimax-m2 ``` #### Benchmark @@ -206,7 +208,7 @@ python3 -m sglang.launch_server \ --enable-prefill-delayer \ --prefill-max-requests 10 \ --chunked-prefill-size 67072 \ - --max-prefill-token 67000 \ + --max-prefill-tokens 67000 \ --cuda-graph-bs 2 4 8 12 16 18 20 22 24 26 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ @@ -317,7 +319,7 @@ python3 -m sglang.launch_server \ --enable-prefill-delayer \ --prefill-max-requests 4 \ --chunked-prefill-size 160000 \ - --max-prefill-token 80000 \ + --max-prefill-tokens 80000 \ --cuda-graph-bs 2 4 6 8 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ @@ -429,7 +431,7 @@ python3 -m sglang.launch_server \ --enable-prefill-delayer \ --prefill-max-requests 3 \ --chunked-prefill-size -1 \ - --max-prefill-token 8192 \ + --max-prefill-tokens 8192 \ --cuda-graph-bs 1 2 3 4 5 6 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ @@ -533,7 +535,7 @@ python3 -m sglang.launch_server \ --prefill-delayer-max-delay-passes 500 \ --enable-prefill-delayer \ --chunked-prefill-size -1 \ - --max-prefill-token 8192 \ + --max-prefill-tokens 8192 \ --cuda-graph-bs 1 2 4 8 12 16 20 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx index 952658b20..733187a9d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3-8b.mdx @@ -101,7 +101,9 @@ python3 -m sglang.launch_server \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -195,7 +197,9 @@ python3 -m sglang.launch_server \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 4 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 + --speculative-num-draft-tokens 5 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -289,7 +293,9 @@ python3 -m sglang.launch_server \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 4 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 5 + --speculative-num-draft-tokens 5 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx index 30ff661d0..654e2c82f 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_235b_a22b.mdx @@ -99,7 +99,9 @@ python3 -m sglang.launch_server \ --enable-dp-lm-head \ --tp 16 \ --mem-fraction-static 0.78 \ - --cuda-graph-bs 1 + --cuda-graph-bs 1 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen25 ``` #### Benchmark @@ -173,8 +175,6 @@ export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_FUSED_MOE_MODE=2 -export SGLANG_NPU_PROFILING=0 -export SGLANG_NPU_PROFILING_BS=27 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 @@ -207,7 +207,9 @@ python3 -m sglang.launch_server \ --enable-dp-attention \ --enable-dp-lm-head \ --mem-fraction-static 0.8 \ - --cuda-graph-bs 1 2 4 8 16 20 24 26 27 + --cuda-graph-bs 1 2 4 8 16 20 24 26 27 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen25 ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx index 7020ed265..235e78959 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_30b_a3b.mdx @@ -108,7 +108,9 @@ python3 -m sglang.launch_server \ --dp-size 2 \ --mem-fraction-static 0.85 \ --cuda-graph-bs 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -206,7 +208,9 @@ python3 -m sglang.launch_server \ --tp-size 2 \ --mem-fraction-static 0.87 \ --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -304,7 +308,9 @@ python3 -m sglang.launch_server \ --tp-size 2 \ --mem-fraction-static 0.87 \ --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -401,7 +407,9 @@ python3 -m sglang.launch_server \ --tp-size 2 \ --mem-fraction-static 0.6 \ --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx index 4be7a162e..4ea15ae91 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_32b.mdx @@ -100,7 +100,9 @@ python3 -m sglang.launch_server \ --tp-size 16 \ --mem-fraction-static 0.72 \ --cuda-graph-bs 1 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -197,7 +199,9 @@ python3 -m sglang.launch_server \ --tp-size 4 \ --mem-fraction-static 0.845 \ --cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -294,7 +298,9 @@ python3 -m sglang.launch_server \ --tp-size 4 \ --mem-fraction-static 0.845 \ --cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ - --dtype bfloat16 + --dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx index f027ebae7..ed49c7473 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_5_397b.mdx @@ -119,7 +119,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -228,7 +230,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -297,20 +301,15 @@ export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=0 +export HCCL_BUFFSIZE=2200 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 -export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648 export STREAMS_PER_DEVICE=32 -export ZBAL_ENABLE_GRAPH=1 -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ @@ -326,7 +325,6 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --max-running-requests 64 \ --mem-fraction-static 0.6 \ - --cuda-graph-bs 2 4 6 8 10 12 16 20 24 32 40 48 56 64 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -338,7 +336,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -455,7 +455,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -568,7 +570,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -581,10 +585,10 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 144 \ + --max-concurrency 132 \ --random-input-len 16384 \ --random-output-len 1024 \ - --num-prompts 144 \ + --num-prompts 132 \ --random-range-ratio 1 \ --request-rate inf \ --warmup-requests 8 @@ -680,7 +684,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -792,7 +798,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -905,7 +913,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -918,13 +928,13 @@ python -m sglang.bench_serving \ --backend sglang \ --host 127.0.0.1 \ --port 6688 \ - --max-concurrency 8 \ + --max-concurrency 6 \ --random-input-len 65536 \ --random-output-len 1024 \ - --num-prompts 8 \ + --num-prompts 6 \ --random-range-ratio 1 \ --request-rate inf \ - --warmup-requests 8 + --warmup-requests 6 ``` ### Qwen3.5-397B W4A8 8P IN64K OUT1K 50ms @@ -1018,7 +1028,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -1087,21 +1099,15 @@ export DEEPEP_NORMAL_LONG_SEQ_ROUND=20 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export GDN_ATTN_BACKEND_TRITON=1 export GLOO_SOCKET_IFNAME= -export HCCL_BUFFSIZE=64 +export HCCL_BUFFSIZE=2200 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 -export SGLANG_ZBAL_LOCAL_MEM_SIZE=58672 export STREAMS_PER_DEVICE=32 -export ZBAL_ENABLE_GRAPH=1 -export ZBAL_HCCL_OP=allreduce,_allgather_base,allgather,broadcast,scatter,reduce_scatter,_reduce_scatter_base,alltoall_base -export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True python3 -m sglang.launch_server \ --model-path $MODEL_PATH \ @@ -1117,7 +1123,6 @@ python3 -m sglang.launch_server \ --max-running-requests 128 \ --mem-fraction-static 0.6 \ --max-total-tokens 1310720 \ - --cuda-graph-bs 2 4 6 8 10 12 16 20 24 32 40 48 56 64 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -1132,7 +1137,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --speculative-draft-model-quantization unquant + --speculative-draft-model-quantization unquant \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx index 8871b1876..00d7b5b60 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_27b.mdx @@ -16,9 +16,9 @@ This page focuses on optimal configuration and benchmark results for Qwen3.6-27B | Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 1080p_30+256 | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-1p-in1080p-30-out256-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K (90% prefix cache hit rate) | 50ms | BF16 | [Optimal Configuration](#qwen3-6-27b-2p-in64k-out1k-prefix90-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 3.5K+1.5K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in3k5-out1k5-50ms) | +| Qwen3.6-27B | Atlas 800I A3 | 1 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-1p-in64k-out1k-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 128K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in128k-out1k-50ms) | | Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 16K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in16k-out1k-50ms) | -| Qwen3.6-27B | Atlas 800I A3 | 2 | PD Mixed | 64K+1K | 50ms | W8A8 INT8 | [Optimal Configuration](#qwen3-6-27b-w8a8-2p-in64k-out1k-50ms) | ## Optimal Configuration @@ -74,8 +74,6 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_NPU_PROFILING=1 -export SGLANG_NPU_PROFILING_STAGE=prefill export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SET_CPU_AFFINITY=1 @@ -105,7 +103,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --mm-enable-dp-encoder + --mm-enable-dp-encoder \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -173,8 +173,6 @@ export HCCL_SOCKET_IFNAME= export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_NPU_PROFILING=1 -export SGLANG_NPU_PROFILING_STAGE=prefill export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SET_CPU_AFFINITY=1 @@ -204,7 +202,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --mm-enable-dp-encoder + --mm-enable-dp-encoder \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -288,18 +288,20 @@ python3 -m sglang.launch_server \ --mamba-scheduler-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 20 \ - --max-mamba-cache-size 108 \ - --mem-fraction-static 0.7 \ - --cuda-graph-bs 1 2 4 8 12 15 17 19 20 \ + --max-mamba-cache-size 160 \ + --mem-fraction-static 0.82 \ + --cuda-graph-bs 1 2 5 10 15 17 19 20 \ --enable-prefill-delayer \ - --prefill-delayer-queue-min-ratio 0.8 \ - --prefill-delayer-max-delay-ms 30000 \ + --prefill-delayer-queue-min-ratio 0.7 \ + --prefill-delayer-max-delay-ms 20000 \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -401,7 +403,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -421,6 +425,98 @@ python -m sglang.bench_serving \ --random-range-ratio 1 ``` +### Qwen3.6-27B W8A8 1P IN64K OUT1K 50ms + +**Model**: Qwen3.6-27B + +**Hardware**: Atlas 800I A3 + +**Cards**: 1 + +**Deploy Mode**: PD Mixed + +**Quantization**: W8A8 INT8 + +**Dataset**: 64K+1K + +**TPOT**: 50ms + +#### Model Deployment + +```bash Command +# ============================================================ +# Before running, update the following variables: +# MODEL_PATH: path to the model weights directory +# HCCL_SOCKET_IFNAME: network interface name for HCCL +# GLOO_SOCKET_IFNAME: network interface name for Gloo +# ============================================================ + +MODEL_PATH=/path/to/model-weights +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export GLOO_SOCKET_IFNAME= +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME= +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH \ + --host 127.0.0.1 --port 6688 \ + --tp-size 2 \ + --nnodes 1 \ + --attention-backend ascend \ + --device npu \ + --chunked-prefill-size -1 \ + --max-prefill-tokens 48000 \ + --disable-radix-cache \ + --trust-remote-code \ + --max-running-requests 6 \ + --max-mamba-cache-size 16 \ + --mem-fraction-static 0.6 \ + --cuda-graph-bs 1 2 4 5 6 \ + --quantization modelslim \ + --dtype bfloat16 \ + --mamba-ssm-dtype bfloat16 \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```shell Command +python -m sglang.bench_serving \ + --dataset-name random \ + --backend sglang \ + --host 127.0.0.1 \ + --port 6688 \ + --max-concurrency 6 \ + --random-input-len 64000 \ + --random-output-len 1000 \ + --num-prompts 12 \ + --random-range-ratio 1 +``` + ### Qwen3.6-27B W8A8 2P IN128K OUT1K 50ms **Model**: Qwen3.6-27B @@ -493,7 +589,9 @@ python3 -m sglang.launch_server \ --quantization modelslim \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ - --mamba-ssm-dtype bfloat16 + --mamba-ssm-dtype bfloat16 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -587,7 +685,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -606,99 +706,3 @@ python -m sglang.bench_serving \ --num-prompts 116 \ --random-range-ratio 1 ``` - -### Qwen3.6-27B W8A8 2P IN64K OUT1K 50ms - -**Model**: Qwen3.6-27B - -**Hardware**: Atlas 800I A3 - -**Cards**: 2 - -**Deploy Mode**: PD Mixed - -**Quantization**: W8A8 INT8 - -**Dataset**: 64K+1K - -**TPOT**: 50ms - -#### Model Deployment - -```bash Command -# ============================================================ -# Before running, update the following variables: -# MODEL_PATH: path to the model weights directory -# HCCL_SOCKET_IFNAME: network interface name for HCCL -# GLOO_SOCKET_IFNAME: network interface name for Gloo -# ============================================================ - -MODEL_PATH=/path/to/model-weights -echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -sysctl -w vm.swappiness=0 -sysctl -w kernel.numa_balancing=0 -sysctl -w kernel.sched_migration_cost_ns=50000 - -unset https_proxy -unset http_proxy -unset HTTPS_PROXY -unset HTTP_PROXY -unset ASCEND_LAUNCH_BLOCKING - -source /usr/local/Ascend/ascend-toolkit/set_env.sh -source /usr/local/Ascend/nnal/atb/set_env.sh - -export ASCEND_USE_FIA=1 -export GLOO_SOCKET_IFNAME= -export HCCL_OP_EXPANSION_MODE=AIV -export HCCL_SOCKET_IFNAME= -export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True -export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 -export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 -export SGLANG_SET_CPU_AFFINITY=1 -export STREAMS_PER_DEVICE=32 - -python3 -m sglang.launch_server \ - --model-path $MODEL_PATH \ - --host 127.0.0.1 --port 6688 \ - --tp-size 4 \ - --nnodes 1 \ - --attention-backend ascend \ - --device npu \ - --chunked-prefill-size -1 \ - --max-prefill-tokens 50000 \ - --disable-radix-cache \ - --trust-remote-code \ - --max-running-requests 28 \ - --max-mamba-cache-size 50 \ - --mem-fraction-static 0.7 \ - --cuda-graph-bs 2 4 6 \ - --enable-multimodal \ - --quantization modelslim \ - --mm-attention-backend ascend_attn \ - --dtype bfloat16 \ - --mamba-ssm-dtype bfloat16 \ - --speculative-algorithm NEXTN \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 -``` - -#### Benchmark - -We tested it based on the `RANDOM` dataset. - -```shell Command -python -m sglang.bench_serving \ - --dataset-name random \ - --backend sglang \ - --host 127.0.0.1 \ - --port 6688 \ - --max-concurrency 6 \ - --random-input-len 64000 \ - --random-output-len 1000 \ - --num-prompts 24 \ - --random-range-ratio 1 -``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx index 4fdeab6ea..5f70d7323 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_6_35b_a3b.mdx @@ -108,7 +108,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -204,7 +206,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -287,12 +291,13 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ + --max-total-tokens 520960 \ --max-prefill-tokens 128000 \ --disable-radix-cache \ --trust-remote-code \ --max-running-requests 3 \ --max-mamba-cache-size 10 \ - --mem-fraction-static 0.63 \ + --mem-fraction-static 0.9 \ --cuda-graph-bs 1 2 3 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ @@ -301,7 +306,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -396,7 +403,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -495,7 +504,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -575,13 +586,14 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ + --max-total-tokens 659840 \ --max-prefill-tokens 43400 \ --disable-radix-cache \ --trust-remote-code \ --prefill-max-requests 12 \ --max-running-requests 122 \ --max-mamba-cache-size 122 \ - --mem-fraction-static 0.8 \ + --mem-fraction-static 0.9 \ --cuda-graph-bs 4 16 32 64 96 116 120 122 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ @@ -590,7 +602,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -686,7 +700,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -767,12 +783,13 @@ python3 -m sglang.launch_server \ --attention-backend ascend \ --device npu \ --chunked-prefill-size -1 \ + --max-total-tokens 470784 \ --max-prefill-tokens 65536 \ --trust-remote-code \ --mamba-scheduler-strategy extra_buffer \ --max-running-requests 40 \ - --max-mamba-cache-size 210 \ - --mem-fraction-static 0.71 \ + --max-mamba-cache-size 200 \ + --mem-fraction-static 0.9 \ --cuda-graph-bs 2 8 16 24 32 36 40 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ @@ -781,7 +798,9 @@ python3 -m sglang.launch_server \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 + --speculative-num-draft-tokens 4 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark @@ -882,7 +901,9 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --context-length 1010000 + --context-length 1010000 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx index c72bedd38..6ff160e01 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/qwen3_next_80b_a3b_instruct.mdx @@ -116,7 +116,9 @@ python3 -m sglang.launch_server \ --max-running-requests 2 \ --cuda-graph-bs 2 \ --mamba-ssm-dtype bfloat16 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -230,7 +232,9 @@ python3 -m sglang.launch_server \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 + --cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen ``` #### Benchmark @@ -344,7 +348,9 @@ python3 -m sglang.launch_server \ --max-running-requests 16 \ --cuda-graph-bs 2 4 8 \ --mamba-ssm-dtype bfloat16 \ - --speculative-draft-model-path $DRAFT_MODEL_PATH + --speculative-draft-model-path $DRAFT_MODEL_PATH \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder ``` #### Benchmark