[DOC] [NPU] add qwen3.5-397b best practice to doc_new (#26709)

2026-05-30 14:20:24 +08:00
parent 1c79015434
commit 23a825c694
1 changed files with 194 additions and 0 deletions
@@ -404,6 +404,16 @@ you encounter issues or have any questions, please [open an issue](https://githu
      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#qwen35-27b-64k-1k-20ms-on-a3-1-cards-mixed-mode">Optimal Configuration</a></td>
    </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen3.5-397B-A17B</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>3.5K+1.5K</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>22ms</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W4A8</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#qwen35-397b-a17b-3_5k-1_5k-22ms-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
+    </tr>
 </tbody>
 </table>

@@ -613,6 +623,16 @@ you encounter issues or have any questions, please [open an issue](https://githu
      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#qwen35-27b-64k-1k-50ms-on-a3-2-cards-mixed-mode">Optimal Configuration</a></td>
    </tr>
+    <tr>
+      <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen3.5-397B-A17B</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>3.5K+1.5K</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W4A8</td>
+      <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#qwen35-397b-a17b-3_5k-1_5k-50ms-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
+    </tr>
 </tbody>
 </table>

@@ -4658,3 +4678,177 @@ We tested it based on the `RANDOM` dataset.
 ```bash Command
 python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port 9000 --dataset-name random  --max-concurrency 9 --num-prompts 36 --random-range-ratio 1  --random-output-len 1000 --random-input-len 64000
 ```
+
+### Qwen3.5-397B-A17B 3_5K-1_5K 22ms on A3 8 Cards Mixed Mode
+
+Model: Qwen3.5-397B-A17B
+
+Hardware: Atlas 800I A3 8Card
+
+DeployMode: PD Mixed
+
+Dataset: random
+
+Input Output Length: 3.5K+1.5K
+
+TPOT: 22ms
+
+#### Model Deployment
+
+```bash Command
+# high performance cpu
+echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+sysctl -w vm.swappiness=0
+sysctl -w kernel.numa_balancing=0
+sysctl -w kernel.sched_migration_cost_ns=50000
+# bind cpu
+export SGLANG_SET_CPU_AFFINITY=1
+
+unset https_proxy
+unset http_proxy
+unset HTTPS_PROXY
+unset HTTP_PROXY
+unset ASCEND_LAUNCH_BLOCKING
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+source /usr/local/Ascend/nnal/atb/set_env.sh
+source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
+export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
+
+export ASCEND_USE_FIA=1
+export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
+export HCCL_BUFFSIZE=3000
+export DEEPEP_NORMAL_LONG_SEQ_ROUND=32
+export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
+export STREAMS_PER_DEVICE=32
+export HCCL_OP_EXPANSION_MODE=AIV
+export HCCL_SOCKET_IFNAME=lo
+export GLOO_SOCKET_IFNAME=lo
+export SGLANG_ENABLE_SPEC_V2=1
+export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
+export SGLANG_NPU_USE_MULTI_STREAM=1
+
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export SGLANG_ZBAL_LOCAL_MEM_SIZE=58624
+export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
+export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669"
+export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
+export ZBAL_ENABLE_GRAPH=1
+
+MODEL_PATH=xxx
+
+python3 -m sglang.launch_server \
+--model-path $MODEL_PATH \
+--attention-backend ascend \
+--device npu \
+--tp-size 16 \
+--chunked-prefill-size -1 --max-prefill-tokens 35000 \
+--disable-radix-cache \
+--trust-remote-code \
+--host 127.0.0.1 --max-running-requests 160 \
+--mem-fraction-static 0.8 \
+--port 6699 \
+--cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 \
+--quantization modelslim \
+--enable-multimodal --moe-a2a-backend deepep --deepep-mode auto \
+--mm-attention-backend ascend_attn \
+--dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 128000 \
+--speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \
+--speculative-draft-model-quantization unquant \
+--dp-size 8 --enable-dp-attention --enable-dp-lm-head \
+--enable-prefill-delayer --prefill-delayer-max-delay-passes 100
+```
+
+#### Benchmark
+
+We tested it based on the `RANDOM` dataset.
+
+```bash Command
+python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 120 --random-output-len 1500 --random-input-len 3500 --num-prompts 480
+```
+
+### Qwen3.5-397B-A17B 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode
+
+Model: Qwen3.5-397B-A17B
+
+Hardware: Atlas 800I A3 8Card
+
+DeployMode: PD Mixed
+
+Dataset: random
+
+Input Output Length: 3.5K+1.5K
+
+TPOT: 50ms
+
+#### Model Deployment
+
+```bash Command
+# high performance cpu
+echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+sysctl -w vm.swappiness=0
+sysctl -w kernel.numa_balancing=0
+sysctl -w kernel.sched_migration_cost_ns=50000
+# bind cpu
+export SGLANG_SET_CPU_AFFINITY=1
+
+unset https_proxy
+unset http_proxy
+unset HTTPS_PROXY
+unset HTTP_PROXY
+unset ASCEND_LAUNCH_BLOCKING
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+source /usr/local/Ascend/nnal/atb/set_env.sh
+source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
+export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
+
+export ASCEND_USE_FIA=1
+export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
+export HCCL_BUFFSIZE=3000
+export DEEPEP_NORMAL_LONG_SEQ_ROUND=32
+export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
+export STREAMS_PER_DEVICE=32
+export HCCL_OP_EXPANSION_MODE=AIV
+export HCCL_SOCKET_IFNAME=lo
+export GLOO_SOCKET_IFNAME=lo
+export SGLANG_ENABLE_SPEC_V2=1
+export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
+export SGLANG_NPU_USE_MULTI_STREAM=1
+
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export SGLANG_ZBAL_LOCAL_MEM_SIZE=59648
+export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
+export SGLANG_ZBAL_BOOTSTRAP_URL="tcp://127.0.0.1:24669"
+export ZBAL_NPU_ALLOC_CONF=use_vmm_for_static_memory:True
+export ZBAL_ENABLE_GRAPH=1
+
+MODEL_PATH=xxx
+
+python3 -m sglang.launch_server \
+--model-path $MODEL_PATH \
+--attention-backend ascend \
+--device npu \
+--tp-size 16 \
+--chunked-prefill-size -1 --max-prefill-tokens 17500 \
+--disable-radix-cache \
+--trust-remote-code \
+--host 127.0.0.1 --max-running-requests 432 \
+--mem-fraction-static 0.75 \
+--port 6699 \
+--cuda-graph-bs 2 4 6 8 12 16 20 24 28 32 36 40 44 48 52 56 \
+--quantization modelslim \
+--enable-multimodal --moe-a2a-backend deepep --deepep-mode auto \
+--mm-attention-backend ascend_attn \
+--dtype bfloat16 --mamba-ssm-dtype bfloat16 --max-total-tokens 280000 \
+--dp-size 8 --enable-dp-attention --enable-dp-lm-head \
+--speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \
+--speculative-draft-model-quantization unquant \
+--enable-prefill-delayer --prefill-delayer-max-delay-passes 200
+```
+
+#### Benchmark
+
+We tested it based on the `RANDOM` dataset.
+
+```bash Command
+python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 352 --random-output-len 1500 --random-input-len 3500 --num-prompts 1408
+```