minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

【NPU】add MiniMax2.5 best practice docs (#26725)

Browse Source
This commit is contained in:
shadowxz109
2026-06-01 10:09:12 +08:00
committed by GitHub
parent 1ee189831f
commit 4d20dc44fc
1 changed files with 923 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx
+923
View File
@@ -636,6 +636,156 @@ you encounter issues or have any questions, please [open an issue](https://githu
</tbody>
</table>
## MiniMax Series Models
### Low Latency
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "13%"}} />
<col style={{width: "13%"}} />
<col style={{width: "13%"}} />
<col style={{width: "13%"}} />
<col style={{width: "12%"}} />
<col style={{width: "12%"}} />
<col style={{width: "12%"}} />
<col style={{width: "12%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Model</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Hardware</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Cards</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Deploy Mode</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Dataset</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>TPOT</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Quantization</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Configuration</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>3.5K+1.5K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>20ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-3_5k-1_5k-low-latency-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>128K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>20ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-128k-1k-low-latency-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
</tbody>
</table>
### High Throughput
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "13%"}} />
<col style={{width: "13%"}} />
<col style={{width: "13%"}} />
<col style={{width: "13%"}} />
<col style={{width: "12%"}} />
<col style={{width: "12%"}} />
<col style={{width: "12%"}} />
<col style={{width: "12%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Model</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Hardware</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Cards</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Deploy Mode</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Dataset</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>TPOT</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Quantization</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Configuration</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>3.5K+1.5K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-3_5k-1_5k-high-throughput-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>32K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-32k-1k-high-throughput-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>64K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-64k-1k-high-throughput-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>128K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-128k-1k-high-throughput-on-a3-8-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>4</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Mixed</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>64K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-64k-1k-high-throughput-on-a3-4-cards-mixed-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>16</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Disaggregation</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>64K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-64k-1k-high-throughput-on-a3-16-cards-disaggregation-mode">Optimal Configuration</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>MiniMax-M2.5</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Atlas 800I A3</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>16</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>PD Disaggregation</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>128K+1K</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>50ms</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>W8A8 INT8</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><a href="#minimax-m25-128k-1k-high-throughput-on-a3-16-cards-disaggregation-mode">Optimal Configuration</a></td>
</tr>
</tbody>
</table>
## Optimal Configuration
### DeepSeek-R1 3_5K-1_5K 50ms on A3 32 Cards Disaggregation Mode
@@ -4852,3 +5002,776 @@ We tested it based on the `RANDOM` dataset.
```bash Command
python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 352 --random-output-len 1500 --random-input-len 3500 --num-prompts 1408
```
### MiniMax-M2.5 3_5K-1_5K Low Latency on A3 8 Cards Mixed Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 8Card
DeployMode: PD Mixed
Dataset: random
Input Output Length: 3.5K+1.5K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export HCCL_OP_EXPANSION_MODE=AIV
export TASK_QUEUE_ENABLE=1
export HCCL_BUFFSIZE=1500
export ASCEND_USE_FIA=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=224000
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
python -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 \
--port 32001 \
--tp-size 16 \
--dp-size 16 \
--enable-dp-attention \
--mem-fraction-static 0.75 \
--max-running-requests 128 \
--disable-radix-cache \
--chunked-prefill-size -1 --max-prefill-token 8192 \
--cuda-graph-bs 2 4 6 8 \
--moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16 \
--tokenizer-worker-num 2 \
--prefill-delayer-max-delay-passes 500 \
--enable-prefill-delayer
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32001 --random-input-len 3500 --random-output-len 1500 --num-prompts 320 --random-range-ratio 1 --max-concurrency 80
```
### MiniMax-M2.5 128K-1K Low Latency on A3 8 Cards Mixed Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 8Card
DeployMode: PD Mixed
Dataset: random
Input Output Length: 128K+1K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export TASK_QUEUE_ENABLE=1
export ASCEND_USE_FIA=1
export HCCL_BUFFSIZE=1600
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1
export SGLANG_NPU_FUSEEP_DECODE_ONLY=1
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
python -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 \
--port 32000 \
--tp-size 16 \
--dp-size 2 \
--enable-dp-attention \
--prefill-delayer-max-delay-passes 100 \
--enable-prefill-delayer \
--mem-fraction-static 0.65 \
--max-running-requests 8 \
--chunked-prefill-size -1 --max-prefill-token 130000 \
--cuda-graph-bs 1 2 4 \
--moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16 \
--trust-remote-code \
--tokenizer-worker-num 8
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 131072 --random-output-len 1024 --num-prompts 8 --random-range-ratio 1 --max-concurrency 2
```
### MiniMax-M2.5 3_5K-1_5K High Throughput on A3 8 Cards Mixed Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 8Card
DeployMode: PD Mixed
Dataset: random
Input Output Length: 3.5K+1.5K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export HCCL_OP_EXPANSION_MODE=AIV
export TASK_QUEUE_ENABLE=1
export HCCL_BUFFSIZE=800
export ASCEND_USE_FIA=1
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
python -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 \
--port 32001 \
--tp-size 16 \
--enable-dp-attention \
--dp-size 16 \
--mem-fraction-static 0.75 \
--max-running-requests 480 \
--disable-radix-cache \
--prefill-delayer-max-delay-passes 500 \
--enable-prefill-delayer \
--chunked-prefill-size -1 --max-prefill-token 8192 \
--cuda-graph-bs 8 16 24 32 48 64 80 \
--moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32001 --random-input-len 3500 --random-output-len 1500 --num-prompts 1280 --random-range-ratio 1 --max-concurrency 320
```
### MiniMax-M2.5 64K-1K High Throughput on A3 8 Cards Mixed Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 8Card
DeployMode: PD Mixed
Dataset: random
Input Output Length: 64K+1K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export TASK_QUEUE_ENABLE=1
export ASCEND_USE_FIA=1
export HCCL_BUFFSIZE=1600
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1
export SGLANG_NPU_FUSEEP_DECODE_ONLY=1
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
python -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 \
--port 32000 \
--tp-size 16 \
--dp-size 2 \
--enable-dp-attention \
--prefill-delayer-max-delay-passes 100 \
--enable-prefill-delayer \
--mem-fraction-static 0.65 \
--max-running-requests 72 \
--chunked-prefill-size -1 --max-prefill-token 180000 \
--cuda-graph-bs 8 16 24 32 40 \
--moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16 \
--trust-remote-code \
--tokenizer-worker-num 8
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 65536 --random-output-len 1024 --num-prompts 144 --random-range-ratio 1 --max-concurrency 36
```
### MiniMax-M2.5 128K-1K High Throughput on A3 8 Cards Mixed Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 8Card
DeployMode: PD Mixed
Dataset: random
Input Output Length: 128K+1K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export TASK_QUEUE_ENABLE=1
export ASCEND_USE_FIA=1
export HCCL_BUFFSIZE=1600
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1
export SGLANG_NPU_FUSEEP_DECODE_ONLY=1
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
python -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 \
--port 32000 \
--tp-size 16 \
--dp-size 2 \
--enable-dp-attention \
--prefill-delayer-max-delay-passes 100 \
--enable-prefill-delayer \
--mem-fraction-static 0.65 \
--max-running-requests 36 \
--chunked-prefill-size -1 --max-prefill-token 130000 \
--cuda-graph-bs 8 16 24 \
--moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16 \
--trust-remote-code \
--tokenizer-worker-num 8
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 131072 --random-output-len 1024 --num-prompts 128 --random-range-ratio 1 --max-concurrency 32
```
### MiniMax-M2.5 64K-1K High Throughput on A3 4 Cards Mixed Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 4Card
DeployMode: PD Mixed
Dataset: random
Input Output Length: 64K+1K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export TASK_QUEUE_ENABLE=1
export ASCEND_USE_FIA=0
export HCCL_BUFFSIZE=1600
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_NPU_DEEPEP_USE_FUSED_MOE_DECODE=1
export SGLANG_NPU_FUSEEP_DECODE_ONLY=1
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
python -m sglang.launch_server \
--model-path $MODEL_PATH \
--host 127.0.0.1 \
--port 32000 \
--tp-size 8 \
--enable-dp-attention \
--prefill-delayer-max-delay-passes 500 \
--enable-prefill-delayer \
--mem-fraction-static 0.65 \
--max-running-requests 36 \
--chunked-prefill-size -1 --max-prefill-token 150000 \
--cuda-graph-bs 8 16 24 32 40 \
--moe-a2a-backend ascend_fuseep --deepep-mode auto --quantization modelslim \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dtype bfloat16 \
--trust-remote-code \
--tokenizer-worker-num 8
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 32000 --random-input-len 65536 --random-output-len 1024 --num-prompts 144 --random-range-ratio 1 --max-concurrency 36
```
### MiniMax-M2.5 64K-1K High Throughput on A3 16 Cards Disaggregation Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 16Card
DeployMode: PD Disaggregation
Dataset: random
Input Output Length: 64K+1K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
export SGLANG_SET_CPU_AFFINITY=1
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export ASCEND_MF_STORE_URL="tcp://your_prefill_ip:24667"
P_IP=('your_prefill_ip')
D_IP=('your_decode_ip')
D_MASTER="${D_IP[0]}:8001"
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export HCCL_SOCKET_IFNAME=your_nic
export GLOO_SOCKET_IFNAME=your_nic
export ASCEND_USE_FIA=1
export HCCL_BUFFSIZE=2500
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \
--port 32000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \
--tp-size 16 --mem-fraction-static 0.43 --attention-backend ascend --device npu --quantization modelslim \
--disaggregation-transfer-backend ascend --max-running-requests 128 \
--chunked-prefill-size -1 --max-prefill-tokens 58000 --moe-a2a-backend deepep --deepep-mode normal \
--tokenizer-worker-num 16 \
--dp-size 2 --enable-dp-attention --dtype bfloat16 --load-balance-method round_robin \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant --skip-server-warmup
NODE_RANK=$i
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export HCCL_BUFFSIZE=1600
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export HCCL_SOCKET_IFNAME=your_nic
export GLOO_SOCKET_IFNAME=your_nic
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=96
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \
--cuda-graph-bs 8 16 24 32 40 \
--port 33000 --trust-remote-code \
--tp-size 16 --mem-fraction-static 0.76 --attention-backend ascend --device npu --quantization modelslim \
--nnodes 1 --node-rank $i --dist-init-addr $D_MASTER \
--disaggregation-transfer-backend ascend --max-running-requests 80 \
--chunked-prefill-size -1 --moe-a2a-backend ascend_fuseep --deepep-mode low_latency \
--tokenizer-worker-num 16 \
--dp-size 2 --enable-dp-attention --dtype bfloat16 \
--load-balance-method round_robin \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant
NODE_RANK=$i
break
fi
done
```
```shell Command
python -m sglang_router.launch_router \
--pd-disaggregation \
--policy round_robin \
--prefill http://your_prefill_ip:32000 8998 \
--decode http://your_decode_ip:33000 \
--host 127.0.0.1 \
--mini-lb \
--port 6688
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --random-input-len 65536 --random-output-len 1024 --num-prompts 640 --random-range-ratio 1 --max-concurrency 160
```
### MiniMax-M2.5 128K-1K High Throughput on A3 16 Cards Disaggregation Mode
Model: MiniMax-M2.5
Hardware: Atlas 800I A3 16Card
DeployMode: PD Disaggregation
Dataset: random
Input Output Length: 128K+1K
#### Model Deployment
```bash Command
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
export SGLANG_SET_CPU_AFFINITY=1
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export ASCEND_MF_STORE_URL="tcp://your_prefill_ip:24667"
P_IP=('your_prefill_ip')
D_IP=('your_decode_ip')
D_MASTER="${D_IP[0]}:8001"
MODEL_PATH=/path/to/MiniMax-M2.5-w8a8-QuaRot
EAGLE_MODEL_PATH=/path/to/MiniMax-M2.5-eagle-model
export PYTHONPATH=${EAGLE_MODEL_PATH}:$PYTHONPATH
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
# prefill
for i in "${!P_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${P_IP[$i]}" || "$LOCAL_HOST2" == "${P_IP[$i]}" ]];
then
echo "${P_IP[$i]}"
export HCCL_SOCKET_IFNAME=your_nic
export GLOO_SOCKET_IFNAME=your_nic
export ASCEND_USE_FIA=1
export HCCL_BUFFSIZE=2500
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \
--port 32000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \
--tp-size 16 --mem-fraction-static 0.43 --attention-backend ascend --device npu --quantization modelslim \
--disaggregation-transfer-backend ascend --max-running-requests 128 \
--chunked-prefill-size -1 --max-prefill-tokens 130000 --moe-a2a-backend deepep --deepep-mode normal \
--tokenizer-worker-num 16 \
--dp-size 2 --enable-dp-attention --dtype bfloat16 --load-balance-method round_robin \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 2 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 3 \
--speculative-draft-model-quantization unquant --skip-server-warmup
NODE_RANK=$i
break
fi
done
# decode
for i in "${!D_IP[@]}";
do
if [[ "$LOCAL_HOST1" == "${D_IP[$i]}" || "$LOCAL_HOST2" == "${D_IP[$i]}" ]];
then
echo "${D_IP[$i]}"
export HCCL_BUFFSIZE=1600
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export HCCL_SOCKET_IFNAME=your_nic
export GLOO_SOCKET_IFNAME=your_nic
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=96
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \
--cuda-graph-bs 2 4 8 \
--port 33000 --trust-remote-code \
--tp-size 16 --mem-fraction-static 0.76 --attention-backend ascend --device npu --quantization modelslim \
--nnodes 1 --node-rank $i --dist-init-addr $D_MASTER \
--disaggregation-transfer-backend ascend --max-running-requests 80 \
--chunked-prefill-size -1 --moe-a2a-backend ascend_fuseep --deepep-mode low_latency \
--tokenizer-worker-num 8 \
--dp-size 2 --enable-dp-attention --dtype bfloat16 \
--load-balance-method round_robin \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $EAGLE_MODEL_PATH \
--speculative-num-steps 2 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 3 \
--speculative-draft-model-quantization unquant
NODE_RANK=$i
break
fi
done
```
```shell Command
python -m sglang_router.launch_router \
--pd-disaggregation \
--policy round_robin \
--prefill http://your_prefill_ip:32000 8998 \
--decode http://your_decode_ip:33000 \
--host 127.0.0.1 \
--mini-lb \
--port 6688
```
#### Benchmark
We tested it based on the `RANDOM` dataset.
```shell Command
python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --random-input-len 131072 --random-output-len 1024 --num-prompts 192 --random-range-ratio 1 --max-concurrency 48
```