From ce0ff154a50d47ff0cf808ad87fdc068637a9983 Mon Sep 17 00:00:00 2001 From: ming_wang <68357922+sigama-w@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:15:31 +0800 Subject: [PATCH] add mimo best practice (#27665) --- .../ascend-npus/ascend_npu_best_practice.mdx | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx index d87cd5b10..14154242d 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx @@ -776,6 +776,106 @@ you encounter issues or have any questions, please [open an issue](https://githu +## MiMo-V2-Flash Series Models + +### Low Latency + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelHardwareCardsDeploy ModeDatasetTTFTQuantizationConfiguration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation16K+1974msW8A8 INT8Optimal Configuration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation32K+12,147msW8A8 INT8Optimal Configuration
+ +### High Throughput + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation16K+1K16msW8A8 INT8Optimal Configuration
MiMo-V2-FlashAtlas 800I A324PD Disaggregation32K+1K12msW8A8 INT8Optimal Configuration
+ ## Kimi Series Models ### Low Latency @@ -6655,3 +6755,229 @@ We tested it based on the `RANDOM` dataset (90% cache hit), this dataset is gene ```bash Command python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 8003 --random-range-ratio 1 --random-output-len 1000 --random-input-len 131072 --num-prompts 192 ``` + +### MiMo-V2-Flash 16K+1 974ms on A3 24 Cards Disaggregation Mode + +**Model:** MiMo-V2-Flash + +**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) + +**Deploy Mode:** PD Disaggregation + +**Dataset:** random + +**Input/Output Length:** 16K+1 + +**TTFT:** 974ms (P99: 2,068ms) + +**Quantization:** W8A8 INT8 + +#### Model Deployment + +**Prefill Node:** + +```bash +# High performance CPU configuration +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +# Bind CPU +export SGLANG_SET_CPU_AFFINITY=1 + +# Proxy settings (unset if needed) +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +# Performance environment variables +export STREAMS_PER_DEVICE=32 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 +export HCCL_BUFFSIZE=1024 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export SGLANG_NPU_PROFILING=0 +export SGLANG_NPU_PROFILING_STAGE="prefill" +export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 +export ASCEND_MF_STORE_URL="tcp://:24669" +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_CONNECT_TIMEOUT=1800 +export ASCEND_USE_FIA=1 + +# Launch prefill server +python3 -m sglang.launch_server \ + --model-path /path/to/MiMo-V2-Flash-w8a8-all-0512 \ + --attention-backend ascend \ + --device npu \ + --tp-size 8 --nnodes 1 --node-rank 0 \ + --chunked-prefill-size 8192 \ + --trust-remote-code --port 10000 \ + --host --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8996 \ + --disable-radix-cache \ + --disable-cuda-graph \ + --disable-piecewise-cuda-graph \ + --dp-size 2 +``` + +**Decode Node:** + +```bash +# High performance CPU configuration +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +# Bind CPU +export SGLANG_SET_CPU_AFFINITY=1 + +# Proxy settings (unset if needed) +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING + +# Performance environment variables +export STREAMS_PER_DEVICE=32 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 +export HCCL_BUFFSIZE=800 +export HCCL_OP_EXPANSION_MODE=AIV +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export SGLANG_NPU_PROFILING=0 +export SGLANG_NPU_PROFILING_STAGE="prefill" +export DEEPEP_NORMAL_LONG_SEQ_ROUND=32 +export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 +export ASCEND_MF_STORE_URL="tcp://:24669" +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 +export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_BF16_DISPATCH=0 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_CONNECT_TIMEOUT=1800 +export SGLANG_PROFILE_WITH_STACK=True +export ASCEND_USE_FIA=1 + +# Launch decode server +python3 -m sglang.launch_server \ + --model-path /path/to/MiMo-V2-Flash-w8a8-all-0512/ \ + --attention-backend ascend \ + --device npu \ + --tp-size 16 --nnodes 1 --node-rank 0 \ + --trust-remote-code --port 10001 \ + --host --max-running-requests 64 \ + --mem-fraction-static 0.8 \ + --swa-full-tokens-ratio 0.3 \ + --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --disaggregation-mode decode --disaggregation-transfer-backend ascend \ + --disaggregation-bootstrap-port 8996 \ + --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ + --enable-multi-layer-eagle \ + --disable-radix-cache \ + --dp-size 2 --enable-dp-attention --enable-dp-lm-head \ + --moe-a2a-backend deepep --deepep-mode low_latency +``` + +**Router:** + +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://:10000 \ + --decode http://:10001 \ + --host \ + --port 9903 \ + --health-check-interval-secs 3600 \ + --mini-lb +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset with throughput focus (infinite request rate). + +```bash +python3 -m sglang.bench_serving --backend sglang --host --port 9903 --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input-len 16000 --random-output-len 1 --request-rate inf --random-range-ratio 1 --num-prompts 128 --max-concurrency 64 +``` + +### MiMo-V2-Flash 32K+1 2,147ms on A3 24 Cards Disaggregation Mode + +**Model:** MiMo-V2-Flash + +**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) + +**Deploy Mode:** PD Disaggregation + +**Dataset:** random + +**Input/Output Length:** 32K+1 + +**TTFT:** 2,147ms (P99: 4,546ms) + +**Quantization:** W8A8 INT8 + +#### Model Deployment + +Same deployment configuration as [MiMo-V2-Flash 16K+1 974ms](#mimo-v2-flash-16k-1-974ms-on-a3-24-cards-disaggregation-mode). + +### MiMo-V2-Flash 16K+1K 16ms on A3 24 Cards Disaggregation Mode + +**Model:** MiMo-V2-Flash + +**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) + +**Deploy Mode:** PD Disaggregation + +**Dataset:** random + +**Input/Output Length:** 16K+1K + +**TTFT:** 11,326ms (P99: 28,160ms) + +**TPOT:** 16ms (P99: 23ms) + +**Quantization:** W8A8 INT8 + +#### Model Deployment + +Same deployment configuration as [MiMo-V2-Flash 16K+1 974ms](#mimo-v2-flash-16k-1-974ms-on-a3-24-cards-disaggregation-mode). + +### MiMo-V2-Flash 32K+1K 12ms on A3 24 Cards Disaggregation Mode + +**Model:** MiMo-V2-Flash + +**Hardware:** Atlas 800I A3 24 Cards (8 cards for prefill node, 16 cards for decode node) + +**Deploy Mode:** PD Disaggregation + +**Dataset:** random + +**Input/Output Length:** 32K+1K + +**TTFT:** 35,144ms (P99: 55,004ms) + +**TPOT:** 12ms (P99: 17ms) + +**Quantization:** W8A8 INT8 + +#### Model Deployment + +Same deployment configuration as [MiMo-V2-Flash 16K+1 974ms](#mimo-v2-flash-16k-1-974ms-on-a3-24-cards-disaggregation-mode).