From f27fa0da93d47f028910ca5d78b427573cd34e97 Mon Sep 17 00:00:00 2001 From: Kurkur <102506892+litmei@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:14:14 +0800 Subject: [PATCH] [NPU][Docs] Kimi-K2.5 best practice (#26774) --- .../ascend-npus/ascend_npu_best_practice.mdx | 230 ++++++++++++++++++ .../ascend_npu_kimi_k2.5_examples.mdx | 8 +- 2 files changed, 234 insertions(+), 4 deletions(-) diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx index b39552a39..1dd298559 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx @@ -786,6 +786,86 @@ you encounter issues or have any questions, please [open an issue](https://githu +## Kimi Series Models + +### Low Latency + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Kimi-K2.5-w4a8Atlas 800I A38PD Mixed3.5K+1.5K20msW4A8 INT8Optimal Configuration
+ +### High Throughput + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelHardwareCardsDeploy ModeDatasetTPOTQuantizationConfiguration
Kimi-K2.5-w4a8Atlas 800I A38PD Mixed3.5K+1.5K50msW4A8 INT8Optimal Configuration
+ ## Optimal Configuration ### DeepSeek-R1 3_5K-1_5K 50ms on A3 32 Cards Disaggregation Mode @@ -5775,3 +5855,153 @@ We tested it based on the `RANDOM` dataset. ```shell Command python -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6688 --random-input-len 131072 --random-output-len 1024 --num-prompts 192 --random-range-ratio 1 --max-concurrency 48 ``` + + +### Kimi K2.5 w4a8 3_5K-1_5K 20ms on A3 8 Cards Mixed Mode + +Model: Kimi-K2.5-w4a8 + +Hardware: Atlas 800I A3 8Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 20ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export STREAMS_PER_DEVICE=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48 +export HCCL_BUFFSIZE=1200 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_NPU_USE_MLAPO=1 +export SGLANG_NPU_USE_MULTI_STREAM=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 + +MODEL_PATH=xxx +DRAFT_PATH=xxx + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH --quantization modelslim --dtype bfloat16 \ + --model-loader-extra-config '{"enable_multithread_load": true}' \ + --host 0.0.0.0 --port 6699 \ + --trust-remote-code --device npu --attention-backend ascend \ + --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.78 --max-running-requests 64 \ + --chunked-prefill-size 32768 --context-length 8192 --max-prefill-tokens 16384 \ + --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ + --enable-dp-attention --dp-size 16 \ + --moe-a2a-backend deepep --deepep-mode auto \ + --cuda-graph-bs 1 2 3 4 --disable-radix-cache \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_PATH \ + --speculative-num-steps 4 --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --speculative-draft-model-quantization unquant +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 64 --random-output-len 1500 --random-input-len 3500 --num-prompts 64 +``` + + +### Kimi K2.5 w4a8 3_5K-1_5K 50ms on A3 8 Cards Mixed Mode + +Model: Kimi-K2.5-w4a8 + +Hardware: Atlas 800I A3 8Card + +DeployMode: PD Mixed + +Dataset: random + +Input Output Length: 3.5K+1.5K + +TPOT: 50ms + +#### Model Deployment + +```bash Command +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh +source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash + +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo +export STREAMS_PER_DEVICE=32 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 +export HCCL_BUFFSIZE=1200 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 + +MODEL_PATH=xxx +DRAFT_PATH=xxx + +python3 -m sglang.launch_server \ + --model-path $MODEL_PATH --quantization modelslim --dtype bfloat16 \ + --model-loader-extra-config '{"enable_multithread_load": true}' \ + --host 0.0.0.0 --port 6699 \ + --trust-remote-code --device npu --attention-backend ascend \ + --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.7 --max-running-requests 120 \ + --chunked-prefill-size 32768 --context-length 8192 --max-prefill-tokens 16384 \ + --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ + --enable-dp-attention --dp-size 16 \ + --moe-a2a-backend deepep --deepep-mode auto \ + --cuda-graph-bs 1 2 4 8 12 16 24 32 48 64 96 120 --disable-radix-cache \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path $DRAFT_PATH \ + --speculative-num-steps 4 --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --speculative-draft-model-quantization unquant +``` + +#### Benchmark + +We tested it based on the `RANDOM` dataset. + +```bash Command +python3 -m sglang.bench_serving --dataset-name random --backend sglang --host 127.0.0.1 --port 6699 --random-range-ratio 1 --max-concurrency 120 --random-output-len 1500 --random-input-len 3500 --num-prompts 120 +``` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx index dac84c9b3..445f8fe36 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx @@ -300,10 +300,10 @@ SGLang Model Gateway (former Router) python -m sglang_router.launch_router \ --pd-disaggregation \ --policy cache_aware \ - --prefill http://'your prefill ip1':8000 8998 \ - --prefill http://'your prefill ip2':8000 8999 \ - --prefill http://'your prefill ip3':8000 9000 \ - --decode http://'your decode ip1':8001 \ + --prefill http://:8000 8998 \ + --prefill http://:8000 8999 \ + --prefill http://:8000 9000 \ + --decode http://:8001 \ --host 127.0.0.1 \ --port 6688 \ ```