From d028697d17b3bb586395ae15e64eb22277fb3d5f Mon Sep 17 00:00:00 2001 From: Kurkur <102506892+litmei@users.noreply.github.com> Date: Tue, 19 May 2026 09:08:28 +0800 Subject: [PATCH] [NPU][Docs] Add Kimi-K2.5-W4A8 instance doc on NPU (#25269) --- .../ascend_npu_kimi_k2.5_examples.mdx | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx new file mode 100644 index 000000000..dac84c9b3 --- /dev/null +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx @@ -0,0 +1,314 @@ +--- +title: "Kimi K2.5 examples" +metatags: + description: "Documentation for Kimi K2.5 examples" +--- +## Introduction + +Kimi K2.5 is an open-source, native multimodal agentic model built through continual pretraining on approximately 15 trillion mixed visual and text tokens atop Kimi-K2-Base. It seamlessly integrates vision and language understanding with advanced agentic capabilities, instant and thinking modes, as well as conversational and agentic paradigms. + +## Environment Preparation + +### Model Weight + +- `Kimi-K2.5`: Download Kimi-K2.5 from [modelers](https://modelers.cn/models/moonshotai/Kimi-K2.5), [modelscope](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.5-w4a8) or [huggingface](https://huggingface.co/moonshotai/Kimi-K2.5). +- `Kimi-K2.5-w4a8`: Download Kimi-K2.5-w4a8 from [modelers](https://modelers.cn/models/Eco-Tech/Kimi-K2.5-w4a8), [modelscope](https://www.modelscope.cn/models/Eco-Tech/Kimi-K2.5-w4a8). +- You can use [msmodelslim](https://gitcode.com/Ascend/msmodelslim) to quantize `Kimi-K2.5-w4a8` from `Kimi-K2.5`. + + +### Installation + +The dependencies required for the NPU runtime environment have been integrated into a Docker image and uploaded to the online platform. You can directly pull it. + +```bash Command +# Atlas 800 A3 +docker pull swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-openEuler-20260427 + +# Start container +docker run -itd --shm-size=16g --privileged=true --name ${NAME} --net=host \ + -v /var/queue_schedule:/var/queue_schedule \ + -v /etc/ascend_install.info:/etc/ascend_install.info \ + -v /usr/local/sbin:/usr/local/sbin \ + -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + -v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ + --device=/dev/davinci0:/dev/davinci0 \ + --device=/dev/davinci1:/dev/davinci1 \ + --device=/dev/davinci2:/dev/davinci2 \ + --device=/dev/davinci3:/dev/davinci3 \ + --device=/dev/davinci4:/dev/davinci4 \ + --device=/dev/davinci5:/dev/davinci5 \ + --device=/dev/davinci6:/dev/davinci6 \ + --device=/dev/davinci7:/dev/davinci7 \ + --device=/dev/davinci8:/dev/davinci8 \ + --device=/dev/davinci9:/dev/davinci9 \ + --device=/dev/davinci10:/dev/davinci10 \ + --device=/dev/davinci11:/dev/davinci11 \ + --device=/dev/davinci12:/dev/davinci12 \ + --device=/dev/davinci13:/dev/davinci13 \ + --device=/dev/davinci14:/dev/davinci14 \ + --device=/dev/davinci15:/dev/davinci15 \ + --device=/dev/davinci_manager:/dev/davinci_manager \ + --device=/dev/hisi_hdc:/dev/hisi_hdc \ + --entrypoint=bash \ + swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann8.5.0-a3-openEuler-20260427 +``` + +### Best Practices +Note: Using this image for **best practices**, switch to the more stable version 4.57.1 of transformers. +``` +# reinstall transformers + +# Install transformers version 4.57.1 from PyPI +pip install transformers==4.57.1 + +# Install from GitHub v4.57.1 tag from GitHub +pip install git+https://github.com/huggingface/transformers.git@v4.57.1 +``` + +## Deployment + +### Single-node Deployment + +- Quantized model `Kimi-K2.5-w4a8` can be deployed on 1 Atlas 800 A3 (64G x 16) . + +Run the following script to execute online inference. + +```shell Launch Server +# high performance cpu +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 +export HCCL_BUFFSIZE=1500 +export HCCL_SOCKET_IFNAME=lo +export GLOO_SOCKET_IFNAME=lo + +python -m sglang.launch_server \ + --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ + --model-loader-extra-config '{"enable_multithread_load": true}' \ + --host 0.0.0.0 --port 8000 \ + --trust-remote-code --device npu --attention-backend ascend \ + --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.77 --max-running-requests 256 \ + --chunked-prefill-size 32768 --context-length 8192 --max-prefill-tokens 16384 \ + --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ + --enable-dp-attention --dp-size 16 \ + --moe-a2a-backend deepep --deepep-mode auto \ + --cuda-graph-bs 1 2 4 8 9 10 12 16 --disable-radix-cache \ + --enable-prefill-delayer --prefill-delayer-max-delay-passes 200 \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path ${DRAFT_MODEL_PATH} \ + --speculative-num-steps 4 --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --speculative-draft-model-quantization unquant +``` + +### Multi-node Deployment + +- Quantized model `Kimi-K2.5-w4a8` also can be deployed on 2 Atlas 800 A3 (64G x 16) . + +**A3 series** + +Modify the IP of 2 nodes, then run the same scripts on two nodes. + +**node 0/1** + +```shell Launch Multi-node Server +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=0 +sysctl -w kernel.numa_balancing=0 +sysctl -w kernel.sched_migration_cost_ns=50000 +# bind cpu +export SGLANG_SET_CPU_AFFINITY=1 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 +export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 +export HCCL_BUFFSIZE=2400 +export SGLANG_ENABLE_SPEC_V2=1 +export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 +export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 +export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 +export SGLANG_NPU_USE_MLAPO=1 + +# Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here +export HCCL_SOCKET_IFNAME=xxx +export GLOO_SOCKET_IFNAME=xxx + +NODES_IP=('your ip1' 'your ip2') + +LOCAL_IPS=$(hostname -I) +for i in "${!NODES_IP[@]}"; +do + if [[ " ${LOCAL_IPS} " =~ " ${NODES_IP[$i]} " ]]; + then + echo "${NODES_IP[$i]}" + python -m sglang.launch_server \ + --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ + --model-loader-extra-config '{"enable_multithread_load": true}' \ + --host 0.0.0.0 --port 8880 --dist-init-addr ${NODES_IP[0]}:5000 --nnodes 2 --node-rank $i \ + --trust-remote-code --device npu --attention-backend ascend \ + --tp-size 32 --base-gpu-id 0 --mem-fraction-static 0.62 --max-running-requests 386 \ + --chunked-prefill-size 65536 --context-length 8192 --max-prefill-tokens 16384 \ + --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ + --enable-dp-attention --dp-size 32 \ + --moe-a2a-backend deepep --deepep-mode auto \ + --cuda-graph-bs 1 2 4 6 8 10 12 --disable-radix-cache \ + --speculative-algorithm EAGLE3 \ + --speculative-draft-model-path ${DRAFT_MODEL_PATH} \ + --speculative-num-steps 4 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 5 \ + --speculative-draft-model-quantization unquant + NODE_RANK=$i + break + fi +done + +``` + +### Prefill-Decode Disaggregation + +Here is an example of a large scale EP scenario (3P1D) for `Kimi-K2.5-w4a8` on 5 Atlas 800 A3 (64G x 16) . + +Modify the IP of 5 nodes, then run the same scripts on 5 nodes. Then start an additional Router on the first P node. + +```shell +# System Settings +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +sysctl -w vm.swappiness=10 +sysctl -w kernel.numa_balancing=0 + +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export SGLANG_SET_CPU_AFFINITY=1 +export STREAMS_PER_DEVICE=32 + +unset https_proxy +unset http_proxy +unset HTTPS_PROXY +unset HTTP_PROXY +unset ASCEND_LAUNCH_BLOCKING +# cann +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +P_IP=('your prefill ip1' 'your prefill ip2' 'your prefill ip3') +D_IP=('your decode ip1' 'your decode ip2') +LOCAL_IPS=$(hostname -I) + +export ASCEND_MF_STORE_URL="tcp://${P_IP[0]}:24669" + +for i in "${!P_IP[@]}"; +do + if [[ " ${LOCAL_IPS} " =~ " ${P_IP[$i]} " ]]; + then + echo "Prefill -> ${P_IP[$i]}" + + export HCCL_SOCKET_IFNAME=lo + export GLOO_SOCKET_IFNAME=lo + + export HCCL_BUFFSIZE=3600 + + export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 + + python -m sglang.launch_server \ + --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ + --disaggregation-mode prefill --disaggregation-transfer-backend ascend \ + --host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port $((8998+$i)) --nnodes 1 --node-rank 0 \ + --trust-remote-code --device npu --attention-backend ascend \ + --tp-size 16 --base-gpu-id 0 --mem-fraction-static 0.68 --max-running-requests 8 \ + --chunked-prefill-size 32768 --context-length 81920 \ + --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ + --enable-dp-attention --dp-size 4 \ + --disable-radix-cache \ + --moe-a2a-backend deepep --deepep-mode auto + NODE_RANK=$i + exit 0 + fi +done + + +for i in "${!D_IP[@]}"; +do + if [[ " ${LOCAL_IPS} " =~ " ${D_IP[$i]} " ]]; + then + echo "Decode -> ${D_IP[$i]}" + + # Run command ifconfig on two nodes, find out which inet addr has same IP with your node IP. That is your public interface, which should be added here + export HCCL_SOCKET_IFNAME=xxx + export GLOO_SOCKET_IFNAME=xxx + + export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 + export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 + export HCCL_BUFFSIZE=2400 + + export SGLANG_ENABLE_SPEC_V2=1 + export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + + export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 + + python -m sglang.launch_server \ + --model-path ${MODEL_PATH} --quantization modelslim --dtype bfloat16 \ + --disaggregation-mode decode --disaggregation-transfer-backend ascend \ + --host ${D_IP[$i]} --port 8001 --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i \ + --trust-remote-code --device npu --attention-backend ascend \ + --tp-size 32 --base-gpu-id 0 --mem-fraction-static 0.74 --max-running-requests 128 \ + --chunked-prefill-size 65536 --context-length 81920 \ + --disable-radix-cache \ + --enable-multimodal --mm-attention-backend ascend_attn --sampling-backend ascend \ + --enable-dp-attention --dp-size 32 \ + --moe-a2a-backend deepep --deepep-mode auto \ + --cuda-graph-bs 4 + NODE_RANK=$i + exit 0 + fi +done +``` + +SGLang Model Gateway (former Router) + +```shell +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://'your prefill ip1':8000 8998 \ + --prefill http://'your prefill ip2':8000 8999 \ + --prefill http://'your prefill ip3':8000 9000 \ + --decode http://'your decode ip1':8001 \ + --host 127.0.0.1 \ + --port 6688 \ +``` + + +### Using Benchmark + +Refer to [Benchmark and Profiling](../../developer_guide/benchmark_and_profiling) for details.