[MoE Refactor] [NPU] Refactor Ascend MoE implementation to reduce code duplication and align with community design (#25663)

Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
Артем Савкин
2026-07-15 14:59:42 +03:00
committed by GitHub
co-authored by ronnie_zheng
parent c9b17403e7
commit 8ed82afcc8
54 changed files with 2851 additions and 2375 deletions
@@ -304,7 +304,7 @@ Speculative decoding reduces per-token latency by generating draft tokens that a
<tbody>
<tr>
<td>`--quantization modelslim`</td>
<td>Uses the Ascend ModelSlim quantization tool to load W8A8 pre-quantized model weights. This reduces model weight footprint by approximately 50% compared to BF16, allowing larger models to fit in NPU memory with minimal accuracy degradation. The quantization method is auto-detected from the model's `quant_model_description.json` file.</td>
<td>Uses the Ascend ModelSlim quantization tool to load W4A4/W4A8/W8A8/W4A16 pre-quantized model weights. This reduces model weight footprint by approximately 50% (for w8) or 75% (for w4) compared to BF16, allowing larger models to fit in NPU memory with minimal accuracy degradation. The quantization method is auto-detected from the model's `quant_model_description.json` file.</td>
<td>`modelslim`</td>
</tr>
</tbody>
@@ -357,11 +357,6 @@ The following environment variables are used in other best practice configuratio
<td>Configures the HCCL communication algorithm scheduling to use AIV (Ascend Intelligent Vision) expansion mode, which can improve communication efficiency for certain collective operations.</td>
<td>Used in Qwen MoE and R1 non-DSA configurations</td>
</tr>
<tr>
<td>`SGLANG_NPU_FUSED_MOE_MODE`</td>
<td>Controls the fused MoE optimization mode on Ascend NPU. `1` is default; `2` enables a more aggressive fusion strategy (`DISPATCH_FFN_COMBINE`) that can improve MoE dispatch throughput. Mode `2` requires `--quantization modelslim`. Used primarily with DeepSeek R1 models.</td>
<td>`1` or `2`</td>
</tr>
<tr>
<td>`SGLANG_NPU_USE_MLAPO=1`</td>
<td><strong>(DeepSeek MLA-specific)</strong> Adopts the `MLAPO` fusion operator in the MLA (Multi-Head Latent Attention) preprocessing stage for DeepSeek models with MLA architecture.</td>
@@ -611,7 +611,6 @@ do
export HCCL_BUFFSIZE=800
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_USE_AG_AFTER_QLORA=1
export TASK_QUEUE_ENABLE=2
@@ -625,13 +624,12 @@ do
--nnodes 1 \
--tp-size 16 \
--mem-fraction-static 0.778 \
--quantization modelslim \
--max-running-requests 16 \
--disable-radix-cache \
--chunked-prefill-size -1 \
--max-prefill-tokens 60000 \
--moe-a2a-backend ascend_fuseep \
--deepep-mode normal \
--fuseep-mode 2 \
--speculative-algorithm NEXTN \
--speculative-num-steps 1 \
--speculative-eagle-topk 1 \
@@ -664,7 +662,6 @@ do
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_LM_HEAD_TP=8
export SGLANG_NPU_FUSED_MOE_MODE=1
export TASK_QUEUE_ENABLE=1
python3 -m sglang.launch_server \
@@ -679,8 +676,8 @@ do
--dp-size 32 \
--mem-fraction-static 0.82 \
--max-running-requests 1024 \
--quantization modelslim \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 1 \
--enable-dp-attention \
--deepep-mode low_latency \
--moe-dense-tp 1 \
@@ -192,7 +192,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
@@ -211,8 +210,7 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 67000 \
--cuda-graph-bs 2 4 8 12 16 18 20 22 24 26 \
--moe-a2a-backend ascend_fuseep \
--deepep-mode auto \
--quantization modelslim \
--fuseep-mode 2 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
@@ -300,7 +298,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
export TASK_QUEUE_ENABLE=1
@@ -321,8 +318,7 @@ python3 -m sglang.launch_server \
--max-prefill-tokens 80000 \
--cuda-graph-bs 2 4 6 8 \
--moe-a2a-backend ascend_fuseep \
--deepep-mode auto \
--quantization modelslim \
--fuseep-mode 2 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \
@@ -174,7 +174,7 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -186,7 +186,6 @@ python3 -m sglang.launch_server \
--node-rank 0 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--max-running-requests 432 \
--context-length 8192 \
--dtype bfloat16 \
@@ -196,6 +195,7 @@ python3 -m sglang.launch_server \
--ep-dispatch-algorithm static \
--disable-radix-cache \
--moe-a2a-backend ascend_fuseep \
--fuseep-mode 2 \
--speculative-algorithm EAGLE3 \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-num-steps 3 \