[MoE Refactor] [NPU] Refactor Ascend MoE implementation to reduce code duplication and align with community design (#25663)
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
co-authored by
ronnie_zheng
parent
c9b17403e7
commit
8ed82afcc8
@@ -304,7 +304,7 @@ Speculative decoding reduces per-token latency by generating draft tokens that a
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>`--quantization modelslim`</td>
|
||||
<td>Uses the Ascend ModelSlim quantization tool to load W8A8 pre-quantized model weights. This reduces model weight footprint by approximately 50% compared to BF16, allowing larger models to fit in NPU memory with minimal accuracy degradation. The quantization method is auto-detected from the model's `quant_model_description.json` file.</td>
|
||||
<td>Uses the Ascend ModelSlim quantization tool to load W4A4/W4A8/W8A8/W4A16 pre-quantized model weights. This reduces model weight footprint by approximately 50% (for w8) or 75% (for w4) compared to BF16, allowing larger models to fit in NPU memory with minimal accuracy degradation. The quantization method is auto-detected from the model's `quant_model_description.json` file.</td>
|
||||
<td>`modelslim`</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
@@ -357,11 +357,6 @@ The following environment variables are used in other best practice configuratio
|
||||
<td>Configures the HCCL communication algorithm scheduling to use AIV (Ascend Intelligent Vision) expansion mode, which can improve communication efficiency for certain collective operations.</td>
|
||||
<td>Used in Qwen MoE and R1 non-DSA configurations</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>`SGLANG_NPU_FUSED_MOE_MODE`</td>
|
||||
<td>Controls the fused MoE optimization mode on Ascend NPU. `1` is default; `2` enables a more aggressive fusion strategy (`DISPATCH_FFN_COMBINE`) that can improve MoE dispatch throughput. Mode `2` requires `--quantization modelslim`. Used primarily with DeepSeek R1 models.</td>
|
||||
<td>`1` or `2`</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>`SGLANG_NPU_USE_MLAPO=1`</td>
|
||||
<td><strong>(DeepSeek MLA-specific)</strong> Adopts the `MLAPO` fusion operator in the MLA (Multi-Head Latent Attention) preprocessing stage for DeepSeek models with MLA architecture.</td>
|
||||
|
||||
@@ -611,7 +611,6 @@ do
|
||||
export HCCL_BUFFSIZE=800
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_USE_AG_AFTER_QLORA=1
|
||||
export TASK_QUEUE_ENABLE=2
|
||||
|
||||
@@ -625,13 +624,12 @@ do
|
||||
--nnodes 1 \
|
||||
--tp-size 16 \
|
||||
--mem-fraction-static 0.778 \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 16 \
|
||||
--disable-radix-cache \
|
||||
--chunked-prefill-size -1 \
|
||||
--max-prefill-tokens 60000 \
|
||||
--moe-a2a-backend ascend_fuseep \
|
||||
--deepep-mode normal \
|
||||
--fuseep-mode 2 \
|
||||
--speculative-algorithm NEXTN \
|
||||
--speculative-num-steps 1 \
|
||||
--speculative-eagle-topk 1 \
|
||||
@@ -664,7 +662,6 @@ do
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_LM_HEAD_TP=8
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
@@ -679,8 +676,8 @@ do
|
||||
--dp-size 32 \
|
||||
--mem-fraction-static 0.82 \
|
||||
--max-running-requests 1024 \
|
||||
--quantization modelslim \
|
||||
--moe-a2a-backend ascend_fuseep \
|
||||
--fuseep-mode 1 \
|
||||
--enable-dp-attention \
|
||||
--deepep-mode low_latency \
|
||||
--moe-dense-tp 1 \
|
||||
|
||||
@@ -192,7 +192,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
@@ -211,8 +210,7 @@ python3 -m sglang.launch_server \
|
||||
--max-prefill-tokens 67000 \
|
||||
--cuda-graph-bs 2 4 8 12 16 18 20 22 24 26 \
|
||||
--moe-a2a-backend ascend_fuseep \
|
||||
--deepep-mode auto \
|
||||
--quantization modelslim \
|
||||
--fuseep-mode 2 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
@@ -300,7 +298,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
@@ -321,8 +318,7 @@ python3 -m sglang.launch_server \
|
||||
--max-prefill-tokens 80000 \
|
||||
--cuda-graph-bs 2 4 6 8 \
|
||||
--moe-a2a-backend ascend_fuseep \
|
||||
--deepep-mode auto \
|
||||
--quantization modelslim \
|
||||
--fuseep-mode 2 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
|
||||
@@ -174,7 +174,7 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -186,7 +186,6 @@ python3 -m sglang.launch_server \
|
||||
--node-rank 0 \
|
||||
--attention-backend ascend \
|
||||
--device npu \
|
||||
--quantization modelslim \
|
||||
--max-running-requests 432 \
|
||||
--context-length 8192 \
|
||||
--dtype bfloat16 \
|
||||
@@ -196,6 +195,7 @@ python3 -m sglang.launch_server \
|
||||
--ep-dispatch-algorithm static \
|
||||
--disable-radix-cache \
|
||||
--moe-a2a-backend ascend_fuseep \
|
||||
--fuseep-mode 2 \
|
||||
--speculative-algorithm EAGLE3 \
|
||||
--speculative-draft-model-path $DRAFT_MODEL_PATH \
|
||||
--speculative-num-steps 3 \
|
||||
|
||||
Reference in New Issue
Block a user