minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Refactor] Refactor DeepEP dispatcher (#22822)

Browse Source 
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
Артем Савкин
2026-05-18 04:36:42 +03:00
committed by GitHub
co-authored by gemini-code-assist[bot] Cheng Wan
parent 5147de26e4
commit a080358cac
30 changed files with 302 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/docs/advanced_features/server_arguments.mdx
+6
View File
@@ -1472,6 +1472,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>auto</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>normal</code>, <code>low_latency</code>, <code>auto</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>--deepep-dispatcher-output-dtype</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Select DeepEP dispather output dtype, could be <code>bf16</code>, <code>fp8</code>, <code>int8</code> (only Ascend A2/A3 NPU), <code>nvfp4</code> or <code>auto</code>. Default is <code>auto</code>, which follows a priority order (server argument → deprecated env var → input_global_scale check → dispatcher_output_dtype from quant_config → flashinfer/cutlass backend → NPU BF16 default → GPU FP8 default).</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>auto</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>bf16</code>, <code>fp8</code>, <code>int8</code>, <code>nvfp4</code>, <code>auto</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>--ep-num-redundant-experts</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Allocate this number of redundant experts in expert parallel.</td>
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx
+8 -26
View File
@@ -622,7 +622,6 @@ do
echo "${P_IP[$i]}"
export SGLANG_USE_AG_AFTER_QLORA=1
export HCCL_BUFFSIZE=800
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=131072
@@ -743,7 +742,6 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=1600
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export SGLANG_USE_AG_AFTER_QLORA=1
export HCCL_SOCKET_IFNAME=lo
@@ -870,7 +868,6 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=lo
@@ -996,7 +993,6 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
@@ -1185,7 +1181,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512
MODEL_PATH=xxx
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
@@ -1276,7 +1271,6 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=2600
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=lo
@@ -1383,7 +1377,6 @@ export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56
export HCCL_BUFFSIZE=1200
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
@@ -1476,7 +1469,6 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=3500
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=lo
@@ -1594,7 +1586,6 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=1200
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=xxx
export GLOO_SOCKET_IFNAME=xxx
@@ -1761,7 +1752,6 @@ do
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
# P节点
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \
@@ -2697,7 +2687,6 @@ do
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill \
--host ${P_IP[$i]} --port 8000 --disaggregation-bootstrap-port 8995 --trust-remote-code \
@@ -2794,7 +2783,6 @@ source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env
export SGLANG_SET_CPU_AFFINITY=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=72
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
MODEL_PATH=xxx
@@ -3458,7 +3446,7 @@ export SGLANG_ENABLE_SPEC_V2=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--max-running-requests 78 \
--disable-radix-cache --speculative-draft-model-quantization unquant \
--chunked-prefill-size -1 --max-prefill-tokens 65536 \
@@ -3527,7 +3515,7 @@ export SGLANG_ENABLE_SPEC_V2=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--max-running-requests 120 \
--disable-radix-cache \
--speculative-algorithm EAGLE3 --speculative-draft-model-path xxx \
@@ -3590,7 +3578,7 @@ export SGLANG_ENABLE_SPEC_V2=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--max-running-requests 16 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
@@ -3654,7 +3642,7 @@ export SGLANG_ENABLE_SPEC_V2=1
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--max-running-requests 8 \
--disable-radix-cache \
--speculative-draft-model-quantization unquant \
@@ -3708,7 +3696,6 @@ export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
export DEEPEP_NORMAL_LONG_SEQ_ROUND=5
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000
@@ -3755,7 +3742,6 @@ python3 -m sglang.launch_server --model-path ${MODEL_PATH} \
--mamba-ssm-dtype bfloat16 \
--base-gpu-id 0 \
--speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \
--quantization modelslim \
--moe-a2a-backend deepep --deepep-mode auto \
```
@@ -3803,7 +3789,6 @@ export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
export DEEPEP_NORMAL_LONG_SEQ_ROUND=5
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3000
@@ -3850,7 +3835,6 @@ python3 -m sglang.launch_server --model-path ${MODEL_PATH} \
--mamba-ssm-dtype bfloat16 \
--base-gpu-id 0 \
--speculative-draft-model-path /home/weights/Qwen3-Next-80B-A3B-Instruct \
--quantization modelslim \
--moe-a2a-backend deepep --deepep-mode auto \
```
@@ -3915,7 +3899,7 @@ echo "${LOCAL_HOST2}"
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--disable-radix-cache --mem-fraction-static 0.8 \
--tp-size 1 --dp-size 1 \
--sampling-backend ascend --max-running-requests 8 \
@@ -3992,7 +3976,7 @@ echo "${LOCAL_HOST2}"
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--disable-radix-cache --mem-fraction-static 0.89 \
--tp-size 1 --dp-size 2 \
--sampling-backend ascend --max-running-requests 144 \
@@ -4068,7 +4052,7 @@ echo "${LOCAL_HOST2}"
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--disable-radix-cache --mem-fraction-static 0.9 \
--tp-size 1 \
--max-running-requests 70 \
@@ -4141,7 +4125,7 @@ echo "${LOCAL_HOST2}"
python -m sglang.launch_server --model-path $MODEL_PATH \
--host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \
--attention-backend ascend --device npu --quantization modelslim \
--attention-backend ascend --device npu \
--disable-radix-cache --mem-fraction-static 0.894 \
--tp-size 2 \
--max-running-requests 1 \
@@ -4198,7 +4182,6 @@ export PATH=/usr/local/Ascend/8.5.0/compiler/bishengir/bin:$PATH
export SGLANG_SET_CPU_AFFINITY=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export DEEPEP_NORMAL_LONG_SEQ_ROUND=10
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
@@ -4235,7 +4218,6 @@ python3 -m sglang.launch_server --model-path ${MODEL_PATH} \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--watchdog-timeout 9000 \
--host 127.0.0.1 --port 6699 \
--mem-fraction-static 0.85 \
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx
+2 -8
View File
@@ -15,7 +15,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
#Deepep communication settings
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export HCCL_BUFFSIZE=1600
@@ -33,7 +32,6 @@ python3 -m sglang.launch_server \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--watchdog-timeout 9000 \
--cuda-graph-bs 8 16 24 28 32 \
--mem-fraction-static 0.68 \
@@ -68,7 +66,6 @@ export STREAMS_PER_DEVICE=32
export ASCEND_MF_STORE_URL="tcp://<PREFILL_HOST_IP>:<PORT>"
#Deepep communication settings
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export HCCL_BUFFSIZE=1536
#npu acceleration operator
@@ -90,7 +87,6 @@ python -m sglang.launch_server \
--mem-fraction-static 0.6 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--load-balance-method round_robin \
--max-running-requests 8 \
--context-length 8192 \
@@ -146,7 +142,6 @@ python -m sglang.launch_server \
--max-running-requests 352 \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--moe-a2a-backend deepep \
--enable-dp-attention \
--deepep-mode low_latency \
@@ -218,14 +213,13 @@ do
then
echo "${P_IP[$i]}"
export HCCL_BUFFSIZE=1536
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export TASK_QUEUE_ENABLE=2
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode prefill --host ${P_IP[$i]} \
--port 8000 --disaggregation-bootstrap-port $((8998+$i)) --trust-remote-code --nnodes 1 --node-rank 0 \
--tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu --quantization modelslim \
--tp-size 16 --mem-fraction-static 0.81 --attention-backend ascend --device npu \
--disaggregation-transfer-backend ascend --max-running-requests 8 --context-length 8192 --disable-radix-cache \
--chunked-prefill-size -1 --max-prefill-tokens 28680 --moe-a2a-backend deepep --deepep-mode normal \
--speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \
@@ -251,7 +245,7 @@ do
export GLOO_SOCKET_IFNAME=xxx
python -m sglang.launch_server --model-path ${MODEL_PATH} --disaggregation-mode decode --host ${D_IP[$i]} \
--port 8001 --trust-remote-code --dist-init-addr ${D_IP[0]}:5000 --nnodes 2 --node-rank $i --tp-size 32 --dp-size 32 \
--mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu --quantization modelslim \
--mem-fraction-static 0.815 --max-running-requests 832 --attention-backend ascend --device npu \
--moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head --moe-dense-tp 1 \
--cuda-graph-bs 12 14 16 18 20 22 24 26 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \
--speculative-algorithm NEXTN --speculative-num-steps 2 --speculative-eagle-topk 1 --speculative-num-draft-tokens 3 \
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_environment_variables.mdx
-5
View File
@@ -76,11 +76,6 @@ This document provides a list of commonly used environment variables and aims to
<td>Needs to be enabled when the expert ID to be processed by <br/> DEEPEP contains -1.</td>
<td><code>0</code></td>
</tr>
<tr>
<td><code>DEEP_NORMAL_MODE_USE_INT8_QUANT</code></td>
<td>Quantizes x to int8 and returns (tensor, scales) in dispatch operator.</td>
<td><code>0</code></td>
</tr>
</tbody>
</table>
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx
-1
View File
@@ -116,7 +116,6 @@ python3 -m sglang.launch_server \
--port 8000 \
--served-model-name glm-5 \
--cuda-graph-bs 16 \
--quantization modelslim \
--moe-a2a-backend deepep --deepep-mode auto
```
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx
-3
View File
@@ -93,7 +93,6 @@ python3 -m sglang.launch_server \
--mem-fraction-static 0.7 \
--port 8000 \
--cuda-graph-bs 16 \
--quantization modelslim \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16
@@ -141,7 +140,6 @@ python3 -m sglang.launch_server \
--mem-fraction-static 0.7 \
--port 8000 \
--cuda-graph-bs 16 \
--quantization modelslim \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16
@@ -189,7 +187,6 @@ python3 -m sglang.launch_server \
--mem-fraction-static 0.7 \
--port 8000 \
--cuda-graph-bs 16 \
--quantization modelslim \
--enable-multimodal \
--mm-attention-backend ascend_attn \
--dtype bfloat16
docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx
-6
View File
@@ -66,7 +66,6 @@ export STREAMS_PER_DEVICE=32
export HCCL_BUFFSIZE=1536
export HCCL_OP_EXPANSION_MODE=AIV
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DEEPEP_BF16_DISPATCH=1
python -m sglang.launch_server \
--device npu \
@@ -87,7 +86,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export HCCL_BUFFSIZE=1536
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DEEPEP_BF16_DISPATCH=1
python -m sglang.launch_server \
--model-path Qwen/Qwen3-235B-A22B-Instruct-2507 \
@@ -119,7 +117,6 @@ MODEL_PATH=/root/.cache/modelscope/hub/models/zcgy26/Qwen3-235B-A22B-Instruct-25
```bash Command
export ASCEND_LAUNCH_BLOCKING=1
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export HCCL_BUFFSIZE=1500
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=1024
export DEEPEP_NORMAL_LONG_SEQ_ROUND=128
@@ -132,7 +129,6 @@ python3 -m sglang.launch_server \
--disaggregation-bootstrap-port 8995 \
--attention-backend ascend \
--disable-radix-cache \
--quantization modelslim \
--chunked-prefill-size -1 \
--skip-server-warmup \
--device npu \
@@ -151,7 +147,6 @@ python3 -m sglang.launch_server \
**Decode node:**
```bash Command
export SGLANG_DEEPEP_BF16_DISPATCH=0
export HCCL_BUFFSIZE=4000
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=4096
export DEEPEP_NORMAL_LONG_SEQ_ROUND=16
@@ -165,7 +160,6 @@ python3 -m sglang.launch_server \
--disable-cuda-graph \
--device npu \
--disable-radix-cache \
--quantization modelslim \
--chunked-prefill-size 8192 \
--skip-server-warmup \
--tp-size 16 \
docs_new/docs/references/environment_variables.mdx
-5
View File
@@ -306,11 +306,6 @@ SGLang supports various environment variables that can be used to configure its
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`SGLANG_DEEPEP_BF16_DISPATCH`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use Bfloat16 for dispatch</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`"false"`</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>The maximum number of dispatched tokens on each GPU</td>