[Spec] Remove dead padded_static_len and stale SGLANG_ENABLE_SPEC_V2 references (#30998)

This commit is contained in:
Liangsheng Yin
2026-07-13 15:30:31 -05:00
committed by GitHub
parent 47030b28be
commit e2728ac504
65 changed files with 12 additions and 166 deletions
@@ -26,7 +26,7 @@ your dispatch prompt, or ask for it.
1. **Never modernize.** Env vars, flags, TP values, docker tags, version strings 1. **Never modernize.** Env vars, flags, TP values, docker tags, version strings
are copied verbatim from the legacy page — even when today's defaults differ are copied verbatim from the legacy page — even when today's defaults differ
(e.g. `SGLANG_ENABLE_SPEC_V2=1` is now default; keep it anyway). The recipe (e.g. a flag whose behavior has since become the default; keep it anyway). The recipe
that was verified is the recipe as written. Allowed normalizations are ONLY that was verified is the recipe as written. Allowed normalizations are ONLY
the five alias rewrites in dimension-mapping.md §2 (`launch_server`→`sglang the five alias rewrites in dimension-mapping.md §2 (`launch_server`→`sglang
serve`, `--model`→`--model-path`, `--tp-size`→`--tp`, abbreviated serve`, `--model`→`--model-path`, `--tp-size`→`--tp`, abbreviated
@@ -493,7 +493,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669" export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_BF16_DISPATCH=0 export SGLANG_DEEPEP_BF16_DISPATCH=0
@@ -553,7 +552,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669" export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_BF16_DISPATCH=0 export SGLANG_DEEPEP_BF16_DISPATCH=0
@@ -678,7 +676,6 @@ do
then then
echo "${D_IP[$i]}" echo "${D_IP[$i]}"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=900 export HCCL_BUFFSIZE=900
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
@@ -151,7 +151,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
python3 -m sglang.launch_server \ python3 -m sglang.launch_server \
@@ -279,7 +278,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MLAPO=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -457,7 +455,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
@@ -666,7 +663,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_LM_HEAD_TP=8 export SGLANG_LM_HEAD_TP=8
export SGLANG_NPU_FUSED_MOE_MODE=1 export SGLANG_NPU_FUSED_MOE_MODE=1
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
@@ -871,7 +867,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
@@ -1076,7 +1071,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
@@ -1280,7 +1274,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1 export TASK_QUEUE_ENABLE=1
@@ -148,7 +148,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -347,7 +346,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -546,7 +544,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -84,7 +84,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -280,7 +279,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -481,7 +479,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -690,7 +687,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -893,7 +889,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -1094,7 +1089,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -1295,7 +1289,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -1505,7 +1498,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -1718,7 +1710,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0 export TASK_QUEUE_ENABLE=0
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -278,7 +277,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -478,7 +476,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -678,7 +675,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -873,7 +869,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -1009,7 +1004,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -1121,7 +1115,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -1237,7 +1230,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -1350,7 +1342,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -83,7 +83,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -269,7 +268,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -455,7 +453,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -641,7 +638,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -85,7 +85,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -192,7 +191,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -301,7 +299,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -411,7 +408,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -516,7 +512,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3 export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -175,7 +174,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \ python3 -m sglang.launch_server \
--model-path $MODEL_PATH \ --model-path $MODEL_PATH \
@@ -271,7 +269,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \ python3 -m sglang.launch_server \
--model-path $MODEL_PATH \ --model-path $MODEL_PATH \
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \ python3 -m sglang.launch_server \
--model-path $MODEL_PATH \ --model-path $MODEL_PATH \
@@ -175,7 +174,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -81,7 +81,6 @@ export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_USE_MAX_DP_ATT=1 export SGLANG_USE_MAX_DP_ATT=1
@@ -184,7 +183,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -284,7 +282,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -383,7 +380,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export TRANSFORMERS_VERBOSITY=error export TRANSFORMERS_VERBOSITY=error
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -174,7 +173,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_DEEPGEMM=1 export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -273,7 +271,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_DEEPGEMM=1 export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -198,7 +197,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -309,7 +307,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -419,7 +416,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -534,7 +530,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -649,7 +644,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -763,7 +757,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -877,7 +870,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -992,7 +984,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669 export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -1107,7 +1098,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -75,7 +75,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -174,7 +173,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -274,7 +272,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -376,7 +373,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -472,7 +468,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -566,7 +561,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -660,7 +654,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -82,7 +82,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -180,7 +179,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -280,7 +278,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -377,7 +374,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -480,7 +476,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -576,7 +571,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -673,7 +667,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -773,7 +766,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface> export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -876,7 +868,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
@@ -82,7 +82,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -194,7 +193,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1 export SGLANG_SET_CPU_AFFINITY=1
@@ -312,7 +310,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_WARMUP_TIMEOUT=3600 export SGLANG_WARMUP_TIMEOUT=3600
@@ -200,7 +200,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_SPEC_V2=1
# MTP OVERLAP # MTP OVERLAP
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -332,7 +331,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32 export STREAMS_PER_DEVICE=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
# MTP OVERLAP # MTP OVERLAP
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -478,7 +476,6 @@ do
echo "${D_IP[$i]}" echo "${D_IP[$i]}"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=650 export HCCL_BUFFSIZE=650
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
@@ -122,7 +122,6 @@ export const DeepSeekR1BasicDeployment = () => {
command += ' \\\n --ep 8'; command += ' \\\n --ep 8';
} }
if (strategyValues.includes('mtp')) { if (strategyValues.includes('mtp')) {
command = 'SGLANG_ENABLE_SPEC_V2=1 ' + command;
command += command +=
' \\\n --speculative-algorithm EAGLE' + ' \\\n --speculative-algorithm EAGLE' +
' \\\n --speculative-num-steps 3' + ' \\\n --speculative-num-steps 3' +
@@ -154,7 +154,6 @@ export const DeepSeekV3Deployment = () => {
if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention'; if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention';
if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8'; if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8';
if (strategyArray.includes('mtp')) { if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4'; cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4';
} }
@@ -127,7 +127,6 @@ export const GLM45Deployment = () => {
cmd += ` \\\n --ep 8`; cmd += ` \\\n --ep 8`;
} }
if (strategyArray.includes('mtp')) { if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`; cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
} }
@@ -141,7 +141,6 @@ export const GLM46Deployment = () => {
cmd += ` \\\n --ep 8`; cmd += ` \\\n --ep 8`;
} }
if (strategyArray.includes('mtp')) { if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`; cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
} }
@@ -203,7 +203,6 @@ export const GLM47Deployment = () => {
// MTP / EAGLE speculative decoding (all platforms) // MTP / EAGLE speculative decoding (all platforms)
if (strategyArray.includes('mtp')) { if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`; cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
} }
@@ -117,7 +117,6 @@ export const GLM47FlashDeployment = () => {
cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`; cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`;
} }
if (strategyArray.includes('mtp')) { if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
if (hardware === 'b200') { if (hardware === 'b200') {
cmd += ` \\\n --speculative-draft-attention-backend triton`; cmd += ` \\\n --speculative-draft-attention-backend triton`;
@@ -152,7 +152,7 @@ export const GPTOSSDeployment = () => {
} }
if (speculative === 'enabled') { if (speculative === 'enabled') {
cmd += 'SGLANG_ENABLE_SPEC_V2=1 SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 '; cmd += 'SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 ';
} }
cmd += 'python -m sglang.launch_server \\\n'; cmd += 'python -m sglang.launch_server \\\n';
@@ -117,7 +117,6 @@ export const Hunyuan3PreviewDeployment = () => {
const enableSpec = values.speculative === 'enabled' && !isXeon; const enableSpec = values.speculative === 'enabled' && !isXeon;
let cmd = ''; let cmd = '';
if (enableSpec) cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
cmd += 'sglang serve \\\n'; cmd += 'sglang serve \\\n';
cmd += ` --model-path ${modelName}`; cmd += ` --model-path ${modelName}`;
cmd += ` \\\n --tp ${tpValue}`; cmd += ` \\\n --tp ${tpValue}`;
@@ -87,8 +87,7 @@ export const InternS2PreviewDeployment = () => {
flags.push(' --host 0.0.0.0'); flags.push(' --host 0.0.0.0');
flags.push(' --port 30000'); flags.push(' --port 30000');
const envPrefix = mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 \\\n' : ''; return `sglang serve \\\n${flags.join(' \\\n')}`;
return `${envPrefix}sglang serve \\\n${flags.join(' \\\n')}`;
}; };
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' }; const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
@@ -102,7 +102,7 @@ export const MiMoV2FlashDeployment = () => {
const commandPrefix = isMI355X const commandPrefix = isMI355X
? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0 ' ? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0 '
: 'SGLANG_ENABLE_SPEC_V2=1'; : '';
const tpSize = isMI355X ? 4 : 8; const tpSize = isMI355X ? 4 : 8;
let cmd = `${commandPrefix}sglang serve \\\n`; let cmd = `${commandPrefix}sglang serve \\\n`;
@@ -15,7 +15,7 @@ export const MiMoV25Deployment = () => {
// GB300 → tp=4, dp=1, single-node, FP8 (Blackwell: vision fa4) // GB300 → tp=4, dp=1, single-node, FP8 (Blackwell: vision fa4)
// //
// Optional toggles: // Optional toggles:
// EAGLE MTP — adds --speculative-* flags + SGLANG_ENABLE_SPEC_V2=1. // EAGLE MTP — adds --speculative-* flags.
// DeepEP — Hopper only (Blackwell uses flashinfer_trtllm). Adds // DeepEP — Hopper only (Blackwell uses flashinfer_trtllm). Adds
// --moe-a2a-backend deepep + --moe-dense-tp-size 1 // --moe-a2a-backend deepep + --moe-dense-tp-size 1
// (and --ep on Pro) + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256. // (and --ep on Pro) + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256.
@@ -300,7 +300,6 @@ export const MiMoV25Deployment = () => {
if (isPro && blackwell && multinode) { if (isPro && blackwell && multinode) {
envVars.push("NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1"); envVars.push("NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1");
} }
if (useMtp) envVars.push("SGLANG_ENABLE_SPEC_V2=1");
if (useDeepep) envVars.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"); if (useDeepep) envVars.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
// ---- flags ---- // ---- flags ----
@@ -79,8 +79,7 @@ export const Nemotron3SuperDeployment = () => {
const modelPath = MODEL_PATHS[model] || MODEL_PATHS['bf16']; const modelPath = MODEL_PATHS[model] || MODEL_PATHS['bf16'];
const specV2Env = values.mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 ' : ''; let cmd = `sglang serve \\\n`;
let cmd = `${specV2Env}sglang serve \\\n`;
cmd += ` --model-path ${modelPath} \\\n`; cmd += ` --model-path ${modelPath} \\\n`;
cmd += ` --trust-remote-code \\\n`; cmd += ` --trust-remote-code \\\n`;
cmd += ` --tp ${tp} \\\n`; cmd += ` --tp ${tp} \\\n`;
@@ -177,8 +177,8 @@ export const Qwen36Deployment = () => {
// NVFP4: nvidia/Qwen3.6-27B-NVFP4 on Blackwell (B200/B300). Follows the exact command // NVFP4: nvidia/Qwen3.6-27B-NVFP4 on Blackwell (B200/B300). Follows the exact command
// shape from the checkpoint's docs — explicit --tp-size 1, --attention-backend trtllm_mha, // shape from the checkpoint's docs — explicit --tp-size 1, --attention-backend trtllm_mha,
// new-style --mamba-radix-cache-strategy, and explicit --host/--port (no --mem-fraction-static // new-style --mamba-radix-cache-strategy, and explicit --host/--port (no
// or SGLANG_ENABLE_SPEC_V2 prefix). Reasoning / tool-call parsers still follow their toggles. // --mem-fraction-static). Reasoning / tool-call parsers still follow their toggles.
if (quantization === 'nvfp4') { if (quantization === 'nvfp4') {
let cmd = `sglang serve --model-path nvidia/Qwen3.6-${sizeConfig.baseName}-NVFP4`; let cmd = `sglang serve --model-path nvidia/Qwen3.6-${sizeConfig.baseName}-NVFP4`;
cmd += ` \\\n --tp-size ${hwConfig.tp} --attention-backend trtllm_mha`; cmd += ` \\\n --tp-size ${hwConfig.tp} --attention-backend trtllm_mha`;
@@ -199,12 +199,7 @@ export const Qwen36Deployment = () => {
const quantSuffix = quantization === 'fp8' ? '-FP8' : ''; const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
const modelName = `Qwen/Qwen3.6-${sizeConfig.baseName}${quantSuffix}`; const modelName = `Qwen/Qwen3.6-${sizeConfig.baseName}${quantSuffix}`;
let cmd = ''; let cmd = `sglang serve --model-path ${modelName}`;
if (speculative === 'enabled') {
cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
}
cmd += `sglang serve --model-path ${modelName}`;
if (hardware === 'xeon') { if (hardware === 'xeon') {
cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`; cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`;
} }
@@ -732,7 +732,6 @@ class TboForwardBatchPreparer:
"is_prefill_only", "is_prefill_only",
"spec_algorithm", "spec_algorithm",
"capture_hidden_mode", "capture_hidden_mode",
"padded_static_len",
"split_index", # for split prefill "split_index", # for split prefill
"orig_seq_lens", # only used by qwen-1m, thus not care "orig_seq_lens", # only used by qwen-1m, thus not care
"return_pooled_hidden_states", "return_pooled_hidden_states",
@@ -230,8 +230,6 @@ class LogitsMetadata:
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None
# The gather mode for DP attention # The gather mode for DP attention
dp_padding_mode: Optional[DpPaddingMode] = None dp_padding_mode: Optional[DpPaddingMode] = None
# for padding
padded_static_len: int = -1
# Whether this batch is prefill-only (no token generation needed) # Whether this batch is prefill-only (no token generation needed)
is_prefill_only: bool = False is_prefill_only: bool = False
@@ -279,7 +277,6 @@ class LogitsMetadata:
top_logprobs_nums=forward_batch.top_logprobs_nums, top_logprobs_nums=forward_batch.top_logprobs_nums,
token_ids_logprobs=forward_batch.token_ids_logprobs, token_ids_logprobs=forward_batch.token_ids_logprobs,
extend_input_logprob_token_ids_gpu=forward_batch.extend_input_logprob_token_ids_gpu, extend_input_logprob_token_ids_gpu=forward_batch.extend_input_logprob_token_ids_gpu,
padded_static_len=forward_batch.padded_static_len,
is_prefill_only=forward_batch.is_prefill_only, is_prefill_only=forward_batch.is_prefill_only,
global_num_tokens_gpu=forward_batch.global_num_tokens_gpu, global_num_tokens_gpu=forward_batch.global_num_tokens_gpu,
dp_local_start_pos=forward_batch.dp_local_start_pos, dp_local_start_pos=forward_batch.dp_local_start_pos,
@@ -527,21 +524,7 @@ class LogitsProcessor(nn.Module):
and not logits_metadata.extend_return_logprob and not logits_metadata.extend_return_logprob
): ):
# Prefill without input logprobs. # Prefill without input logprobs.
if logits_metadata.padded_static_len < 0:
last_index = torch.cumsum(logits_metadata.extend_seq_lens, dim=0) - 1 last_index = torch.cumsum(logits_metadata.extend_seq_lens, dim=0) - 1
else:
# If padding_static length is 5 and extended_seq_lens is [2, 3],
# then our batch looks like [t00, t01, p, p, p, t10, t11, t12, p, p]
# and this retrieves t01 and t12, which are the valid last tokens
idx = torch.arange(
len(logits_metadata.extend_seq_lens),
device=logits_metadata.extend_seq_lens.device,
)
last_index = (
idx * logits_metadata.padded_static_len
+ logits_metadata.extend_seq_lens
- 1
)
pruned_states = hidden_states[last_index] pruned_states = hidden_states[last_index]
if hidden_states_before_norm is not None: if hidden_states_before_norm is not None:
pruned_states_before_norm = hidden_states_before_norm[last_index] pruned_states_before_norm = hidden_states_before_norm[last_index]
@@ -502,9 +502,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime
global_dp_buffer_len: Optional[int] = None global_dp_buffer_len: Optional[int] = None
# For padding
padded_static_len: int = -1 # -1 if not padded
# For Qwen2-VL # For Qwen2-VL
mrope_positions: torch.Tensor = None mrope_positions: torch.Tensor = None
@@ -130,7 +130,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
self.capture_hidden_mode = CaptureHiddenMode.LAST self.capture_hidden_mode = CaptureHiddenMode.LAST
self.capture_bs, _ = get_batch_sizes_to_capture(model_runner) self.capture_bs, _ = get_batch_sizes_to_capture(model_runner)
self.padded_static_len = -1
# Size cuda-graph buffers by num_draft_tokens (full tree width), not # Size cuda-graph buffers by num_draft_tokens (full tree width), not
# num_steps + 1, or topk > 1 draft-extend overflows them. # num_steps + 1, or topk > 1 draft-extend overflows them.
@@ -395,7 +394,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
spec_algorithm=self.model_runner.spec_algorithm, spec_algorithm=self.model_runner.spec_algorithm,
spec_info=spec_info, spec_info=spec_info,
capture_hidden_mode=CaptureHiddenMode.LAST, capture_hidden_mode=CaptureHiddenMode.LAST,
padded_static_len=self.padded_static_len,
) )
if self.buffers.dsa_seed_topk_capture is not None: if self.buffers.dsa_seed_topk_capture is not None:
@@ -156,7 +156,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
self.capture_hidden_mode = CaptureHiddenMode.FULL self.capture_hidden_mode = CaptureHiddenMode.FULL
self.capture_bs, _ = get_batch_sizes_to_capture(model_runner) self.capture_bs, _ = get_batch_sizes_to_capture(model_runner)
self.padded_static_len = -1
# Fixed window: every step extends each request by the same number of # Fixed window: every step extends each request by the same number of
# tokens, which lets all steps share one buffer set. # tokens, which lets all steps share one buffer set.
@@ -301,7 +300,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
capture_hidden_mode=capture_mode, capture_hidden_mode=capture_mode,
extend_seq_lens=extend_seq_lens, extend_seq_lens=extend_seq_lens,
extend_seq_lens_cpu=extend_seq_lens_cpu, extend_seq_lens_cpu=extend_seq_lens_cpu,
padded_static_len=self.padded_static_len,
extend_start_loc=extend_start_loc, extend_start_loc=extend_start_loc,
extend_num_tokens=self.num_tokens_per_req * bs, extend_num_tokens=self.num_tokens_per_req * bs,
num_token_non_padded_cpu=self.num_tokens_per_req * bs, num_token_non_padded_cpu=self.num_tokens_per_req * bs,
@@ -229,7 +229,6 @@ def _make_static_forward_batch(raw_batch, static_num_tokens: int, device: str):
input_ids=input_ids, input_ids=input_ids,
positions=positions, positions=positions,
out_cache_loc=out_cache_loc, out_cache_loc=out_cache_loc,
padded_static_len=static_num_tokens,
num_token_non_padded_cpu=raw_num_tokens, num_token_non_padded_cpu=raw_num_tokens,
) )
@@ -125,7 +125,6 @@ class TestDeepSeekR1MXFP4TP4MTPMI35x(unittest.TestCase):
"SGLANG_USE_AITER": "1", "SGLANG_USE_AITER": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
# Retired on current main, but kept to mirror the reported launch. # Retired on current main, but kept to mirror the reported launch.
"SGLANG_ENABLE_SPEC_V2": "1",
"ROCM_QUICK_REDUCE_QUANTIZATION": "NONE", "ROCM_QUICK_REDUCE_QUANTIZATION": "NONE",
"SGLANG_AITER_FP8_PREFILL_ATTN": "1", "SGLANG_AITER_FP8_PREFILL_ATTN": "1",
"SGLANG_AITER_MLA_PERSIST": "1", "SGLANG_AITER_MLA_PERSIST": "1",
@@ -23,7 +23,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
} }
@@ -22,7 +22,6 @@ QWEN3_6_27B_64K_PREFIX_ENVS = {
"HCCL_SOCKET_IFNAME": "lo", "HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1", "GDN_ATTN_BACKEND_TRITON": "1",
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1", "GDN_ATTN_BACKEND_TRITON": "1",
@@ -23,7 +23,6 @@ ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400", "HCCL_BUFFSIZE": "400",
@@ -44,7 +44,6 @@ GLM_5_1_PD_SEP_DECODE_ENVS = {
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200", "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200",
"SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1", "SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"HCCL_BUFFSIZE": "200", "HCCL_BUFFSIZE": "200",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16",
"TASK_QUEUE_ENABLE": "0", "TASK_QUEUE_ENABLE": "0",
@@ -27,7 +27,6 @@ ENVS = {
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
"HCCL_BUFFSIZE": "4400", "HCCL_BUFFSIZE": "4400",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME, "HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME, "GLOO_SOCKET_IFNAME": NIC_NAME,
@@ -26,7 +26,6 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"TASK_QUEUE_ENABLE": "1", "TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_FUSED_MOE_MODE": "2", "SGLANG_NPU_FUSED_MOE_MODE": "2",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000",
@@ -29,7 +29,6 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
"HCCL_BUFFSIZE": "1024", "HCCL_BUFFSIZE": "1024",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_FUSED_MOE_MODE": "2", "SGLANG_NPU_FUSED_MOE_MODE": "2",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800", "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800",
@@ -26,7 +26,6 @@ QWEN3_8B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
} }
@@ -24,7 +24,6 @@ QWEN3_8B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
} }
QWEN3_8B_OTHER_ARGS = [ QWEN3_8B_OTHER_ARGS = [
@@ -27,7 +27,6 @@ QWEN3_30B_A3B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400", "HCCL_BUFFSIZE": "400",
@@ -23,7 +23,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
} }
@@ -26,7 +26,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1", "SGLANG_NPU_USE_DEEPGEMM": "1",
@@ -26,7 +26,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1", "SGLANG_NPU_USE_DEEPGEMM": "1",
@@ -23,7 +23,6 @@ QWEN3_6_27B_1024_ENVS = {
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1", "SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_NPU_PROFILING": "1", "SGLANG_NPU_PROFILING": "1",
"SGLANG_NPU_PROFILING_STAGE": "prefill", "SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
@@ -22,7 +22,6 @@ QWEN3_6_27B_1080P_ENVS = {
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1", "SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_NPU_PROFILING": "0", "SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill", "SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
@@ -21,7 +21,6 @@ QWEN3_6_27B_64K_PREFIX_ENVS = {
"HCCL_SOCKET_IFNAME": "lo", "HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1", "GDN_ATTN_BACKEND_TRITON": "1",
@@ -25,7 +25,6 @@ QWEN3_6_27B_3K5_1K5_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130",
@@ -21,7 +21,6 @@ QWEN3_6_27B_64K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
} }
@@ -22,7 +22,6 @@ QWEN3_6_27B_128K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
@@ -21,7 +21,6 @@ QWEN3_6_27B_16K_1k_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
@@ -22,7 +22,6 @@ QWEN3_6_27B_64K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_128K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
@@ -21,7 +21,6 @@ QWEN3_6_35B_A3B_128K_PREFIX_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
} }
@@ -22,7 +22,6 @@ QWEN3_6_35B_A3B_64K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "1", "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "1",
@@ -26,7 +26,6 @@ QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
"GLOO_SOCKET_IFNAME": "lo", "GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1", "SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1", "GDN_ATTN_BACKEND_TRITON": "1",
@@ -31,7 +31,6 @@ QWEN3_NEXT_80B_A3B_ENVS = {
"ASCEND_USE_FIA": "1", "ASCEND_USE_FIA": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "0", "SGLANG_NPU_USE_MULTI_STREAM": "0",
"SGLANG_WARMUP_TIMEOUT": "3600", "SGLANG_WARMUP_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"FORCE_DRAFT_MODEL_NON_QUANT": "1", "FORCE_DRAFT_MODEL_NON_QUANT": "1",
"HCCL_BUFFSIZE": "2000", "HCCL_BUFFSIZE": "2000",