[Spec] Remove dead padded_static_len and stale SGLANG_ENABLE_SPEC_V2 references (#30998)

This commit is contained in:
Liangsheng Yin
2026-07-13 15:30:31 -05:00
committed by GitHub
parent 47030b28be
commit e2728ac504
65 changed files with 12 additions and 166 deletions
@@ -26,7 +26,7 @@ your dispatch prompt, or ask for it.
1. **Never modernize.** Env vars, flags, TP values, docker tags, version strings
are copied verbatim from the legacy page — even when today's defaults differ
(e.g. `SGLANG_ENABLE_SPEC_V2=1` is now default; keep it anyway). The recipe
(e.g. a flag whose behavior has since become the default; keep it anyway). The recipe
that was verified is the recipe as written. Allowed normalizations are ONLY
the five alias rewrites in dimension-mapping.md §2 (`launch_server``sglang
serve`, `--model``--model-path`, `--tp-size``--tp`, abbreviated
@@ -493,7 +493,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
@@ -553,7 +552,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_BF16_DISPATCH=0
@@ -678,7 +676,6 @@ do
then
echo "${D_IP[$i]}"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=900
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
export TASK_QUEUE_ENABLE=1
@@ -151,7 +151,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export TASK_QUEUE_ENABLE=1
python3 -m sglang.launch_server \
@@ -279,7 +278,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -457,7 +455,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -666,7 +663,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_LM_HEAD_TP=8
export SGLANG_NPU_FUSED_MOE_MODE=1
export TASK_QUEUE_ENABLE=1
@@ -871,7 +867,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -1076,7 +1071,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -1280,7 +1274,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=1
@@ -148,7 +148,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
@@ -347,7 +346,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
@@ -546,7 +544,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export TASK_QUEUE_ENABLE=0
@@ -84,7 +84,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -280,7 +279,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -481,7 +479,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -690,7 +687,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -893,7 +889,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1094,7 +1089,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1295,7 +1289,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1505,7 +1498,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -1718,7 +1710,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
export TASK_QUEUE_ENABLE=0
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -278,7 +277,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -478,7 +476,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -678,7 +675,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -873,7 +869,6 @@ do
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -1009,7 +1004,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -1121,7 +1115,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -1237,7 +1230,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_MLAPO=1
export SGLANG_NPU_USE_MULTI_STREAM=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -1350,7 +1342,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -83,7 +83,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -269,7 +268,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -455,7 +453,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -641,7 +638,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -85,7 +85,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_SET_CPU_AFFINITY=1
@@ -192,7 +191,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
@@ -301,7 +299,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
@@ -411,7 +408,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
@@ -516,7 +512,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_SET_CPU_AFFINITY=1
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -175,7 +174,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
@@ -271,7 +269,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
python3 -m sglang.launch_server \
--model-path $MODEL_PATH \
@@ -175,7 +174,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_FUSED_MOE_MODE=2
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -81,7 +81,6 @@ export INF_NAN_MODE_FORCE_DISABLE=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_USE_MAX_DP_ATT=1
@@ -184,7 +183,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -284,7 +282,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -383,7 +380,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export TRANSFORMERS_VERBOSITY=error
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -174,7 +173,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -273,7 +271,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_NPU_USE_DEEPGEMM=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -198,7 +197,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -309,7 +307,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -419,7 +416,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -534,7 +530,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -649,7 +644,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -763,7 +757,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -877,7 +870,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -992,7 +984,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
@@ -1107,7 +1098,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -75,7 +75,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -174,7 +173,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -274,7 +272,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -376,7 +373,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -472,7 +468,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -566,7 +561,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -660,7 +654,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_SET_CPU_AFFINITY=1
@@ -82,7 +82,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -180,7 +179,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -280,7 +278,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -377,7 +374,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -480,7 +476,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -576,7 +571,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -673,7 +667,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -773,7 +766,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -876,7 +868,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_SET_CPU_AFFINITY=1
export STREAMS_PER_DEVICE=32
@@ -82,7 +82,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1
@@ -194,7 +193,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_SET_CPU_AFFINITY=1
@@ -312,7 +310,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
export SGLANG_NPU_USE_MULTI_STREAM=0
export SGLANG_WARMUP_TIMEOUT=3600
@@ -200,7 +200,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
export SGLANG_ENABLE_SPEC_V2=1
# MTP OVERLAP
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -332,7 +331,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
export STREAMS_PER_DEVICE=32
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
# MTP OVERLAP
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_NPU_USE_MULTI_STREAM=1
@@ -478,7 +476,6 @@ do
echo "${D_IP[$i]}"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export HCCL_BUFFSIZE=650
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
@@ -122,7 +122,6 @@ export const DeepSeekR1BasicDeployment = () => {
command += ' \\\n --ep 8';
}
if (strategyValues.includes('mtp')) {
command = 'SGLANG_ENABLE_SPEC_V2=1 ' + command;
command +=
' \\\n --speculative-algorithm EAGLE' +
' \\\n --speculative-num-steps 3' +
@@ -154,7 +154,6 @@ export const DeepSeekV3Deployment = () => {
if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention';
if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8';
if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4';
}
@@ -127,7 +127,6 @@ export const GLM45Deployment = () => {
cmd += ` \\\n --ep 8`;
}
if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
}
@@ -141,7 +141,6 @@ export const GLM46Deployment = () => {
cmd += ` \\\n --ep 8`;
}
if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
}
@@ -203,7 +203,6 @@ export const GLM47Deployment = () => {
// MTP / EAGLE speculative decoding (all platforms)
if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
}
@@ -117,7 +117,6 @@ export const GLM47FlashDeployment = () => {
cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`;
}
if (strategyArray.includes('mtp')) {
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
if (hardware === 'b200') {
cmd += ` \\\n --speculative-draft-attention-backend triton`;
@@ -152,7 +152,7 @@ export const GPTOSSDeployment = () => {
}
if (speculative === 'enabled') {
cmd += 'SGLANG_ENABLE_SPEC_V2=1 SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 ';
cmd += 'SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 ';
}
cmd += 'python -m sglang.launch_server \\\n';
@@ -117,7 +117,6 @@ export const Hunyuan3PreviewDeployment = () => {
const enableSpec = values.speculative === 'enabled' && !isXeon;
let cmd = '';
if (enableSpec) cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
cmd += 'sglang serve \\\n';
cmd += ` --model-path ${modelName}`;
cmd += ` \\\n --tp ${tpValue}`;
@@ -87,8 +87,7 @@ export const InternS2PreviewDeployment = () => {
flags.push(' --host 0.0.0.0');
flags.push(' --port 30000');
const envPrefix = mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 \\\n' : '';
return `${envPrefix}sglang serve \\\n${flags.join(' \\\n')}`;
return `sglang serve \\\n${flags.join(' \\\n')}`;
};
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
@@ -101,11 +101,11 @@ export const MiMoV2FlashDeployment = () => {
}
const commandPrefix = isMI355X
? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0'
: 'SGLANG_ENABLE_SPEC_V2=1';
? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0 '
: '';
const tpSize = isMI355X ? 4 : 8;
let cmd = `${commandPrefix} sglang serve \\\n`;
let cmd = `${commandPrefix}sglang serve \\\n`;
cmd += ` --model-path ${modelPath} \\\n`;
cmd += ` --trust-remote-code \\\n`;
cmd += ` --tp-size ${tpSize}`;
@@ -15,7 +15,7 @@ export const MiMoV25Deployment = () => {
// GB300 → tp=4, dp=1, single-node, FP8 (Blackwell: vision fa4)
//
// Optional toggles:
// EAGLE MTP — adds --speculative-* flags + SGLANG_ENABLE_SPEC_V2=1.
// EAGLE MTP — adds --speculative-* flags.
// DeepEP — Hopper only (Blackwell uses flashinfer_trtllm). Adds
// --moe-a2a-backend deepep + --moe-dense-tp-size 1
// (and --ep on Pro) + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256.
@@ -300,7 +300,6 @@ export const MiMoV25Deployment = () => {
if (isPro && blackwell && multinode) {
envVars.push("NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1");
}
if (useMtp) envVars.push("SGLANG_ENABLE_SPEC_V2=1");
if (useDeepep) envVars.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
// ---- flags ----
@@ -79,8 +79,7 @@ export const Nemotron3SuperDeployment = () => {
const modelPath = MODEL_PATHS[model] || MODEL_PATHS['bf16'];
const specV2Env = values.mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 ' : '';
let cmd = `${specV2Env}sglang serve \\\n`;
let cmd = `sglang serve \\\n`;
cmd += ` --model-path ${modelPath} \\\n`;
cmd += ` --trust-remote-code \\\n`;
cmd += ` --tp ${tp} \\\n`;
@@ -177,8 +177,8 @@ export const Qwen36Deployment = () => {
// NVFP4: nvidia/Qwen3.6-27B-NVFP4 on Blackwell (B200/B300). Follows the exact command
// shape from the checkpoint's docs — explicit --tp-size 1, --attention-backend trtllm_mha,
// new-style --mamba-radix-cache-strategy, and explicit --host/--port (no --mem-fraction-static
// or SGLANG_ENABLE_SPEC_V2 prefix). Reasoning / tool-call parsers still follow their toggles.
// new-style --mamba-radix-cache-strategy, and explicit --host/--port (no
// --mem-fraction-static). Reasoning / tool-call parsers still follow their toggles.
if (quantization === 'nvfp4') {
let cmd = `sglang serve --model-path nvidia/Qwen3.6-${sizeConfig.baseName}-NVFP4`;
cmd += ` \\\n --tp-size ${hwConfig.tp} --attention-backend trtllm_mha`;
@@ -199,12 +199,7 @@ export const Qwen36Deployment = () => {
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
const modelName = `Qwen/Qwen3.6-${sizeConfig.baseName}${quantSuffix}`;
let cmd = '';
if (speculative === 'enabled') {
cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
}
cmd += `sglang serve --model-path ${modelName}`;
let cmd = `sglang serve --model-path ${modelName}`;
if (hardware === 'xeon') {
cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`;
}
@@ -732,7 +732,6 @@ class TboForwardBatchPreparer:
"is_prefill_only",
"spec_algorithm",
"capture_hidden_mode",
"padded_static_len",
"split_index", # for split prefill
"orig_seq_lens", # only used by qwen-1m, thus not care
"return_pooled_hidden_states",
@@ -230,8 +230,6 @@ class LogitsMetadata:
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None
# The gather mode for DP attention
dp_padding_mode: Optional[DpPaddingMode] = None
# for padding
padded_static_len: int = -1
# Whether this batch is prefill-only (no token generation needed)
is_prefill_only: bool = False
@@ -279,7 +277,6 @@ class LogitsMetadata:
top_logprobs_nums=forward_batch.top_logprobs_nums,
token_ids_logprobs=forward_batch.token_ids_logprobs,
extend_input_logprob_token_ids_gpu=forward_batch.extend_input_logprob_token_ids_gpu,
padded_static_len=forward_batch.padded_static_len,
is_prefill_only=forward_batch.is_prefill_only,
global_num_tokens_gpu=forward_batch.global_num_tokens_gpu,
dp_local_start_pos=forward_batch.dp_local_start_pos,
@@ -527,21 +524,7 @@ class LogitsProcessor(nn.Module):
and not logits_metadata.extend_return_logprob
):
# Prefill without input logprobs.
if logits_metadata.padded_static_len < 0:
last_index = torch.cumsum(logits_metadata.extend_seq_lens, dim=0) - 1
else:
# If padding_static length is 5 and extended_seq_lens is [2, 3],
# then our batch looks like [t00, t01, p, p, p, t10, t11, t12, p, p]
# and this retrieves t01 and t12, which are the valid last tokens
idx = torch.arange(
len(logits_metadata.extend_seq_lens),
device=logits_metadata.extend_seq_lens.device,
)
last_index = (
idx * logits_metadata.padded_static_len
+ logits_metadata.extend_seq_lens
- 1
)
pruned_states = hidden_states[last_index]
if hidden_states_before_norm is not None:
pruned_states_before_norm = hidden_states_before_norm[last_index]
@@ -502,9 +502,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime
global_dp_buffer_len: Optional[int] = None
# For padding
padded_static_len: int = -1 # -1 if not padded
# For Qwen2-VL
mrope_positions: torch.Tensor = None
@@ -130,7 +130,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
self.capture_hidden_mode = CaptureHiddenMode.LAST
self.capture_bs, _ = get_batch_sizes_to_capture(model_runner)
self.padded_static_len = -1
# Size cuda-graph buffers by num_draft_tokens (full tree width), not
# num_steps + 1, or topk > 1 draft-extend overflows them.
@@ -395,7 +394,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
spec_algorithm=self.model_runner.spec_algorithm,
spec_info=spec_info,
capture_hidden_mode=CaptureHiddenMode.LAST,
padded_static_len=self.padded_static_len,
)
if self.buffers.dsa_seed_topk_capture is not None:
@@ -156,7 +156,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
self.capture_hidden_mode = CaptureHiddenMode.FULL
self.capture_bs, _ = get_batch_sizes_to_capture(model_runner)
self.padded_static_len = -1
# Fixed window: every step extends each request by the same number of
# tokens, which lets all steps share one buffer set.
@@ -301,7 +300,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
capture_hidden_mode=capture_mode,
extend_seq_lens=extend_seq_lens,
extend_seq_lens_cpu=extend_seq_lens_cpu,
padded_static_len=self.padded_static_len,
extend_start_loc=extend_start_loc,
extend_num_tokens=self.num_tokens_per_req * bs,
num_token_non_padded_cpu=self.num_tokens_per_req * bs,
@@ -229,7 +229,6 @@ def _make_static_forward_batch(raw_batch, static_num_tokens: int, device: str):
input_ids=input_ids,
positions=positions,
out_cache_loc=out_cache_loc,
padded_static_len=static_num_tokens,
num_token_non_padded_cpu=raw_num_tokens,
)
@@ -125,7 +125,6 @@ class TestDeepSeekR1MXFP4TP4MTPMI35x(unittest.TestCase):
"SGLANG_USE_AITER": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
# Retired on current main, but kept to mirror the reported launch.
"SGLANG_ENABLE_SPEC_V2": "1",
"ROCM_QUICK_REDUCE_QUANTIZATION": "NONE",
"SGLANG_AITER_FP8_PREFILL_ATTN": "1",
"SGLANG_AITER_MLA_PERSIST": "1",
@@ -23,7 +23,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
}
@@ -22,7 +22,6 @@ QWEN3_6_27B_64K_PREFIX_ENVS = {
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
@@ -23,7 +23,6 @@ ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400",
@@ -44,7 +44,6 @@ GLM_5_1_PD_SEP_DECODE_ENVS = {
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200",
"SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"HCCL_BUFFSIZE": "200",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16",
"TASK_QUEUE_ENABLE": "0",
@@ -27,7 +27,6 @@ ENVS = {
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
"HCCL_BUFFSIZE": "4400",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"HCCL_SOCKET_IFNAME": NIC_NAME,
"GLOO_SOCKET_IFNAME": NIC_NAME,
@@ -26,7 +26,6 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
"TASK_QUEUE_ENABLE": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_FUSED_MOE_MODE": "2",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000",
@@ -29,7 +29,6 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
"HCCL_BUFFSIZE": "1024",
"ASCEND_USE_FIA": "1",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_NPU_FUSED_MOE_MODE": "2",
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800",
@@ -26,7 +26,6 @@ QWEN3_8B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
}
@@ -24,7 +24,6 @@ QWEN3_8B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
}
QWEN3_8B_OTHER_ARGS = [
@@ -27,7 +27,6 @@ QWEN3_30B_A3B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
"HCCL_BUFFSIZE": "400",
@@ -23,7 +23,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
}
@@ -26,7 +26,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1",
@@ -26,7 +26,6 @@ QWEN3_32B_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
"SGLANG_NPU_USE_DEEPGEMM": "1",
@@ -23,7 +23,6 @@ QWEN3_6_27B_1024_ENVS = {
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_NPU_PROFILING": "1",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
@@ -22,7 +22,6 @@ QWEN3_6_27B_1080P_ENVS = {
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_NPU_PROFILING": "0",
"SGLANG_NPU_PROFILING_STAGE": "prefill",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
@@ -21,7 +21,6 @@ QWEN3_6_27B_64K_PREFIX_ENVS = {
"HCCL_SOCKET_IFNAME": "lo",
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
@@ -25,7 +25,6 @@ QWEN3_6_27B_3K5_1K5_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130",
@@ -21,7 +21,6 @@ QWEN3_6_27B_64K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
}
@@ -22,7 +22,6 @@ QWEN3_6_27B_128K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
@@ -21,7 +21,6 @@ QWEN3_6_27B_16K_1k_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
@@ -22,7 +22,6 @@ QWEN3_6_27B_64K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_128K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
@@ -21,7 +21,6 @@ QWEN3_6_35B_A3B_128K_PREFIX_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
}
@@ -22,7 +22,6 @@ QWEN3_6_35B_A3B_64K_1K_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"ASCEND_USE_FIA": "1",
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "1",
@@ -26,7 +26,6 @@ QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
"GLOO_SOCKET_IFNAME": "lo",
"HCCL_OP_EXPANSION_MODE": "AIV",
"SGLANG_SET_CPU_AFFINITY": "1",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
"ASCEND_USE_FIA": "1",
"GDN_ATTN_BACKEND_TRITON": "1",
@@ -31,7 +31,6 @@ QWEN3_NEXT_80B_A3B_ENVS = {
"ASCEND_USE_FIA": "1",
"SGLANG_NPU_USE_MULTI_STREAM": "0",
"SGLANG_WARMUP_TIMEOUT": "3600",
"SGLANG_ENABLE_SPEC_V2": "1",
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
"HCCL_BUFFSIZE": "2000",