[Spec] Remove dead padded_static_len and stale SGLANG_ENABLE_SPEC_V2 references (#30998)
This commit is contained in:
@@ -26,7 +26,7 @@ your dispatch prompt, or ask for it.
|
||||
|
||||
1. **Never modernize.** Env vars, flags, TP values, docker tags, version strings
|
||||
are copied verbatim from the legacy page — even when today's defaults differ
|
||||
(e.g. `SGLANG_ENABLE_SPEC_V2=1` is now default; keep it anyway). The recipe
|
||||
(e.g. a flag whose behavior has since become the default; keep it anyway). The recipe
|
||||
that was verified is the recipe as written. Allowed normalizations are ONLY
|
||||
the five alias rewrites in dimension-mapping.md §2 (`launch_server`→`sglang
|
||||
serve`, `--model`→`--model-path`, `--tp-size`→`--tp`, abbreviated
|
||||
|
||||
@@ -493,7 +493,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
|
||||
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
@@ -553,7 +552,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
|
||||
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
@@ -678,7 +676,6 @@ do
|
||||
then
|
||||
echo "${D_IP[$i]}"
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export HCCL_BUFFSIZE=900
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -151,7 +151,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
@@ -279,7 +278,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
@@ -457,7 +455,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -666,7 +663,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_LM_HEAD_TP=8
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
@@ -871,7 +867,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -1076,7 +1071,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -1280,7 +1274,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
|
||||
@@ -148,7 +148,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -347,7 +346,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -546,7 +544,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
|
||||
@@ -84,7 +84,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -280,7 +279,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -481,7 +479,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -690,7 +687,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -893,7 +889,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -1094,7 +1089,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -1295,7 +1289,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
@@ -1505,7 +1498,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -1718,7 +1710,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
|
||||
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -278,7 +277,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -478,7 +476,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -678,7 +675,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -873,7 +869,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -1009,7 +1004,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -1121,7 +1115,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -1237,7 +1230,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -1350,7 +1342,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
@@ -83,7 +83,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -269,7 +268,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -455,7 +453,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -641,7 +638,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
@@ -85,7 +85,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -192,7 +191,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -301,7 +299,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -411,7 +408,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -516,7 +512,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
|
||||
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -175,7 +174,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
@@ -271,7 +269,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
|
||||
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
@@ -175,7 +174,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -81,7 +81,6 @@ export INF_NAN_MODE_FORCE_DISABLE=1
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
|
||||
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_USE_MAX_DP_ATT=1
|
||||
@@ -184,7 +183,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -284,7 +282,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -383,7 +380,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export TRANSFORMERS_VERBOSITY=error
|
||||
|
||||
|
||||
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -174,7 +173,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_DEEPGEMM=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
@@ -273,7 +271,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_DEEPGEMM=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -198,7 +197,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -309,7 +307,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -419,7 +416,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -534,7 +530,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -649,7 +644,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -763,7 +757,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -877,7 +870,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -992,7 +984,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -1107,7 +1098,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
@@ -75,7 +75,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -174,7 +173,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -274,7 +272,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -376,7 +373,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -472,7 +468,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -566,7 +561,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -660,7 +654,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
|
||||
@@ -82,7 +82,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -180,7 +179,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -280,7 +278,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -377,7 +374,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -480,7 +476,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -576,7 +571,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -673,7 +667,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -773,7 +766,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -876,7 +868,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
-3
@@ -82,7 +82,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -194,7 +193,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -312,7 +310,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_WARMUP_TIMEOUT=3600
|
||||
|
||||
@@ -200,7 +200,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
# MTP OVERLAP
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
@@ -332,7 +331,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
# MTP OVERLAP
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
@@ -478,7 +476,6 @@ do
|
||||
echo "${D_IP[$i]}"
|
||||
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export HCCL_BUFFSIZE=650
|
||||
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
|
||||
@@ -122,7 +122,6 @@ export const DeepSeekR1BasicDeployment = () => {
|
||||
command += ' \\\n --ep 8';
|
||||
}
|
||||
if (strategyValues.includes('mtp')) {
|
||||
command = 'SGLANG_ENABLE_SPEC_V2=1 ' + command;
|
||||
command +=
|
||||
' \\\n --speculative-algorithm EAGLE' +
|
||||
' \\\n --speculative-num-steps 3' +
|
||||
|
||||
@@ -154,7 +154,6 @@ export const DeepSeekV3Deployment = () => {
|
||||
if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention';
|
||||
if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8';
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4';
|
||||
}
|
||||
|
||||
|
||||
@@ -127,7 +127,6 @@ export const GLM45Deployment = () => {
|
||||
cmd += ` \\\n --ep 8`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
|
||||
@@ -141,7 +141,6 @@ export const GLM46Deployment = () => {
|
||||
cmd += ` \\\n --ep 8`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
|
||||
@@ -203,7 +203,6 @@ export const GLM47Deployment = () => {
|
||||
|
||||
// MTP / EAGLE speculative decoding (all platforms)
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
|
||||
@@ -117,7 +117,6 @@ export const GLM47FlashDeployment = () => {
|
||||
cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
|
||||
if (hardware === 'b200') {
|
||||
cmd += ` \\\n --speculative-draft-attention-backend triton`;
|
||||
|
||||
@@ -152,7 +152,7 @@ export const GPTOSSDeployment = () => {
|
||||
}
|
||||
|
||||
if (speculative === 'enabled') {
|
||||
cmd += 'SGLANG_ENABLE_SPEC_V2=1 SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 ';
|
||||
cmd += 'SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 ';
|
||||
}
|
||||
|
||||
cmd += 'python -m sglang.launch_server \\\n';
|
||||
|
||||
@@ -117,7 +117,6 @@ export const Hunyuan3PreviewDeployment = () => {
|
||||
const enableSpec = values.speculative === 'enabled' && !isXeon;
|
||||
|
||||
let cmd = '';
|
||||
if (enableSpec) cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
|
||||
cmd += 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
@@ -87,8 +87,7 @@ export const InternS2PreviewDeployment = () => {
|
||||
flags.push(' --host 0.0.0.0');
|
||||
flags.push(' --port 30000');
|
||||
|
||||
const envPrefix = mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 \\\n' : '';
|
||||
return `${envPrefix}sglang serve \\\n${flags.join(' \\\n')}`;
|
||||
return `sglang serve \\\n${flags.join(' \\\n')}`;
|
||||
};
|
||||
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
|
||||
@@ -101,11 +101,11 @@ export const MiMoV2FlashDeployment = () => {
|
||||
}
|
||||
|
||||
const commandPrefix = isMI355X
|
||||
? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0'
|
||||
: 'SGLANG_ENABLE_SPEC_V2=1';
|
||||
? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0 '
|
||||
: '';
|
||||
const tpSize = isMI355X ? 4 : 8;
|
||||
|
||||
let cmd = `${commandPrefix} sglang serve \\\n`;
|
||||
let cmd = `${commandPrefix}sglang serve \\\n`;
|
||||
cmd += ` --model-path ${modelPath} \\\n`;
|
||||
cmd += ` --trust-remote-code \\\n`;
|
||||
cmd += ` --tp-size ${tpSize}`;
|
||||
|
||||
@@ -15,7 +15,7 @@ export const MiMoV25Deployment = () => {
|
||||
// GB300 → tp=4, dp=1, single-node, FP8 (Blackwell: vision fa4)
|
||||
//
|
||||
// Optional toggles:
|
||||
// EAGLE MTP — adds --speculative-* flags + SGLANG_ENABLE_SPEC_V2=1.
|
||||
// EAGLE MTP — adds --speculative-* flags.
|
||||
// DeepEP — Hopper only (Blackwell uses flashinfer_trtllm). Adds
|
||||
// --moe-a2a-backend deepep + --moe-dense-tp-size 1
|
||||
// (and --ep on Pro) + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256.
|
||||
@@ -300,7 +300,6 @@ export const MiMoV25Deployment = () => {
|
||||
if (isPro && blackwell && multinode) {
|
||||
envVars.push("NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1");
|
||||
}
|
||||
if (useMtp) envVars.push("SGLANG_ENABLE_SPEC_V2=1");
|
||||
if (useDeepep) envVars.push("SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256");
|
||||
|
||||
// ---- flags ----
|
||||
|
||||
@@ -79,8 +79,7 @@ export const Nemotron3SuperDeployment = () => {
|
||||
|
||||
const modelPath = MODEL_PATHS[model] || MODEL_PATHS['bf16'];
|
||||
|
||||
const specV2Env = values.mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 ' : '';
|
||||
let cmd = `${specV2Env}sglang serve \\\n`;
|
||||
let cmd = `sglang serve \\\n`;
|
||||
cmd += ` --model-path ${modelPath} \\\n`;
|
||||
cmd += ` --trust-remote-code \\\n`;
|
||||
cmd += ` --tp ${tp} \\\n`;
|
||||
|
||||
@@ -177,8 +177,8 @@ export const Qwen36Deployment = () => {
|
||||
|
||||
// NVFP4: nvidia/Qwen3.6-27B-NVFP4 on Blackwell (B200/B300). Follows the exact command
|
||||
// shape from the checkpoint's docs — explicit --tp-size 1, --attention-backend trtllm_mha,
|
||||
// new-style --mamba-radix-cache-strategy, and explicit --host/--port (no --mem-fraction-static
|
||||
// or SGLANG_ENABLE_SPEC_V2 prefix). Reasoning / tool-call parsers still follow their toggles.
|
||||
// new-style --mamba-radix-cache-strategy, and explicit --host/--port (no
|
||||
// --mem-fraction-static). Reasoning / tool-call parsers still follow their toggles.
|
||||
if (quantization === 'nvfp4') {
|
||||
let cmd = `sglang serve --model-path nvidia/Qwen3.6-${sizeConfig.baseName}-NVFP4`;
|
||||
cmd += ` \\\n --tp-size ${hwConfig.tp} --attention-backend trtllm_mha`;
|
||||
@@ -199,12 +199,7 @@ export const Qwen36Deployment = () => {
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `Qwen/Qwen3.6-${sizeConfig.baseName}${quantSuffix}`;
|
||||
|
||||
let cmd = '';
|
||||
if (speculative === 'enabled') {
|
||||
cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
|
||||
}
|
||||
|
||||
cmd += `sglang serve --model-path ${modelName}`;
|
||||
let cmd = `sglang serve --model-path ${modelName}`;
|
||||
if (hardware === 'xeon') {
|
||||
cmd += ` \\\n --device cpu \\\n --disable-overlap-schedule`;
|
||||
}
|
||||
|
||||
@@ -732,7 +732,6 @@ class TboForwardBatchPreparer:
|
||||
"is_prefill_only",
|
||||
"spec_algorithm",
|
||||
"capture_hidden_mode",
|
||||
"padded_static_len",
|
||||
"split_index", # for split prefill
|
||||
"orig_seq_lens", # only used by qwen-1m, thus not care
|
||||
"return_pooled_hidden_states",
|
||||
|
||||
@@ -230,8 +230,6 @@ class LogitsMetadata:
|
||||
global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None
|
||||
# The gather mode for DP attention
|
||||
dp_padding_mode: Optional[DpPaddingMode] = None
|
||||
# for padding
|
||||
padded_static_len: int = -1
|
||||
|
||||
# Whether this batch is prefill-only (no token generation needed)
|
||||
is_prefill_only: bool = False
|
||||
@@ -279,7 +277,6 @@ class LogitsMetadata:
|
||||
top_logprobs_nums=forward_batch.top_logprobs_nums,
|
||||
token_ids_logprobs=forward_batch.token_ids_logprobs,
|
||||
extend_input_logprob_token_ids_gpu=forward_batch.extend_input_logprob_token_ids_gpu,
|
||||
padded_static_len=forward_batch.padded_static_len,
|
||||
is_prefill_only=forward_batch.is_prefill_only,
|
||||
global_num_tokens_gpu=forward_batch.global_num_tokens_gpu,
|
||||
dp_local_start_pos=forward_batch.dp_local_start_pos,
|
||||
@@ -527,21 +524,7 @@ class LogitsProcessor(nn.Module):
|
||||
and not logits_metadata.extend_return_logprob
|
||||
):
|
||||
# Prefill without input logprobs.
|
||||
if logits_metadata.padded_static_len < 0:
|
||||
last_index = torch.cumsum(logits_metadata.extend_seq_lens, dim=0) - 1
|
||||
else:
|
||||
# If padding_static length is 5 and extended_seq_lens is [2, 3],
|
||||
# then our batch looks like [t00, t01, p, p, p, t10, t11, t12, p, p]
|
||||
# and this retrieves t01 and t12, which are the valid last tokens
|
||||
idx = torch.arange(
|
||||
len(logits_metadata.extend_seq_lens),
|
||||
device=logits_metadata.extend_seq_lens.device,
|
||||
)
|
||||
last_index = (
|
||||
idx * logits_metadata.padded_static_len
|
||||
+ logits_metadata.extend_seq_lens
|
||||
- 1
|
||||
)
|
||||
pruned_states = hidden_states[last_index]
|
||||
if hidden_states_before_norm is not None:
|
||||
pruned_states_before_norm = hidden_states_before_norm[last_index]
|
||||
|
||||
@@ -502,9 +502,6 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime
|
||||
global_dp_buffer_len: Optional[int] = None
|
||||
|
||||
# For padding
|
||||
padded_static_len: int = -1 # -1 if not padded
|
||||
|
||||
# For Qwen2-VL
|
||||
mrope_positions: torch.Tensor = None
|
||||
|
||||
|
||||
@@ -130,7 +130,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self.capture_hidden_mode = CaptureHiddenMode.LAST
|
||||
|
||||
self.capture_bs, _ = get_batch_sizes_to_capture(model_runner)
|
||||
self.padded_static_len = -1
|
||||
|
||||
# Size cuda-graph buffers by num_draft_tokens (full tree width), not
|
||||
# num_steps + 1, or topk > 1 draft-extend overflows them.
|
||||
@@ -395,7 +394,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
spec_algorithm=self.model_runner.spec_algorithm,
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=CaptureHiddenMode.LAST,
|
||||
padded_static_len=self.padded_static_len,
|
||||
)
|
||||
|
||||
if self.buffers.dsa_seed_topk_capture is not None:
|
||||
|
||||
@@ -156,7 +156,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self.capture_hidden_mode = CaptureHiddenMode.FULL
|
||||
|
||||
self.capture_bs, _ = get_batch_sizes_to_capture(model_runner)
|
||||
self.padded_static_len = -1
|
||||
|
||||
# Fixed window: every step extends each request by the same number of
|
||||
# tokens, which lets all steps share one buffer set.
|
||||
@@ -301,7 +300,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
capture_hidden_mode=capture_mode,
|
||||
extend_seq_lens=extend_seq_lens,
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu,
|
||||
padded_static_len=self.padded_static_len,
|
||||
extend_start_loc=extend_start_loc,
|
||||
extend_num_tokens=self.num_tokens_per_req * bs,
|
||||
num_token_non_padded_cpu=self.num_tokens_per_req * bs,
|
||||
|
||||
@@ -229,7 +229,6 @@ def _make_static_forward_batch(raw_batch, static_num_tokens: int, device: str):
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
out_cache_loc=out_cache_loc,
|
||||
padded_static_len=static_num_tokens,
|
||||
num_token_non_padded_cpu=raw_num_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -125,7 +125,6 @@ class TestDeepSeekR1MXFP4TP4MTPMI35x(unittest.TestCase):
|
||||
"SGLANG_USE_AITER": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
# Retired on current main, but kept to mirror the reported launch.
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"ROCM_QUICK_REDUCE_QUANTIZATION": "NONE",
|
||||
"SGLANG_AITER_FP8_PREFILL_ATTN": "1",
|
||||
"SGLANG_AITER_MLA_PERSIST": "1",
|
||||
|
||||
@@ -23,7 +23,6 @@ QWEN3_32B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ QWEN3_6_27B_64K_PREFIX_ENVS = {
|
||||
"HCCL_SOCKET_IFNAME": "lo",
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"GDN_ATTN_BACKEND_TRITON": "1",
|
||||
|
||||
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"GDN_ATTN_BACKEND_TRITON": "1",
|
||||
|
||||
-1
@@ -23,7 +23,6 @@ ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
|
||||
"HCCL_BUFFSIZE": "400",
|
||||
|
||||
-1
@@ -44,7 +44,6 @@ GLM_5_1_PD_SEP_DECODE_ENVS = {
|
||||
"SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200",
|
||||
"SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"HCCL_BUFFSIZE": "200",
|
||||
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16",
|
||||
"TASK_QUEUE_ENABLE": "0",
|
||||
|
||||
-1
@@ -27,7 +27,6 @@ ENVS = {
|
||||
"DEEP_NORMAL_MODE_USE_INT8_QUANT": "1",
|
||||
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64",
|
||||
"HCCL_BUFFSIZE": "4400",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"HCCL_SOCKET_IFNAME": NIC_NAME,
|
||||
"GLOO_SOCKET_IFNAME": NIC_NAME,
|
||||
|
||||
-1
@@ -26,7 +26,6 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = {
|
||||
"TASK_QUEUE_ENABLE": "1",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_NPU_FUSED_MOE_MODE": "2",
|
||||
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000",
|
||||
|
||||
-1
@@ -29,7 +29,6 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = {
|
||||
"HCCL_BUFFSIZE": "1024",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_NPU_FUSED_MOE_MODE": "2",
|
||||
"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800",
|
||||
|
||||
-1
@@ -26,7 +26,6 @@ QWEN3_8B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50",
|
||||
}
|
||||
|
||||
-1
@@ -24,7 +24,6 @@ QWEN3_8B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
}
|
||||
|
||||
QWEN3_8B_OTHER_ARGS = [
|
||||
|
||||
-1
@@ -27,7 +27,6 @@ QWEN3_30B_A3B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
|
||||
"HCCL_BUFFSIZE": "400",
|
||||
|
||||
-1
@@ -23,7 +23,6 @@ QWEN3_32B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200",
|
||||
}
|
||||
|
||||
-1
@@ -26,7 +26,6 @@ QWEN3_32B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
|
||||
"SGLANG_NPU_USE_DEEPGEMM": "1",
|
||||
|
||||
-1
@@ -26,7 +26,6 @@ QWEN3_32B_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
|
||||
"SGLANG_NPU_USE_DEEPGEMM": "1",
|
||||
|
||||
-1
@@ -23,7 +23,6 @@ QWEN3_6_27B_1024_ENVS = {
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_NPU_PROFILING": "1",
|
||||
"SGLANG_NPU_PROFILING_STAGE": "prefill",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
|
||||
-1
@@ -22,7 +22,6 @@ QWEN3_6_27B_1080P_ENVS = {
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_VIT_ENABLE_CUDA_GRAPH": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_NPU_PROFILING": "0",
|
||||
"SGLANG_NPU_PROFILING_STAGE": "prefill",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
|
||||
-1
@@ -21,7 +21,6 @@ QWEN3_6_27B_64K_PREFIX_ENVS = {
|
||||
"HCCL_SOCKET_IFNAME": "lo",
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"GDN_ATTN_BACKEND_TRITON": "1",
|
||||
|
||||
-1
@@ -25,7 +25,6 @@ QWEN3_6_27B_3K5_1K5_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130",
|
||||
|
||||
-1
@@ -21,7 +21,6 @@ QWEN3_6_27B_64K_1K_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
}
|
||||
|
||||
|
||||
-1
@@ -22,7 +22,6 @@ QWEN3_6_27B_128K_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
|
||||
|
||||
-1
@@ -21,7 +21,6 @@ QWEN3_6_27B_16K_1k_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100",
|
||||
|
||||
-1
@@ -22,7 +22,6 @@ QWEN3_6_27B_64K_1K_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
|
||||
|
||||
-1
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_128K_1K_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "20",
|
||||
|
||||
-1
@@ -21,7 +21,6 @@ QWEN3_6_35B_A3B_128K_PREFIX_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "30",
|
||||
|
||||
-1
@@ -23,7 +23,6 @@ QWEN3_6_35B_A3B_3K5_1K5_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
}
|
||||
|
||||
-1
@@ -22,7 +22,6 @@ QWEN3_6_35B_A3B_64K_1K_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "1",
|
||||
|
||||
-1
@@ -26,7 +26,6 @@ QWEN3_6_35B_A3B_64K_PREFIX_ENVS = {
|
||||
"GLOO_SOCKET_IFNAME": "lo",
|
||||
"HCCL_OP_EXPANSION_MODE": "AIV",
|
||||
"SGLANG_SET_CPU_AFFINITY": "1",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0",
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"GDN_ATTN_BACKEND_TRITON": "1",
|
||||
|
||||
-1
@@ -31,7 +31,6 @@ QWEN3_NEXT_80B_A3B_ENVS = {
|
||||
"ASCEND_USE_FIA": "1",
|
||||
"SGLANG_NPU_USE_MULTI_STREAM": "0",
|
||||
"SGLANG_WARMUP_TIMEOUT": "3600",
|
||||
"SGLANG_ENABLE_SPEC_V2": "1",
|
||||
"SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1",
|
||||
"FORCE_DRAFT_MODEL_NON_QUANT": "1",
|
||||
"HCCL_BUFFSIZE": "2000",
|
||||
|
||||
Reference in New Issue
Block a user