[Spec] Remove dead padded_static_len and stale SGLANG_ENABLE_SPEC_V2 references (#30998)
This commit is contained in:
@@ -493,7 +493,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
|
||||
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
@@ -553,7 +552,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584
|
||||
export ASCEND_MF_STORE_URL="tcp://127.0.0.1:24669"
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
|
||||
export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
@@ -678,7 +676,6 @@ do
|
||||
then
|
||||
echo "${D_IP[$i]}"
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export HCCL_BUFFSIZE=900
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -151,7 +151,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
@@ -279,7 +278,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=56
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
@@ -457,7 +455,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -666,7 +663,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_LM_HEAD_TP=8
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
@@ -871,7 +867,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -1076,7 +1071,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
@@ -1280,7 +1274,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=1
|
||||
|
||||
|
||||
@@ -148,7 +148,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -347,7 +346,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -546,7 +544,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=8
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
|
||||
@@ -84,7 +84,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -280,7 +279,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -481,7 +479,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -690,7 +687,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -893,7 +889,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -1094,7 +1089,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -1295,7 +1289,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
@@ -1505,7 +1498,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
@@ -1718,7 +1710,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1
|
||||
export TASK_QUEUE_ENABLE=0
|
||||
|
||||
|
||||
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -278,7 +277,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -478,7 +476,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -678,7 +675,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -873,7 +869,6 @@ do
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
|
||||
@@ -1009,7 +1004,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -1121,7 +1115,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -1237,7 +1230,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_MLAPO=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -1350,7 +1342,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
@@ -83,7 +83,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -269,7 +268,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -455,7 +453,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -641,7 +638,6 @@ export SGLANG_DEEPEP_BF16_DISPATCH=0
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
|
||||
export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
@@ -85,7 +85,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -192,7 +191,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=140000
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -301,7 +299,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=160000
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -411,7 +408,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -516,7 +512,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_EXTERNAL_MODEL_PACKAGE=custom_eagle3
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
|
||||
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -175,7 +174,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
@@ -271,7 +269,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
|
||||
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
|
||||
python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH \
|
||||
@@ -175,7 +174,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=188416
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_FUSED_MOE_MODE=2
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -81,7 +81,6 @@ export INF_NAN_MODE_FORCE_DISABLE=1
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:False
|
||||
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_USE_MAX_DP_ATT=1
|
||||
@@ -184,7 +183,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -284,7 +282,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -383,7 +380,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export TRANSFORMERS_VERBOSITY=error
|
||||
|
||||
|
||||
@@ -77,7 +77,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -174,7 +173,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_DEEPGEMM=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
@@ -273,7 +271,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_NPU_USE_DEEPGEMM=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
|
||||
@@ -87,7 +87,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -198,7 +197,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -309,7 +307,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -419,7 +416,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -534,7 +530,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -649,7 +644,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -763,7 +757,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -877,7 +870,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -992,7 +984,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export SGLANG_ZBAL_BOOTSTRAP_URL=tcp://127.0.0.1:24669
|
||||
@@ -1107,7 +1098,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
@@ -75,7 +75,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -174,7 +173,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=150
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -274,7 +272,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -376,7 +373,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=130
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -472,7 +468,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -566,7 +561,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -660,7 +654,6 @@ export GLOO_SOCKET_IFNAME=<network-interface>
|
||||
export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100
|
||||
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
|
||||
@@ -82,7 +82,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -180,7 +179,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=10
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -280,7 +278,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=20
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -377,7 +374,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -480,7 +476,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -576,7 +571,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -673,7 +667,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
@@ -773,7 +766,6 @@ export HCCL_OP_EXPANSION_MODE=AIV
|
||||
export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
@@ -876,7 +868,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
export STREAMS_PER_DEVICE=32
|
||||
|
||||
|
||||
-3
@@ -82,7 +82,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -194,7 +193,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_SET_CPU_AFFINITY=1
|
||||
@@ -312,7 +310,6 @@ export HCCL_SOCKET_IFNAME=<network-interface>
|
||||
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=400
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=0
|
||||
export SGLANG_WARMUP_TIMEOUT=3600
|
||||
|
||||
@@ -200,7 +200,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
# MTP OVERLAP
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
@@ -332,7 +331,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh
|
||||
export STREAMS_PER_DEVICE=32
|
||||
export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600
|
||||
# MTP OVERLAP
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
|
||||
export SGLANG_NPU_USE_MULTI_STREAM=1
|
||||
@@ -478,7 +476,6 @@ do
|
||||
echo "${D_IP[$i]}"
|
||||
|
||||
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
|
||||
export SGLANG_ENABLE_SPEC_V2=1
|
||||
export HCCL_BUFFSIZE=650
|
||||
|
||||
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32
|
||||
|
||||
Reference in New Issue
Block a user