[Refactor] Let model MLPs select their parallel group (#42468)

This commit is contained in:
Cheng Wan
2026-10-07 16:18:14 -07:00
committed by GitHub
parent 4c5130ac5a
commit 961d7290fe
15 changed files with 281 additions and 222 deletions
+12 -19
View File
@@ -47,6 +47,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
@@ -102,11 +103,11 @@ class BailingMoEMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: Optional[bool] = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
config.hidden_size,
@@ -114,8 +115,7 @@ class BailingMoEMLP(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -124,8 +124,7 @@ class BailingMoEMLP(nn.Module):
reduce_results=reduce_results,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if config.hidden_act != "silu":
@@ -137,7 +136,7 @@ class BailingMoEMLP(nn.Module):
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
) -> torch.Tensor:
if (self.tp_size == 1) and hidden_states.shape[0] == 0:
if self.is_replicated and hidden_states.shape[0] == 0:
return hidden_states
gate_up, _ = self.gate_up_proj(hidden_states)
@@ -325,11 +324,9 @@ class BailingMoESparseMoeBlock(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
if get_moe_a2a_backend().is_deepep()
else {}
),
parallel_group="replicated"
if get_moe_a2a_backend().is_deepep()
else "tp",
)
# dispatcher
if get_moe_a2a_backend().is_deepep():
@@ -674,17 +671,13 @@ class BailingMoEBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = BailingMoEMLP(
intermediate_size=config.intermediate_size,
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+11 -18
View File
@@ -36,6 +36,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
@@ -304,14 +305,13 @@ class BailingMLP(nn.Module):
prefix: str = "",
swiglu_limit: Optional[float] = None,
padded_intermediate_size: Optional[int] = None,
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.config = config
self.swiglu_limit = swiglu_limit
self.tp_size = tp_size if tp_size is not None else get_parallel().tp_size
self.intermediate_size = intermediate_size
self.padded_intermediate_size = padded_intermediate_size or intermediate_size
@@ -322,8 +322,7 @@ class BailingMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
self.padded_intermediate_size,
@@ -332,10 +331,11 @@ class BailingMLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.tp_size = self.gate_up_proj.tp_size
if self.padded_intermediate_size > self.intermediate_size:
self.padded_size_per_partition = (
self.padded_intermediate_size // self.tp_size
@@ -628,10 +628,8 @@ class BailingMoE(nn.Module):
# because MoE output is already complete after EP combine.
# Using tp_size=1 ensures shared output is also complete,
# so no all-reduce is needed at the MoE level.
shared_tp_kwargs = {}
shared_tp_size = self.tp_size
if self._enable_a2a_moe:
shared_tp_kwargs = dict(tp_rank=0, tp_size=1)
shared_tp_size = 1
padded_intermediate_size = self._compute_padded_intermediate_size(
intermediate_size, quant_config, shared_tp_size
@@ -645,7 +643,7 @@ class BailingMoE(nn.Module):
prefix=f"{prefix}.shared_experts",
swiglu_limit=self.share_expert_swiglu_limit,
padded_intermediate_size=padded_intermediate_size,
**shared_tp_kwargs,
parallel_group="replicated" if self._enable_a2a_moe else "tp",
)
else:
self.shared_experts = None
@@ -1088,10 +1086,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
is_next_layer_sparse = not (self.expert_num == 1) and (
self.layer_id + 1 >= config.first_k_dense_replace
)
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
if self.expert_num == 1:
self.mlp = BailingMLP(
@@ -1100,8 +1095,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
config=config,
quant_config=quant_config,
prefix=prefix,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
else:
@@ -1121,8 +1115,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
config=config,
quant_config=quant_config,
prefix=prefix,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
+10 -15
View File
@@ -97,6 +97,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
@@ -282,13 +283,13 @@ class DeepseekV2MLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
swiglu_limit: Optional[float] = None,
allow_fused_down: Optional[bool] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.swiglu_limit = swiglu_limit
# The fused down path calls deepgemm directly and runs no all-reduce.
self.allow_fused_down = (
@@ -301,8 +302,7 @@ class DeepseekV2MLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -311,8 +311,7 @@ class DeepseekV2MLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if not hasattr(self.gate_up_proj, "weight") and hasattr(
self.gate_up_proj, "weight_packed"
@@ -338,7 +337,7 @@ class DeepseekV2MLP(nn.Module):
gemm_output_zero_allocator: BumpAllocator = None,
gateup_pre_quant: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
):
if (self.tp_size == 1) and x.shape[0] == 0:
if self.is_replicated and x.shape[0] == 0:
return x
if (
@@ -827,7 +826,7 @@ class DeepseekV2MoE(nn.Module):
reduce_results=False,
swiglu_limit=getattr(config, "swiglu_limit", None),
prefix=add_prefix("shared_experts", prefix),
**(dict(tp_rank=0, tp_size=1) if _shared_expert_use_tp1 else {}),
parallel_group="replicated" if _shared_expert_use_tp1 else "tp",
)
# Flags must be set before weight load so
# process_weights_after_loading sees them and builds the
@@ -2611,18 +2610,14 @@ class DeepseekV2DecoderLayer(nn.Module):
reduce_results=False,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = DeepseekV2MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
swiglu_limit=getattr(config, "swiglu_limit", None),
reduce_results=False,
allow_fused_down=False,
@@ -65,6 +65,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
@@ -194,11 +195,11 @@ class Dots3MLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -206,8 +207,7 @@ class Dots3MLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -216,8 +216,7 @@ class Dots3MLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -226,7 +225,7 @@ class Dots3MLP(nn.Module):
self.act_fn = SiluAndMul()
def forward(self, x, forward_batch=None):
if (self.tp_size == 1) and x.shape[0] == 0:
if self.is_replicated and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
@@ -337,12 +336,10 @@ class Dots3MoE(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
if get_moe_a2a_backend().is_deepep()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else {}
),
parallel_group="replicated"
if get_moe_a2a_backend().is_deepep()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else "tp",
)
is_packed_weight = quant_config is not None and (
quant_config.get_name()
@@ -1396,18 +1393,14 @@ class Dots3DecoderLayer(nn.Module):
is_nextn=is_nextn,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = Dots3MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+2 -6
View File
@@ -198,18 +198,14 @@ class Ernie4DecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = Ernie4MLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+19 -26
View File
@@ -52,6 +52,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
@@ -164,11 +165,11 @@ class Glm4MoeMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -176,8 +177,7 @@ class Glm4MoeMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -186,8 +186,7 @@ class Glm4MoeMLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -200,7 +199,7 @@ class Glm4MoeMLP(nn.Module):
x,
forward_batch=None,
):
if (self.tp_size == 1) and x.shape[0] == 0:
if self.is_replicated and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
@@ -479,18 +478,16 @@ class Glm4MoeSparseMoeBlock(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
if get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mooncake()
or get_moe_a2a_backend().is_nixl()
or get_moe_a2a_backend().is_mori()
or get_moe_a2a_backend().is_ascend_fuseep()
or get_moe_a2a_backend().is_flashinfer()
or get_moe_a2a_backend().is_flashinfer_megamoe()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else {}
),
parallel_group="replicated"
if get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mooncake()
or get_moe_a2a_backend().is_nixl()
or get_moe_a2a_backend().is_mori()
or get_moe_a2a_backend().is_ascend_fuseep()
or get_moe_a2a_backend().is_flashinfer()
or get_moe_a2a_backend().is_flashinfer_megamoe()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else "tp",
)
is_packed_weight = (
hasattr(self.shared_experts.gate_up_proj.quant_method, "quant_config")
@@ -749,18 +746,14 @@ class Glm4MoeDecoderLayer(nn.Module):
alt_stream=alt_stream,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = Glm4MoeMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+18 -22
View File
@@ -46,7 +46,11 @@ from sglang.srt.layers.layer_boundary import (
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
@@ -97,11 +101,11 @@ class Glm4MoeLiteMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -109,8 +113,7 @@ class Glm4MoeLiteMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -119,8 +122,7 @@ class Glm4MoeLiteMLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -133,7 +135,7 @@ class Glm4MoeLiteMLP(nn.Module):
x,
forward_batch=None,
):
if (self.tp_size == 1) and x.shape[0] == 0:
if self.is_replicated and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
@@ -249,13 +251,11 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
if get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mooncake()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else {}
),
parallel_group="replicated"
if get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mooncake()
or should_use_flashinfer_cutlass_moe_fp4_allgather()
else "tp",
)
is_packed_weight = hasattr(
self.shared_experts.gate_up_proj.quant_method, "quant_config"
@@ -460,18 +460,14 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
is_nextn=is_nextn,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = Glm4MoeLiteMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+2 -6
View File
@@ -742,18 +742,14 @@ class Glm5NextDecoderLayer(nn.Module):
reduce_results=False,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = Glm5NextMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
swiglu_limit=config.swiglu_limit,
reduce_results=False,
allow_fused_down=False,
+12 -19
View File
@@ -43,6 +43,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
@@ -106,11 +107,11 @@ class LLaDA2MoeMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: Optional[bool] = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
config.hidden_size,
@@ -118,8 +119,7 @@ class LLaDA2MoeMLP(nn.Module):
bias=config.use_bias,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -128,8 +128,7 @@ class LLaDA2MoeMLP(nn.Module):
reduce_results=reduce_results,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if config.hidden_act != "silu":
@@ -141,7 +140,7 @@ class LLaDA2MoeMLP(nn.Module):
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
) -> torch.Tensor:
if (self.tp_size == 1) and hidden_states.shape[0] == 0:
if self.is_replicated and hidden_states.shape[0] == 0:
return hidden_states
gate_up, _ = self.gate_up_proj(hidden_states)
@@ -294,11 +293,9 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
if get_moe_a2a_backend().is_deepep()
else {}
),
parallel_group="replicated"
if get_moe_a2a_backend().is_deepep()
else "tp",
)
# dispatcher
if get_moe_a2a_backend().is_deepep():
@@ -611,17 +608,13 @@ class LLaDA2MoeBlock(nn.Module):
prefix=add_prefix("mlp", prefix),
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = LLaDA2MoeMLP(
intermediate_size=config.intermediate_size,
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+9 -14
View File
@@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
@@ -293,11 +294,11 @@ class MiMoV2MLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.tp_size = tp_size
self.is_replicated = parallel_group == "replicated"
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
@@ -305,8 +306,7 @@ class MiMoV2MLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -315,8 +315,7 @@ class MiMoV2MLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -329,7 +328,7 @@ class MiMoV2MLP(nn.Module):
x,
forward_batch: ForwardBatch = None,
):
if (self.tp_size == 1) and x.shape[0] == 0:
if self.is_replicated and x.shape[0] == 0:
return x
gate_up, _ = self.gate_up_proj(x)
@@ -830,18 +829,14 @@ class MiMoV2DecoderLayer(nn.Module):
layer_id=layer_id,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = MiMoV2MLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+2 -6
View File
@@ -101,18 +101,14 @@ class MiMoV2MTPLayer(nn.Module):
self.is_layer_sparse = False
is_next_layer_sparse = False
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = MiMoV2MLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
+8 -13
View File
@@ -48,6 +48,7 @@ from sglang.srt.layers.layer_boundary import (
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
from sglang.srt.layers.linear import (
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
@@ -273,8 +274,8 @@ class MiniMaxM3MLP(nn.Module):
prefix: str = "",
reduce_results: bool = True,
intermediate_size: int = None,
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
hidden_size = config.hidden_size
@@ -286,8 +287,7 @@ class MiniMaxM3MLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -296,8 +296,7 @@ class MiniMaxM3MLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act == "silu":
self.act_fn = SiluAndMul()
@@ -406,7 +405,7 @@ class MiniMaxM3MoE(nn.Module):
prefix=add_prefix("shared_experts", prefix),
reduce_results=False,
intermediate_size=intermediate_size,
**(dict(tp_rank=0, tp_size=1) if shared_experts_tp1 else {}),
parallel_group="replicated" if shared_experts_tp1 else "tp",
)
else:
self.shared_experts = None
@@ -1482,17 +1481,13 @@ class MiniMaxM3DecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = MiniMaxM3MLP(
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
intermediate_size=config.dense_intermediate_size,
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+10 -17
View File
@@ -29,6 +29,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
@@ -180,8 +181,8 @@ class SarvamMoEMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
reduce_results: bool = True,
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -190,8 +191,7 @@ class SarvamMoEMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -200,8 +200,7 @@ class SarvamMoEMLP(nn.Module):
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
reduce_results=reduce_results,
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -307,11 +306,9 @@ class SarvamMoESparseMoeBlock(nn.Module):
reduce_results=False,
# The shared output joins the routed output's TP sum; where that
# output is already complete on each rank, it is not TP-sharded.
**(
dict(tp_rank=0, tp_size=1)
if post_experts_output_is_complete(is_tp_path=True)
else {}
),
parallel_group="replicated"
if post_experts_output_is_complete(is_tp_path=True)
else "tp",
)
else:
self.shared_experts = None
@@ -1018,18 +1015,14 @@ class SarvamMoEMLADecoderLayer(nn.Module):
alt_stream=alt_stream,
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = SarvamMoEMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)
+14 -21
View File
@@ -50,6 +50,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
LinearParallelGroup,
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
@@ -828,8 +829,8 @@ class XllmMLP(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
reduce_results: bool = True,
prefix: str = "",
tp_rank: Optional[int] = None,
tp_size: Optional[int] = None,
*,
parallel_group: LinearParallelGroup = "tp",
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -838,8 +839,7 @@ class XllmMLP(nn.Module):
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
self.down_proj = RowParallelLinear(
intermediate_size,
@@ -848,8 +848,7 @@ class XllmMLP(nn.Module):
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("down_proj", prefix),
tp_rank=tp_rank,
tp_size=tp_size,
parallel_group=parallel_group,
)
if hidden_act != "silu":
raise ValueError(
@@ -954,15 +953,13 @@ class XllmSparseMoeBlock(nn.Module):
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("shared_experts", prefix),
**(
dict(tp_rank=0, tp_size=1)
if (
get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mori()
or get_moe_a2a_backend().is_flashinfer()
)
else {}
),
parallel_group="replicated"
if (
get_moe_a2a_backend().is_deepep()
or get_moe_a2a_backend().is_mori()
or get_moe_a2a_backend().is_flashinfer()
)
else "tp",
)
else:
self.shared_experts = None
@@ -1502,18 +1499,14 @@ class XllmDecoderLayer(nn.Module):
prefix=add_prefix("mlp", prefix),
)
else:
if is_dense_ffn_fully_dp():
mlp_tp_rank, mlp_tp_size = 0, 1
else:
mlp_tp_rank, mlp_tp_size = None, None
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
self.mlp = XllmMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
tp_rank=mlp_tp_rank,
tp_size=mlp_tp_size,
parallel_group=mlp_parallel_group,
reduce_results=False,
)