diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 8e01dc98df..25e7ed8ca0 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -47,6 +47,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -102,11 +103,11 @@ class BailingMoEMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: Optional[bool] = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.gate_up_proj = MergedColumnParallelLinear( config.hidden_size, @@ -114,8 +115,7 @@ class BailingMoEMLP(nn.Module): bias=config.use_bias, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -124,8 +124,7 @@ class BailingMoEMLP(nn.Module): reduce_results=reduce_results, quant_config=quant_config, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if config.hidden_act != "silu": @@ -137,7 +136,7 @@ class BailingMoEMLP(nn.Module): hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, ) -> torch.Tensor: - if (self.tp_size == 1) and hidden_states.shape[0] == 0: + if self.is_replicated and hidden_states.shape[0] == 0: return hidden_states gate_up, _ = self.gate_up_proj(hidden_states) @@ -325,11 +324,9 @@ class BailingMoESparseMoeBlock(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_experts", prefix), - **( - dict(tp_rank=0, tp_size=1) - if get_moe_a2a_backend().is_deepep() - else {} - ), + parallel_group="replicated" + if get_moe_a2a_backend().is_deepep() + else "tp", ) # dispatcher if get_moe_a2a_backend().is_deepep(): @@ -674,17 +671,13 @@ class BailingMoEBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = BailingMoEMLP( intermediate_size=config.intermediate_size, config=config, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index d7506e5e0a..6df9f75eca 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -36,6 +36,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -304,14 +305,13 @@ class BailingMLP(nn.Module): prefix: str = "", swiglu_limit: Optional[float] = None, padded_intermediate_size: Optional[int] = None, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() self.config = config self.swiglu_limit = swiglu_limit - self.tp_size = tp_size if tp_size is not None else get_parallel().tp_size self.intermediate_size = intermediate_size self.padded_intermediate_size = padded_intermediate_size or intermediate_size @@ -322,8 +322,7 @@ class BailingMLP(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_up_proj", - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( self.padded_intermediate_size, @@ -332,10 +331,11 @@ class BailingMLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=f"{prefix}.down_proj", - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) + self.tp_size = self.gate_up_proj.tp_size + if self.padded_intermediate_size > self.intermediate_size: self.padded_size_per_partition = ( self.padded_intermediate_size // self.tp_size @@ -628,10 +628,8 @@ class BailingMoE(nn.Module): # because MoE output is already complete after EP combine. # Using tp_size=1 ensures shared output is also complete, # so no all-reduce is needed at the MoE level. - shared_tp_kwargs = {} shared_tp_size = self.tp_size if self._enable_a2a_moe: - shared_tp_kwargs = dict(tp_rank=0, tp_size=1) shared_tp_size = 1 padded_intermediate_size = self._compute_padded_intermediate_size( intermediate_size, quant_config, shared_tp_size @@ -645,7 +643,7 @@ class BailingMoE(nn.Module): prefix=f"{prefix}.shared_experts", swiglu_limit=self.share_expert_swiglu_limit, padded_intermediate_size=padded_intermediate_size, - **shared_tp_kwargs, + parallel_group="replicated" if self._enable_a2a_moe else "tp", ) else: self.shared_experts = None @@ -1088,10 +1086,7 @@ class BailingMoELinearDecoderLayer(nn.Module): is_next_layer_sparse = not (self.expert_num == 1) and ( self.layer_id + 1 >= config.first_k_dense_replace ) - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" if self.expert_num == 1: self.mlp = BailingMLP( @@ -1100,8 +1095,7 @@ class BailingMoELinearDecoderLayer(nn.Module): config=config, quant_config=quant_config, prefix=prefix, - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) else: @@ -1121,8 +1115,7 @@ class BailingMoELinearDecoderLayer(nn.Module): config=config, quant_config=quant_config, prefix=prefix, - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5)) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index de12799843..108f876745 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -97,6 +97,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, @@ -282,13 +283,13 @@ class DeepseekV2MLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, swiglu_limit: Optional[float] = None, allow_fused_down: Optional[bool] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.swiglu_limit = swiglu_limit # The fused down path calls deepgemm directly and runs no all-reduce. self.allow_fused_down = ( @@ -301,8 +302,7 @@ class DeepseekV2MLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -311,8 +311,7 @@ class DeepseekV2MLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if not hasattr(self.gate_up_proj, "weight") and hasattr( self.gate_up_proj, "weight_packed" @@ -338,7 +337,7 @@ class DeepseekV2MLP(nn.Module): gemm_output_zero_allocator: BumpAllocator = None, gateup_pre_quant: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, ): - if (self.tp_size == 1) and x.shape[0] == 0: + if self.is_replicated and x.shape[0] == 0: return x if ( @@ -827,7 +826,7 @@ class DeepseekV2MoE(nn.Module): reduce_results=False, swiglu_limit=getattr(config, "swiglu_limit", None), prefix=add_prefix("shared_experts", prefix), - **(dict(tp_rank=0, tp_size=1) if _shared_expert_use_tp1 else {}), + parallel_group="replicated" if _shared_expert_use_tp1 else "tp", ) # Flags must be set before weight load so # process_weights_after_loading sees them and builds the @@ -2611,18 +2610,14 @@ class DeepseekV2DecoderLayer(nn.Module): reduce_results=False, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = DeepseekV2MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, swiglu_limit=getattr(config, "swiglu_limit", None), reduce_results=False, allow_fused_down=False, diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 2488ce139f..5b6d68745e 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -65,6 +65,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, @@ -194,11 +195,11 @@ class Dots3MLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, @@ -206,8 +207,7 @@ class Dots3MLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -216,8 +216,7 @@ class Dots3MLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -226,7 +225,7 @@ class Dots3MLP(nn.Module): self.act_fn = SiluAndMul() def forward(self, x, forward_batch=None): - if (self.tp_size == 1) and x.shape[0] == 0: + if self.is_replicated and x.shape[0] == 0: return x gate_up, _ = self.gate_up_proj(x) @@ -337,12 +336,10 @@ class Dots3MoE(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_experts", prefix), - **( - dict(tp_rank=0, tp_size=1) - if get_moe_a2a_backend().is_deepep() - or should_use_flashinfer_cutlass_moe_fp4_allgather() - else {} - ), + parallel_group="replicated" + if get_moe_a2a_backend().is_deepep() + or should_use_flashinfer_cutlass_moe_fp4_allgather() + else "tp", ) is_packed_weight = quant_config is not None and ( quant_config.get_name() @@ -1396,18 +1393,14 @@ class Dots3DecoderLayer(nn.Module): is_nextn=is_nextn, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = Dots3MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/ernie4.py b/python/sglang/srt/models/ernie4.py index 71d71b4819..5d30b1e77e 100644 --- a/python/sglang/srt/models/ernie4.py +++ b/python/sglang/srt/models/ernie4.py @@ -198,18 +198,14 @@ class Ernie4DecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = Ernie4MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index ae4b349d9b..7ab4603f01 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -52,6 +52,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -164,11 +165,11 @@ class Glm4MoeMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, @@ -176,8 +177,7 @@ class Glm4MoeMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -186,8 +186,7 @@ class Glm4MoeMLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -200,7 +199,7 @@ class Glm4MoeMLP(nn.Module): x, forward_batch=None, ): - if (self.tp_size == 1) and x.shape[0] == 0: + if self.is_replicated and x.shape[0] == 0: return x gate_up, _ = self.gate_up_proj(x) @@ -479,18 +478,16 @@ class Glm4MoeSparseMoeBlock(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_experts", prefix), - **( - dict(tp_rank=0, tp_size=1) - if get_moe_a2a_backend().is_deepep() - or get_moe_a2a_backend().is_mooncake() - or get_moe_a2a_backend().is_nixl() - or get_moe_a2a_backend().is_mori() - or get_moe_a2a_backend().is_ascend_fuseep() - or get_moe_a2a_backend().is_flashinfer() - or get_moe_a2a_backend().is_flashinfer_megamoe() - or should_use_flashinfer_cutlass_moe_fp4_allgather() - else {} - ), + parallel_group="replicated" + if get_moe_a2a_backend().is_deepep() + or get_moe_a2a_backend().is_mooncake() + or get_moe_a2a_backend().is_nixl() + or get_moe_a2a_backend().is_mori() + or get_moe_a2a_backend().is_ascend_fuseep() + or get_moe_a2a_backend().is_flashinfer() + or get_moe_a2a_backend().is_flashinfer_megamoe() + or should_use_flashinfer_cutlass_moe_fp4_allgather() + else "tp", ) is_packed_weight = ( hasattr(self.shared_experts.gate_up_proj.quant_method, "quant_config") @@ -749,18 +746,14 @@ class Glm4MoeDecoderLayer(nn.Module): alt_stream=alt_stream, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = Glm4MoeMLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 30dca657f7..ba81f79366 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -46,7 +46,11 @@ from sglang.srt.layers.layer_boundary import ( ) from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm -from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear +from sglang.srt.layers.linear import ( + LinearParallelGroup, + MergedColumnParallelLinear, + RowParallelLinear, +) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_moe_a2a_backend, @@ -97,11 +101,11 @@ class Glm4MoeLiteMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, @@ -109,8 +113,7 @@ class Glm4MoeLiteMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -119,8 +122,7 @@ class Glm4MoeLiteMLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -133,7 +135,7 @@ class Glm4MoeLiteMLP(nn.Module): x, forward_batch=None, ): - if (self.tp_size == 1) and x.shape[0] == 0: + if self.is_replicated and x.shape[0] == 0: return x gate_up, _ = self.gate_up_proj(x) @@ -249,13 +251,11 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_experts", prefix), - **( - dict(tp_rank=0, tp_size=1) - if get_moe_a2a_backend().is_deepep() - or get_moe_a2a_backend().is_mooncake() - or should_use_flashinfer_cutlass_moe_fp4_allgather() - else {} - ), + parallel_group="replicated" + if get_moe_a2a_backend().is_deepep() + or get_moe_a2a_backend().is_mooncake() + or should_use_flashinfer_cutlass_moe_fp4_allgather() + else "tp", ) is_packed_weight = hasattr( self.shared_experts.gate_up_proj.quant_method, "quant_config" @@ -460,18 +460,14 @@ class Glm4MoeLiteDecoderLayer(nn.Module): is_nextn=is_nextn, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = Glm4MoeLiteMLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 1cfdfefd28..9bdbda858b 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -742,18 +742,14 @@ class Glm5NextDecoderLayer(nn.Module): reduce_results=False, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = Glm5NextMLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, swiglu_limit=config.swiglu_limit, reduce_results=False, allow_fused_down=False, diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 4fe4565764..e0fa95b6be 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -43,6 +43,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -106,11 +107,11 @@ class LLaDA2MoeMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: Optional[bool] = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.gate_up_proj = MergedColumnParallelLinear( config.hidden_size, @@ -118,8 +119,7 @@ class LLaDA2MoeMLP(nn.Module): bias=config.use_bias, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -128,8 +128,7 @@ class LLaDA2MoeMLP(nn.Module): reduce_results=reduce_results, quant_config=quant_config, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if config.hidden_act != "silu": @@ -141,7 +140,7 @@ class LLaDA2MoeMLP(nn.Module): hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, ) -> torch.Tensor: - if (self.tp_size == 1) and hidden_states.shape[0] == 0: + if self.is_replicated and hidden_states.shape[0] == 0: return hidden_states gate_up, _ = self.gate_up_proj(hidden_states) @@ -294,11 +293,9 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_experts", prefix), - **( - dict(tp_rank=0, tp_size=1) - if get_moe_a2a_backend().is_deepep() - else {} - ), + parallel_group="replicated" + if get_moe_a2a_backend().is_deepep() + else "tp", ) # dispatcher if get_moe_a2a_backend().is_deepep(): @@ -611,17 +608,13 @@ class LLaDA2MoeBlock(nn.Module): prefix=add_prefix("mlp", prefix), ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = LLaDA2MoeMLP( intermediate_size=config.intermediate_size, config=config, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 56338884cf..6476ec8695 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, RowParallelLinear, @@ -293,11 +294,11 @@ class MiMoV2MLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() - self.tp_size = tp_size + self.is_replicated = parallel_group == "replicated" self.gate_up_proj = MergedColumnParallelLinear( hidden_size, @@ -305,8 +306,7 @@ class MiMoV2MLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -315,8 +315,7 @@ class MiMoV2MLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -329,7 +328,7 @@ class MiMoV2MLP(nn.Module): x, forward_batch: ForwardBatch = None, ): - if (self.tp_size == 1) and x.shape[0] == 0: + if self.is_replicated and x.shape[0] == 0: return x gate_up, _ = self.gate_up_proj(x) @@ -830,18 +829,14 @@ class MiMoV2DecoderLayer(nn.Module): layer_id=layer_id, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = MiMoV2MLP( hidden_size=self.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 75a043fad9..5abeac0448 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -101,18 +101,14 @@ class MiMoV2MTPLayer(nn.Module): self.is_layer_sparse = False is_next_layer_sparse = False - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = MiMoV2MLP( hidden_size=self.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 1f40a8481d..cd0be2109d 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -48,6 +48,7 @@ from sglang.srt.layers.layer_boundary import ( from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.linear import ( + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, ReplicatedLinear, @@ -273,8 +274,8 @@ class MiniMaxM3MLP(nn.Module): prefix: str = "", reduce_results: bool = True, intermediate_size: int = None, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() hidden_size = config.hidden_size @@ -286,8 +287,7 @@ class MiniMaxM3MLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -296,8 +296,7 @@ class MiniMaxM3MLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act == "silu": self.act_fn = SiluAndMul() @@ -406,7 +405,7 @@ class MiniMaxM3MoE(nn.Module): prefix=add_prefix("shared_experts", prefix), reduce_results=False, intermediate_size=intermediate_size, - **(dict(tp_rank=0, tp_size=1) if shared_experts_tp1 else {}), + parallel_group="replicated" if shared_experts_tp1 else "tp", ) else: self.shared_experts = None @@ -1482,17 +1481,13 @@ class MiniMaxM3DecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = MiniMaxM3MLP( config=config, quant_config=quant_config, prefix=add_prefix("mlp", prefix), intermediate_size=config.dense_intermediate_size, - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 8b470d7558..9c91225bd7 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -29,6 +29,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, @@ -180,8 +181,8 @@ class SarvamMoEMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", reduce_results: bool = True, - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -190,8 +191,7 @@ class SarvamMoEMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -200,8 +200,7 @@ class SarvamMoEMLP(nn.Module): quant_config=quant_config, prefix=add_prefix("down_proj", prefix), reduce_results=reduce_results, - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -307,11 +306,9 @@ class SarvamMoESparseMoeBlock(nn.Module): reduce_results=False, # The shared output joins the routed output's TP sum; where that # output is already complete on each rank, it is not TP-sharded. - **( - dict(tp_rank=0, tp_size=1) - if post_experts_output_is_complete(is_tp_path=True) - else {} - ), + parallel_group="replicated" + if post_experts_output_is_complete(is_tp_path=True) + else "tp", ) else: self.shared_experts = None @@ -1018,18 +1015,14 @@ class SarvamMoEMLADecoderLayer(nn.Module): alt_stream=alt_stream, ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = SarvamMoEMLP( hidden_size=self.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 093400a550..3f46f3433b 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -50,6 +50,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, + LinearParallelGroup, MergedColumnParallelLinear, QKVParallelLinear, ReplicatedLinear, @@ -828,8 +829,8 @@ class XllmMLP(nn.Module): quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, + *, + parallel_group: LinearParallelGroup = "tp", ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( @@ -838,8 +839,7 @@ class XllmMLP(nn.Module): bias=False, quant_config=quant_config, prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) self.down_proj = RowParallelLinear( intermediate_size, @@ -848,8 +848,7 @@ class XllmMLP(nn.Module): quant_config=quant_config, reduce_results=reduce_results, prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, + parallel_group=parallel_group, ) if hidden_act != "silu": raise ValueError( @@ -954,15 +953,13 @@ class XllmSparseMoeBlock(nn.Module): quant_config=quant_config, reduce_results=False, prefix=add_prefix("shared_experts", prefix), - **( - dict(tp_rank=0, tp_size=1) - if ( - get_moe_a2a_backend().is_deepep() - or get_moe_a2a_backend().is_mori() - or get_moe_a2a_backend().is_flashinfer() - ) - else {} - ), + parallel_group="replicated" + if ( + get_moe_a2a_backend().is_deepep() + or get_moe_a2a_backend().is_mori() + or get_moe_a2a_backend().is_flashinfer() + ) + else "tp", ) else: self.shared_experts = None @@ -1502,18 +1499,14 @@ class XllmDecoderLayer(nn.Module): prefix=add_prefix("mlp", prefix), ) else: - if is_dense_ffn_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None + mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp" self.mlp = XllmMLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + parallel_group=mlp_parallel_group, reduce_results=False, ) diff --git a/test/registered/unit/layers/test_mlp_parallel_groups.py b/test/registered/unit/layers/test_mlp_parallel_groups.py new file mode 100644 index 0000000000..6c05c990d4 --- /dev/null +++ b/test/registered/unit/layers/test_mlp_parallel_groups.py @@ -0,0 +1,139 @@ +"""Model MLP weight layouts stay fixed across construction and reload scopes.""" + +import importlib +import inspect +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import torch.nn.functional as F + +from sglang.srt.runtime_context import SpawnRanks, get_parallel, publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +MODELS = { + "bailing_moe": "BailingMoEMLP", + "bailing_moe_v3": "BailingMLP", + "deepseek_v2": "DeepseekV2MLP", + "dots3_common.modeling": "Dots3MLP", + "glm4_moe": "Glm4MoeMLP", + "glm4_moe_lite": "Glm4MoeLiteMLP", + "llada2": "LLaDA2MoeMLP", + "mimo_v2": "MiMoV2MLP", + "minimax_m3": "MiniMaxM3MLP", + "sarvam_moe": "SarvamMoEMLP", + "xllm": "XllmMLP", +} + + +class TestMLPParallelGroups(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + publish( + ServerArgs(model_path="dummy", device="cpu", tp_size=4, attn_dp_size=2), + role="test", + ranks=SpawnRanks(world_rank=3), + ) + self.x = torch.arange(16, dtype=torch.float32).reshape(2, 8) / 16 + + def test_model_layout_math_reload_and_row_policy(self): + tp = Mock() + tp.all_reduce.side_effect = lambda x: x * 2 + for module_name, class_name in MODELS.items(): + cls = getattr( + importlib.import_module("sglang.srt.models." + module_name), class_name + ) + for group, rank, size in ((None, 3, 4), ("tp", 3, 4), ("replicated", 0, 1)): + for reduce in (False, True): + for padded in ( + (False, True) if class_name == "BailingMLP" else (False,) + ): + with self.subTest( + model=module_name, group=group, reduce=reduce, padded=padded + ): + config = SimpleNamespace( + hidden_size=8, hidden_act="silu", use_bias=True + ) + options = dict( + hidden_size=8, + intermediate_size=8, + hidden_act="silu", + config=config, + parallel_group=group, + reduce_results=reduce, + padded_intermediate_size=16 if padded else None, + ) + if group is None: + options.pop("parallel_group") + signature = inspect.signature(cls).parameters + mlp = cls( + **{k: v for k, v in options.items() if k in signature} + ) + mlp.act_fn._forward_method = mlp.act_fn.forward_native + up, down = mlp.gate_up_proj, mlp.down_proj + width = 16 if padded else 8 + gate = ( + torch.arange(width * 8, dtype=torch.float32).reshape( + width, 8 + ) + / 128 + ) + other = gate + 0.25 + weight = gate.T.contiguous() + 0.5 + with get_parallel().override( + tp_rank=0, attn_dp_rank=0, attn_tp_rank=0 + ): + up.weight.weight_loader(up.weight, gate, 0) + up.weight.weight_loader(up.weight, other, 1) + down.weight.weight_loader(down.weight, weight) + if up.bias is not None: + up.bias.data.zero_() + down.bias.data.zero_() + local_gate = gate.chunk(size)[rank] + local_up = other.chunk(size)[rank] + projected = F.silu(F.linear(self.x, local_gate)) * F.linear( + self.x, local_up + ) + if padded: + projected = F.pad( + projected[:, : 8 // size], (0, 8 // size) + ) + expected = F.linear( + projected, weight.chunk(size, dim=1)[rank] + ) + if reduce and size > 1: + expected = expected * 2 + tp.all_reduce.reset_mock() + with ( + get_parallel().override(tp_group=tp), + patch( + "sglang.srt.layers.linear.is_allocation_symmetric", + return_value=True, + ), + patch( + "sglang.srt.layers.linear.use_symmetric_memory", + return_value=nullcontext(), + ) as allocator, + ): + torch.testing.assert_close(mlp(self.x), expected) + allocator.assert_called_once_with(tp, disabled=False) + self.assertEqual( + tp.all_reduce.call_count, + 1 if reduce and size > 1 else 0, + ) + torch.testing.assert_close(mlp(self.x[:0]), self.x[:0]) + self.assertEqual((up.tp_rank, up.tp_size), (rank, size)) + self.assertEqual((down.tp_rank, down.tp_size), (rank, size)) + self.assertEqual(down.reduce_results, reduce) + self.assertFalse(down.use_dp_attention_reduce) + + +if __name__ == "__main__": + unittest.main()