[Refactor] Let model MLPs select their parallel group (#42468)
This commit is contained in:
@@ -47,6 +47,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -102,11 +103,11 @@ class BailingMoEMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: Optional[bool] = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
@@ -114,8 +115,7 @@ class BailingMoEMLP(nn.Module):
|
||||
bias=config.use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -124,8 +124,7 @@ class BailingMoEMLP(nn.Module):
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
if config.hidden_act != "silu":
|
||||
@@ -137,7 +136,7 @@ class BailingMoEMLP(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
) -> torch.Tensor:
|
||||
if (self.tp_size == 1) and hidden_states.shape[0] == 0:
|
||||
if self.is_replicated and hidden_states.shape[0] == 0:
|
||||
return hidden_states
|
||||
|
||||
gate_up, _ = self.gate_up_proj(hidden_states)
|
||||
@@ -325,11 +324,9 @@ class BailingMoESparseMoeBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
else "tp",
|
||||
)
|
||||
# dispatcher
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
@@ -674,17 +671,13 @@ class BailingMoEBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = BailingMoEMLP(
|
||||
intermediate_size=config.intermediate_size,
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -304,14 +305,13 @@ class BailingMLP(nn.Module):
|
||||
prefix: str = "",
|
||||
swiglu_limit: Optional[float] = None,
|
||||
padded_intermediate_size: Optional[int] = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.config = config
|
||||
self.swiglu_limit = swiglu_limit
|
||||
self.tp_size = tp_size if tp_size is not None else get_parallel().tp_size
|
||||
|
||||
self.intermediate_size = intermediate_size
|
||||
self.padded_intermediate_size = padded_intermediate_size or intermediate_size
|
||||
@@ -322,8 +322,7 @@ class BailingMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
self.padded_intermediate_size,
|
||||
@@ -332,10 +331,11 @@ class BailingMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
self.tp_size = self.gate_up_proj.tp_size
|
||||
|
||||
if self.padded_intermediate_size > self.intermediate_size:
|
||||
self.padded_size_per_partition = (
|
||||
self.padded_intermediate_size // self.tp_size
|
||||
@@ -628,10 +628,8 @@ class BailingMoE(nn.Module):
|
||||
# because MoE output is already complete after EP combine.
|
||||
# Using tp_size=1 ensures shared output is also complete,
|
||||
# so no all-reduce is needed at the MoE level.
|
||||
shared_tp_kwargs = {}
|
||||
shared_tp_size = self.tp_size
|
||||
if self._enable_a2a_moe:
|
||||
shared_tp_kwargs = dict(tp_rank=0, tp_size=1)
|
||||
shared_tp_size = 1
|
||||
padded_intermediate_size = self._compute_padded_intermediate_size(
|
||||
intermediate_size, quant_config, shared_tp_size
|
||||
@@ -645,7 +643,7 @@ class BailingMoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
swiglu_limit=self.share_expert_swiglu_limit,
|
||||
padded_intermediate_size=padded_intermediate_size,
|
||||
**shared_tp_kwargs,
|
||||
parallel_group="replicated" if self._enable_a2a_moe else "tp",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
@@ -1088,10 +1086,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
is_next_layer_sparse = not (self.expert_num == 1) and (
|
||||
self.layer_id + 1 >= config.first_k_dense_replace
|
||||
)
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
|
||||
if self.expert_num == 1:
|
||||
self.mlp = BailingMLP(
|
||||
@@ -1100,8 +1095,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
@@ -1121,8 +1115,7 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
|
||||
|
||||
@@ -97,6 +97,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
@@ -282,13 +283,13 @@ class DeepseekV2MLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
swiglu_limit: Optional[float] = None,
|
||||
allow_fused_down: Optional[bool] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
self.swiglu_limit = swiglu_limit
|
||||
# The fused down path calls deepgemm directly and runs no all-reduce.
|
||||
self.allow_fused_down = (
|
||||
@@ -301,8 +302,7 @@ class DeepseekV2MLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -311,8 +311,7 @@ class DeepseekV2MLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if not hasattr(self.gate_up_proj, "weight") and hasattr(
|
||||
self.gate_up_proj, "weight_packed"
|
||||
@@ -338,7 +337,7 @@ class DeepseekV2MLP(nn.Module):
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
gateup_pre_quant: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
):
|
||||
if (self.tp_size == 1) and x.shape[0] == 0:
|
||||
if self.is_replicated and x.shape[0] == 0:
|
||||
return x
|
||||
|
||||
if (
|
||||
@@ -827,7 +826,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
reduce_results=False,
|
||||
swiglu_limit=getattr(config, "swiglu_limit", None),
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(dict(tp_rank=0, tp_size=1) if _shared_expert_use_tp1 else {}),
|
||||
parallel_group="replicated" if _shared_expert_use_tp1 else "tp",
|
||||
)
|
||||
# Flags must be set before weight load so
|
||||
# process_weights_after_loading sees them and builds the
|
||||
@@ -2611,18 +2610,14 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = DeepseekV2MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
swiglu_limit=getattr(config, "swiglu_limit", None),
|
||||
reduce_results=False,
|
||||
allow_fused_down=False,
|
||||
|
||||
@@ -65,6 +65,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
@@ -194,11 +195,11 @@ class Dots3MLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
@@ -206,8 +207,7 @@ class Dots3MLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -216,8 +216,7 @@ class Dots3MLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -226,7 +225,7 @@ class Dots3MLP(nn.Module):
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x, forward_batch=None):
|
||||
if (self.tp_size == 1) and x.shape[0] == 0:
|
||||
if self.is_replicated and x.shape[0] == 0:
|
||||
return x
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
@@ -337,12 +336,10 @@ class Dots3MoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else "tp",
|
||||
)
|
||||
is_packed_weight = quant_config is not None and (
|
||||
quant_config.get_name()
|
||||
@@ -1396,18 +1393,14 @@ class Dots3DecoderLayer(nn.Module):
|
||||
is_nextn=is_nextn,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = Dots3MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -198,18 +198,14 @@ class Ernie4DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = Ernie4MLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
)
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -52,6 +52,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -164,11 +165,11 @@ class Glm4MoeMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
@@ -176,8 +177,7 @@ class Glm4MoeMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -186,8 +186,7 @@ class Glm4MoeMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -200,7 +199,7 @@ class Glm4MoeMLP(nn.Module):
|
||||
x,
|
||||
forward_batch=None,
|
||||
):
|
||||
if (self.tp_size == 1) and x.shape[0] == 0:
|
||||
if self.is_replicated and x.shape[0] == 0:
|
||||
return x
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
@@ -479,18 +478,16 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or get_moe_a2a_backend().is_nixl()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
or get_moe_a2a_backend().is_flashinfer()
|
||||
or get_moe_a2a_backend().is_flashinfer_megamoe()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or get_moe_a2a_backend().is_nixl()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
or get_moe_a2a_backend().is_flashinfer()
|
||||
or get_moe_a2a_backend().is_flashinfer_megamoe()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else "tp",
|
||||
)
|
||||
is_packed_weight = (
|
||||
hasattr(self.shared_experts.gate_up_proj.quant_method, "quant_config")
|
||||
@@ -749,18 +746,14 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = Glm4MoeMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -46,7 +46,11 @@ from sglang.srt.layers.layer_boundary import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
@@ -97,11 +101,11 @@ class Glm4MoeLiteMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
@@ -109,8 +113,7 @@ class Glm4MoeLiteMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -119,8 +122,7 @@ class Glm4MoeLiteMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -133,7 +135,7 @@ class Glm4MoeLiteMLP(nn.Module):
|
||||
x,
|
||||
forward_batch=None,
|
||||
):
|
||||
if (self.tp_size == 1) and x.shape[0] == 0:
|
||||
if self.is_replicated and x.shape[0] == 0:
|
||||
return x
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
@@ -249,13 +251,11 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or should_use_flashinfer_cutlass_moe_fp4_allgather()
|
||||
else "tp",
|
||||
)
|
||||
is_packed_weight = hasattr(
|
||||
self.shared_experts.gate_up_proj.quant_method, "quant_config"
|
||||
@@ -460,18 +460,14 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
is_nextn=is_nextn,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = Glm4MoeLiteMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -742,18 +742,14 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = Glm5NextMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
swiglu_limit=config.swiglu_limit,
|
||||
reduce_results=False,
|
||||
allow_fused_down=False,
|
||||
|
||||
@@ -43,6 +43,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -106,11 +107,11 @@ class LLaDA2MoeMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: Optional[bool] = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
@@ -118,8 +119,7 @@ class LLaDA2MoeMLP(nn.Module):
|
||||
bias=config.use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -128,8 +128,7 @@ class LLaDA2MoeMLP(nn.Module):
|
||||
reduce_results=reduce_results,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
if config.hidden_act != "silu":
|
||||
@@ -141,7 +140,7 @@ class LLaDA2MoeMLP(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
) -> torch.Tensor:
|
||||
if (self.tp_size == 1) and hidden_states.shape[0] == 0:
|
||||
if self.is_replicated and hidden_states.shape[0] == 0:
|
||||
return hidden_states
|
||||
|
||||
gate_up, _ = self.gate_up_proj(hidden_states)
|
||||
@@ -294,11 +293,9 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
else "tp",
|
||||
)
|
||||
# dispatcher
|
||||
if get_moe_a2a_backend().is_deepep():
|
||||
@@ -611,17 +608,13 @@ class LLaDA2MoeBlock(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = LLaDA2MoeMLP(
|
||||
intermediate_size=config.intermediate_size,
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -293,11 +294,11 @@ class MiMoV2MLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_size = tp_size
|
||||
self.is_replicated = parallel_group == "replicated"
|
||||
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
@@ -305,8 +306,7 @@ class MiMoV2MLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -315,8 +315,7 @@ class MiMoV2MLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -329,7 +328,7 @@ class MiMoV2MLP(nn.Module):
|
||||
x,
|
||||
forward_batch: ForwardBatch = None,
|
||||
):
|
||||
if (self.tp_size == 1) and x.shape[0] == 0:
|
||||
if self.is_replicated and x.shape[0] == 0:
|
||||
return x
|
||||
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
@@ -830,18 +829,14 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = MiMoV2MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -101,18 +101,14 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
self.is_layer_sparse = False
|
||||
is_next_layer_sparse = False
|
||||
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = MiMoV2MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
||||
|
||||
@@ -48,6 +48,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -273,8 +274,8 @@ class MiniMaxM3MLP(nn.Module):
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
intermediate_size: int = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
hidden_size = config.hidden_size
|
||||
@@ -286,8 +287,7 @@ class MiniMaxM3MLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -296,8 +296,7 @@ class MiniMaxM3MLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act == "silu":
|
||||
self.act_fn = SiluAndMul()
|
||||
@@ -406,7 +405,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
reduce_results=False,
|
||||
intermediate_size=intermediate_size,
|
||||
**(dict(tp_rank=0, tp_size=1) if shared_experts_tp1 else {}),
|
||||
parallel_group="replicated" if shared_experts_tp1 else "tp",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
@@ -1482,17 +1481,13 @@ class MiniMaxM3DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = MiniMaxM3MLP(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
intermediate_size=config.dense_intermediate_size,
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
@@ -180,8 +181,8 @@ class SarvamMoEMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -190,8 +191,7 @@ class SarvamMoEMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -200,8 +200,7 @@ class SarvamMoEMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -307,11 +306,9 @@ class SarvamMoESparseMoeBlock(nn.Module):
|
||||
reduce_results=False,
|
||||
# The shared output joins the routed output's TP sum; where that
|
||||
# output is already complete on each rank, it is not TP-sharded.
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if post_experts_output_is_complete(is_tp_path=True)
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if post_experts_output_is_complete(is_tp_path=True)
|
||||
else "tp",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
@@ -1018,18 +1015,14 @@ class SarvamMoEMLADecoderLayer(nn.Module):
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = SarvamMoEMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -828,8 +829,8 @@ class XllmMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -838,8 +839,7 @@ class XllmMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -848,8 +848,7 @@ class XllmMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -954,15 +953,13 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
if (
|
||||
get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
or get_moe_a2a_backend().is_flashinfer()
|
||||
)
|
||||
else {}
|
||||
),
|
||||
parallel_group="replicated"
|
||||
if (
|
||||
get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
or get_moe_a2a_backend().is_flashinfer()
|
||||
)
|
||||
else "tp",
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
@@ -1502,18 +1499,14 @@ class XllmDecoderLayer(nn.Module):
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
else:
|
||||
if is_dense_ffn_fully_dp():
|
||||
mlp_tp_rank, mlp_tp_size = 0, 1
|
||||
else:
|
||||
mlp_tp_rank, mlp_tp_size = None, None
|
||||
mlp_parallel_group = "replicated" if is_dense_ffn_fully_dp() else "tp"
|
||||
self.mlp = XllmMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
tp_rank=mlp_tp_rank,
|
||||
tp_size=mlp_tp_size,
|
||||
parallel_group=mlp_parallel_group,
|
||||
reduce_results=False,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user