Super tiny remove unused MiniMaxM2MLP class (#13659)
This commit is contained in:
@@ -31,7 +31,6 @@ from sglang.srt.distributed import (
|
|||||||
)
|
)
|
||||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||||
from sglang.srt.layers.activation import SiluAndMul
|
|
||||||
from sglang.srt.layers.communicator import (
|
from sglang.srt.layers.communicator import (
|
||||||
LayerCommunicator,
|
LayerCommunicator,
|
||||||
LayerScatterModes,
|
LayerScatterModes,
|
||||||
@@ -39,7 +38,6 @@ from sglang.srt.layers.communicator import (
|
|||||||
)
|
)
|
||||||
from sglang.srt.layers.layernorm import RMSNorm
|
from sglang.srt.layers.layernorm import RMSNorm
|
||||||
from sglang.srt.layers.linear import (
|
from sglang.srt.layers.linear import (
|
||||||
MergedColumnParallelLinear,
|
|
||||||
QKVParallelLinear,
|
QKVParallelLinear,
|
||||||
ReplicatedLinear,
|
ReplicatedLinear,
|
||||||
RowParallelLinear,
|
RowParallelLinear,
|
||||||
@@ -127,40 +125,6 @@ class MiniMaxM2RMSNormTP(nn.Module):
|
|||||||
return x
|
return x
|
||||||
|
|
||||||
|
|
||||||
class MiniMaxM2MLP(nn.Module):
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
hidden_size: int,
|
|
||||||
intermediate_size: int,
|
|
||||||
quant_config: Optional[QuantizationConfig] = None,
|
|
||||||
prefix: str = "mlp",
|
|
||||||
) -> None:
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
self.gate_up_proj = MergedColumnParallelLinear(
|
|
||||||
hidden_size,
|
|
||||||
[intermediate_size] * 2,
|
|
||||||
bias=False,
|
|
||||||
quant_config=quant_config,
|
|
||||||
prefix=add_prefix("gate_up_proj", prefix),
|
|
||||||
)
|
|
||||||
self.down_proj = RowParallelLinear(
|
|
||||||
intermediate_size,
|
|
||||||
hidden_size,
|
|
||||||
bias=False,
|
|
||||||
quant_config=quant_config,
|
|
||||||
prefix=add_prefix("down_proj", prefix),
|
|
||||||
)
|
|
||||||
self.act_fn = SiluAndMul()
|
|
||||||
return
|
|
||||||
|
|
||||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
||||||
gate_up, _ = self.gate_up_proj(x)
|
|
||||||
x = self.act_fn(gate_up)
|
|
||||||
x, _ = self.down_proj(x)
|
|
||||||
return x
|
|
||||||
|
|
||||||
|
|
||||||
class MiniMaxM2MoE(nn.Module):
|
class MiniMaxM2MoE(nn.Module):
|
||||||
"""MiniMax MoE implementation using DeepEP for Expert Parallel support."""
|
"""MiniMax MoE implementation using DeepEP for Expert Parallel support."""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user