[refactor] remove unused op_mlp (#26673)
This commit is contained in:
@@ -2114,19 +2114,6 @@ class DeepseekV2DecoderLayer(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def op_mlp(self, state):
|
|
||||||
hidden_states = state.pop("hidden_states_mlp_input")
|
|
||||||
if not (
|
|
||||||
enable_moe_dense_fully_dp()
|
|
||||||
and (not self.is_layer_sparse)
|
|
||||||
and hidden_states.shape[0] == 0
|
|
||||||
):
|
|
||||||
state.hidden_states_mlp_output = self.mlp(
|
|
||||||
hidden_states, state.forward_batch
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
state.hidden_states_mlp_output = hidden_states
|
|
||||||
|
|
||||||
def op_comm_postprocess_layer(self, state):
|
def op_comm_postprocess_layer(self, state):
|
||||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||||
state.pop("hidden_states_mlp_output"),
|
state.pop("hidden_states_mlp_output"),
|
||||||
|
|||||||
@@ -1017,19 +1017,6 @@ class Glm4MoeDecoderLayer(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def op_mlp(self, state):
|
|
||||||
hidden_states = state.pop("hidden_states_mlp_input")
|
|
||||||
if not (
|
|
||||||
enable_moe_dense_fully_dp()
|
|
||||||
and (not self.is_layer_sparse)
|
|
||||||
and hidden_states.shape[0] == 0
|
|
||||||
):
|
|
||||||
state.hidden_states_mlp_output = self.mlp(
|
|
||||||
hidden_states, state.forward_batch
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
state.hidden_states_mlp_output = hidden_states
|
|
||||||
|
|
||||||
def op_comm_postprocess_layer(self, state):
|
def op_comm_postprocess_layer(self, state):
|
||||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||||
state.pop("hidden_states_mlp_output"),
|
state.pop("hidden_states_mlp_output"),
|
||||||
|
|||||||
@@ -737,19 +737,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def op_mlp(self, state):
|
|
||||||
hidden_states = state.pop("hidden_states_mlp_input")
|
|
||||||
if not (
|
|
||||||
enable_moe_dense_fully_dp()
|
|
||||||
and (not self.is_layer_sparse)
|
|
||||||
and hidden_states.shape[0] == 0
|
|
||||||
):
|
|
||||||
state.hidden_states_mlp_output = self.mlp(
|
|
||||||
hidden_states, state.forward_batch
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
state.hidden_states_mlp_output = hidden_states
|
|
||||||
|
|
||||||
def op_comm_postprocess_layer(self, state):
|
def op_comm_postprocess_layer(self, state):
|
||||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||||
state.pop("hidden_states_mlp_output"),
|
state.pop("hidden_states_mlp_output"),
|
||||||
|
|||||||
@@ -808,10 +808,6 @@ class MiMoV2DecoderLayer(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def op_mlp(self, state):
|
|
||||||
hidden_states = state.pop("hidden_states_mlp_input")
|
|
||||||
state.hidden_states_mlp_output = self.mlp(hidden_states, state.forward_batch)
|
|
||||||
|
|
||||||
def op_comm_postprocess_layer(self, state):
|
def op_comm_postprocess_layer(self, state):
|
||||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||||
state.pop("hidden_states_mlp_output"),
|
state.pop("hidden_states_mlp_output"),
|
||||||
|
|||||||
@@ -1069,12 +1069,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def op_mlp(self, state):
|
|
||||||
hidden_states = state.pop("hidden_states_mlp_input")
|
|
||||||
state.hidden_states_mlp_output = self.block_sparse_moe(
|
|
||||||
hidden_states, state.forward_batch
|
|
||||||
)
|
|
||||||
|
|
||||||
def op_comm_postprocess_layer(self, state):
|
def op_comm_postprocess_layer(self, state):
|
||||||
"""Communication postprocess for layer - TBO operation"""
|
"""Communication postprocess for layer - TBO operation"""
|
||||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||||
|
|||||||
@@ -891,10 +891,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def op_mlp(self, state):
|
|
||||||
hidden_states = state.pop("hidden_states_mlp_input")
|
|
||||||
state.hidden_states_mlp_output = self.mlp(hidden_states, state.forward_batch)
|
|
||||||
|
|
||||||
def op_comm_postprocess_layer(self, state):
|
def op_comm_postprocess_layer(self, state):
|
||||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||||
state.pop("hidden_states_mlp_output"),
|
state.pop("hidden_states_mlp_output"),
|
||||||
|
|||||||
Reference in New Issue
Block a user