[Refactor] Take a layer's last-layer fact from its scatter-mode plan (#41199)

This commit is contained in:
Cheng Wan
2026-09-25 01:49:33 -07:00
committed by GitHub
parent 8f5a636f8d
commit 26a3214cda
28 changed files with 46 additions and 57 deletions
+5 -3
View File
@@ -443,6 +443,8 @@ class LayerScatterModes:
middle_residual_mode: ScatterMode
layer_output_mode: ScatterMode
is_layer_sparse: bool = False
# The model's last layer: its output goes to the final norm, not a next layer.
is_last_layer: bool = False
@classmethod
def init_new(cls, **kwargs):
@@ -454,6 +456,7 @@ class LayerScatterModes:
middle_residual_mode=cls._compute_middle_residual_mode(context),
layer_output_mode=cls._compute_layer_output_mode(context),
is_layer_sparse=context.is_layer_sparse,
is_last_layer=context.layer_id == context.num_layers - 1,
)
@classmethod
@@ -705,7 +708,6 @@ class LayerCommunicator:
post_attention_layernorm: torch.nn.Module,
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
allow_reduce_scatter: bool = False,
is_last_layer: bool = False,
qkv_latent_func: Optional[Callable] = None,
force_layernorm_before_dp_gather: bool = False,
enable_fused_ar_quant: bool = False,
@@ -716,7 +718,7 @@ class LayerCommunicator:
self.input_layernorm = input_layernorm
self.post_attention_layernorm = post_attention_layernorm
self.allow_reduce_scatter = allow_reduce_scatter
self.is_last_layer = is_last_layer
self.is_last_layer = layer_scatter_modes.is_last_layer
self.qkv_latent_func = qkv_latent_func
self.force_layernorm_before_dp_gather = force_layernorm_before_dp_gather
self.enable_fused_ar_quant = enable_fused_ar_quant
@@ -743,11 +745,11 @@ class LayerCommunicator:
mlp_mode=ScatterMode.SCATTERED,
middle_residual_mode=ScatterMode.SCATTERED,
layer_output_mode=ScatterMode.SCATTERED,
is_last_layer=layer_scatter_modes.is_last_layer,
),
input_layernorm=input_layernorm,
post_attention_layernorm=post_attention_layernorm,
allow_reduce_scatter=allow_reduce_scatter,
is_last_layer=is_last_layer,
qkv_latent_func=qkv_latent_func,
force_layernorm_before_dp_gather=force_layernorm_before_dp_gather,
enable_fused_ar_quant=enable_fused_ar_quant,
@@ -92,7 +92,6 @@ class DSACPLayerCommunicator(LayerCommunicator):
post_attention_layernorm: torch.nn.Module,
# Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator.
allow_reduce_scatter: bool = False,
is_last_layer: bool = False,
qkv_latent_func: Optional[Callable] = None,
):
super().__init__(
@@ -100,7 +99,6 @@ class DSACPLayerCommunicator(LayerCommunicator):
input_layernorm,
post_attention_layernorm,
allow_reduce_scatter,
is_last_layer,
qkv_latent_func,
)
@@ -416,7 +416,6 @@ class MHCLayerCommunicator(LayerCommunicator):
input_layernorm: torch.nn.Module,
post_attention_layernorm: torch.nn.Module,
allow_reduce_scatter: bool = False,
is_last_layer: bool = False,
qkv_latent_func: Optional[Callable] = None,
*,
is_first_layer: bool,
@@ -440,7 +439,6 @@ class MHCLayerCommunicator(LayerCommunicator):
input_layernorm,
post_attention_layernorm,
allow_reduce_scatter,
is_last_layer,
qkv_latent_func,
)
-3
View File
@@ -678,8 +678,6 @@ class BailingMoEBlock(nn.Module):
is_next_layer_sparse=is_next_layer_sparse,
)
self.is_last_layer = is_nextn or (self.layer_id == config.num_hidden_layers - 1)
if self.is_layer_sparse:
self.mlp = BailingMoESparseMoeBlock(
layer_id=layer_id,
@@ -709,7 +707,6 @@ class BailingMoEBlock(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=self.is_last_layer,
)
def _is_layer_sparse(
@@ -832,7 +832,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=False,
is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1),
qkv_latent_func=qkv_latent_func,
)
@@ -1164,7 +1164,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1),
qkv_latent_func=(
self.attention.prepare_qkv_latent
if self.attention_type == 1 and self.use_mla
-3
View File
@@ -2631,9 +2631,6 @@ class DeepseekV2DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(
is_nextn or (self.layer_id == self.config.num_hidden_layers - 1)
),
qkv_latent_func=self.self_attn.prepare_qkv_latent,
)
@@ -1583,9 +1583,6 @@ class Dots3DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(
is_nextn or (self.layer_id == self.config.num_hidden_layers - 1)
),
)
def _is_layer_sparse(self, layer_id: int, is_nextn: bool) -> bool:
-3
View File
@@ -877,9 +877,6 @@ class Glm4MoeDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(
is_nextn or (self.layer_id == self.config.num_hidden_layers - 1)
),
)
# Detect if QKV uses aiter FP8 per-token quant so we can fuse
@@ -592,9 +592,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(
is_nextn or (self.layer_id == self.config.num_hidden_layers - 1)
),
qkv_latent_func=self.self_attn.prepare_qkv_latent,
)
-3
View File
@@ -788,9 +788,6 @@ class Glm5NextDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(
is_nextn or (self.layer_id == self.config.num_hidden_layers - 1)
),
qkv_latent_func=(
self.self_attn.prepare_qkv_latent if not self.is_linear_attn else None
),
+1 -5
View File
@@ -564,9 +564,8 @@ class GptOssDecoderLayer(nn.Module):
self.attn_tp_size = get_parallel().attn_tp_size
self.attn_tp_rank = get_parallel().attn_tp_rank
# GptOss all layers are sparse and have no nextn now
# GptOss all layers are sparse
self.is_layer_sparse = True
self.is_nextn = False
is_previous_layer_sparse = True
is_next_layer_sparse = True
@@ -611,9 +610,6 @@ class GptOssDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(
self.is_nextn or (self.layer_id == self.config.num_hidden_layers - 1)
),
)
def forward(
@@ -505,7 +505,6 @@ class InternS2MobiusLinearDecoderLayer(_InternS2MobiusDecoderMixin, nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
)
@@ -639,7 +638,6 @@ class InternS2MobiusAttentionDecoderLayer(
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=False,
)
-1
View File
@@ -467,7 +467,6 @@ class LagunaDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
def forward(
-2
View File
@@ -620,8 +620,6 @@ class LLaDA2MoeBlock(nn.Module):
is_next_layer_sparse=is_next_layer_sparse,
)
self.is_last_layer = self.layer_id == config.num_hidden_layers - 1
if self.is_layer_sparse:
self.mlp = LLaDA2MoeSparseMoeBlock(
layer_id=layer_id,
-1
View File
@@ -458,7 +458,6 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == cfg.num_hidden_layers - 1),
)
-1
View File
@@ -869,7 +869,6 @@ class MiMoV2DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(self.layer_id == self.config.num_hidden_layers - 1),
)
def forward(
-1
View File
@@ -989,7 +989,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
def forward(
-1
View File
@@ -1361,7 +1361,6 @@ class MiniMaxM3DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
def forward(
+5 -3
View File
@@ -23,7 +23,9 @@ def feeds_mlp_layer(pattern: str, layer_idx: int) -> bool:
return next_idx < len(pattern) and not is_attn_layer(pattern[next_idx])
def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes:
def _build_layer_scatter_modes(
is_sparse: bool = False, is_last_layer: bool = False
) -> LayerScatterModes:
scatter_mlp = is_sparse and not get_moe_a2a_backend().is_none()
mlp_mode = ScatterMode.SCATTERED if scatter_mlp else ScatterMode.FULL
middle_residual_mode = (
@@ -36,6 +38,7 @@ def _build_layer_scatter_modes(is_sparse: bool = False) -> LayerScatterModes:
middle_residual_mode=middle_residual_mode,
layer_output_mode=ScatterMode.TP_ATTN_FULL,
is_layer_sparse=is_sparse,
is_last_layer=is_last_layer,
)
@@ -48,12 +51,11 @@ def make_layer_communicator(
is_last_layer: bool = False,
) -> LayerCommunicator:
return LayerCommunicator(
layer_scatter_modes=_build_layer_scatter_modes(is_sparse),
layer_scatter_modes=_build_layer_scatter_modes(is_sparse, is_last_layer),
input_layernorm=layer_norm if for_attn else nn.Identity(),
post_attention_layernorm=nn.Identity() if for_attn else layer_norm,
# With attention TP > 1, the default gather adds the residual to one
# rank's partial in bf16 before the cross-rank sum.
force_layernorm_before_dp_gather=True,
allow_reduce_scatter=allow_reduce_scatter,
is_last_layer=is_last_layer,
)
-2
View File
@@ -1129,7 +1129,6 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=enable_fused_ar_quant,
)
@@ -1333,7 +1332,6 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
enable_fused_ar_quant=enable_fused_ar_quant,
fused_ar_quant_keep_bf16=False,
)
-1
View File
@@ -873,7 +873,6 @@ class Qwen3MoeDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(self.layer_id == self.config.num_hidden_layers - 1),
)
def forward(
-2
View File
@@ -543,7 +543,6 @@ class Qwen3HybridLinearDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
def forward(
@@ -716,7 +715,6 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
self.alt_stream = alt_stream
-1
View File
@@ -1060,7 +1060,6 @@ class SarvamMoEMLADecoderLayer(nn.Module):
post_attention_layernorm=self.post_attention_layernorm,
qkv_latent_func=self.self_attn.prepare_qkv_latent,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
def forward(
-1
View File
@@ -368,7 +368,6 @@ class SDARMoeBlock(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(layer_id == config.num_hidden_layers - 1),
)
def forward(
-1
View File
@@ -572,7 +572,6 @@ class Step3p5DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(is_nextn or layer_id == config.num_hidden_layers - 1),
)
self.layer_id = layer_id
-1
View File
@@ -1542,7 +1542,6 @@ class XllmDecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
is_last_layer=(self.layer_id == config.num_hidden_layers - 1),
)
def forward(