[Refactor] Step-3.5: complete the dense MLP's sum through ffn_exit (#41437)

This commit is contained in:
Cheng Wan
2026-09-27 16:53:57 -07:00
committed by GitHub
parent 0971450c16
commit 55d2ee5546
3 changed files with 31 additions and 17 deletions
+5 -11
View File
@@ -572,6 +572,9 @@ class Step3p5DecoderLayer(nn.Module):
input_layernorm=self.input_layernorm,
post_attention_layernorm=self.post_attention_layernorm,
allow_reduce_scatter=True,
# The dense MLP all-reduces its own output unless postprocess
# reduce-scatters it; it never leaves the sum to the next layer.
allow_deferred_ffn_reduction=self.use_moe,
)
self.layer_id = layer_id
@@ -618,18 +621,9 @@ class Step3p5DecoderLayer(nn.Module):
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
return ffn_exit.finish(hidden_states, residual)
# The dense MLP all-reduces its own output unless postprocess
# reduce-scatters it; it never leaves the sum to the next layer.
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
forward_batch
)
with get_forward().scoped(
fuse_mlp_allreduce=False, mlp_reduce_scatter=mlp_reduce_scatter
):
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return self.layer_communicator.postprocess_layer(
hidden_states, residual, forward_batch
)
return ffn_exit.finish(hidden_states, residual)
class Step3p5Model(nn.Module):