[Fix] Offer the fused MoE finalize all-reduce only to a block that owes the sum (#42479)
This commit is contained in:
@@ -1106,6 +1106,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
# The mHC post-split consumes the reduced row without an RMSNorm.
|
||||
use_fused_finalize_all_reduce = (
|
||||
self._fuse_finalize_all_reduce
|
||||
# The kernel completes the tensor-parallel sum itself.
|
||||
and self.reduce_results
|
||||
and has_shared_output
|
||||
and hidden_states.shape[-1] == 5120
|
||||
and not self._shared_expert_tp1
|
||||
|
||||
Reference in New Issue
Block a user