[Refactor] Choose MoE layers' boundaries from declarations when moe_dp_size equals attn_cp_size (#41435)

This commit is contained in:
Cheng Wan
2026-09-27 16:53:15 -07:00
committed by GitHub
parent ecb8e9fc8f
commit 4fac02fe5d
4 changed files with 119 additions and 6 deletions
+11 -1
View File
@@ -335,11 +335,14 @@ def decoder_layer_sides(
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
hands_on_attention_rows: bool = False,
ffn_shards_over_cp: bool = False,
) -> DecoderLayerSides:
"""An attention followed by an FFN, derived from the groups each computes
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
DP shard, which completes its own combine, or a dense MLP on every rank).
A MoE whose data-parallel groups are the CP ranks (``ffn_shards_over_cp``)
computes each CP shard on its own ranks: its rows stay sharded over CP.
A layer on local rows hands on the attention's rows when it is the last, or
with ``hands_on_attention_rows``."""
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
@@ -354,7 +357,14 @@ def decoder_layer_sides(
axis_sizes=axis_sizes,
)
# The TP group spans every token axis, so an FFN on it needs every row.
ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes)
ffn = (
local
if ffn_on_local_rows
else Layout.sharded_over(
*((TokenAxis.ATTN_CP,) if ffn_shards_over_cp else ()),
axis_sizes=axis_sizes,
)
)
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
input_rows=local if previous_on_local_rows else attention,
+25 -5
View File
@@ -1004,6 +1004,15 @@ class LayerCommunicator:
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
dense_on_local_rows = enable_moe_dense_fully_dp()
# A MoE whose data-parallel groups are the CP ranks computes each CP
# shard of a GQA prefill CP on its own ranks.
ffn_on_cp_shards = (
modes.is_layer_sparse
and parallel.attn_cp_size > 1
and parallel.moe_dp_size == parallel.attn_cp_size
and not _gathers_over_attention_cp()
)
def on_local_rows(sparse: bool) -> bool:
return moe_on_local_rows if sparse else dense_on_local_rows
@@ -1039,11 +1048,15 @@ class LayerCommunicator:
may_leave = not cp_active
may_leave_to_reduce_scatter = True
else:
# Otherwise under CP the FFN completes its own sum.
may_leave = may_leave_to_reduce_scatter = parallel.attn_cp_size == 1
# Otherwise under CP an FFN over every CP shard completes its own sum;
# one on its own CP shard hands on its rows as without CP.
may_leave = may_leave_to_reduce_scatter = (
parallel.attn_cp_size == 1 or ffn_on_cp_shards
)
return decoder_layer_sides(
axis_sizes=token_axis_sizes(cp_active=cp_active),
ffn_on_local_rows=on_local_rows(modes.is_layer_sparse),
ffn_shards_over_cp=ffn_on_cp_shards,
previous_on_local_rows=(
not modes.is_first_layer
and on_local_rows(modes.is_previous_layer_sparse)
@@ -2304,10 +2317,17 @@ def tbo_split_moves(layer_input_rows: Layout) -> Tuple[Callable, Callable]:
def _cp_on_declarations() -> bool:
"""Whether attention CP is one the declarations cover: a prefill CP that
shards tokens, with DSA or MLA attention, or with the FFN input gathered
over a MoE-CP group that is the whole CP group."""
shards tokens, with DSA or MLA attention, with the FFN input gathered over a
MoE-CP group that is the whole CP group, or, without attention DP, with the
MoE's data-parallel groups the CP ranks (a MoE then computes each CP shard
on its own ranks)."""
parallel = get_parallel()
return _generic_prefill_cp_shards_tokens() and (
_gathers_over_attention_cp() or get_parallel().moe_dp_size == 1
_gathers_over_attention_cp()
or parallel.moe_dp_size == 1
or (
parallel.moe_dp_size == parallel.attn_cp_size and parallel.attn_dp_size == 1
)
)