[Refactor] Choose the boundaries of plain-TP dense layers and MoE layers from declarations (#41417)

This commit is contained in:
Cheng Wan
2026-09-27 16:46:50 -07:00
committed by GitHub
parent 583fada954
commit 446d181e23
5 changed files with 490 additions and 157 deletions
+50 -15
View File
@@ -56,6 +56,9 @@ class StageInput(msgspec.Struct, frozen=True):
compute group does not span."""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
class StageOutput(msgspec.Struct, frozen=True):
@@ -87,32 +90,58 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
"""The declarations both sides of a decoder layer's boundaries are chosen
from."""
# The rows the layer takes and hands on; the residual stays on them.
layer_rows: Layout
# The rows the layer's input and residual arrive on.
input_rows: Layout
attention: StageInput
attention_output: StageOutput
ffn: StageInput
ffn_output: StageOutput
# The rows the residual is on while the FFN runs.
ffn_residual_rows: Layout
# The rows the layer hands on.
output_rows: Layout
def dense_decoder_layer_sides(
def decoder_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_on_local_rows: bool,
previous_on_local_rows: bool,
is_last_layer: bool,
attention_gathers_local_rows: bool,
ffn_group: SumGroup,
leaves_for_next_layer: bool,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
) -> DecoderLayerSides:
"""An attention followed by a dense MLP on the TP group, derived from the
groups each computes over."""
"""An attention followed by an FFN, derived from the groups each computes
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
DP shard, which completes its own combine)."""
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
# The TP group spans every token axis, so its MLP needs every row.
ffn = Layout.sharded_over(axis_sizes=axis_sizes)
# Each attention-TP rank's own slice of those rows.
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
# The TP group spans every token axis, so an FFN on it needs every row.
ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes)
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
layer_rows=attention,
attention=StageInput(attention),
input_rows=local if previous_on_local_rows else attention,
attention=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
),
),
# The output projection leaves the attention-TP sum to prepare_mlp.
attention_output=StageOutput(
attention,
@@ -120,11 +149,17 @@ def dense_decoder_layer_sides(
always_leaves=attention_tp,
),
ffn=StageInput(ffn),
ffn_output=StageOutput(
ffn,
group=SumGroup.TP,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatter,
ffn_output=(
StageOutput(ffn)
if ffn_on_local_rows
else StageOutput(
ffn,
group=ffn_group,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
)
),
ffn_residual_rows=local if ffn_on_local_rows else attention,
output_rows=local if ffn_on_local_rows and not is_last_layer else attention,
)
+114 -45
View File
@@ -44,7 +44,7 @@ from sglang.srt.layers.boundary_layout import (
StageOutput,
SumGroup,
TokenAxis,
dense_decoder_layer_sides,
decoder_layer_sides,
)
from sglang.srt.layers.cp.utils import (
is_mla_cp_active,
@@ -75,6 +75,7 @@ from sglang.srt.layers.flashinfer_comm_fusion import (
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
is_moe_input_scattered_across_dp_ranks,
post_experts_output_is_complete,
post_experts_reduction_group,
should_use_dp_reduce_scatterv,
@@ -773,50 +774,75 @@ class LayerCommunicator:
def _declared_sides(self) -> Optional[DecoderLayerSides]:
"""The declarations this layer's boundaries are chosen from: an
attention and a dense MLP on the TP group under attention DP, without CP
or LayerNorm SP, in a layer that takes the rows a dense layer or the
embedding hands on. None when the steps come from the scatter modes."""
attention and an FFN, without CP, LayerNorm SP or input-scattered
attention, in a layer whose previous layer is known. None when the
steps come from the scatter modes."""
modes = self.layer_scatter_modes
parallel = get_parallel()
# A MoE dispatched per DP shard computes on this rank's local rows and
# hands its layer's output on there.
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
if not (
self._takes_declared_boundaries
and parallel.attn_dp_size > 1
and parallel.attn_cp_size == 1
and not layernorm_sp.layernorm_sp_enabled()
and not modes.is_layer_sparse
and not parallel.enable_attn_tp_input_scattered
and not enable_moe_dense_fully_dp()
and (modes.is_first_layer or modes.is_previous_layer_sparse is False)
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
):
return None
return dense_decoder_layer_sides(
return decoder_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: parallel.attn_dp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size,
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
},
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
previous_on_local_rows=(
not modes.is_first_layer
and modes.is_previous_layer_sparse
and moe_on_local_rows
),
is_last_layer=modes.is_last_layer,
attention_gathers_local_rows=_use_ag_after_qlora,
ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
leaves_for_reduce_scatter=self.allow_reduce_scatter,
# A MoE block leaves its sum to reduce_scatterv whenever that combine
# applies (should_skip_post_experts_all_reduce).
leaves_for_reduce_scatterv=(
self.allow_reduce_scatter or modes.is_layer_sparse
),
)
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
"""Choose the three boundary sides this layer owns from the
declarations: the attention input, the attention -> FFN steps, and the
FFN output's way back to the layer's rows."""
FFN output's way on to the rows the layer hands on."""
self._communicate_simple_fn = _select_attention_input_move(
sides.layer_rows, sides.attention
sides.input_rows, sides.attention
)
self._mlp_input = _select_ffn_input(
sides.attention_output,
residual=sides.layer_rows,
residual=sides.input_rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
fusions=self._select_mlp_input_fusions(),
)
self._mlp_input_may_fuse = (
self._mlp_input_reduce_output_and_update_and_read_residual
in getattr(self._mlp_input, "keywords", {}).get("fusions", ())
)
# Neither fused kernel runs under attention DP.
self._mlp_input_may_fuse = False
self._ffn_output = sides.ffn_output
self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp(
sides.ffn_output, sides.layer_rows
(
self._postprocess_scatters_to_local_tokens,
postprocess,
) = _select_ffn_output_move(
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
)
if postprocess is not None:
self._communicate_summable_tensor_pair_fn = postprocess
self._ffn_sum_is_movable = sides.ffn_output.group is not None
def _select_boundaries_from_scatter_modes(self) -> None:
@@ -1097,9 +1123,7 @@ class LayerCommunicator:
)
# Neither fused kernel runs under attention DP.
fusions = (
self._select_mlp_input_fusions(residual_input_mode)
if self._context.attn_dp_size == 1
else ()
self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else ()
)
steps = partial(
_mlp_input_gather,
@@ -1112,9 +1136,7 @@ class LayerCommunicator:
self._mlp_input_reduce_output_and_update_and_read_residual in fusions,
)
def _select_mlp_input_fusions(
self, residual_input_mode: ScatterMode
) -> Tuple[Callable, ...]:
def _select_mlp_input_fusions(self) -> Tuple[Callable, ...]:
"""The fused kernels that complete the attention output's all-reduce
together with the residual update and the post-attention norm, in the
order they are tried. Each takes (hidden_states, residual, forward_batch)
@@ -1803,29 +1825,34 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
"""How the rows a layer takes become its attention's input."""
if rows == need.layout:
return CommunicateSimpleFn._trivial
raise NotImplementedError(f"{rows=} {need=}")
"""How the rows a layer takes become its attention's input: as they are,
or gathered over attention TP from each rank's slice, unless the attention
gathers them itself."""
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
if not need.layout.sharded <= rows.sharded or gathered not in (
frozenset(),
{TokenAxis.ATTN_TP_SCATTER},
):
raise NotImplementedError(f"{rows=} {need=}")
if gathered:
return CommunicateSimpleFn._scattered_to_tp_attn_full
return CommunicateSimpleFn._trivial
def _select_ffn_input(
produced: StageOutput,
*,
residual: Layout,
residual_to: Layout,
need: StageInput,
force_layernorm_before_gather: bool,
fusions: Tuple[Callable, ...],
) -> Callable:
"""The steps from the attention output to the FFN input: complete the
attention-TP sum, add the residual and normalize, and gather the rows the
FFN's group needs, with the residual left on the attention's rows."""
gathered = produced.layout.sharded - need.layout.sharded
if (
residual != produced.layout
or not need.layout.sharded <= produced.layout.sharded
or gathered != {TokenAxis.ATTN_DP}
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
attention-TP sum, move the residual to the rows it has while the FFN runs,
add it and normalize, and bring the rows to what the FFN's group needs: a
gather over attention DP, or each rank's own slice. The fused kernels in
``fusions`` are tried first when nothing is gathered or sliced."""
# What the attention output owes decides the steps: the attention-TP sum,
# always left by the output projection, or nothing.
owes_attention_tp = produced.group is SumGroup.ATTN_TP
@@ -1834,28 +1861,70 @@ def _select_ffn_input(
SumGroup.ATTN_TP,
):
raise NotImplementedError(f"{produced=}")
gathered = produced.layout.sharded - need.layout.sharded
sliced = need.layout.sharded - produced.layout.sharded
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
# completes the attention-TP sum and slices in one collective.
if (
sliced != {TokenAxis.ATTN_TP_SCATTER}
or gathered
or not owes_attention_tp
or residual_to != need.layout
or residual not in (produced.layout, need.layout)
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_scatter, scatters_residual=residual != residual_to)
if (
residual_to != produced.layout
or gathered not in (frozenset(), {TokenAxis.ATTN_DP})
or residual.sharded - residual_to.sharded
not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER})
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
# A residual arriving on each rank's slice is gathered back first.
gathers_residual = residual != residual_to
if not gathered:
if not owes_attention_tp:
return _mlp_input_norm
return partial(
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
)
# The partial order adds the residual on attention-TP rank 0 before the DP
# gather's collective completes that sum, which only a plain residual add
# allows.
if owes_attention_tp and not force_layernorm_before_gather:
return partial(_mlp_input_dp_partial, gathers_residual=False)
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
return partial(
_mlp_input_dp_replicate,
gathers_residual=False,
gathers_residual=gathers_residual,
reduces_attention_tp=owes_attention_tp,
)
def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool:
"""Whether the FFN output goes back to the layer's rows by undoing the
attention-DP gather."""
returned = rows.sharded - produced.layout.sharded
if not produced.layout.sharded <= rows.sharded or returned not in (
frozenset(),
{TokenAxis.ATTN_DP},
def _select_ffn_output_move(
produced: StageOutput, *, residual: Layout, to: Layout
) -> Tuple[bool, Optional[Callable]]:
"""How the FFN output reaches the rows the layer hands on: whether it goes
back by undoing the attention-DP gather (the FFN exit and postprocess run
that step), or else the postprocess that moves it, None when there is none
to choose here."""
if produced.layout == residual:
if to == residual:
return False, CommunicateSummableTensorPairFn._trivial
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice back to the attention's rows: fold the residual
# into the output, then gather over attention TP.
return False, CommunicateSummableTensorPairFn._gather
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if (
to != residual
or not produced.layout.sharded <= residual.sharded
or returned != {TokenAxis.ATTN_DP}
):
raise NotImplementedError(f"{produced=} {rows=}")
return bool(returned)
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return True, None
class MlpInputKind(Enum):
@@ -239,11 +239,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
gamma=_fused_norm_gamma(self.input_layernorm),
)
def _select_mlp_input_fusions(self, residual_input_mode):
fusions = super()._select_mlp_input_fusions(residual_input_mode)
def _select_mlp_input_fusions(self):
fusions = super()._select_mlp_input_fusions()
parallel = get_parallel()
if (
residual_input_mode is ScatterMode.TP_ATTN_FULL
self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL
and parallel.attn_tp_size == parallel.tp_size
and _fused_norm_gamma(self.post_attention_layernorm) is not None
):