[Refactor] Choose the boundaries of plain-TP dense layers and MoE layers from declarations (#41417)
This commit is contained in:
@@ -56,6 +56,9 @@ class StageInput(msgspec.Struct, frozen=True):
|
||||
compute group does not span."""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
@@ -87,32 +90,58 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
"""The declarations both sides of a decoder layer's boundaries are chosen
|
||||
from."""
|
||||
|
||||
# The rows the layer takes and hands on; the residual stays on them.
|
||||
layer_rows: Layout
|
||||
# The rows the layer's input and residual arrive on.
|
||||
input_rows: Layout
|
||||
attention: StageInput
|
||||
attention_output: StageOutput
|
||||
ffn: StageInput
|
||||
ffn_output: StageOutput
|
||||
# The rows the residual is on while the FFN runs.
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
|
||||
|
||||
def dense_decoder_layer_sides(
|
||||
def decoder_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_on_local_rows: bool,
|
||||
previous_on_local_rows: bool,
|
||||
is_last_layer: bool,
|
||||
attention_gathers_local_rows: bool,
|
||||
ffn_group: SumGroup,
|
||||
leaves_for_next_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""An attention followed by a dense MLP on the TP group, derived from the
|
||||
groups each computes over."""
|
||||
"""An attention followed by an FFN, derived from the groups each computes
|
||||
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
|
||||
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
|
||||
DP shard, which completes its own combine)."""
|
||||
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
# The TP group spans every token axis, so its MLP needs every row.
|
||||
ffn = Layout.sharded_over(axis_sizes=axis_sizes)
|
||||
# Each attention-TP rank's own slice of those rows.
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
# The TP group spans every token axis, so an FFN on it needs every row.
|
||||
ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes)
|
||||
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return DecoderLayerSides(
|
||||
layer_rows=attention,
|
||||
attention=StageInput(attention),
|
||||
input_rows=local if previous_on_local_rows else attention,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
attention_output=StageOutput(
|
||||
attention,
|
||||
@@ -120,11 +149,17 @@ def dense_decoder_layer_sides(
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
ffn=StageInput(ffn),
|
||||
ffn_output=StageOutput(
|
||||
ffn,
|
||||
group=SumGroup.TP,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatter,
|
||||
ffn_output=(
|
||||
StageOutput(ffn)
|
||||
if ffn_on_local_rows
|
||||
else StageOutput(
|
||||
ffn,
|
||||
group=ffn_group,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
|
||||
)
|
||||
),
|
||||
ffn_residual_rows=local if ffn_on_local_rows else attention,
|
||||
output_rows=local if ffn_on_local_rows and not is_last_layer else attention,
|
||||
)
|
||||
|
||||
@@ -44,7 +44,7 @@ from sglang.srt.layers.boundary_layout import (
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
dense_decoder_layer_sides,
|
||||
decoder_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
is_mla_cp_active,
|
||||
@@ -75,6 +75,7 @@ from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
post_experts_output_is_complete,
|
||||
post_experts_reduction_group,
|
||||
should_use_dp_reduce_scatterv,
|
||||
@@ -773,50 +774,75 @@ class LayerCommunicator:
|
||||
|
||||
def _declared_sides(self) -> Optional[DecoderLayerSides]:
|
||||
"""The declarations this layer's boundaries are chosen from: an
|
||||
attention and a dense MLP on the TP group under attention DP, without CP
|
||||
or LayerNorm SP, in a layer that takes the rows a dense layer or the
|
||||
embedding hands on. None when the steps come from the scatter modes."""
|
||||
attention and an FFN, without CP, LayerNorm SP or input-scattered
|
||||
attention, in a layer whose previous layer is known. None when the
|
||||
steps come from the scatter modes."""
|
||||
modes = self.layer_scatter_modes
|
||||
parallel = get_parallel()
|
||||
# A MoE dispatched per DP shard computes on this rank's local rows and
|
||||
# hands its layer's output on there.
|
||||
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
|
||||
if not (
|
||||
self._takes_declared_boundaries
|
||||
and parallel.attn_dp_size > 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and not layernorm_sp.layernorm_sp_enabled()
|
||||
and not modes.is_layer_sparse
|
||||
and not parallel.enable_attn_tp_input_scattered
|
||||
and not enable_moe_dense_fully_dp()
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is False)
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
|
||||
):
|
||||
return None
|
||||
return dense_decoder_layer_sides(
|
||||
return decoder_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
},
|
||||
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
|
||||
previous_on_local_rows=(
|
||||
not modes.is_first_layer
|
||||
and modes.is_previous_layer_sparse
|
||||
and moe_on_local_rows
|
||||
),
|
||||
is_last_layer=modes.is_last_layer,
|
||||
attention_gathers_local_rows=_use_ag_after_qlora,
|
||||
ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
# A MoE block leaves its sum to reduce_scatterv whenever that combine
|
||||
# applies (should_skip_post_experts_all_reduce).
|
||||
leaves_for_reduce_scatterv=(
|
||||
self.allow_reduce_scatter or modes.is_layer_sparse
|
||||
),
|
||||
)
|
||||
|
||||
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
|
||||
"""Choose the three boundary sides this layer owns from the
|
||||
declarations: the attention input, the attention -> FFN steps, and the
|
||||
FFN output's way back to the layer's rows."""
|
||||
FFN output's way on to the rows the layer hands on."""
|
||||
self._communicate_simple_fn = _select_attention_input_move(
|
||||
sides.layer_rows, sides.attention
|
||||
sides.input_rows, sides.attention
|
||||
)
|
||||
self._mlp_input = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=sides.layer_rows,
|
||||
residual=sides.input_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
)
|
||||
self._mlp_input_may_fuse = (
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual
|
||||
in getattr(self._mlp_input, "keywords", {}).get("fusions", ())
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
self._mlp_input_may_fuse = False
|
||||
self._ffn_output = sides.ffn_output
|
||||
self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp(
|
||||
sides.ffn_output, sides.layer_rows
|
||||
(
|
||||
self._postprocess_scatters_to_local_tokens,
|
||||
postprocess,
|
||||
) = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if postprocess is not None:
|
||||
self._communicate_summable_tensor_pair_fn = postprocess
|
||||
self._ffn_sum_is_movable = sides.ffn_output.group is not None
|
||||
|
||||
def _select_boundaries_from_scatter_modes(self) -> None:
|
||||
@@ -1097,9 +1123,7 @@ class LayerCommunicator:
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
fusions = (
|
||||
self._select_mlp_input_fusions(residual_input_mode)
|
||||
if self._context.attn_dp_size == 1
|
||||
else ()
|
||||
self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else ()
|
||||
)
|
||||
steps = partial(
|
||||
_mlp_input_gather,
|
||||
@@ -1112,9 +1136,7 @@ class LayerCommunicator:
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual in fusions,
|
||||
)
|
||||
|
||||
def _select_mlp_input_fusions(
|
||||
self, residual_input_mode: ScatterMode
|
||||
) -> Tuple[Callable, ...]:
|
||||
def _select_mlp_input_fusions(self) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete the attention output's all-reduce
|
||||
together with the residual update and the post-attention norm, in the
|
||||
order they are tried. Each takes (hidden_states, residual, forward_batch)
|
||||
@@ -1803,29 +1825,34 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input."""
|
||||
if rows == need.layout:
|
||||
return CommunicateSimpleFn._trivial
|
||||
raise NotImplementedError(f"{rows=} {need=}")
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
gathers them itself."""
|
||||
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
|
||||
if not need.layout.sharded <= rows.sharded or gathered not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_TP_SCATTER},
|
||||
):
|
||||
raise NotImplementedError(f"{rows=} {need=}")
|
||||
if gathered:
|
||||
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
return CommunicateSimpleFn._trivial
|
||||
|
||||
|
||||
def _select_ffn_input(
|
||||
produced: StageOutput,
|
||||
*,
|
||||
residual: Layout,
|
||||
residual_to: Layout,
|
||||
need: StageInput,
|
||||
force_layernorm_before_gather: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
) -> Callable:
|
||||
"""The steps from the attention output to the FFN input: complete the
|
||||
attention-TP sum, add the residual and normalize, and gather the rows the
|
||||
FFN's group needs, with the residual left on the attention's rows."""
|
||||
gathered = produced.layout.sharded - need.layout.sharded
|
||||
if (
|
||||
residual != produced.layout
|
||||
or not need.layout.sharded <= produced.layout.sharded
|
||||
or gathered != {TokenAxis.ATTN_DP}
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
attention-TP sum, move the residual to the rows it has while the FFN runs,
|
||||
add it and normalize, and bring the rows to what the FFN's group needs: a
|
||||
gather over attention DP, or each rank's own slice. The fused kernels in
|
||||
``fusions`` are tried first when nothing is gathered or sliced."""
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
# always left by the output projection, or nothing.
|
||||
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
||||
@@ -1834,28 +1861,70 @@ def _select_ffn_input(
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
# completes the attention-TP sum and slices in one collective.
|
||||
if (
|
||||
sliced != {TokenAxis.ATTN_TP_SCATTER}
|
||||
or gathered
|
||||
or not owes_attention_tp
|
||||
or residual_to != need.layout
|
||||
or residual not in (produced.layout, need.layout)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_scatter, scatters_residual=residual != residual_to)
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
or gathered not in (frozenset(), {TokenAxis.ATTN_DP})
|
||||
or residual.sharded - residual_to.sharded
|
||||
not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER})
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
# A residual arriving on each rank's slice is gathered back first.
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return _mlp_input_norm
|
||||
return partial(
|
||||
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
|
||||
)
|
||||
# The partial order adds the residual on attention-TP rank 0 before the DP
|
||||
# gather's collective completes that sum, which only a plain residual add
|
||||
# allows.
|
||||
if owes_attention_tp and not force_layernorm_before_gather:
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=False)
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
|
||||
return partial(
|
||||
_mlp_input_dp_replicate,
|
||||
gathers_residual=False,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
)
|
||||
|
||||
|
||||
def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool:
|
||||
"""Whether the FFN output goes back to the layer's rows by undoing the
|
||||
attention-DP gather."""
|
||||
returned = rows.sharded - produced.layout.sharded
|
||||
if not produced.layout.sharded <= rows.sharded or returned not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_DP},
|
||||
def _select_ffn_output_move(
|
||||
produced: StageOutput, *, residual: Layout, to: Layout
|
||||
) -> Tuple[bool, Optional[Callable]]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
that step), or else the postprocess that moves it, None when there is none
|
||||
to choose here."""
|
||||
if produced.layout == residual:
|
||||
if to == residual:
|
||||
return False, CommunicateSummableTensorPairFn._trivial
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: fold the residual
|
||||
# into the output, then gather over attention TP.
|
||||
return False, CommunicateSummableTensorPairFn._gather
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if (
|
||||
to != residual
|
||||
or not produced.layout.sharded <= residual.sharded
|
||||
or returned != {TokenAxis.ATTN_DP}
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {rows=}")
|
||||
return bool(returned)
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return True, None
|
||||
|
||||
|
||||
class MlpInputKind(Enum):
|
||||
|
||||
@@ -239,11 +239,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
gamma=_fused_norm_gamma(self.input_layernorm),
|
||||
)
|
||||
|
||||
def _select_mlp_input_fusions(self, residual_input_mode):
|
||||
fusions = super()._select_mlp_input_fusions(residual_input_mode)
|
||||
def _select_mlp_input_fusions(self):
|
||||
fusions = super()._select_mlp_input_fusions()
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
residual_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(self.post_attention_layernorm) is not None
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user