[Refactor] Choose the boundaries of plain-TP dense layers and MoE layers from declarations (#41417)

This commit is contained in:
Cheng Wan
2026-09-27 16:46:50 -07:00
committed by GitHub
parent 583fada954
commit 446d181e23
5 changed files with 490 additions and 157 deletions
+50 -15
View File
@@ -56,6 +56,9 @@ class StageInput(msgspec.Struct, frozen=True):
compute group does not span."""
layout: Layout
# Token axes the consumer gathers over itself when its input arrives
# sharded over them.
gathers_itself: FrozenSet[TokenAxis] = frozenset()
class StageOutput(msgspec.Struct, frozen=True):
@@ -87,32 +90,58 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
"""The declarations both sides of a decoder layer's boundaries are chosen
from."""
# The rows the layer takes and hands on; the residual stays on them.
layer_rows: Layout
# The rows the layer's input and residual arrive on.
input_rows: Layout
attention: StageInput
attention_output: StageOutput
ffn: StageInput
ffn_output: StageOutput
# The rows the residual is on while the FFN runs.
ffn_residual_rows: Layout
# The rows the layer hands on.
output_rows: Layout
def dense_decoder_layer_sides(
def decoder_layer_sides(
*,
axis_sizes: Mapping[TokenAxis, int],
ffn_on_local_rows: bool,
previous_on_local_rows: bool,
is_last_layer: bool,
attention_gathers_local_rows: bool,
ffn_group: SumGroup,
leaves_for_next_layer: bool,
leaves_for_reduce_scatter: bool,
leaves_for_reduce_scatterv: bool,
) -> DecoderLayerSides:
"""An attention followed by a dense MLP on the TP group, derived from the
groups each computes over."""
"""An attention followed by an FFN, derived from the groups each computes
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
DP shard, which completes its own combine)."""
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
attention = Layout.sharded_over(
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
)
# The TP group spans every token axis, so its MLP needs every row.
ffn = Layout.sharded_over(axis_sizes=axis_sizes)
# Each attention-TP rank's own slice of those rows.
local = Layout.sharded_over(
TokenAxis.ATTN_DP,
TokenAxis.ATTN_CP,
TokenAxis.ATTN_TP_SCATTER,
axis_sizes=axis_sizes,
)
# The TP group spans every token axis, so an FFN on it needs every row.
ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes)
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
return DecoderLayerSides(
layer_rows=attention,
attention=StageInput(attention),
input_rows=local if previous_on_local_rows else attention,
attention=StageInput(
attention,
gathers_itself=(
frozenset({TokenAxis.ATTN_TP_SCATTER})
if attention_gathers_local_rows
else frozenset()
),
),
# The output projection leaves the attention-TP sum to prepare_mlp.
attention_output=StageOutput(
attention,
@@ -120,11 +149,17 @@ def dense_decoder_layer_sides(
always_leaves=attention_tp,
),
ffn=StageInput(ffn),
ffn_output=StageOutput(
ffn,
group=SumGroup.TP,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatter,
ffn_output=(
StageOutput(ffn)
if ffn_on_local_rows
else StageOutput(
ffn,
group=ffn_group,
leaves_for_next_layer=leaves_for_next_layer,
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
)
),
ffn_residual_rows=local if ffn_on_local_rows else attention,
output_rows=local if ffn_on_local_rows and not is_last_layer else attention,
)
+114 -45
View File
@@ -44,7 +44,7 @@ from sglang.srt.layers.boundary_layout import (
StageOutput,
SumGroup,
TokenAxis,
dense_decoder_layer_sides,
decoder_layer_sides,
)
from sglang.srt.layers.cp.utils import (
is_mla_cp_active,
@@ -75,6 +75,7 @@ from sglang.srt.layers.flashinfer_comm_fusion import (
from sglang.srt.layers.moe import (
can_merge_post_experts_all_reduce,
get_moe_a2a_backend,
is_moe_input_scattered_across_dp_ranks,
post_experts_output_is_complete,
post_experts_reduction_group,
should_use_dp_reduce_scatterv,
@@ -773,50 +774,75 @@ class LayerCommunicator:
def _declared_sides(self) -> Optional[DecoderLayerSides]:
"""The declarations this layer's boundaries are chosen from: an
attention and a dense MLP on the TP group under attention DP, without CP
or LayerNorm SP, in a layer that takes the rows a dense layer or the
embedding hands on. None when the steps come from the scatter modes."""
attention and an FFN, without CP, LayerNorm SP or input-scattered
attention, in a layer whose previous layer is known. None when the
steps come from the scatter modes."""
modes = self.layer_scatter_modes
parallel = get_parallel()
# A MoE dispatched per DP shard computes on this rank's local rows and
# hands its layer's output on there.
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
if not (
self._takes_declared_boundaries
and parallel.attn_dp_size > 1
and parallel.attn_cp_size == 1
and not layernorm_sp.layernorm_sp_enabled()
and not modes.is_layer_sparse
and not parallel.enable_attn_tp_input_scattered
and not enable_moe_dense_fully_dp()
and (modes.is_first_layer or modes.is_previous_layer_sparse is False)
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
):
return None
return dense_decoder_layer_sides(
return decoder_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: parallel.attn_dp_size,
TokenAxis.ATTN_CP: parallel.attn_cp_size,
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
},
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
previous_on_local_rows=(
not modes.is_first_layer
and modes.is_previous_layer_sparse
and moe_on_local_rows
),
is_last_layer=modes.is_last_layer,
attention_gathers_local_rows=_use_ag_after_qlora,
ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
leaves_for_reduce_scatter=self.allow_reduce_scatter,
# A MoE block leaves its sum to reduce_scatterv whenever that combine
# applies (should_skip_post_experts_all_reduce).
leaves_for_reduce_scatterv=(
self.allow_reduce_scatter or modes.is_layer_sparse
),
)
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
"""Choose the three boundary sides this layer owns from the
declarations: the attention input, the attention -> FFN steps, and the
FFN output's way back to the layer's rows."""
FFN output's way on to the rows the layer hands on."""
self._communicate_simple_fn = _select_attention_input_move(
sides.layer_rows, sides.attention
sides.input_rows, sides.attention
)
self._mlp_input = _select_ffn_input(
sides.attention_output,
residual=sides.layer_rows,
residual=sides.input_rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
fusions=self._select_mlp_input_fusions(),
)
self._mlp_input_may_fuse = (
self._mlp_input_reduce_output_and_update_and_read_residual
in getattr(self._mlp_input, "keywords", {}).get("fusions", ())
)
# Neither fused kernel runs under attention DP.
self._mlp_input_may_fuse = False
self._ffn_output = sides.ffn_output
self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp(
sides.ffn_output, sides.layer_rows
(
self._postprocess_scatters_to_local_tokens,
postprocess,
) = _select_ffn_output_move(
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
)
if postprocess is not None:
self._communicate_summable_tensor_pair_fn = postprocess
self._ffn_sum_is_movable = sides.ffn_output.group is not None
def _select_boundaries_from_scatter_modes(self) -> None:
@@ -1097,9 +1123,7 @@ class LayerCommunicator:
)
# Neither fused kernel runs under attention DP.
fusions = (
self._select_mlp_input_fusions(residual_input_mode)
if self._context.attn_dp_size == 1
else ()
self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else ()
)
steps = partial(
_mlp_input_gather,
@@ -1112,9 +1136,7 @@ class LayerCommunicator:
self._mlp_input_reduce_output_and_update_and_read_residual in fusions,
)
def _select_mlp_input_fusions(
self, residual_input_mode: ScatterMode
) -> Tuple[Callable, ...]:
def _select_mlp_input_fusions(self) -> Tuple[Callable, ...]:
"""The fused kernels that complete the attention output's all-reduce
together with the residual update and the post-attention norm, in the
order they are tried. Each takes (hidden_states, residual, forward_batch)
@@ -1803,29 +1825,34 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
"""How the rows a layer takes become its attention's input."""
if rows == need.layout:
return CommunicateSimpleFn._trivial
raise NotImplementedError(f"{rows=} {need=}")
"""How the rows a layer takes become its attention's input: as they are,
or gathered over attention TP from each rank's slice, unless the attention
gathers them itself."""
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
if not need.layout.sharded <= rows.sharded or gathered not in (
frozenset(),
{TokenAxis.ATTN_TP_SCATTER},
):
raise NotImplementedError(f"{rows=} {need=}")
if gathered:
return CommunicateSimpleFn._scattered_to_tp_attn_full
return CommunicateSimpleFn._trivial
def _select_ffn_input(
produced: StageOutput,
*,
residual: Layout,
residual_to: Layout,
need: StageInput,
force_layernorm_before_gather: bool,
fusions: Tuple[Callable, ...],
) -> Callable:
"""The steps from the attention output to the FFN input: complete the
attention-TP sum, add the residual and normalize, and gather the rows the
FFN's group needs, with the residual left on the attention's rows."""
gathered = produced.layout.sharded - need.layout.sharded
if (
residual != produced.layout
or not need.layout.sharded <= produced.layout.sharded
or gathered != {TokenAxis.ATTN_DP}
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
attention-TP sum, move the residual to the rows it has while the FFN runs,
add it and normalize, and bring the rows to what the FFN's group needs: a
gather over attention DP, or each rank's own slice. The fused kernels in
``fusions`` are tried first when nothing is gathered or sliced."""
# What the attention output owes decides the steps: the attention-TP sum,
# always left by the output projection, or nothing.
owes_attention_tp = produced.group is SumGroup.ATTN_TP
@@ -1834,28 +1861,70 @@ def _select_ffn_input(
SumGroup.ATTN_TP,
):
raise NotImplementedError(f"{produced=}")
gathered = produced.layout.sharded - need.layout.sharded
sliced = need.layout.sharded - produced.layout.sharded
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
# completes the attention-TP sum and slices in one collective.
if (
sliced != {TokenAxis.ATTN_TP_SCATTER}
or gathered
or not owes_attention_tp
or residual_to != need.layout
or residual not in (produced.layout, need.layout)
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
return partial(_mlp_input_scatter, scatters_residual=residual != residual_to)
if (
residual_to != produced.layout
or gathered not in (frozenset(), {TokenAxis.ATTN_DP})
or residual.sharded - residual_to.sharded
not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER})
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
# A residual arriving on each rank's slice is gathered back first.
gathers_residual = residual != residual_to
if not gathered:
if not owes_attention_tp:
return _mlp_input_norm
return partial(
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
)
# The partial order adds the residual on attention-TP rank 0 before the DP
# gather's collective completes that sum, which only a plain residual add
# allows.
if owes_attention_tp and not force_layernorm_before_gather:
return partial(_mlp_input_dp_partial, gathers_residual=False)
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
return partial(
_mlp_input_dp_replicate,
gathers_residual=False,
gathers_residual=gathers_residual,
reduces_attention_tp=owes_attention_tp,
)
def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool:
"""Whether the FFN output goes back to the layer's rows by undoing the
attention-DP gather."""
returned = rows.sharded - produced.layout.sharded
if not produced.layout.sharded <= rows.sharded or returned not in (
frozenset(),
{TokenAxis.ATTN_DP},
def _select_ffn_output_move(
produced: StageOutput, *, residual: Layout, to: Layout
) -> Tuple[bool, Optional[Callable]]:
"""How the FFN output reaches the rows the layer hands on: whether it goes
back by undoing the attention-DP gather (the FFN exit and postprocess run
that step), or else the postprocess that moves it, None when there is none
to choose here."""
if produced.layout == residual:
if to == residual:
return False, CommunicateSummableTensorPairFn._trivial
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
# Each rank's slice back to the attention's rows: fold the residual
# into the output, then gather over attention TP.
return False, CommunicateSummableTensorPairFn._gather
raise NotImplementedError(f"{produced=} {residual=} {to=}")
returned = residual.sharded - produced.layout.sharded
if (
to != residual
or not produced.layout.sharded <= residual.sharded
or returned != {TokenAxis.ATTN_DP}
):
raise NotImplementedError(f"{produced=} {rows=}")
return bool(returned)
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return True, None
class MlpInputKind(Enum):
@@ -239,11 +239,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
gamma=_fused_norm_gamma(self.input_layernorm),
)
def _select_mlp_input_fusions(self, residual_input_mode):
fusions = super()._select_mlp_input_fusions(residual_input_mode)
def _select_mlp_input_fusions(self):
fusions = super()._select_mlp_input_fusions()
parallel = get_parallel()
if (
residual_input_mode is ScatterMode.TP_ATTN_FULL
self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL
and parallel.attn_tp_size == parallel.tp_size
and _fused_norm_gamma(self.post_attention_layernorm) is not None
):
@@ -119,12 +119,15 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel():
f"{_MODULE}.get_parallel",
return_value=SimpleNamespace(attn_tp_size=2, tp_size=2),
):
assert comm._select_mlp_input_fusions(ScatterMode.TP_ATTN_FULL) == (
cutedsl,
base,
comm.layer_scatter_modes = SimpleNamespace(
layer_input_mode=ScatterMode.TP_ATTN_FULL
)
assert comm._select_mlp_input_fusions() == (cutedsl, base)
# A scattered residual is gathered first, which the workspace does not do.
assert comm._select_mlp_input_fusions(ScatterMode.SCATTERED) == (base,)
comm.layer_scatter_modes = SimpleNamespace(
layer_input_mode=ScatterMode.SCATTERED
)
assert comm._select_mlp_input_fusions() == (base,)
def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible):
@@ -1,5 +1,7 @@
"""The boundaries of an attention and a dense MLP on the TP group under attention
DP, chosen from both sides' declarations.
"""The boundaries of an attention and an FFN, chosen from both sides'
declarations. The FFN runs on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on each attention-TP rank's slice of its DP shard's
rows (a MoE dispatched per DP shard).
The numeric checks run every rank of a DP x attention-TP world as a thread over
fake collectives that wait for all members of their group, build the layers'
@@ -21,7 +23,7 @@ from sglang.srt.layers import communicator as comm
from sglang.srt.layers.boundary_layout import (
SumGroup,
TokenAxis,
dense_decoder_layer_sides,
decoder_layer_sides,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
@@ -57,6 +59,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
moe_dp_size=1,
dwdp_size=1,
enable_dp_attention=attn_dp > 1,
enable_attn_tp_input_scattered=False,
tp_group=SimpleNamespace(name="tp"),
attn_tp_group=SimpleNamespace(name="attn_tp"),
)
@@ -65,7 +68,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
@contextmanager
def planning(parallel, *, sp=False):
def planning(parallel, *, sp=False, a2a=False):
"""What layer planning and communicator construction read, without the
process-wide parallel state. ``parallel`` may be a callable, for a
per-thread parallel state."""
@@ -80,8 +83,11 @@ def planning(parallel, *, sp=False):
comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None)
),
patch.object(
comm, "get_moe_a2a_backend", lambda: SimpleNamespace(is_none=lambda: True)
comm,
"get_moe_a2a_backend",
lambda: SimpleNamespace(is_none=lambda: not a2a),
),
patch.object(comm, "is_moe_input_scattered_across_dp_ranks", lambda: a2a),
patch.object(
comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False
),
@@ -102,6 +108,13 @@ class Norm:
return s * 2, s
class FusableNorm(Norm):
"""A norm with the fused all-reduce + add + norm method."""
def forward_with_allreduce_fusion(self, *args, **kwargs):
raise AssertionError("not run here")
class FactsOnly:
"""Scatter modes whose mode fields fail when read: only the layer facts
are there."""
@@ -113,17 +126,40 @@ class FactsOnly:
raise AssertionError(f"read scatter mode {name!r}")
def dense_layer_facts(layer_id, num_layers, *, previous_sparse=False):
def layer_facts(layer_id, num_layers, *, sparse=False, previous_sparse=False):
return FactsOnly(
is_layer_sparse=False,
is_layer_sparse=sparse,
is_first_layer=layer_id == 0,
is_last_layer=layer_id == num_layers - 1,
is_previous_layer_sparse=None if layer_id == 0 else previous_sparse,
)
def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs):
with planning(parallel, sp=sp):
def sides_of(
axis_sizes,
*,
sparse=False,
a2a=False,
previous_a2a=False,
last=False,
leaves_next=False,
leaves_rs=False,
):
return decoder_layer_sides(
axis_sizes=axis_sizes,
ffn_on_local_rows=sparse and a2a,
previous_on_local_rows=previous_a2a,
is_last_layer=last,
attention_gathers_local_rows=False,
ffn_group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
leaves_for_next_layer=leaves_next,
leaves_for_reduce_scatter=leaves_rs,
leaves_for_reduce_scatterv=leaves_rs or sparse,
)
def build(modes, parallel, *, cls=LayerCommunicator, sp=False, a2a=False, **kwargs):
with planning(parallel, sp=sp, a2a=a2a):
return cls(
layer_scatter_modes=modes,
input_layernorm=Norm(),
@@ -132,8 +168,10 @@ def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs):
)
def planned_modes(layer_id, num_layers, *, sparse, previous_sparse, parallel):
with planning(parallel):
def planned_modes(
layer_id, num_layers, *, sparse, previous_sparse, parallel, a2a=False
):
with planning(parallel, a2a=a2a):
return LayerScatterModes.init_new(
layer_id=layer_id,
num_layers=num_layers,
@@ -148,13 +186,37 @@ class TestDeclarationsMatchScatterModes(CustomTestCase):
layouts of the modes layer planning gives the same layers."""
def test_every_attention_dp_and_tp(self):
for attn_dp, attn_tp, layer_id in itertools.product(
(2, 4, 8), (1, 2, 4), (0, 1, 3)
for (
attn_dp,
attn_tp,
layer_id,
sparse,
previous_sparse,
a2a,
) in itertools.product(
(1, 2, 4, 8),
(1, 2, 4),
(0, 1, 3),
(False, True),
(False, True),
(False, True),
):
with self.subTest(attn_dp=attn_dp, attn_tp=attn_tp, layer_id=layer_id):
with self.subTest(
attn_dp=attn_dp,
attn_tp=attn_tp,
layer_id=layer_id,
sparse=sparse,
previous_sparse=previous_sparse,
a2a=a2a,
):
parallel = parallel_of(attn_dp=attn_dp, attn_tp=attn_tp)
modes = planned_modes(
layer_id, 4, sparse=False, previous_sparse=False, parallel=parallel
layer_id,
4,
sparse=sparse,
previous_sparse=previous_sparse,
parallel=parallel,
a2a=a2a,
)
sizes = {
TokenAxis.ATTN_DP: attn_dp,
@@ -164,29 +226,37 @@ class TestDeclarationsMatchScatterModes(CustomTestCase):
layouts = scatter_mode_layouts(
attn_dp_size=attn_dp, attn_cp_size=1, attn_tp_size=attn_tp
)
sides = dense_decoder_layer_sides(
axis_sizes=sizes,
leaves_for_next_layer=True,
leaves_for_reduce_scatter=True,
sides = sides_of(
sizes,
sparse=sparse,
a2a=a2a,
previous_a2a=layer_id > 0 and previous_sparse and a2a,
last=layer_id == 3,
leaves_next=True,
leaves_rs=True,
)
self.assertEqual(sides.layer_rows, layouts[modes.layer_input_mode])
self.assertEqual(sides.layer_rows, layouts[modes.middle_residual_mode])
self.assertEqual(sides.layer_rows, layouts[modes.layer_output_mode])
self.assertEqual(sides.input_rows, layouts[modes.layer_input_mode])
self.assertEqual(
sides.ffn_residual_rows, layouts[modes.middle_residual_mode]
)
self.assertEqual(sides.output_rows, layouts[modes.layer_output_mode])
self.assertEqual(sides.attention.layout, layouts[modes.attn_mode])
self.assertEqual(sides.ffn.layout, layouts[modes.mlp_mode])
self.assertEqual(sides.attention_output.layout, sides.layer_rows)
self.assertEqual(
sides.attention_output.layout, layouts[modes.attn_mode]
)
self.assertEqual(sides.ffn_output.layout, sides.ffn.layout)
# A MoE dispatched per DP shard hands on a complete output.
self.assertIs(sides.ffn_output.group is None, sparse and a2a)
def test_the_attention_output_owes_the_attention_tp_sum(self):
for attn_tp in (1, 2):
sides = dense_decoder_layer_sides(
axis_sizes={
sides = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: attn_tp,
},
leaves_for_next_layer=False,
leaves_for_reduce_scatter=False,
}
)
owed = attn_tp > 1
self.assertIs(
@@ -202,16 +272,18 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
declarations, or none do."""
def declared(self, communicator):
return communicator._mlp_input.func in (
return getattr(communicator._mlp_input, "func", None) in (
comm._mlp_input_dp_partial,
comm._mlp_input_dp_replicate,
comm._mlp_input_without_dp,
comm._mlp_input_scatter,
)
def test_a_dense_model(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
communicator = build(dense_layer_facts(layer_id, 3), parallel)
communicator = build(layer_facts(layer_id, 3), parallel)
self.assertTrue(self.declared(communicator))
self.assertIs(
communicator._communicate_simple_fn,
@@ -219,6 +291,36 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
)
self.assertTrue(communicator._postprocess_scatters_to_local_tokens)
def test_plain_tp(self):
for norm, fuses in ((Norm, False), (FusableNorm, True)):
with self.subTest(norm=norm.__name__):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
communicator = LayerCommunicator(
layer_scatter_modes=layer_facts(1, 3),
input_layernorm=norm(),
post_attention_layernorm=norm(),
)
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
entry = (
communicator._mlp_input_reduce_output_and_update_and_read_residual
)
self.assertEqual(
communicator._mlp_input.keywords["fusions"],
(entry,) if fuses else (),
)
self.assertIs(communicator._mlp_input_may_fuse, fuses)
self.assertFalse(communicator._postprocess_scatters_to_local_tokens)
self.assertIs(
communicator._communicate_summable_tensor_pair_fn,
comm.CommunicateSummableTensorPairFn._trivial,
)
# One rank: the attention output is complete, only the norm is left.
one_rank = parallel_of(attn_dp=1, attn_tp=1)
single = build(layer_facts(1, 3), one_rank)
with planning(one_rank):
self.assertIsNotNone(single._declared_sides())
self.assertIs(single._mlp_input, comm._mlp_input_norm)
def test_the_order_follows_the_attention_output(self):
for attn_tp, force, order in (
(2, False, comm._mlp_input_dp_partial),
@@ -227,45 +329,99 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
):
with self.subTest(attn_tp=attn_tp, force=force):
communicator = build(
dense_layer_facts(1, 3),
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=attn_tp),
force_layernorm_before_dp_gather=force,
)
self.assertIs(communicator._mlp_input.func, order)
def test_a_dense_first_layer_before_sparse_ones(self):
# DeepSeek-V2-Lite: layer 0 is dense, the rest sparse.
parallel = parallel_of(attn_dp=2, attn_tp=2)
first = build(
planned_modes(0, 3, sparse=False, previous_sparse=False, parallel=parallel),
for a2a in (False, True):
with self.subTest(a2a=a2a):
layers = [
build(
planned_modes(
i,
4,
sparse=sparse,
previous_sparse=previous,
parallel=parallel,
a2a=a2a,
),
parallel,
a2a=a2a,
allow_reduce_scatter=True,
)
for i, (sparse, previous) in enumerate(
((False, False), (True, False), (True, True), (False, True))
)
]
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
if not a2a:
moe = layers[1]._ffn_output
self.assertIs(moe.group, SumGroup.MOE_OUTPUT)
self.assertTrue(moe.leaves_for_reduce_scatterv)
continue
# The first a2a layer slices the residual it takes from a
# dense layer; the next one takes it already sliced.
self.assertEqual(
[layers[i]._mlp_input.func for i in (1, 2)],
[comm._mlp_input_scatter] * 2,
)
self.assertEqual(
[
layers[i]._mlp_input.keywords["scatters_residual"]
for i in (1, 2)
],
[True, False],
)
self.assertIsNone(layers[1]._ffn_output.group)
self.assertIs(
layers[2]._communicate_simple_fn,
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
)
self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial)
self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"])
def test_the_last_a2a_layer_folds_the_residual_back(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
last = build(
planned_modes(
3, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
),
parallel,
allow_reduce_scatter=True,
a2a=True,
)
second = build(
planned_modes(1, 3, sparse=True, previous_sparse=False, parallel=parallel),
parallel,
allow_reduce_scatter=True,
self.assertIs(
last._communicate_summable_tensor_pair_fn,
comm.CommunicateSummableTensorPairFn._gather,
)
self.assertTrue(self.declared(first))
self.assertFalse(self.declared(second))
self.assertIs(second._mlp_input.func, comm._mlp_input_gather)
def test_an_attention_that_gathers_its_slice_itself(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
modes = planned_modes(
2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
)
with patch.object(comm, "_use_ag_after_qlora", True):
layer = build(modes, parallel, a2a=True)
self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial)
def test_layers_that_keep_the_scatter_modes(self):
dp = parallel_of(attn_dp=2, attn_tp=2)
cases = {
"no attention DP": (
dense_layer_facts(1, 3),
parallel_of(attn_dp=1, attn_tp=4),
"input-scattered attention": (
layer_facts(1, 3),
parallel_of(attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True),
),
"attention CP": (
dense_layer_facts(1, 3),
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=1, attn_cp=2),
),
"dense MLP fully DP": (
dense_layer_facts(1, 3),
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1),
),
"after a sparse layer": (dense_layer_facts(1, 3, previous_sparse=True), dp),
}
for name, (facts, parallel) in cases.items():
with self.subTest(name):
@@ -287,7 +443,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
self.assertFalse(self.declared(build(direct, dp)))
with planning(dp, sp=True):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator.layer_scatter_modes = dense_layer_facts(1, 3)
communicator.layer_scatter_modes = layer_facts(1, 3)
self.assertIsNone(communicator._declared_sides())
def test_subclasses_that_pick_their_own_steps(self):
@@ -311,16 +467,14 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
sides = dense_decoder_layer_sides(
axis_sizes=sizes,
leaves_for_next_layer=False,
leaves_for_reduce_scatter=False,
)
sides = sides_of(sizes)
steps = comm._select_ffn_input(
produced,
residual=sides.layer_rows,
residual=sides.input_rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=force,
fusions=(),
)
reduced = []
context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0)
@@ -349,15 +503,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
return len(reduced), hidden, residual
def test_a_complete_output_is_not_summed_again(self):
layout = dense_decoder_layer_sides(
axis_sizes={
layout = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
},
leaves_for_next_layer=False,
leaves_for_reduce_scatter=False,
).layer_rows
}
).input_rows
complete = comm.StageOutput(layout)
for force in (False, True):
with self.subTest(force=force):
@@ -367,15 +519,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
torch.testing.assert_close(hidden, torch.full((1, HIDDEN), 20.0))
def test_an_owed_output_is_summed_once(self):
layout = dense_decoder_layer_sides(
axis_sizes={
layout = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
},
leaves_for_next_layer=False,
leaves_for_reduce_scatter=False,
).layer_rows
}
).input_rows
owed = comm.StageOutput(layout, group=SumGroup.ATTN_TP, always_leaves=True)
reductions, hidden, residual = self.run_steps(owed, force=True)
self.assertEqual(reductions, 1)
@@ -383,15 +533,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
torch.testing.assert_close(residual, torch.full((1, HIDDEN), 17.0))
def test_declarations_the_steps_cannot_run_are_rejected(self):
layout = dense_decoder_layer_sides(
axis_sizes={
layout = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
},
leaves_for_next_layer=False,
leaves_for_reduce_scatter=False,
).layer_rows
}
).input_rows
for produced in (
# Leaves the attention-TP sum only when asked, like a mixer exit.
comm.StageOutput(layout, group=SumGroup.ATTN_TP),
@@ -408,7 +556,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
# ---------------------------------------------------------------------------
# Two consecutive dense layers, rank by rank.
# Two consecutive layers, rank by rank.
class World:
@@ -535,7 +683,7 @@ def state():
@contextmanager
def running(*, reduce_scatterv):
def running(*, reduce_scatterv, a2a=False):
replaced = {
"get_forward": lambda: state().flags,
"attention_tensor_model_parallel_all_reduce": lambda x: (
@@ -551,10 +699,16 @@ def running(*, reduce_scatterv):
"dp_gather_partial": partial(fake_dp_gather, is_partial=True),
"dp_scatter": fake_dp_scatter,
"dp_reduce_scatter_tensor": fake_dp_reduce_scatter_tensor,
"attn_tp_reduce_scatter_tensor": lambda out, x: (
state().parallel.attn_tp_group.reduce_scatter_tensor(out, x)
),
"attn_tp_all_gather_into_tensor": lambda out, x: (
state().parallel.attn_tp_group.all_gather_into_tensor(out, x)
),
"get_dp_global_num_tokens": lambda: state().dp_rows,
"should_use_dp_reduce_scatterv": lambda: reduce_scatterv,
"can_use_dp_reduce_scatter": lambda: True,
"is_dp_attention_enabled": lambda: True,
"is_dp_attention_enabled": lambda: state().parallel.attn_dp_size > 1,
"apply_flashinfer_allreduce_fusion": lambda n: False,
"post_experts_output_is_complete": lambda **kw: False,
"post_experts_reduction_group": lambda: state().parallel.tp_group,
@@ -569,7 +723,7 @@ def running(*, reduce_scatterv):
"is_allocation_symmetric": lambda: False,
}
with ExitStack() as stack:
stack.enter_context(planning(lambda: state().parallel))
stack.enter_context(planning(lambda: state().parallel, a2a=a2a))
for name, value in replaced.items():
stack.enter_context(patch.object(comm, name, value))
yield
@@ -593,8 +747,22 @@ def dense_mlp(x, state):
return state.parallel.tp_group.all_reduce(partial_sum)
def moe(x, state):
"""A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE
output's group unless a published flag asks it to leave the sum, or the
attention-DP reduce_scatterv takes it, as the MoE blocks do."""
partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
if (
state.flags.fuse_mlp_allreduce
or state.flags.mlp_reduce_scatter
or comm.should_use_dp_reduce_scatterv()
):
return partial_sum
return comm.post_experts_reduction_group().all_reduce(partial_sum)
def reference(x):
"""(hidden, residual) after two layers of Norm / attention / dense MLP."""
"""(hidden, residual) after two layers of Norm / attention / FFN."""
residual = x
for _ in range(2):
hidden = 2 * residual
@@ -605,8 +773,19 @@ def reference(x):
return ffn, residual - ffn
class TestTwoDenseLayers(CustomTestCase):
def run_world(self, *, attn_dp, attn_tp, rows, padding, reduce_scatterv, leaves):
class TestTwoLayers(CustomTestCase):
def run_world(
self,
*,
attn_dp,
attn_tp,
rows,
padding,
reduce_scatterv,
leaves,
sparse=(False, False),
a2a=False,
):
tp = attn_dp * attn_tp
world = World(tp)
WORLD[0] = world
@@ -638,6 +817,8 @@ class TestTwoDenseLayers(CustomTestCase):
),
dp_padding_mode=SimpleNamespace(is_max_len=lambda: max_len),
global_dp_buffer_len=global_rows,
# Without attention DP the FFN exit counts this rank's tokens.
input_ids=torch.zeros(rows[0]),
)
states = []
for rank in range(tp):
@@ -668,7 +849,12 @@ class TestTwoDenseLayers(CustomTestCase):
s = states[rank]
layers = [
LayerCommunicator(
layer_scatter_modes=dense_layer_facts(i, 2),
layer_scatter_modes=layer_facts(
i,
2,
sparse=sparse[i],
previous_sparse=i > 0 and sparse[i - 1],
),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
allow_reduce_scatter=leaves,
@@ -680,46 +866,67 @@ class TestTwoDenseLayers(CustomTestCase):
hidden[: s.rows] = embeddings[s.dp]
residual = None
handed_on = []
for layer in layers:
for layer_index, layer in enumerate(layers):
hidden, residual = layer.prepare_attn(hidden, residual, forward_batch)
hidden = attention(hidden, s)
hidden, residual = layer.prepare_mlp(hidden, residual, forward_batch)
with layer.ffn_exit(forward_batch) as ffn_exit:
hidden = dense_mlp(hidden, s)
if not sparse[layer_index]:
hidden = dense_mlp(hidden, s)
elif a2a:
# On this rank's slice; the combine completes the sum.
hidden = 5 * hidden
else:
hidden = moe(hidden, s)
hidden, residual = ffn_exit.finish(hidden, residual)
handed_on.append(type(hidden))
hidden, residual = layers[-1].finish_layer_stack(
hidden, residual, forward_batch
)
return hidden[: s.rows], residual[: s.rows], handed_on
# The last a2a layer folds the residual into its output.
if residual is not None:
residual = residual[: s.rows]
return hidden[: s.rows], residual, handed_on
with running(reduce_scatterv=reduce_scatterv):
with running(reduce_scatterv=reduce_scatterv, a2a=a2a):
results, errors = world.run(states, forward)
for rank, error in enumerate(errors):
if error is not None:
raise AssertionError(f"rank {rank} raised") from error
for rank, (hidden, residual, handed_on) in enumerate(results):
want_hidden, want_residual = reference(embeddings[states[rank].dp])
if residual is None:
want_hidden, want_residual = want_hidden + want_residual, None
torch.testing.assert_close(hidden, want_hidden, rtol=0, atol=0)
torch.testing.assert_close(residual, want_residual, rtol=0, atol=0)
self.assertIs(residual is None, want_residual is None)
if residual is not None:
torch.testing.assert_close(residual, want_residual, rtol=0, atol=0)
# Every rank ran the same collectives.
self.assertEqual(len({tuple(s.calls) for s in states if s.dp == 0}), 1)
return results
def test_rows_come_back_to_each_rank(self):
for (attn_dp, attn_tp), rows, padding, leaves in itertools.product(
((2, 2), (4, 1), (2, 1)),
for (attn_dp, attn_tp), rows, padding, leaves, sparse in itertools.product(
((2, 2), (4, 1), (2, 1), (1, 2), (1, 4)),
("ragged", "idle"),
("sum_len", "max_len"),
(False, True),
((False, False), (False, True), (True, True)),
):
token_rows = {
("ragged", 1): [3],
("idle", 1): [0],
("ragged", 2): [3, 1],
("idle", 2): [2, 0],
("ragged", 4): [2, 3, 1, 1],
("idle", 4): [2, 0, 3, 1],
}[rows, attn_dp]
for reduce_scatterv in (False, True) if attn_tp == 1 else (False,):
for reduce_scatterv, a2a in itertools.product(
(False, True) if attn_tp == 1 and attn_dp > 1 else (False,),
(False, True) if any(sparse) else (False,),
):
if reduce_scatterv and a2a:
continue
with self.subTest(
attn_dp=attn_dp,
attn_tp=attn_tp,
@@ -727,6 +934,8 @@ class TestTwoDenseLayers(CustomTestCase):
padding=padding,
leaves=leaves,
reduce_scatterv=reduce_scatterv,
sparse=sparse,
a2a=a2a,
):
results = self.run_world(
attn_dp=attn_dp,
@@ -735,13 +944,30 @@ class TestTwoDenseLayers(CustomTestCase):
padding=padding,
reduce_scatterv=reduce_scatterv,
leaves=leaves,
sparse=sparse,
a2a=a2a,
)
first_layer_hands_on = results[0][2][0]
# A producer that may leave its sum leaves it for the next
# layer's input; one that may not hands on a complete value.
# Without attention DP an empty batch completes its own sum.
# A MoE dispatched per DP shard hands on a complete value.
# Otherwise the reduce-scatter back to this rank's tokens
# can be left to the next layer; the all-reduce only
# without an a2a backend.
has_tokens = attn_dp > 1 or sum(token_rows) > 0
reduce_scatter_left = attn_dp > 1 and (
padding == "max_len" or reduce_scatterv
)
owes = (
leaves
and has_tokens
and not (sparse[0] and a2a)
and (reduce_scatter_left or not a2a)
)
self.assertIs(
first_layer_hands_on,
UnreducedOutput if leaves else torch.Tensor,
UnreducedOutput if owes else torch.Tensor,
)