[Refactor] Choose the boundaries of plain-TP dense layers and MoE layers from declarations (#41417)
This commit is contained in:
@@ -56,6 +56,9 @@ class StageInput(msgspec.Struct, frozen=True):
|
||||
compute group does not span."""
|
||||
|
||||
layout: Layout
|
||||
# Token axes the consumer gathers over itself when its input arrives
|
||||
# sharded over them.
|
||||
gathers_itself: FrozenSet[TokenAxis] = frozenset()
|
||||
|
||||
|
||||
class StageOutput(msgspec.Struct, frozen=True):
|
||||
@@ -87,32 +90,58 @@ class DecoderLayerSides(msgspec.Struct, frozen=True):
|
||||
"""The declarations both sides of a decoder layer's boundaries are chosen
|
||||
from."""
|
||||
|
||||
# The rows the layer takes and hands on; the residual stays on them.
|
||||
layer_rows: Layout
|
||||
# The rows the layer's input and residual arrive on.
|
||||
input_rows: Layout
|
||||
attention: StageInput
|
||||
attention_output: StageOutput
|
||||
ffn: StageInput
|
||||
ffn_output: StageOutput
|
||||
# The rows the residual is on while the FFN runs.
|
||||
ffn_residual_rows: Layout
|
||||
# The rows the layer hands on.
|
||||
output_rows: Layout
|
||||
|
||||
|
||||
def dense_decoder_layer_sides(
|
||||
def decoder_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_on_local_rows: bool,
|
||||
previous_on_local_rows: bool,
|
||||
is_last_layer: bool,
|
||||
attention_gathers_local_rows: bool,
|
||||
ffn_group: SumGroup,
|
||||
leaves_for_next_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
leaves_for_reduce_scatterv: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""An attention followed by a dense MLP on the TP group, derived from the
|
||||
groups each computes over."""
|
||||
"""An attention followed by an FFN, derived from the groups each computes
|
||||
over. The FFN runs either on the TP group (a dense MLP, or a MoE not
|
||||
dispatched per DP shard) or on this rank's local rows (a MoE dispatched per
|
||||
DP shard, which completes its own combine)."""
|
||||
# Attention computes over the attention-TP ranks of one DP (and CP) shard.
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
# The TP group spans every token axis, so its MLP needs every row.
|
||||
ffn = Layout.sharded_over(axis_sizes=axis_sizes)
|
||||
# Each attention-TP rank's own slice of those rows.
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
# The TP group spans every token axis, so an FFN on it needs every row.
|
||||
ffn = local if ffn_on_local_rows else Layout.sharded_over(axis_sizes=axis_sizes)
|
||||
attention_tp = axis_sizes[TokenAxis.ATTN_TP_SCATTER] > 1
|
||||
return DecoderLayerSides(
|
||||
layer_rows=attention,
|
||||
attention=StageInput(attention),
|
||||
input_rows=local if previous_on_local_rows else attention,
|
||||
attention=StageInput(
|
||||
attention,
|
||||
gathers_itself=(
|
||||
frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
if attention_gathers_local_rows
|
||||
else frozenset()
|
||||
),
|
||||
),
|
||||
# The output projection leaves the attention-TP sum to prepare_mlp.
|
||||
attention_output=StageOutput(
|
||||
attention,
|
||||
@@ -120,11 +149,17 @@ def dense_decoder_layer_sides(
|
||||
always_leaves=attention_tp,
|
||||
),
|
||||
ffn=StageInput(ffn),
|
||||
ffn_output=StageOutput(
|
||||
ffn,
|
||||
group=SumGroup.TP,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatter,
|
||||
ffn_output=(
|
||||
StageOutput(ffn)
|
||||
if ffn_on_local_rows
|
||||
else StageOutput(
|
||||
ffn,
|
||||
group=ffn_group,
|
||||
leaves_for_next_layer=leaves_for_next_layer,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=leaves_for_reduce_scatterv,
|
||||
)
|
||||
),
|
||||
ffn_residual_rows=local if ffn_on_local_rows else attention,
|
||||
output_rows=local if ffn_on_local_rows and not is_last_layer else attention,
|
||||
)
|
||||
|
||||
@@ -44,7 +44,7 @@ from sglang.srt.layers.boundary_layout import (
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
dense_decoder_layer_sides,
|
||||
decoder_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
is_mla_cp_active,
|
||||
@@ -75,6 +75,7 @@ from sglang.srt.layers.flashinfer_comm_fusion import (
|
||||
from sglang.srt.layers.moe import (
|
||||
can_merge_post_experts_all_reduce,
|
||||
get_moe_a2a_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
post_experts_output_is_complete,
|
||||
post_experts_reduction_group,
|
||||
should_use_dp_reduce_scatterv,
|
||||
@@ -773,50 +774,75 @@ class LayerCommunicator:
|
||||
|
||||
def _declared_sides(self) -> Optional[DecoderLayerSides]:
|
||||
"""The declarations this layer's boundaries are chosen from: an
|
||||
attention and a dense MLP on the TP group under attention DP, without CP
|
||||
or LayerNorm SP, in a layer that takes the rows a dense layer or the
|
||||
embedding hands on. None when the steps come from the scatter modes."""
|
||||
attention and an FFN, without CP, LayerNorm SP or input-scattered
|
||||
attention, in a layer whose previous layer is known. None when the
|
||||
steps come from the scatter modes."""
|
||||
modes = self.layer_scatter_modes
|
||||
parallel = get_parallel()
|
||||
# A MoE dispatched per DP shard computes on this rank's local rows and
|
||||
# hands its layer's output on there.
|
||||
moe_on_local_rows = is_moe_input_scattered_across_dp_ranks()
|
||||
if not (
|
||||
self._takes_declared_boundaries
|
||||
and parallel.attn_dp_size > 1
|
||||
and parallel.attn_cp_size == 1
|
||||
and not layernorm_sp.layernorm_sp_enabled()
|
||||
and not modes.is_layer_sparse
|
||||
and not parallel.enable_attn_tp_input_scattered
|
||||
and not enable_moe_dense_fully_dp()
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is False)
|
||||
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)
|
||||
):
|
||||
return None
|
||||
return dense_decoder_layer_sides(
|
||||
return decoder_layer_sides(
|
||||
axis_sizes={
|
||||
TokenAxis.ATTN_DP: parallel.attn_dp_size,
|
||||
TokenAxis.ATTN_CP: parallel.attn_cp_size,
|
||||
TokenAxis.ATTN_TP_SCATTER: parallel.attn_tp_size,
|
||||
},
|
||||
ffn_on_local_rows=modes.is_layer_sparse and moe_on_local_rows,
|
||||
previous_on_local_rows=(
|
||||
not modes.is_first_layer
|
||||
and modes.is_previous_layer_sparse
|
||||
and moe_on_local_rows
|
||||
),
|
||||
is_last_layer=modes.is_last_layer,
|
||||
attention_gathers_local_rows=_use_ag_after_qlora,
|
||||
ffn_group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
# A MoE block leaves its sum to reduce_scatterv whenever that combine
|
||||
# applies (should_skip_post_experts_all_reduce).
|
||||
leaves_for_reduce_scatterv=(
|
||||
self.allow_reduce_scatter or modes.is_layer_sparse
|
||||
),
|
||||
)
|
||||
|
||||
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
|
||||
"""Choose the three boundary sides this layer owns from the
|
||||
declarations: the attention input, the attention -> FFN steps, and the
|
||||
FFN output's way back to the layer's rows."""
|
||||
FFN output's way on to the rows the layer hands on."""
|
||||
self._communicate_simple_fn = _select_attention_input_move(
|
||||
sides.layer_rows, sides.attention
|
||||
sides.input_rows, sides.attention
|
||||
)
|
||||
self._mlp_input = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=sides.layer_rows,
|
||||
residual=sides.input_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
)
|
||||
self._mlp_input_may_fuse = (
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual
|
||||
in getattr(self._mlp_input, "keywords", {}).get("fusions", ())
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
self._mlp_input_may_fuse = False
|
||||
self._ffn_output = sides.ffn_output
|
||||
self._postprocess_scatters_to_local_tokens = _ffn_output_returns_over_dp(
|
||||
sides.ffn_output, sides.layer_rows
|
||||
(
|
||||
self._postprocess_scatters_to_local_tokens,
|
||||
postprocess,
|
||||
) = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if postprocess is not None:
|
||||
self._communicate_summable_tensor_pair_fn = postprocess
|
||||
self._ffn_sum_is_movable = sides.ffn_output.group is not None
|
||||
|
||||
def _select_boundaries_from_scatter_modes(self) -> None:
|
||||
@@ -1097,9 +1123,7 @@ class LayerCommunicator:
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
fusions = (
|
||||
self._select_mlp_input_fusions(residual_input_mode)
|
||||
if self._context.attn_dp_size == 1
|
||||
else ()
|
||||
self._select_mlp_input_fusions() if self._context.attn_dp_size == 1 else ()
|
||||
)
|
||||
steps = partial(
|
||||
_mlp_input_gather,
|
||||
@@ -1112,9 +1136,7 @@ class LayerCommunicator:
|
||||
self._mlp_input_reduce_output_and_update_and_read_residual in fusions,
|
||||
)
|
||||
|
||||
def _select_mlp_input_fusions(
|
||||
self, residual_input_mode: ScatterMode
|
||||
) -> Tuple[Callable, ...]:
|
||||
def _select_mlp_input_fusions(self) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete the attention output's all-reduce
|
||||
together with the residual update and the post-attention norm, in the
|
||||
order they are tried. Each takes (hidden_states, residual, forward_batch)
|
||||
@@ -1803,29 +1825,34 @@ def _sum_group(group: SumGroup) -> GroupCoordinator:
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input."""
|
||||
if rows == need.layout:
|
||||
return CommunicateSimpleFn._trivial
|
||||
raise NotImplementedError(f"{rows=} {need=}")
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
gathers them itself."""
|
||||
gathered = rows.sharded - need.layout.sharded - need.gathers_itself
|
||||
if not need.layout.sharded <= rows.sharded or gathered not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_TP_SCATTER},
|
||||
):
|
||||
raise NotImplementedError(f"{rows=} {need=}")
|
||||
if gathered:
|
||||
return CommunicateSimpleFn._scattered_to_tp_attn_full
|
||||
return CommunicateSimpleFn._trivial
|
||||
|
||||
|
||||
def _select_ffn_input(
|
||||
produced: StageOutput,
|
||||
*,
|
||||
residual: Layout,
|
||||
residual_to: Layout,
|
||||
need: StageInput,
|
||||
force_layernorm_before_gather: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
) -> Callable:
|
||||
"""The steps from the attention output to the FFN input: complete the
|
||||
attention-TP sum, add the residual and normalize, and gather the rows the
|
||||
FFN's group needs, with the residual left on the attention's rows."""
|
||||
gathered = produced.layout.sharded - need.layout.sharded
|
||||
if (
|
||||
residual != produced.layout
|
||||
or not need.layout.sharded <= produced.layout.sharded
|
||||
or gathered != {TokenAxis.ATTN_DP}
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
attention-TP sum, move the residual to the rows it has while the FFN runs,
|
||||
add it and normalize, and bring the rows to what the FFN's group needs: a
|
||||
gather over attention DP, or each rank's own slice. The fused kernels in
|
||||
``fusions`` are tried first when nothing is gathered or sliced."""
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
# always left by the output projection, or nothing.
|
||||
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
||||
@@ -1834,28 +1861,70 @@ def _select_ffn_input(
|
||||
SumGroup.ATTN_TP,
|
||||
):
|
||||
raise NotImplementedError(f"{produced=}")
|
||||
gathered = produced.layout.sharded - need.layout.sharded
|
||||
sliced = need.layout.sharded - produced.layout.sharded
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
# completes the attention-TP sum and slices in one collective.
|
||||
if (
|
||||
sliced != {TokenAxis.ATTN_TP_SCATTER}
|
||||
or gathered
|
||||
or not owes_attention_tp
|
||||
or residual_to != need.layout
|
||||
or residual not in (produced.layout, need.layout)
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_scatter, scatters_residual=residual != residual_to)
|
||||
if (
|
||||
residual_to != produced.layout
|
||||
or gathered not in (frozenset(), {TokenAxis.ATTN_DP})
|
||||
or residual.sharded - residual_to.sharded
|
||||
not in (frozenset(), {TokenAxis.ATTN_TP_SCATTER})
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
# A residual arriving on each rank's slice is gathered back first.
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return _mlp_input_norm
|
||||
return partial(
|
||||
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
|
||||
)
|
||||
# The partial order adds the residual on attention-TP rank 0 before the DP
|
||||
# gather's collective completes that sum, which only a plain residual add
|
||||
# allows.
|
||||
if owes_attention_tp and not force_layernorm_before_gather:
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=False)
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
|
||||
return partial(
|
||||
_mlp_input_dp_replicate,
|
||||
gathers_residual=False,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
)
|
||||
|
||||
|
||||
def _ffn_output_returns_over_dp(produced: StageOutput, rows: Layout) -> bool:
|
||||
"""Whether the FFN output goes back to the layer's rows by undoing the
|
||||
attention-DP gather."""
|
||||
returned = rows.sharded - produced.layout.sharded
|
||||
if not produced.layout.sharded <= rows.sharded or returned not in (
|
||||
frozenset(),
|
||||
{TokenAxis.ATTN_DP},
|
||||
def _select_ffn_output_move(
|
||||
produced: StageOutput, *, residual: Layout, to: Layout
|
||||
) -> Tuple[bool, Optional[Callable]]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
that step), or else the postprocess that moves it, None when there is none
|
||||
to choose here."""
|
||||
if produced.layout == residual:
|
||||
if to == residual:
|
||||
return False, CommunicateSummableTensorPairFn._trivial
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: fold the residual
|
||||
# into the output, then gather over attention TP.
|
||||
return False, CommunicateSummableTensorPairFn._gather
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if (
|
||||
to != residual
|
||||
or not produced.layout.sharded <= residual.sharded
|
||||
or returned != {TokenAxis.ATTN_DP}
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {rows=}")
|
||||
return bool(returned)
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return True, None
|
||||
|
||||
|
||||
class MlpInputKind(Enum):
|
||||
|
||||
@@ -239,11 +239,11 @@ class CuteDSLFusionLayerCommunicator(LayerCommunicator):
|
||||
gamma=_fused_norm_gamma(self.input_layernorm),
|
||||
)
|
||||
|
||||
def _select_mlp_input_fusions(self, residual_input_mode):
|
||||
fusions = super()._select_mlp_input_fusions(residual_input_mode)
|
||||
def _select_mlp_input_fusions(self):
|
||||
fusions = super()._select_mlp_input_fusions()
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
residual_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
self.layer_scatter_modes.layer_input_mode is ScatterMode.TP_ATTN_FULL
|
||||
and parallel.attn_tp_size == parallel.tp_size
|
||||
and _fused_norm_gamma(self.post_attention_layernorm) is not None
|
||||
):
|
||||
|
||||
@@ -119,12 +119,15 @@ def test_cutedsl_entries_come_before_the_base_fused_kernel():
|
||||
f"{_MODULE}.get_parallel",
|
||||
return_value=SimpleNamespace(attn_tp_size=2, tp_size=2),
|
||||
):
|
||||
assert comm._select_mlp_input_fusions(ScatterMode.TP_ATTN_FULL) == (
|
||||
cutedsl,
|
||||
base,
|
||||
comm.layer_scatter_modes = SimpleNamespace(
|
||||
layer_input_mode=ScatterMode.TP_ATTN_FULL
|
||||
)
|
||||
assert comm._select_mlp_input_fusions() == (cutedsl, base)
|
||||
# A scattered residual is gathered first, which the workspace does not do.
|
||||
assert comm._select_mlp_input_fusions(ScatterMode.SCATTERED) == (base,)
|
||||
comm.layer_scatter_modes = SimpleNamespace(
|
||||
layer_input_mode=ScatterMode.SCATTERED
|
||||
)
|
||||
assert comm._select_mlp_input_fusions() == (base,)
|
||||
|
||||
|
||||
def test_a_replicated_output_producer_keeps_its_own_all_reduce(eligible):
|
||||
|
||||
+316
-90
@@ -1,5 +1,7 @@
|
||||
"""The boundaries of an attention and a dense MLP on the TP group under attention
|
||||
DP, chosen from both sides' declarations.
|
||||
"""The boundaries of an attention and an FFN, chosen from both sides'
|
||||
declarations. The FFN runs on the TP group (a dense MLP, or a MoE not
|
||||
dispatched per DP shard) or on each attention-TP rank's slice of its DP shard's
|
||||
rows (a MoE dispatched per DP shard).
|
||||
|
||||
The numeric checks run every rank of a DP x attention-TP world as a thread over
|
||||
fake collectives that wait for all members of their group, build the layers'
|
||||
@@ -21,7 +23,7 @@ from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
dense_decoder_layer_sides,
|
||||
decoder_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.communicator import (
|
||||
LayerCommunicator,
|
||||
@@ -57,6 +59,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
|
||||
moe_dp_size=1,
|
||||
dwdp_size=1,
|
||||
enable_dp_attention=attn_dp > 1,
|
||||
enable_attn_tp_input_scattered=False,
|
||||
tp_group=SimpleNamespace(name="tp"),
|
||||
attn_tp_group=SimpleNamespace(name="attn_tp"),
|
||||
)
|
||||
@@ -65,7 +68,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def planning(parallel, *, sp=False):
|
||||
def planning(parallel, *, sp=False, a2a=False):
|
||||
"""What layer planning and communicator construction read, without the
|
||||
process-wide parallel state. ``parallel`` may be a callable, for a
|
||||
per-thread parallel state."""
|
||||
@@ -80,8 +83,11 @@ def planning(parallel, *, sp=False):
|
||||
comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None)
|
||||
),
|
||||
patch.object(
|
||||
comm, "get_moe_a2a_backend", lambda: SimpleNamespace(is_none=lambda: True)
|
||||
comm,
|
||||
"get_moe_a2a_backend",
|
||||
lambda: SimpleNamespace(is_none=lambda: not a2a),
|
||||
),
|
||||
patch.object(comm, "is_moe_input_scattered_across_dp_ranks", lambda: a2a),
|
||||
patch.object(
|
||||
comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False
|
||||
),
|
||||
@@ -102,6 +108,13 @@ class Norm:
|
||||
return s * 2, s
|
||||
|
||||
|
||||
class FusableNorm(Norm):
|
||||
"""A norm with the fused all-reduce + add + norm method."""
|
||||
|
||||
def forward_with_allreduce_fusion(self, *args, **kwargs):
|
||||
raise AssertionError("not run here")
|
||||
|
||||
|
||||
class FactsOnly:
|
||||
"""Scatter modes whose mode fields fail when read: only the layer facts
|
||||
are there."""
|
||||
@@ -113,17 +126,40 @@ class FactsOnly:
|
||||
raise AssertionError(f"read scatter mode {name!r}")
|
||||
|
||||
|
||||
def dense_layer_facts(layer_id, num_layers, *, previous_sparse=False):
|
||||
def layer_facts(layer_id, num_layers, *, sparse=False, previous_sparse=False):
|
||||
return FactsOnly(
|
||||
is_layer_sparse=False,
|
||||
is_layer_sparse=sparse,
|
||||
is_first_layer=layer_id == 0,
|
||||
is_last_layer=layer_id == num_layers - 1,
|
||||
is_previous_layer_sparse=None if layer_id == 0 else previous_sparse,
|
||||
)
|
||||
|
||||
|
||||
def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs):
|
||||
with planning(parallel, sp=sp):
|
||||
def sides_of(
|
||||
axis_sizes,
|
||||
*,
|
||||
sparse=False,
|
||||
a2a=False,
|
||||
previous_a2a=False,
|
||||
last=False,
|
||||
leaves_next=False,
|
||||
leaves_rs=False,
|
||||
):
|
||||
return decoder_layer_sides(
|
||||
axis_sizes=axis_sizes,
|
||||
ffn_on_local_rows=sparse and a2a,
|
||||
previous_on_local_rows=previous_a2a,
|
||||
is_last_layer=last,
|
||||
attention_gathers_local_rows=False,
|
||||
ffn_group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=leaves_next,
|
||||
leaves_for_reduce_scatter=leaves_rs,
|
||||
leaves_for_reduce_scatterv=leaves_rs or sparse,
|
||||
)
|
||||
|
||||
|
||||
def build(modes, parallel, *, cls=LayerCommunicator, sp=False, a2a=False, **kwargs):
|
||||
with planning(parallel, sp=sp, a2a=a2a):
|
||||
return cls(
|
||||
layer_scatter_modes=modes,
|
||||
input_layernorm=Norm(),
|
||||
@@ -132,8 +168,10 @@ def build(modes, parallel, *, cls=LayerCommunicator, sp=False, **kwargs):
|
||||
)
|
||||
|
||||
|
||||
def planned_modes(layer_id, num_layers, *, sparse, previous_sparse, parallel):
|
||||
with planning(parallel):
|
||||
def planned_modes(
|
||||
layer_id, num_layers, *, sparse, previous_sparse, parallel, a2a=False
|
||||
):
|
||||
with planning(parallel, a2a=a2a):
|
||||
return LayerScatterModes.init_new(
|
||||
layer_id=layer_id,
|
||||
num_layers=num_layers,
|
||||
@@ -148,13 +186,37 @@ class TestDeclarationsMatchScatterModes(CustomTestCase):
|
||||
layouts of the modes layer planning gives the same layers."""
|
||||
|
||||
def test_every_attention_dp_and_tp(self):
|
||||
for attn_dp, attn_tp, layer_id in itertools.product(
|
||||
(2, 4, 8), (1, 2, 4), (0, 1, 3)
|
||||
for (
|
||||
attn_dp,
|
||||
attn_tp,
|
||||
layer_id,
|
||||
sparse,
|
||||
previous_sparse,
|
||||
a2a,
|
||||
) in itertools.product(
|
||||
(1, 2, 4, 8),
|
||||
(1, 2, 4),
|
||||
(0, 1, 3),
|
||||
(False, True),
|
||||
(False, True),
|
||||
(False, True),
|
||||
):
|
||||
with self.subTest(attn_dp=attn_dp, attn_tp=attn_tp, layer_id=layer_id):
|
||||
with self.subTest(
|
||||
attn_dp=attn_dp,
|
||||
attn_tp=attn_tp,
|
||||
layer_id=layer_id,
|
||||
sparse=sparse,
|
||||
previous_sparse=previous_sparse,
|
||||
a2a=a2a,
|
||||
):
|
||||
parallel = parallel_of(attn_dp=attn_dp, attn_tp=attn_tp)
|
||||
modes = planned_modes(
|
||||
layer_id, 4, sparse=False, previous_sparse=False, parallel=parallel
|
||||
layer_id,
|
||||
4,
|
||||
sparse=sparse,
|
||||
previous_sparse=previous_sparse,
|
||||
parallel=parallel,
|
||||
a2a=a2a,
|
||||
)
|
||||
sizes = {
|
||||
TokenAxis.ATTN_DP: attn_dp,
|
||||
@@ -164,29 +226,37 @@ class TestDeclarationsMatchScatterModes(CustomTestCase):
|
||||
layouts = scatter_mode_layouts(
|
||||
attn_dp_size=attn_dp, attn_cp_size=1, attn_tp_size=attn_tp
|
||||
)
|
||||
sides = dense_decoder_layer_sides(
|
||||
axis_sizes=sizes,
|
||||
leaves_for_next_layer=True,
|
||||
leaves_for_reduce_scatter=True,
|
||||
sides = sides_of(
|
||||
sizes,
|
||||
sparse=sparse,
|
||||
a2a=a2a,
|
||||
previous_a2a=layer_id > 0 and previous_sparse and a2a,
|
||||
last=layer_id == 3,
|
||||
leaves_next=True,
|
||||
leaves_rs=True,
|
||||
)
|
||||
self.assertEqual(sides.layer_rows, layouts[modes.layer_input_mode])
|
||||
self.assertEqual(sides.layer_rows, layouts[modes.middle_residual_mode])
|
||||
self.assertEqual(sides.layer_rows, layouts[modes.layer_output_mode])
|
||||
self.assertEqual(sides.input_rows, layouts[modes.layer_input_mode])
|
||||
self.assertEqual(
|
||||
sides.ffn_residual_rows, layouts[modes.middle_residual_mode]
|
||||
)
|
||||
self.assertEqual(sides.output_rows, layouts[modes.layer_output_mode])
|
||||
self.assertEqual(sides.attention.layout, layouts[modes.attn_mode])
|
||||
self.assertEqual(sides.ffn.layout, layouts[modes.mlp_mode])
|
||||
self.assertEqual(sides.attention_output.layout, sides.layer_rows)
|
||||
self.assertEqual(
|
||||
sides.attention_output.layout, layouts[modes.attn_mode]
|
||||
)
|
||||
self.assertEqual(sides.ffn_output.layout, sides.ffn.layout)
|
||||
# A MoE dispatched per DP shard hands on a complete output.
|
||||
self.assertIs(sides.ffn_output.group is None, sparse and a2a)
|
||||
|
||||
def test_the_attention_output_owes_the_attention_tp_sum(self):
|
||||
for attn_tp in (1, 2):
|
||||
sides = dense_decoder_layer_sides(
|
||||
axis_sizes={
|
||||
sides = sides_of(
|
||||
{
|
||||
TokenAxis.ATTN_DP: 2,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: attn_tp,
|
||||
},
|
||||
leaves_for_next_layer=False,
|
||||
leaves_for_reduce_scatter=False,
|
||||
}
|
||||
)
|
||||
owed = attn_tp > 1
|
||||
self.assertIs(
|
||||
@@ -202,16 +272,18 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
declarations, or none do."""
|
||||
|
||||
def declared(self, communicator):
|
||||
return communicator._mlp_input.func in (
|
||||
return getattr(communicator._mlp_input, "func", None) in (
|
||||
comm._mlp_input_dp_partial,
|
||||
comm._mlp_input_dp_replicate,
|
||||
comm._mlp_input_without_dp,
|
||||
comm._mlp_input_scatter,
|
||||
)
|
||||
|
||||
def test_a_dense_model(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
for layer_id in range(3):
|
||||
with self.subTest(layer_id=layer_id):
|
||||
communicator = build(dense_layer_facts(layer_id, 3), parallel)
|
||||
communicator = build(layer_facts(layer_id, 3), parallel)
|
||||
self.assertTrue(self.declared(communicator))
|
||||
self.assertIs(
|
||||
communicator._communicate_simple_fn,
|
||||
@@ -219,6 +291,36 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
)
|
||||
self.assertTrue(communicator._postprocess_scatters_to_local_tokens)
|
||||
|
||||
def test_plain_tp(self):
|
||||
for norm, fuses in ((Norm, False), (FusableNorm, True)):
|
||||
with self.subTest(norm=norm.__name__):
|
||||
with planning(parallel_of(attn_dp=1, attn_tp=2)):
|
||||
communicator = LayerCommunicator(
|
||||
layer_scatter_modes=layer_facts(1, 3),
|
||||
input_layernorm=norm(),
|
||||
post_attention_layernorm=norm(),
|
||||
)
|
||||
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
|
||||
entry = (
|
||||
communicator._mlp_input_reduce_output_and_update_and_read_residual
|
||||
)
|
||||
self.assertEqual(
|
||||
communicator._mlp_input.keywords["fusions"],
|
||||
(entry,) if fuses else (),
|
||||
)
|
||||
self.assertIs(communicator._mlp_input_may_fuse, fuses)
|
||||
self.assertFalse(communicator._postprocess_scatters_to_local_tokens)
|
||||
self.assertIs(
|
||||
communicator._communicate_summable_tensor_pair_fn,
|
||||
comm.CommunicateSummableTensorPairFn._trivial,
|
||||
)
|
||||
# One rank: the attention output is complete, only the norm is left.
|
||||
one_rank = parallel_of(attn_dp=1, attn_tp=1)
|
||||
single = build(layer_facts(1, 3), one_rank)
|
||||
with planning(one_rank):
|
||||
self.assertIsNotNone(single._declared_sides())
|
||||
self.assertIs(single._mlp_input, comm._mlp_input_norm)
|
||||
|
||||
def test_the_order_follows_the_attention_output(self):
|
||||
for attn_tp, force, order in (
|
||||
(2, False, comm._mlp_input_dp_partial),
|
||||
@@ -227,45 +329,99 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
):
|
||||
with self.subTest(attn_tp=attn_tp, force=force):
|
||||
communicator = build(
|
||||
dense_layer_facts(1, 3),
|
||||
layer_facts(1, 3),
|
||||
parallel_of(attn_dp=2, attn_tp=attn_tp),
|
||||
force_layernorm_before_dp_gather=force,
|
||||
)
|
||||
self.assertIs(communicator._mlp_input.func, order)
|
||||
|
||||
def test_a_dense_first_layer_before_sparse_ones(self):
|
||||
# DeepSeek-V2-Lite: layer 0 is dense, the rest sparse.
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
first = build(
|
||||
planned_modes(0, 3, sparse=False, previous_sparse=False, parallel=parallel),
|
||||
for a2a in (False, True):
|
||||
with self.subTest(a2a=a2a):
|
||||
layers = [
|
||||
build(
|
||||
planned_modes(
|
||||
i,
|
||||
4,
|
||||
sparse=sparse,
|
||||
previous_sparse=previous,
|
||||
parallel=parallel,
|
||||
a2a=a2a,
|
||||
),
|
||||
parallel,
|
||||
a2a=a2a,
|
||||
allow_reduce_scatter=True,
|
||||
)
|
||||
for i, (sparse, previous) in enumerate(
|
||||
((False, False), (True, False), (True, True), (False, True))
|
||||
)
|
||||
]
|
||||
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
|
||||
if not a2a:
|
||||
moe = layers[1]._ffn_output
|
||||
self.assertIs(moe.group, SumGroup.MOE_OUTPUT)
|
||||
self.assertTrue(moe.leaves_for_reduce_scatterv)
|
||||
continue
|
||||
# The first a2a layer slices the residual it takes from a
|
||||
# dense layer; the next one takes it already sliced.
|
||||
self.assertEqual(
|
||||
[layers[i]._mlp_input.func for i in (1, 2)],
|
||||
[comm._mlp_input_scatter] * 2,
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
layers[i]._mlp_input.keywords["scatters_residual"]
|
||||
for i in (1, 2)
|
||||
],
|
||||
[True, False],
|
||||
)
|
||||
self.assertIsNone(layers[1]._ffn_output.group)
|
||||
self.assertIs(
|
||||
layers[2]._communicate_simple_fn,
|
||||
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
|
||||
)
|
||||
self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial)
|
||||
self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"])
|
||||
|
||||
def test_the_last_a2a_layer_folds_the_residual_back(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
last = build(
|
||||
planned_modes(
|
||||
3, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
|
||||
),
|
||||
parallel,
|
||||
allow_reduce_scatter=True,
|
||||
a2a=True,
|
||||
)
|
||||
second = build(
|
||||
planned_modes(1, 3, sparse=True, previous_sparse=False, parallel=parallel),
|
||||
parallel,
|
||||
allow_reduce_scatter=True,
|
||||
self.assertIs(
|
||||
last._communicate_summable_tensor_pair_fn,
|
||||
comm.CommunicateSummableTensorPairFn._gather,
|
||||
)
|
||||
self.assertTrue(self.declared(first))
|
||||
self.assertFalse(self.declared(second))
|
||||
self.assertIs(second._mlp_input.func, comm._mlp_input_gather)
|
||||
|
||||
def test_an_attention_that_gathers_its_slice_itself(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
modes = planned_modes(
|
||||
2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
|
||||
)
|
||||
with patch.object(comm, "_use_ag_after_qlora", True):
|
||||
layer = build(modes, parallel, a2a=True)
|
||||
self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial)
|
||||
|
||||
def test_layers_that_keep_the_scatter_modes(self):
|
||||
dp = parallel_of(attn_dp=2, attn_tp=2)
|
||||
cases = {
|
||||
"no attention DP": (
|
||||
dense_layer_facts(1, 3),
|
||||
parallel_of(attn_dp=1, attn_tp=4),
|
||||
"input-scattered attention": (
|
||||
layer_facts(1, 3),
|
||||
parallel_of(attn_dp=1, attn_tp=4, enable_attn_tp_input_scattered=True),
|
||||
),
|
||||
"attention CP": (
|
||||
dense_layer_facts(1, 3),
|
||||
layer_facts(1, 3),
|
||||
parallel_of(attn_dp=2, attn_tp=1, attn_cp=2),
|
||||
),
|
||||
"dense MLP fully DP": (
|
||||
dense_layer_facts(1, 3),
|
||||
layer_facts(1, 3),
|
||||
parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1),
|
||||
),
|
||||
"after a sparse layer": (dense_layer_facts(1, 3, previous_sparse=True), dp),
|
||||
}
|
||||
for name, (facts, parallel) in cases.items():
|
||||
with self.subTest(name):
|
||||
@@ -287,7 +443,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
self.assertFalse(self.declared(build(direct, dp)))
|
||||
with planning(dp, sp=True):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator.layer_scatter_modes = dense_layer_facts(1, 3)
|
||||
communicator.layer_scatter_modes = layer_facts(1, 3)
|
||||
self.assertIsNone(communicator._declared_sides())
|
||||
|
||||
def test_subclasses_that_pick_their_own_steps(self):
|
||||
@@ -311,16 +467,14 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
}
|
||||
sides = dense_decoder_layer_sides(
|
||||
axis_sizes=sizes,
|
||||
leaves_for_next_layer=False,
|
||||
leaves_for_reduce_scatter=False,
|
||||
)
|
||||
sides = sides_of(sizes)
|
||||
steps = comm._select_ffn_input(
|
||||
produced,
|
||||
residual=sides.layer_rows,
|
||||
residual=sides.input_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=force,
|
||||
fusions=(),
|
||||
)
|
||||
reduced = []
|
||||
context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0)
|
||||
@@ -349,15 +503,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
|
||||
return len(reduced), hidden, residual
|
||||
|
||||
def test_a_complete_output_is_not_summed_again(self):
|
||||
layout = dense_decoder_layer_sides(
|
||||
axis_sizes={
|
||||
layout = sides_of(
|
||||
{
|
||||
TokenAxis.ATTN_DP: 2,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
},
|
||||
leaves_for_next_layer=False,
|
||||
leaves_for_reduce_scatter=False,
|
||||
).layer_rows
|
||||
}
|
||||
).input_rows
|
||||
complete = comm.StageOutput(layout)
|
||||
for force in (False, True):
|
||||
with self.subTest(force=force):
|
||||
@@ -367,15 +519,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
|
||||
torch.testing.assert_close(hidden, torch.full((1, HIDDEN), 20.0))
|
||||
|
||||
def test_an_owed_output_is_summed_once(self):
|
||||
layout = dense_decoder_layer_sides(
|
||||
axis_sizes={
|
||||
layout = sides_of(
|
||||
{
|
||||
TokenAxis.ATTN_DP: 2,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
},
|
||||
leaves_for_next_layer=False,
|
||||
leaves_for_reduce_scatter=False,
|
||||
).layer_rows
|
||||
}
|
||||
).input_rows
|
||||
owed = comm.StageOutput(layout, group=SumGroup.ATTN_TP, always_leaves=True)
|
||||
reductions, hidden, residual = self.run_steps(owed, force=True)
|
||||
self.assertEqual(reductions, 1)
|
||||
@@ -383,15 +533,13 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
|
||||
torch.testing.assert_close(residual, torch.full((1, HIDDEN), 17.0))
|
||||
|
||||
def test_declarations_the_steps_cannot_run_are_rejected(self):
|
||||
layout = dense_decoder_layer_sides(
|
||||
axis_sizes={
|
||||
layout = sides_of(
|
||||
{
|
||||
TokenAxis.ATTN_DP: 2,
|
||||
TokenAxis.ATTN_CP: 1,
|
||||
TokenAxis.ATTN_TP_SCATTER: 2,
|
||||
},
|
||||
leaves_for_next_layer=False,
|
||||
leaves_for_reduce_scatter=False,
|
||||
).layer_rows
|
||||
}
|
||||
).input_rows
|
||||
for produced in (
|
||||
# Leaves the attention-TP sum only when asked, like a mixer exit.
|
||||
comm.StageOutput(layout, group=SumGroup.ATTN_TP),
|
||||
@@ -408,7 +556,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Two consecutive dense layers, rank by rank.
|
||||
# Two consecutive layers, rank by rank.
|
||||
|
||||
|
||||
class World:
|
||||
@@ -535,7 +683,7 @@ def state():
|
||||
|
||||
|
||||
@contextmanager
|
||||
def running(*, reduce_scatterv):
|
||||
def running(*, reduce_scatterv, a2a=False):
|
||||
replaced = {
|
||||
"get_forward": lambda: state().flags,
|
||||
"attention_tensor_model_parallel_all_reduce": lambda x: (
|
||||
@@ -551,10 +699,16 @@ def running(*, reduce_scatterv):
|
||||
"dp_gather_partial": partial(fake_dp_gather, is_partial=True),
|
||||
"dp_scatter": fake_dp_scatter,
|
||||
"dp_reduce_scatter_tensor": fake_dp_reduce_scatter_tensor,
|
||||
"attn_tp_reduce_scatter_tensor": lambda out, x: (
|
||||
state().parallel.attn_tp_group.reduce_scatter_tensor(out, x)
|
||||
),
|
||||
"attn_tp_all_gather_into_tensor": lambda out, x: (
|
||||
state().parallel.attn_tp_group.all_gather_into_tensor(out, x)
|
||||
),
|
||||
"get_dp_global_num_tokens": lambda: state().dp_rows,
|
||||
"should_use_dp_reduce_scatterv": lambda: reduce_scatterv,
|
||||
"can_use_dp_reduce_scatter": lambda: True,
|
||||
"is_dp_attention_enabled": lambda: True,
|
||||
"is_dp_attention_enabled": lambda: state().parallel.attn_dp_size > 1,
|
||||
"apply_flashinfer_allreduce_fusion": lambda n: False,
|
||||
"post_experts_output_is_complete": lambda **kw: False,
|
||||
"post_experts_reduction_group": lambda: state().parallel.tp_group,
|
||||
@@ -569,7 +723,7 @@ def running(*, reduce_scatterv):
|
||||
"is_allocation_symmetric": lambda: False,
|
||||
}
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(planning(lambda: state().parallel))
|
||||
stack.enter_context(planning(lambda: state().parallel, a2a=a2a))
|
||||
for name, value in replaced.items():
|
||||
stack.enter_context(patch.object(comm, name, value))
|
||||
yield
|
||||
@@ -593,8 +747,22 @@ def dense_mlp(x, state):
|
||||
return state.parallel.tp_group.all_reduce(partial_sum)
|
||||
|
||||
|
||||
def moe(x, state):
|
||||
"""A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE
|
||||
output's group unless a published flag asks it to leave the sum, or the
|
||||
attention-DP reduce_scatterv takes it, as the MoE blocks do."""
|
||||
partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
|
||||
if (
|
||||
state.flags.fuse_mlp_allreduce
|
||||
or state.flags.mlp_reduce_scatter
|
||||
or comm.should_use_dp_reduce_scatterv()
|
||||
):
|
||||
return partial_sum
|
||||
return comm.post_experts_reduction_group().all_reduce(partial_sum)
|
||||
|
||||
|
||||
def reference(x):
|
||||
"""(hidden, residual) after two layers of Norm / attention / dense MLP."""
|
||||
"""(hidden, residual) after two layers of Norm / attention / FFN."""
|
||||
residual = x
|
||||
for _ in range(2):
|
||||
hidden = 2 * residual
|
||||
@@ -605,8 +773,19 @@ def reference(x):
|
||||
return ffn, residual - ffn
|
||||
|
||||
|
||||
class TestTwoDenseLayers(CustomTestCase):
|
||||
def run_world(self, *, attn_dp, attn_tp, rows, padding, reduce_scatterv, leaves):
|
||||
class TestTwoLayers(CustomTestCase):
|
||||
def run_world(
|
||||
self,
|
||||
*,
|
||||
attn_dp,
|
||||
attn_tp,
|
||||
rows,
|
||||
padding,
|
||||
reduce_scatterv,
|
||||
leaves,
|
||||
sparse=(False, False),
|
||||
a2a=False,
|
||||
):
|
||||
tp = attn_dp * attn_tp
|
||||
world = World(tp)
|
||||
WORLD[0] = world
|
||||
@@ -638,6 +817,8 @@ class TestTwoDenseLayers(CustomTestCase):
|
||||
),
|
||||
dp_padding_mode=SimpleNamespace(is_max_len=lambda: max_len),
|
||||
global_dp_buffer_len=global_rows,
|
||||
# Without attention DP the FFN exit counts this rank's tokens.
|
||||
input_ids=torch.zeros(rows[0]),
|
||||
)
|
||||
states = []
|
||||
for rank in range(tp):
|
||||
@@ -668,7 +849,12 @@ class TestTwoDenseLayers(CustomTestCase):
|
||||
s = states[rank]
|
||||
layers = [
|
||||
LayerCommunicator(
|
||||
layer_scatter_modes=dense_layer_facts(i, 2),
|
||||
layer_scatter_modes=layer_facts(
|
||||
i,
|
||||
2,
|
||||
sparse=sparse[i],
|
||||
previous_sparse=i > 0 and sparse[i - 1],
|
||||
),
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
allow_reduce_scatter=leaves,
|
||||
@@ -680,46 +866,67 @@ class TestTwoDenseLayers(CustomTestCase):
|
||||
hidden[: s.rows] = embeddings[s.dp]
|
||||
residual = None
|
||||
handed_on = []
|
||||
for layer in layers:
|
||||
for layer_index, layer in enumerate(layers):
|
||||
hidden, residual = layer.prepare_attn(hidden, residual, forward_batch)
|
||||
hidden = attention(hidden, s)
|
||||
hidden, residual = layer.prepare_mlp(hidden, residual, forward_batch)
|
||||
with layer.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden = dense_mlp(hidden, s)
|
||||
if not sparse[layer_index]:
|
||||
hidden = dense_mlp(hidden, s)
|
||||
elif a2a:
|
||||
# On this rank's slice; the combine completes the sum.
|
||||
hidden = 5 * hidden
|
||||
else:
|
||||
hidden = moe(hidden, s)
|
||||
hidden, residual = ffn_exit.finish(hidden, residual)
|
||||
handed_on.append(type(hidden))
|
||||
hidden, residual = layers[-1].finish_layer_stack(
|
||||
hidden, residual, forward_batch
|
||||
)
|
||||
return hidden[: s.rows], residual[: s.rows], handed_on
|
||||
# The last a2a layer folds the residual into its output.
|
||||
if residual is not None:
|
||||
residual = residual[: s.rows]
|
||||
return hidden[: s.rows], residual, handed_on
|
||||
|
||||
with running(reduce_scatterv=reduce_scatterv):
|
||||
with running(reduce_scatterv=reduce_scatterv, a2a=a2a):
|
||||
results, errors = world.run(states, forward)
|
||||
for rank, error in enumerate(errors):
|
||||
if error is not None:
|
||||
raise AssertionError(f"rank {rank} raised") from error
|
||||
for rank, (hidden, residual, handed_on) in enumerate(results):
|
||||
want_hidden, want_residual = reference(embeddings[states[rank].dp])
|
||||
if residual is None:
|
||||
want_hidden, want_residual = want_hidden + want_residual, None
|
||||
torch.testing.assert_close(hidden, want_hidden, rtol=0, atol=0)
|
||||
torch.testing.assert_close(residual, want_residual, rtol=0, atol=0)
|
||||
self.assertIs(residual is None, want_residual is None)
|
||||
if residual is not None:
|
||||
torch.testing.assert_close(residual, want_residual, rtol=0, atol=0)
|
||||
# Every rank ran the same collectives.
|
||||
self.assertEqual(len({tuple(s.calls) for s in states if s.dp == 0}), 1)
|
||||
return results
|
||||
|
||||
def test_rows_come_back_to_each_rank(self):
|
||||
for (attn_dp, attn_tp), rows, padding, leaves in itertools.product(
|
||||
((2, 2), (4, 1), (2, 1)),
|
||||
for (attn_dp, attn_tp), rows, padding, leaves, sparse in itertools.product(
|
||||
((2, 2), (4, 1), (2, 1), (1, 2), (1, 4)),
|
||||
("ragged", "idle"),
|
||||
("sum_len", "max_len"),
|
||||
(False, True),
|
||||
((False, False), (False, True), (True, True)),
|
||||
):
|
||||
token_rows = {
|
||||
("ragged", 1): [3],
|
||||
("idle", 1): [0],
|
||||
("ragged", 2): [3, 1],
|
||||
("idle", 2): [2, 0],
|
||||
("ragged", 4): [2, 3, 1, 1],
|
||||
("idle", 4): [2, 0, 3, 1],
|
||||
}[rows, attn_dp]
|
||||
for reduce_scatterv in (False, True) if attn_tp == 1 else (False,):
|
||||
for reduce_scatterv, a2a in itertools.product(
|
||||
(False, True) if attn_tp == 1 and attn_dp > 1 else (False,),
|
||||
(False, True) if any(sparse) else (False,),
|
||||
):
|
||||
if reduce_scatterv and a2a:
|
||||
continue
|
||||
with self.subTest(
|
||||
attn_dp=attn_dp,
|
||||
attn_tp=attn_tp,
|
||||
@@ -727,6 +934,8 @@ class TestTwoDenseLayers(CustomTestCase):
|
||||
padding=padding,
|
||||
leaves=leaves,
|
||||
reduce_scatterv=reduce_scatterv,
|
||||
sparse=sparse,
|
||||
a2a=a2a,
|
||||
):
|
||||
results = self.run_world(
|
||||
attn_dp=attn_dp,
|
||||
@@ -735,13 +944,30 @@ class TestTwoDenseLayers(CustomTestCase):
|
||||
padding=padding,
|
||||
reduce_scatterv=reduce_scatterv,
|
||||
leaves=leaves,
|
||||
sparse=sparse,
|
||||
a2a=a2a,
|
||||
)
|
||||
first_layer_hands_on = results[0][2][0]
|
||||
# A producer that may leave its sum leaves it for the next
|
||||
# layer's input; one that may not hands on a complete value.
|
||||
# Without attention DP an empty batch completes its own sum.
|
||||
# A MoE dispatched per DP shard hands on a complete value.
|
||||
# Otherwise the reduce-scatter back to this rank's tokens
|
||||
# can be left to the next layer; the all-reduce only
|
||||
# without an a2a backend.
|
||||
has_tokens = attn_dp > 1 or sum(token_rows) > 0
|
||||
reduce_scatter_left = attn_dp > 1 and (
|
||||
padding == "max_len" or reduce_scatterv
|
||||
)
|
||||
owes = (
|
||||
leaves
|
||||
and has_tokens
|
||||
and not (sparse[0] and a2a)
|
||||
and (reduce_scatter_left or not a2a)
|
||||
)
|
||||
self.assertIs(
|
||||
first_layer_hands_on,
|
||||
UnreducedOutput if leaves else torch.Tensor,
|
||||
UnreducedOutput if owes else torch.Tensor,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user