Files
sglang/test/registered/unit/layers/test_declared_decoder_boundary.py
T

1221 lines
47 KiB
Python

"""The boundaries of an attention and an FFN, chosen from both sides'
declarations. The FFN runs on the TP group (a dense MLP, or a MoE not
dispatched per DP shard) or on each attention-TP rank's slice of its DP shard's
rows (a MoE dispatched per DP shard).
The numeric checks run every rank of a DP x attention-TP world as a thread over
fake collectives that wait for all members of their group, build the layers'
communicators for real, and pass them scatter modes that fail on any read of a
mode field.
"""
import itertools
import threading
import unittest
from contextlib import ExitStack, contextmanager, nullcontext
from functools import partial
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.layers import communicator as comm
from sglang.srt.layers.boundary_layout import (
SumGroup,
TokenAxis,
decoder_layer_sides,
input_scattered_layer_sides,
sequence_parallel_layer_sides,
)
from sglang.srt.layers.communicator import (
LayerCommunicator,
LayerScatterModes,
ScatterMode,
UnreducedOutput,
scatter_mode_layouts,
)
from sglang.srt.layers.communicator_dsa_cp import DSACPLayerCommunicator
from sglang.srt.layers.communicator_mhc import MHCLayerCommunicator
from sglang.srt.layers.moe.cutedsl_ar_fusion import CuteDSLFusionLayerCommunicator
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
HIDDEN = 4
def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides):
fields = dict(
tp_size=attn_dp * attn_cp * attn_tp,
tp_rank=0,
attn_dp_size=attn_dp,
attn_tp_size=attn_tp,
attn_tp_rank=0,
attn_cp_size=attn_cp,
attn_cp_rank=0,
moe_dense_tp_size=None,
enable_prefill_cp=False,
moe_ep_size=1,
moe_tp_size=attn_dp * attn_cp * attn_tp,
moe_dp_size=1,
dwdp_size=1,
enable_dp_attention=attn_dp > 1,
enable_attn_tp_input_scattered=False,
tp_group=SimpleNamespace(name="tp"),
attn_tp_group=SimpleNamespace(name="attn_tp"),
)
fields.update(overrides)
return SimpleNamespace(**fields)
@contextmanager
def planning(parallel, *, sp=False, a2a=False):
"""What layer planning and communicator construction read, without the
process-wide parallel state. ``parallel`` may be a callable, for a
per-thread parallel state."""
get_parallel = parallel if callable(parallel) else (lambda: parallel)
with (
patch.object(comm, "get_parallel", get_parallel),
patch.object(comm, "is_dsa_enable_prefill_cp", lambda: False),
patch.object(comm, "is_mla_cp_enabled", lambda: False),
patch.object(comm, "get_moe_cp_size", lambda: 1),
patch.object(comm.layernorm_sp, "layernorm_sp_enabled", lambda: sp),
patch.object(
comm, "get_spec", lambda: SimpleNamespace(speculative_algorithm=None)
),
patch.object(
comm,
"get_moe_a2a_backend",
lambda: SimpleNamespace(is_none=lambda: not a2a),
),
patch.object(comm, "is_moe_input_scattered_across_dp_ranks", lambda: a2a),
patch.object(
comm, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False
),
patch.object(comm, "is_enable_moe_cp_allgather", lambda: False),
patch.object(comm, "get_lora", lambda: SimpleNamespace(enable_lora=False)),
):
yield
class Norm:
"""add + norm stand-in: (2 * (h + r), h + r); one input: 2 * h."""
def __call__(self, x, residual=None, post_residual_addition=None):
assert post_residual_addition is None
if residual is None:
return x * 2
s = x + residual
return s * 2, s
class FusableNorm(Norm):
"""A norm with the fused all-reduce + add + norm method."""
def forward_with_allreduce_fusion(self, *args, **kwargs):
raise AssertionError("not run here")
class FactsOnly:
"""Scatter modes whose mode fields fail when read: only the layer facts
are there."""
def __init__(self, **facts):
self.__dict__.update(facts)
def __getattr__(self, name):
raise AssertionError(f"read scatter mode {name!r}")
def layer_facts(layer_id, num_layers, *, sparse=False, previous_sparse=False):
return FactsOnly(
is_layer_sparse=sparse,
is_first_layer=layer_id == 0,
is_last_layer=layer_id == num_layers - 1,
is_previous_layer_sparse=None if layer_id == 0 else previous_sparse,
)
def sides_of(
axis_sizes,
*,
sparse=False,
a2a=False,
previous_a2a=False,
last=False,
leaves_next=False,
leaves_rs=False,
):
return decoder_layer_sides(
axis_sizes=axis_sizes,
ffn_on_local_rows=sparse and a2a,
previous_on_local_rows=previous_a2a,
is_last_layer=last,
attention_gathers_local_rows=False,
ffn_group=SumGroup.MOE_OUTPUT if sparse else SumGroup.TP,
leaves_for_next_layer=leaves_next,
leaves_for_reduce_scatter=leaves_rs,
leaves_for_reduce_scatterv=leaves_rs or sparse,
)
def build(modes, parallel, *, cls=LayerCommunicator, sp=False, a2a=False, **kwargs):
with planning(parallel, sp=sp, a2a=a2a):
return cls(
layer_scatter_modes=modes,
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
**kwargs,
)
def planned_modes(
layer_id, num_layers, *, sparse, previous_sparse, parallel, a2a=False
):
with planning(parallel, a2a=a2a):
return LayerScatterModes.init_new(
layer_id=layer_id,
num_layers=num_layers,
is_layer_sparse=sparse,
is_previous_layer_sparse=previous_sparse,
is_next_layer_sparse=False,
)
class TestDeclarationsMatchScatterModes(CustomTestCase):
"""The layouts derived from the groups each side computes over are the
layouts of the modes layer planning gives the same layers."""
def test_every_attention_dp_and_tp(self):
for (
attn_dp,
attn_tp,
layer_id,
sparse,
previous_sparse,
a2a,
) in itertools.product(
(1, 2, 4, 8),
(1, 2, 4),
(0, 1, 3),
(False, True),
(False, True),
(False, True),
):
with self.subTest(
attn_dp=attn_dp,
attn_tp=attn_tp,
layer_id=layer_id,
sparse=sparse,
previous_sparse=previous_sparse,
a2a=a2a,
):
parallel = parallel_of(attn_dp=attn_dp, attn_tp=attn_tp)
modes = planned_modes(
layer_id,
4,
sparse=sparse,
previous_sparse=previous_sparse,
parallel=parallel,
a2a=a2a,
)
sizes = {
TokenAxis.ATTN_DP: attn_dp,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: attn_tp,
}
layouts = scatter_mode_layouts(
attn_dp_size=attn_dp, attn_cp_size=1, attn_tp_size=attn_tp
)
sides = sides_of(
sizes,
sparse=sparse,
a2a=a2a,
previous_a2a=layer_id > 0 and previous_sparse and a2a,
last=layer_id == 3,
leaves_next=True,
leaves_rs=True,
)
self.assertEqual(sides.input_rows, layouts[modes.layer_input_mode])
self.assertEqual(
sides.ffn_residual_rows, layouts[modes.middle_residual_mode]
)
self.assertEqual(sides.output_rows, layouts[modes.layer_output_mode])
self.assertEqual(sides.attention.layout, layouts[modes.attn_mode])
self.assertEqual(sides.ffn.layout, layouts[modes.mlp_mode])
self.assertEqual(
sides.attention_output.layout, layouts[modes.attn_mode]
)
self.assertEqual(sides.ffn_output.layout, sides.ffn.layout)
# A MoE dispatched per DP shard hands on a complete output.
self.assertIs(sides.ffn_output.group is None, sparse and a2a)
def test_the_attention_output_owes_the_attention_tp_sum(self):
for attn_tp in (1, 2):
sides = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: attn_tp,
}
)
owed = attn_tp > 1
self.assertIs(
sides.attention_output.group, SumGroup.ATTN_TP if owed else None
)
self.assertIs(sides.attention_output.always_leaves, owed)
self.assertIs(sides.ffn_output.group, SumGroup.TP)
self.assertFalse(sides.ffn_output.always_leaves)
class TestWhichLayersUseDeclarations(CustomTestCase):
"""The layer is the unit: all three boundary sides it owns follow the
declarations, or none do."""
def declared(self, communicator):
return getattr(communicator._mlp_input, "func", None) in (
comm._mlp_input_dp_partial,
comm._mlp_input_dp_replicate,
comm._mlp_input_without_dp,
comm._mlp_input_scatter,
)
def test_a_dense_model(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
communicator = build(layer_facts(layer_id, 3), parallel)
self.assertTrue(self.declared(communicator))
self.assertIs(
communicator._communicate_simple_fn,
comm.CommunicateSimpleFn._trivial,
)
self.assertTrue(communicator._postprocess_scatters_to_local_tokens)
def test_plain_tp(self):
for norm, fuses in ((Norm, False), (FusableNorm, True)):
with self.subTest(norm=norm.__name__):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
communicator = LayerCommunicator(
layer_scatter_modes=layer_facts(1, 3),
input_layernorm=norm(),
post_attention_layernorm=norm(),
)
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
entry = (
communicator._mlp_input_reduce_output_and_update_and_read_residual
)
self.assertEqual(
communicator._mlp_input.keywords["fusions"],
(entry,) if fuses else (),
)
self.assertIs(communicator._mlp_input_may_return_new_residual, fuses)
self.assertFalse(communicator._postprocess_scatters_to_local_tokens)
self.assertIs(
communicator._communicate_summable_tensor_pair_fn,
comm.CommunicateSummableTensorPairFn._trivial,
)
# One rank: the attention output is complete, only the norm is left.
one_rank = parallel_of(attn_dp=1, attn_tp=1)
single = build(layer_facts(1, 3), one_rank)
with planning(one_rank):
self.assertIsNotNone(single._declared_sides())
self.assertIs(single._mlp_input, comm._mlp_input_norm)
def test_the_order_follows_the_attention_output(self):
for attn_tp, force, order in (
(2, False, comm._mlp_input_dp_partial),
(2, True, comm._mlp_input_dp_replicate),
(1, False, comm._mlp_input_dp_replicate),
):
with self.subTest(attn_tp=attn_tp, force=force):
communicator = build(
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=attn_tp),
force_layernorm_before_dp_gather=force,
)
self.assertIs(communicator._mlp_input.func, order)
def test_a_dense_first_layer_before_sparse_ones(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
for a2a in (False, True):
with self.subTest(a2a=a2a):
layers = [
build(
planned_modes(
i,
4,
sparse=sparse,
previous_sparse=previous,
parallel=parallel,
a2a=a2a,
),
parallel,
a2a=a2a,
allow_reduce_scatter=True,
)
for i, (sparse, previous) in enumerate(
((False, False), (True, False), (True, True), (False, True))
)
]
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
if not a2a:
moe = layers[1]._ffn_output
self.assertIs(moe.group, SumGroup.MOE_OUTPUT)
self.assertTrue(moe.leaves_for_reduce_scatterv)
continue
# The first a2a layer slices the residual it takes from a
# dense layer; the next one takes it already sliced.
self.assertEqual(
[layers[i]._mlp_input.func for i in (1, 2)],
[comm._mlp_input_scatter] * 2,
)
self.assertEqual(
[
layers[i]._mlp_input.keywords["scatters_residual"]
for i in (1, 2)
],
[True, False],
)
self.assertIsNone(layers[1]._ffn_output.group)
self.assertIs(
layers[2]._communicate_simple_fn,
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
)
self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial)
self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"])
def test_the_last_a2a_layer_folds_the_residual_back(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
last = build(
planned_modes(
3, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
),
parallel,
a2a=True,
)
self.assertIs(
last._communicate_summable_tensor_pair_fn,
comm.CommunicateSummableTensorPairFn._gather,
)
def test_an_attention_that_gathers_its_slice_itself(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
modes = planned_modes(
2, 4, sparse=True, previous_sparse=True, parallel=parallel, a2a=True
)
with patch.object(comm, "_use_ag_after_qlora", True):
layer = build(modes, parallel, a2a=True)
self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial)
def test_layers_that_keep_the_scatter_modes(self):
dp = parallel_of(attn_dp=2, attn_tp=2)
cases = {
"attention CP": (
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=1, attn_cp=2),
),
"dense MLP fully DP": (
layer_facts(1, 3),
parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1),
),
}
for name, (facts, parallel) in cases.items():
with self.subTest(name):
with planning(parallel):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator.layer_scatter_modes = facts
communicator.allow_deferred_ffn_reduction = True
communicator.allow_reduce_scatter = False
self.assertIsNone(communicator._declared_sides())
# Modes given directly (Nemotron-H's stages) do not say which rows the
# layer takes.
direct = LayerScatterModes(
layer_input_mode=ScatterMode.TP_ATTN_FULL,
attn_mode=ScatterMode.TP_ATTN_FULL,
mlp_mode=ScatterMode.FULL,
middle_residual_mode=ScatterMode.TP_ATTN_FULL,
layer_output_mode=ScatterMode.TP_ATTN_FULL,
)
self.assertFalse(self.declared(build(direct, dp)))
def test_subclasses_that_pick_their_own_steps(self):
for cls in (
MHCLayerCommunicator,
DSACPLayerCommunicator,
CuteDSLFusionLayerCommunicator,
):
with self.subTest(cls.__name__):
self.assertFalse(cls._takes_declared_boundaries)
self.assertTrue(LayerCommunicator._takes_declared_boundaries)
class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
"""prepare_mlp completes the attention-TP sum exactly when the attention
output's declaration says it is owed, whatever the attention-TP size."""
def run_steps(self, produced, *, force=False):
sizes = {
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
sides = sides_of(sizes)
steps, _ = comm._select_ffn_input(
produced,
residual=sides.input_rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=force,
fusions=(),
)
reduced = []
context = SimpleNamespace(attn_tp_size=2, attn_tp_rank=0)
with (
patch.object(
comm,
"attention_tensor_model_parallel_all_reduce",
lambda x: reduced.append(x) or 2 * x,
),
patch.object(comm, "_redistribute_input_to_dp", lambda h, fb: h),
patch.object(
comm, "get_parallel", lambda: parallel_of(attn_dp=2, attn_tp=2)
),
patch.object(
comm, "use_symmetric_memory", lambda g, disabled=False: nullcontext()
),
patch.object(comm, "is_allocation_symmetric", lambda: False),
):
hidden, residual = steps(
torch.full((1, HIDDEN), 7.0),
torch.full((1, HIDDEN), 3.0),
None,
Norm(),
context,
)
return len(reduced), hidden, residual
def test_a_complete_output_is_not_summed_again(self):
layout = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
).input_rows
complete = comm.StageOutput(layout)
for force in (False, True):
with self.subTest(force=force):
reductions, hidden, residual = self.run_steps(complete, force=force)
self.assertEqual(reductions, 0)
torch.testing.assert_close(residual, torch.full((1, HIDDEN), 10.0))
torch.testing.assert_close(hidden, torch.full((1, HIDDEN), 20.0))
def test_an_owed_output_is_summed_once(self):
layout = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
).input_rows
owed = comm.StageOutput(layout, group=SumGroup.ATTN_TP, always_leaves=True)
reductions, hidden, residual = self.run_steps(owed, force=True)
self.assertEqual(reductions, 1)
# The stand-in all-reduce doubles the one rank's value.
torch.testing.assert_close(residual, torch.full((1, HIDDEN), 17.0))
def test_declarations_the_steps_cannot_run_are_rejected(self):
layout = sides_of(
{
TokenAxis.ATTN_DP: 2,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
).input_rows
for produced in (
# Leaves the attention-TP sum only when asked, like a mixer exit.
comm.StageOutput(layout, group=SumGroup.ATTN_TP),
# Always leaves a sum over no group.
comm.StageOutput(layout, always_leaves=True),
# Owes a sum over a group the steps do not complete.
comm.StageOutput(layout, group=SumGroup.TP, always_leaves=True),
):
with (
self.subTest(produced=produced),
self.assertRaises(NotImplementedError),
):
self.run_steps(produced)
class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
"""A fused kernel replaces the attention-TP sum, the residual add and the
norm only where those are the steps, and only if it completes the sum the
attention output owes."""
def fused(self, completes, may_return_new_residual):
return comm.FusedMlpInput(
completes=completes,
run=lambda h, r, fb: None,
may_return_new_residual=may_return_new_residual,
)
def select(self, *, attn_dp, fusions):
sides = sides_of(
{
TokenAxis.ATTN_DP: attn_dp,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: 2,
}
)
return comm._select_ffn_input(
sides.attention_output,
residual=sides.input_rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=False,
fusions=fusions,
)
def test_a_kernel_over_another_group_is_not_chosen(self):
over_tp = self.fused(SumGroup.TP, True)
over_attention_tp = self.fused(SumGroup.ATTN_TP, False)
steps, chosen = self.select(attn_dp=1, fusions=(over_tp, over_attention_tp))
self.assertIs(steps.func, comm._mlp_input_without_dp)
self.assertEqual(steps.keywords["fusions"], (over_attention_tp.run,))
self.assertEqual(chosen, (over_attention_tp,))
def test_no_kernel_is_chosen_where_rows_are_gathered(self):
steps, chosen = self.select(
attn_dp=2, fusions=(self.fused(SumGroup.ATTN_TP, True),)
)
self.assertIs(steps.func, comm._mlp_input_dp_partial)
self.assertEqual(chosen, ())
def test_aux_capture_follows_the_chosen_kernels(self):
for kernels, expected in (
((self.fused(SumGroup.ATTN_TP, False),), False),
((self.fused(SumGroup.ATTN_TP, True),), True),
((self.fused(SumGroup.TP, True),), False),
):
with self.subTest(kernels=kernels):
class Declaring(LayerCommunicator):
def _select_mlp_input_fusions(self):
return kernels
with planning(parallel_of(attn_dp=1, attn_tp=2)):
communicator = Declaring(
layer_scatter_modes=layer_facts(1, 3),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertIs(communicator._mlp_input_may_return_new_residual, expected)
class TestTheSequenceParallelRegion(CustomTestCase):
"""While a LayerNorm SP region is active, the layer runs the steps its
region declarations choose: the linears gather and reduce-scatter
themselves, so every boundary stays on this rank's slice. Other batches
take the layer's ordinary declared steps."""
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
SP_STEPS = comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output,
)
def test_the_region_declarations_choose_local_steps(self):
for attn_tp in (2, 4):
with self.subTest(attn_tp=attn_tp):
sides = sequence_parallel_layer_sides(
axis_sizes={
TokenAxis.ATTN_DP: 1,
TokenAxis.ATTN_CP: 1,
TokenAxis.ATTN_TP_SCATTER: attn_tp,
}
)
local = frozenset({TokenAxis.ATTN_TP_SCATTER})
self.assertEqual(sides.input_rows.sharded, local)
self.assertEqual(sides.attention.gathers_itself, local)
self.assertEqual(sides.ffn.gathers_itself, local)
self.assertIsNone(sides.attention_output.group)
self.assertIsNone(sides.ffn_output.group)
steps = comm._select_boundary_steps(sides)
self.assertEqual(
(steps.attention_input, steps.ffn_input, steps.ffn_output_move),
(
self.SP_STEPS.attention_input,
self.SP_STEPS.ffn_input,
self.SP_STEPS.ffn_output_move,
),
)
self.assertIsNone(steps.layer_input)
def test_a_layer_under_sp_takes_both_sets_of_steps(self):
parallel = parallel_of(attn_dp=1, attn_tp=2)
for layer_id in range(3):
with self.subTest(layer_id=layer_id):
with planning(parallel, sp=True):
communicator = LayerCommunicator(
layer_scatter_modes=layer_facts(layer_id, 3),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertEqual(communicator._sp_steps, self.SP_STEPS)
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
def test_without_sp_there_is_no_region(self):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
communicator = LayerCommunicator(
layer_scatter_modes=layer_facts(1, 3),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertIsNone(communicator._sp_steps)
def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self):
# A sum the FFN exit may hand to the next layer, or a move back over
# attention DP, waits on a per-batch decision.
for sizes, leaves_next in (
(self.SIZES, True),
({**self.SIZES, TokenAxis.ATTN_DP: 2}, False),
):
with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError):
comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next))
class TestInputScatteredAttention(CustomTestCase):
"""On a batch whose attention input is scattered over attention TP, a layer
runs the steps that batch's declarations choose: its input arrives as a TP
partial that a reduce-scatter completes onto each rank's slice, and the
residual comes back to every row inside the attention output's sum."""
SIZES = {TokenAxis.ATTN_DP: 1, TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP_SCATTER: 2}
def test_the_declarations_choose_the_steps(self):
for hands_on in (False, True):
with self.subTest(hands_on_partial=hands_on):
sides = input_scattered_layer_sides(
axis_sizes=self.SIZES,
ffn_group=SumGroup.TP,
hands_on_partial=hands_on,
)
steps = comm._select_boundary_steps(sides)
self.assertIs(steps.layer_input, comm.tp_reduce_scatter)
self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial)
self.assertIs(steps.ffn_input, comm._mlp_input_residual_into_sum)
self.assertIs(
steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._trivial,
)
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
self.assertFalse(steps.ffn_output.leaves_for_next_layer)
def test_the_order_of_a_residual_on_the_slice_is_declared(self):
# The same layouts take two orders: with a scattered input the residual
# joins the attention output's sum; after a MoE on local rows it is
# gathered first.
sizes = self.SIZES
attention = comm.Layout.sharded_over(axis_sizes=sizes)
local = comm.Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
owed = comm.StageOutput(attention, group=SumGroup.ATTN_TP, always_leaves=True)
for joins, step in (
(True, comm._mlp_input_residual_into_sum),
(False, comm._mlp_input_without_dp),
):
with self.subTest(residual_joins_sum=joins):
steps, _ = comm._select_ffn_input(
owed,
residual=local,
residual_to=attention,
need=comm.StageInput(attention),
force_layernorm_before_gather=False,
fusions=(),
residual_joins_sum=joins,
)
self.assertIs(getattr(steps, "func", steps), step)
def test_which_layers_can_scatter_their_input(self):
configured = dict(enable_attn_tp_input_scattered=True)
for name, parallel, a2a, expected in (
("pure TP", parallel_of(attn_dp=1, attn_tp=2, **configured), False, True),
("not configured", parallel_of(attn_dp=1, attn_tp=2), False, False),
(
"attention DP",
parallel_of(attn_dp=2, attn_tp=2, **configured),
False,
False,
),
("a2a", parallel_of(attn_dp=1, attn_tp=2, **configured), True, False),
):
with self.subTest(name):
communicator = build(
layer_facts(1, 3, sparse=a2a, previous_sparse=a2a),
parallel,
a2a=a2a,
allow_reduce_scatter=True,
)
self.assertIs(communicator._input_scattered_steps is not None, expected)
def test_a_batch_runs_them_only_while_its_input_is_scattered(self):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
)
communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
for scattered in (False, True):
with (
self.subTest(input_scattered=scattered),
patch.object(
comm,
"get_attn_tp_context",
lambda: SimpleNamespace(input_scattered=scattered),
),
patch.object(
comm, "get_forward", lambda: SimpleNamespace(sp_active=False)
),
):
self.assertIs(
communicator._batch_steps(),
communicator._input_scattered_steps if scattered else None,
)
def test_the_last_layer_completes_its_own_sum(self):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
)
for layer_id, allow, hands_on in (
(1, True, True),
(2, True, False),
(1, False, False),
):
with self.subTest(layer_id=layer_id, allow_reduce_scatter=allow):
communicator = build(
layer_facts(layer_id, 3), parallel, allow_reduce_scatter=allow
)
steps = communicator._input_scattered_steps
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
# ---------------------------------------------------------------------------
# Two consecutive layers, rank by rank.
class World:
def __init__(self, size):
self.size = size
self.local = threading.local()
self.groups = []
def state(self):
return self.local.state
def run(self, states, fn):
results, errors = [None] * self.size, [None] * self.size
def body(rank):
self.local.state = states[rank]
try:
results[rank] = fn(rank)
except BaseException as error: # noqa: BLE001
errors[rank] = error
for group in self.groups:
group.barrier.abort()
threads = [threading.Thread(target=body, args=(r,)) for r in range(self.size)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
return results, errors
class Group:
"""A process group whose collectives wait for every member thread."""
def __init__(self, world, name, ranks):
self.world, self.name, self.ranks = world, name, list(ranks)
self.barrier = threading.Barrier(len(self.ranks), timeout=30)
self.slots = {}
world.groups.append(self)
@property
def world_size(self):
return len(self.ranks)
@property
def rank_in_group(self):
return self.ranks.index(self.world.state().rank)
def _exchange(self, x):
self.world.state().calls.append(self.name)
self.slots[self.world.state().rank] = x.clone()
self.barrier.wait()
values = [self.slots[r] for r in self.ranks]
self.barrier.wait()
return values
def all_reduce(self, x):
return torch.stack(self._exchange(x)).sum(0)
def reduce_scatter_tensor(self, output, input):
full = torch.stack(self._exchange(input)).sum(0)
output.copy_(full.tensor_split(self.world_size)[self.rank_in_group])
def all_gather_into_tensor(self, output, input):
output.copy_(torch.cat(self._exchange(input)))
def reduce_scatterv(self, input, output=None, sizes=None):
full = torch.stack(self._exchange(input)).sum(0)
output.copy_(full.split(list(sizes))[self.rank_in_group])
return output
class Flags:
"""The per-forward flags an FFN exit publishes, one set per rank."""
def __init__(self):
self.fuse_mlp_allreduce = False
self.mlp_reduce_scatter = False
self.defer_moe_finalize = False
self.sp_active = False
@contextmanager
def scoped(self, **flags):
saved = {k: getattr(self, k) for k in flags}
self.__dict__.update(flags)
try:
yield
finally:
self.__dict__.update(saved)
def fake_dp_gather(global_tokens, local_tokens, forward_batch, *, is_partial):
state = WORLD[0].state()
global_tokens.zero_()
if is_partial or state.parallel.attn_tp_rank == 0:
global_tokens[state.offset : state.offset + state.rows] += local_tokens[
: state.rows
]
global_tokens.copy_(state.parallel.tp_group.all_reduce(global_tokens))
def fake_dp_scatter(local_tokens, global_tokens, forward_batch):
state = WORLD[0].state()
local_tokens.fill_(0)
local_tokens[: state.rows] = global_tokens[state.offset : state.offset + state.rows]
def fake_dp_reduce_scatter_tensor(output, input):
# dp_attention.dp_reduce_scatter_tensor for max-len padding.
parallel = WORLD[0].state().parallel
if parallel.tp_size == parallel.attn_dp_size:
parallel.tp_group.reduce_scatter_tensor(output, input)
else:
chunk = input.tensor_split(parallel.tp_size)[parallel.tp_rank].clone()
parallel.tp_group.reduce_scatter_tensor(chunk, input)
parallel.attn_tp_group.all_gather_into_tensor(output, chunk)
WORLD = [None]
def state():
return WORLD[0].state()
@contextmanager
def running(*, reduce_scatterv, a2a=False):
replaced = {
"get_forward": lambda: state().flags,
"attention_tensor_model_parallel_all_reduce": lambda x: (
state().parallel.attn_tp_group.all_reduce(x)
),
"get_global_dp_buffer": lambda g: torch.zeros(
state().global_rows, HIDDEN, dtype=torch.double
),
"get_local_dp_buffer": lambda g: torch.zeros(
state().local_rows, HIDDEN, dtype=torch.double
),
"dp_gather_replicate": partial(fake_dp_gather, is_partial=False),
"dp_gather_partial": partial(fake_dp_gather, is_partial=True),
"dp_scatter": fake_dp_scatter,
"dp_reduce_scatter_tensor": fake_dp_reduce_scatter_tensor,
"attn_tp_reduce_scatter_tensor": lambda out, x: (
state().parallel.attn_tp_group.reduce_scatter_tensor(out, x)
),
"attn_tp_all_gather_into_tensor": lambda out, x: (
state().parallel.attn_tp_group.all_gather_into_tensor(out, x)
),
"get_dp_global_num_tokens": lambda: state().dp_rows,
"should_use_dp_reduce_scatterv": lambda: reduce_scatterv,
"can_use_dp_reduce_scatter": lambda: True,
"is_dp_attention_enabled": lambda: state().parallel.attn_dp_size > 1,
"apply_flashinfer_allreduce_fusion": lambda n: False,
"post_experts_output_is_complete": lambda **kw: False,
"post_experts_reduction_group": lambda: state().parallel.tp_group,
"get_lora": lambda: SimpleNamespace(enable_lora=False),
"get_exec": lambda: SimpleNamespace(
comm=SimpleNamespace(enable_quant_communications=False)
),
"get_attn_tp_context": lambda: SimpleNamespace(
input_scattered=False, set_attn_inputs=lambda x: None
),
"use_symmetric_memory": lambda group, disabled=False: nullcontext(),
"is_allocation_symmetric": lambda: False,
}
with ExitStack() as stack:
stack.enter_context(planning(lambda: state().parallel, a2a=a2a))
for name, value in replaced.items():
stack.enter_context(patch.object(comm, name, value))
yield
# Binary fractions, so partial sums add back to the value exactly.
WEIGHTS = {1: [1.0], 2: [0.25, 0.75], 4: [0.125, 0.25, 0.375, 0.25]}
def attention(x, state):
"""The attention output: 3 * x, as this rank's attention-TP partial."""
return 3 * x * WEIGHTS[state.parallel.attn_tp_size][state.parallel.attn_tp_rank]
def dense_mlp(x, state):
"""A dense MLP on the TP group: 5 * x, reduced unless a published flag asks
it to leave the sum, as RowParallelLinear does."""
partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
if state.flags.fuse_mlp_allreduce or state.flags.mlp_reduce_scatter:
return partial_sum
return state.parallel.tp_group.all_reduce(partial_sum)
def moe(x, state):
"""A MoE block not dispatched per DP shard: 5 * x, reduced over the MoE
output's group unless a published flag asks it to leave the sum, or the
attention-DP reduce_scatterv takes it, as the MoE blocks do."""
partial_sum = 5 * x * WEIGHTS[state.parallel.tp_size][state.parallel.tp_rank]
if (
state.flags.fuse_mlp_allreduce
or state.flags.mlp_reduce_scatter
or comm.should_use_dp_reduce_scatterv()
):
return partial_sum
return comm.post_experts_reduction_group().all_reduce(partial_sum)
def reference(x):
"""(hidden, residual) after two layers of Norm / attention / FFN."""
residual = x
for _ in range(2):
hidden = 2 * residual
residual = 3 * hidden + residual
ffn = 5 * (2 * residual)
residual = ffn + residual
# The last layer hands on its FFN output and the residual before it.
return ffn, residual - ffn
class TestTwoLayers(CustomTestCase):
def run_world(
self,
*,
attn_dp,
attn_tp,
rows,
padding,
reduce_scatterv,
leaves,
sparse=(False, False),
a2a=False,
):
tp = attn_dp * attn_tp
world = World(tp)
WORLD[0] = world
tp_group = Group(world, "tp", range(tp))
attn_tp_groups = [
tp_group
if attn_tp == tp
else Group(world, f"attn_tp{d}", range(d * attn_tp, (d + 1) * attn_tp))
for d in range(attn_dp)
]
max_len = padding == "max_len"
# Max-len padding pads every DP rank to one length that attention TP tiles.
local_len = -(-max(rows) // attn_tp) * attn_tp if max_len else None
offsets = (
[d * local_len for d in range(attn_dp)]
if max_len
else [sum(rows[:d]) for d in range(attn_dp)]
)
global_rows = attn_dp * local_len if max_len else sum(rows)
generator = torch.Generator().manual_seed(0)
embeddings = [
torch.randint(-4, 4, (n, HIDDEN), generator=generator).double()
for n in rows
]
forward_batch = SimpleNamespace(
forward_mode=SimpleNamespace(
is_context_parallel_extend=lambda: False,
is_decode_or_idle=lambda: True,
),
dp_padding_mode=SimpleNamespace(is_max_len=lambda: max_len),
global_dp_buffer_len=global_rows,
# Without attention DP the FFN exit counts this rank's tokens.
input_ids=torch.zeros(rows[0]),
)
states = []
for rank in range(tp):
d, t = divmod(rank, attn_tp)
states.append(
SimpleNamespace(
rank=rank,
parallel=parallel_of(
attn_dp=attn_dp,
attn_tp=attn_tp,
tp_rank=rank,
attn_tp_rank=t,
tp_group=tp_group,
attn_tp_group=attn_tp_groups[d],
),
flags=Flags(),
calls=[],
dp=d,
rows=rows[d],
offset=offsets[d],
local_rows=local_len or rows[d],
global_rows=global_rows,
dp_rows=[local_len] * attn_dp if max_len else list(rows),
)
)
def forward(rank):
s = states[rank]
layers = [
LayerCommunicator(
layer_scatter_modes=layer_facts(
i,
2,
sparse=sparse[i],
previous_sparse=i > 0 and sparse[i - 1],
),
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
allow_reduce_scatter=leaves,
allow_deferred_ffn_reduction=leaves,
)
for i in range(2)
]
hidden = torch.zeros(s.local_rows, HIDDEN).double()
hidden[: s.rows] = embeddings[s.dp]
residual = None
handed_on = []
for layer_index, layer in enumerate(layers):
hidden, residual = layer.prepare_attn(hidden, residual, forward_batch)
hidden = attention(hidden, s)
hidden, residual = layer.prepare_mlp(hidden, residual, forward_batch)
with layer.ffn_exit(forward_batch) as ffn_exit:
if not sparse[layer_index]:
hidden = dense_mlp(hidden, s)
elif a2a:
# On this rank's slice; the combine completes the sum.
hidden = 5 * hidden
else:
hidden = moe(hidden, s)
hidden, residual = ffn_exit.finish(hidden, residual)
handed_on.append(type(hidden))
hidden, residual = layers[-1].finish_layer_stack(
hidden, residual, forward_batch
)
# The last a2a layer folds the residual into its output.
if residual is not None:
residual = residual[: s.rows]
return hidden[: s.rows], residual, handed_on
with running(reduce_scatterv=reduce_scatterv, a2a=a2a):
results, errors = world.run(states, forward)
for rank, error in enumerate(errors):
if error is not None:
raise AssertionError(f"rank {rank} raised") from error
for rank, (hidden, residual, handed_on) in enumerate(results):
want_hidden, want_residual = reference(embeddings[states[rank].dp])
if residual is None:
want_hidden, want_residual = want_hidden + want_residual, None
torch.testing.assert_close(hidden, want_hidden, rtol=0, atol=0)
self.assertIs(residual is None, want_residual is None)
if residual is not None:
torch.testing.assert_close(residual, want_residual, rtol=0, atol=0)
# Every rank ran the same collectives.
self.assertEqual(len({tuple(s.calls) for s in states if s.dp == 0}), 1)
return results
def test_rows_come_back_to_each_rank(self):
for (attn_dp, attn_tp), rows, padding, leaves, sparse in itertools.product(
((2, 2), (4, 1), (2, 1), (1, 2), (1, 4)),
("ragged", "idle"),
("sum_len", "max_len"),
(False, True),
((False, False), (False, True), (True, True)),
):
token_rows = {
("ragged", 1): [3],
("idle", 1): [0],
("ragged", 2): [3, 1],
("idle", 2): [2, 0],
("ragged", 4): [2, 3, 1, 1],
("idle", 4): [2, 0, 3, 1],
}[rows, attn_dp]
for reduce_scatterv, a2a in itertools.product(
(False, True) if attn_tp == 1 and attn_dp > 1 else (False,),
(False, True) if any(sparse) else (False,),
):
if reduce_scatterv and a2a:
continue
with self.subTest(
attn_dp=attn_dp,
attn_tp=attn_tp,
rows=token_rows,
padding=padding,
leaves=leaves,
reduce_scatterv=reduce_scatterv,
sparse=sparse,
a2a=a2a,
):
results = self.run_world(
attn_dp=attn_dp,
attn_tp=attn_tp,
rows=token_rows,
padding=padding,
reduce_scatterv=reduce_scatterv,
leaves=leaves,
sparse=sparse,
a2a=a2a,
)
first_layer_hands_on = results[0][2][0]
# A producer that may leave its sum leaves it for the next
# layer's input; one that may not hands on a complete value.
# Without attention DP an empty batch completes its own sum.
# A MoE dispatched per DP shard hands on a complete value.
# Otherwise the reduce-scatter back to this rank's tokens
# can be left to the next layer; the all-reduce only
# without an a2a backend.
has_tokens = attn_dp > 1 or sum(token_rows) > 0
reduce_scatter_left = attn_dp > 1 and (
padding == "max_len" or reduce_scatterv
)
owes = (
leaves
and has_tokens
and not (sparse[0] and a2a)
and (reduce_scatter_left or not a2a)
)
self.assertIs(
first_layer_hands_on,
UnreducedOutput if owes else torch.Tensor,
)
if __name__ == "__main__":
unittest.main()