[Refactor] Run every batch of a layer from one BoundarySteps (#41420)

This commit is contained in:
Cheng Wan
2026-09-27 16:47:55 -07:00
committed by GitHub
parent 9f87d28db6
commit 380a4d0332
12 changed files with 419 additions and 238 deletions
+116 -141
View File
@@ -756,11 +756,17 @@ class LayerCommunicator:
self._context.force_layernorm_before_dp_gather = (
force_layernorm_before_dp_gather
)
# The steps the layer's ordinary batches run.
sides = self._declared_sides()
if sides is not None:
self._select_declared_boundaries(sides)
else:
self._select_boundaries_from_scatter_modes()
self._steps = (
_select_boundary_steps(
sides,
fusions=self._select_mlp_input_fusions(),
force_layernorm_before_gather=force_layernorm_before_dp_gather,
)
if sides is not None
else self._select_boundaries_from_scatter_modes()
)
# The steps a batch with input-scattered attention runs; None where it
# cannot run or the steps come from the scatter modes.
self._input_scattered_steps = (
@@ -846,46 +852,13 @@ class LayerCommunicator:
),
)
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
"""Choose the three boundary sides this layer owns from the
declarations: the attention input, the attention -> FFN steps, and the
FFN output's way on to the rows the layer hands on."""
self._communicate_simple_fn = _select_attention_input_move(
sides.input_rows, sides.attention
)
self._mlp_input, fused = _select_ffn_input(
sides.attention_output,
residual=sides.input_rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
fusions=self._select_mlp_input_fusions(),
)
self._mlp_input_may_return_new_residual = any(
f.may_return_new_residual for f in fused
)
self._ffn_output = sides.ffn_output
(
self._postprocess_scatters_to_local_tokens,
postprocess,
) = _select_ffn_output_move(
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
)
if postprocess is not None:
self._communicate_summable_tensor_pair_fn = postprocess
self._ffn_sum_is_movable = sides.ffn_output.group is not None
def _select_boundaries_from_scatter_modes(self) -> None:
self._post_init_communicate()
# Under attention DP, the base postprocess scatters the FFN output back
# to this rank's tokens, which the next layer's input can run instead;
# the MHC and DSA-CP postprocess do more, so theirs stays here.
self._postprocess_scatters_to_local_tokens = (
self._communicate_summable_tensor_pair_fn
is CommunicateSummableTensorPairFn._scatter_hidden_states
)
def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps":
"""The layer's steps chosen from its scatter modes, for the layers the
declarations do not cover yet."""
attention_input, postprocess = self._post_init_communicate()
ffn_input, fused = self._select_mlp_input()
modes = self.layer_scatter_modes
self._ffn_output = StageOutput(
ffn_output = StageOutput(
self._context.layouts[modes.mlp_mode],
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
@@ -897,30 +870,41 @@ class LayerCommunicator:
self.allow_reduce_scatter or modes.is_layer_sparse
),
)
# Whether the FFN sum is one the next layer's input can take: not when
# the way back is the MoE-CP scatter, nor when a SCATTERED FFN computes
# whole tokens with nothing left to sum.
self._ffn_sum_is_movable = modes.mlp_mode not in (
ScatterMode.MOE_FULL,
ScatterMode.SCATTERED,
)
self._mlp_input, self._mlp_input_may_return_new_residual = (
self._select_mlp_input()
return BoundarySteps(
attention_input=attention_input,
ffn_input=ffn_input,
ffn_output=ffn_output,
# Under attention DP the base postprocess scatters the FFN output
# back to this rank's tokens, which the next layer's input can run
# instead; the MHC and DSA-CP postprocess do more.
ffn_output_move=(
None
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
else postprocess
),
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
# FFN computes whole tokens with nothing left to sum.
ffn_sum_is_movable=modes.mlp_mode
not in (ScatterMode.MOE_FULL, ScatterMode.SCATTERED),
fused=fused,
layer_input=_complete_scattered_input,
)
def _post_init_communicate(self):
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
input_mode=self.layer_scatter_modes.layer_input_mode,
output_mode=self.layer_scatter_modes.attn_mode,
context=self._context,
)
self._communicate_summable_tensor_pair_fn = (
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
"""The attention input move and the postprocess the scatter modes
choose."""
return (
CommunicateSimpleFn.get_fn(
input_mode=self.layer_scatter_modes.layer_input_mode,
output_mode=self.layer_scatter_modes.attn_mode,
context=self._context,
),
CommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
output_mode=self.layer_scatter_modes.layer_output_mode,
context=self._context,
)
),
)
def prepare_attn_and_capture_last_layer_outputs(
@@ -940,7 +924,7 @@ class LayerCommunicator:
post_residual_addition=post_residual_addition,
)
if captured_last_layer_outputs is not None:
gathered_last_layer_output = self._communicate_simple_fn(
gathered_last_layer_output = self._batch_steps().attention_input(
hidden_states=residual,
forward_batch=forward_batch,
context=self._context,
@@ -966,7 +950,7 @@ class LayerCommunicator:
the batch is not input-scattered.
"""
return (
self._mlp_input_may_return_new_residual
any(f.may_return_new_residual for f in self._batch_steps().fused)
and not get_attn_tp_context().input_scattered
and apply_flashinfer_allreduce_fusion(residual.shape[0])
)
@@ -1028,16 +1012,11 @@ class LayerCommunicator:
)
if get_forward().sp_active:
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
steps = self._batch_steps()
if steps is not None and steps.layer_input is not None:
hidden_states, residual = steps.layer_input(
layer_input = self._batch_steps().layer_input
if layer_input is not None:
hidden_states, residual = layer_input(
hidden_states, residual, self._context
)
elif get_attn_tp_context().input_scattered:
hidden_states, residual = self._tp_reduce_scatter(
hidden_states,
residual,
)
if hidden_states.shape[0] == 0:
residual = hidden_states
elif owed is not None:
@@ -1059,9 +1038,9 @@ class LayerCommunicator:
def _in_sp_region(self) -> bool:
return self._sp_steps is not None and get_forward().sp_active
def _batch_steps(self) -> Optional["BoundarySteps"]:
"""The steps this batch runs in place of the layer's ordinary ones: an
active LayerNorm SP region's, or input-scattered attention's."""
def _batch_steps(self) -> "BoundarySteps":
"""The steps this batch runs: an active LayerNorm SP region's,
input-scattered attention's, or the layer's ordinary ones."""
if self._in_sp_region():
return self._sp_steps
if (
@@ -1069,13 +1048,11 @@ class LayerCommunicator:
and get_attn_tp_context().input_scattered
):
return self._input_scattered_steps
return None
return self._steps
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
steps = self._batch_steps()
move = self._communicate_simple_fn if steps is None else steps.attention_input
hidden_states = move(
hidden_states = self._batch_steps().attention_input(
hidden_states=hidden_states,
forward_batch=forward_batch,
context=self._context,
@@ -1145,30 +1122,23 @@ class LayerCommunicator:
hidden_states, residual, use_attn_tp_group=False
)
def _tp_reduce_scatter(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
return tp_reduce_scatter(hidden_states, residual, self._context)
def _select_mlp_input(self) -> Tuple[Callable, bool]:
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
whether they include the aiter / flashinfer fused entry. The MHC and
DSA-CP communicators pick their own implementations."""
the fused kernels they try first. The MHC and DSA-CP communicators pick
their own implementations."""
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
residual_input_mode = self.layer_scatter_modes.layer_input_mode
if kind is MlpInputKind.NORM:
return _mlp_input_norm, False
return _mlp_input_norm, ()
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
return _mlp_input_attn_tp_all_reduce, False
return _mlp_input_attn_tp_all_reduce, ()
if kind is MlpInputKind.SCATTER:
return (
partial(
_mlp_input_scatter,
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
),
False,
(),
)
# Neither fused kernel runs under attention DP.
fusions = (
@@ -1182,7 +1152,7 @@ class LayerCommunicator:
)
if kind is MlpInputKind.GATHER_MOE_CP:
steps = partial(_mlp_input_gather_moe_cp, gather=steps)
return steps, any(f.may_return_new_residual for f in fusions)
return steps, fusions
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
"""The fused kernels that can take the attention -> FFN steps, in the
@@ -1222,20 +1192,10 @@ class LayerCommunicator:
cache=None,
):
self.publish_mlp_lora_layout()
if self._in_sp_region():
return self._sp_steps.ffn_input(
hidden_states,
residual,
forward_batch,
self.post_attention_layernorm,
self._context,
)
if cache is not None:
self._context.cache = cache
steps = self._batch_steps()
mlp_input = self._mlp_input if steps is None else steps.ffn_input
return mlp_input(
return self._batch_steps().ffn_input(
hidden_states,
residual,
forward_batch,
@@ -1257,17 +1217,10 @@ class LayerCommunicator:
forward_batch: ForwardBatch,
):
steps = self._batch_steps()
if steps is not None:
return steps.ffn_output_move(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
context=self._context,
)
if self._postprocess_scatters_to_local_tokens:
if steps.returns_over_dp:
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
return _to_local_tokens(step, forward_batch, hidden_states), residual
return self._communicate_summable_tensor_pair_fn(
return steps.ffn_output_move(
hidden_states=hidden_states,
residual=residual,
forward_batch=forward_batch,
@@ -1278,9 +1231,8 @@ class LayerCommunicator:
def _local_token_move_can_go_to_next_layer(self) -> bool:
"""Whether the next layer's input can run this layer's move of its FFN
output back to this rank's tokens: the base postprocess scatter, outside
an active LayerNorm SP region."""
return self._postprocess_scatters_to_local_tokens and not self._in_sp_region()
output back to this rank's tokens: the base postprocess scatter."""
return self._batch_steps().returns_over_dp
def _postprocess_dp_step(
self, forward_batch: ForwardBatch
@@ -1288,12 +1240,13 @@ class LayerCommunicator:
"""The reduce-scatter that brings this layer's FFN output back to this
rank's tokens under attention DP; None when the base postprocess would
only scatter, or does not move tokens."""
if not self._postprocess_scatters_to_local_tokens:
steps = self._batch_steps()
if not steps.returns_over_dp:
return None
return _reduce_and_redistribute_output_step(
forward_batch,
leaves_for_reduce_scatter=self._ffn_output.leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=self._ffn_output.leaves_for_reduce_scatterv,
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
)
def _ffn_leaves_sum_to_reduce_scatter(
@@ -1302,10 +1255,7 @@ class LayerCommunicator:
"""Whether the FFN leaves its sum out because a reduce-scatter completes
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
one."""
steps = self._batch_steps()
if steps is not None:
return steps.ffn_output.leaves_for_reduce_scatter
if not self._ffn_output.leaves_for_reduce_scatter:
if not self._batch_steps().ffn_output.leaves_for_reduce_scatter:
return False
if dp_step is not None:
return True
@@ -1319,7 +1269,7 @@ class LayerCommunicator:
def ffn_reduction_group(self) -> GroupCoordinator:
"""The group this layer's FFN output owes its sum over: the MoE output's
group on a sparse layer, the TP group a dense MLP reduces over."""
return _sum_group(self._ffn_output.group)
return _sum_group(self._batch_steps().ffn_output.group)
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion":
"""Decide once, before the FFN runs, what it skips and what completes its
@@ -1334,8 +1284,7 @@ class LayerCommunicator:
dp_step=dp_step,
)
steps = self._batch_steps()
ffn_output = self._ffn_output if steps is None else steps.ffn_output
if not ffn_output.leaves_for_next_layer:
if not steps.ffn_output.leaves_for_next_layer:
return FfnCompletion(
defer_moe_finalize=False,
fuse_mlp_allreduce=False,
@@ -1349,7 +1298,7 @@ class LayerCommunicator:
)
if fuse_mlp_allreduce:
group = self.ffn_reduction_group()
if self._postprocess_scatters_to_local_tokens:
if steps.returns_over_dp:
# Under attention DP the next layer also brings the sum back to
# this rank's tokens.
wrap = partial(
@@ -1435,7 +1384,7 @@ class LayerCommunicator:
def _ffn_sum_can_move_to_next_layer(self) -> bool:
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
if is_enable_moe_cp_allgather() or not self._ffn_sum_is_movable:
if is_enable_moe_cp_allgather() or not self._batch_steps().ffn_sum_is_movable:
return False
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
@@ -1914,29 +1863,40 @@ def _token_axis_sizes() -> Dict[TokenAxis, int]:
class BoundarySteps(msgspec.Struct, frozen=True):
"""The steps one set of declarations chooses for a layer's boundaries:
into the attention, from the attention output to the FFN input, and the
FFN output on to the rows the layer hands on."""
"""The steps a batch runs at a layer's boundaries: into the attention,
from the attention output to the FFN input, and the FFN output on to the
rows the layer hands on."""
attention_input: Callable
ffn_input: Callable
ffn_output_move: Callable
# What the FFN exit reads in place of the layer's own FFN output declaration.
# What the FFN exit reads: the FFN output's group and what it may leave.
ffn_output: StageOutput
# The postprocess that moves the FFN output on; None when it goes back over
# attention DP, whose step the FFN exit and postprocess choose per batch.
ffn_output_move: Optional[Callable]
# Whether the next layer's input can take the FFN's sum.
ffn_sum_is_movable: bool
# The fused kernels ffn_input tries first.
fused: Tuple["FusedMlpInput", ...] = ()
# Completes what the layer's input owes before the input norm; None when it
# owes nothing.
layer_input: Optional[Callable] = None
@property
def returns_over_dp(self) -> bool:
return self.ffn_output_move is None
def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
"""The steps for a batch's own declarations: nothing moves back over
attention DP and no fused kernel runs, so the steps do not wait on a
per-batch decision."""
def _select_boundary_steps(
sides: DecoderLayerSides,
*,
fusions: Tuple["FusedMlpInput", ...] = (),
force_layernorm_before_gather: bool = False,
) -> BoundarySteps:
"""The steps a set of declarations chooses."""
returns_over_dp, ffn_output_move = _select_ffn_output_move(
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
)
if returns_over_dp or sides.ffn_output.leaves_for_next_layer:
raise NotImplementedError(f"{sides=}")
rows = sides.input_rows
layer_input = None
if sides.input_owes is not None:
@@ -1950,24 +1910,39 @@ def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
raise NotImplementedError(f"{sides=}")
layer_input = tp_reduce_scatter
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
ffn_input, _ = _select_ffn_input(
ffn_input, fused = _select_ffn_input(
sides.attention_output,
residual=rows,
residual_to=sides.ffn_residual_rows,
need=sides.ffn,
force_layernorm_before_gather=False,
fusions=(),
force_layernorm_before_gather=force_layernorm_before_gather,
fusions=fusions,
residual_joins_sum=sides.residual_joins_attention_sum,
)
return BoundarySteps(
attention_input=_select_attention_input_move(rows, sides.attention),
ffn_input=ffn_input,
ffn_output_move=ffn_output_move,
ffn_output=sides.ffn_output,
ffn_output_move=None if returns_over_dp else ffn_output_move,
ffn_sum_is_movable=sides.ffn_output.group is not None,
fused=fused,
layer_input=layer_input,
)
def _complete_scattered_input(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
context: "CommunicateContext",
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""The scatter-mode path's input completion: with input-scattered
attention the layer's input is a TP partial that a reduce-scatter completes
onto this rank's slice."""
if get_attn_tp_context().input_scattered:
return tp_reduce_scatter(hidden_states, residual, context)
return hidden_states, residual
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
"""How the rows a layer takes become its attention's input: as they are,
or gathered over attention TP from each rank's slice, unless the attention
+13 -11
View File
@@ -93,16 +93,18 @@ class DSACPLayerCommunicator(LayerCommunicator):
assert self._context.attn_dp_size == 1, (
f"dp_size should be 1 when moe_runner_backend is none"
)
self._communicate_simple_fn = DSACPCommunicateSimpleFn.get_fn(
input_mode=ScatterMode.SCATTERED,
output_mode=ScatterMode.SCATTERED,
context=self._context,
)
self._communicate_summable_tensor_pair_fn = DSACPCommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
residual_input_mode=ScatterMode.SCATTERED,
output_mode=ScatterMode.SCATTERED,
context=self._context,
return (
DSACPCommunicateSimpleFn.get_fn(
input_mode=ScatterMode.SCATTERED,
output_mode=ScatterMode.SCATTERED,
context=self._context,
),
DSACPCommunicateSummableTensorPairFn.get_fn(
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
residual_input_mode=ScatterMode.SCATTERED,
output_mode=ScatterMode.SCATTERED,
context=self._context,
),
)
def _select_mlp_input(self):
@@ -113,7 +115,7 @@ class DSACPLayerCommunicator(LayerCommunicator):
residual_output_mode=ScatterMode.SCATTERED,
context=self._context,
)
return fn, False
return fn, ()
class DSACPCommunicateSimpleFn(CommunicateSimpleFn):
+4 -2
View File
@@ -441,6 +441,8 @@ class MHCLayerCommunicator(LayerCommunicator):
context=self._context,
)
)
# MHC's own prepare_attn and postprocess run these.
return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn
def prepare_attn(
self,
@@ -494,14 +496,14 @@ class MHCLayerCommunicator(LayerCommunicator):
residual_input_mode = self.layer_scatter_modes.layer_input_mode
fns = MHCCommunicateWithAllReduceAndLayerNormFn
if kind is MlpInputKind.NORM:
return partial(fns._simple, mhc=self.mhc), False
return partial(fns._simple, mhc=self.mhc), ()
if kind is MlpInputKind.GATHER:
fn = fns._gather_hidden_states_and_residual
elif kind is MlpInputKind.SCATTER:
fn = fns._scatter_hidden_states_and_residual
else:
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), False
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), ()
def postprocess_layer(self, hidden_states, residual, forward_batch):
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
@@ -45,6 +45,20 @@ def make_group(scale=3):
return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale))
def ordinary_steps(ffn_output, *, returns_over_dp=False):
"""A layer's ordinary steps, carrying the FFN output declaration the exit
reads and whether the output goes back over attention DP."""
return comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output=ffn_output,
ffn_output_move=(
None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial
),
ffn_sum_is_movable=True,
)
def make_communicator(
*,
fuse,
@@ -58,8 +72,13 @@ def make_communicator(
"""A communicator whose decisions and postprocess are stubbed, built
without the process-wide parallel state."""
communicator = cls.__new__(cls)
communicator._ffn_output = StageOutput(
Layout(frozenset()), group=SumGroup.TP, leaves_for_next_layer=allow_deferred
communicator._steps = ordinary_steps(
StageOutput(
Layout(frozenset()),
group=SumGroup.TP,
leaves_for_next_layer=allow_deferred,
),
returns_over_dp=scatters_to_local_tokens,
)
if cls is LayerCommunicator:
communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse)
@@ -72,7 +91,6 @@ def make_communicator(
communicator.is_last_layer = False
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
communicator.postprocess_layer = MagicMock(
@@ -306,13 +324,15 @@ class TestSelectFfnCompletion(CustomTestCase):
communicator.is_last_layer = is_last_layer
communicator._sp_steps = sp_region_steps() if sp_region else None
communicator._input_scattered_steps = None
communicator._postprocess_scatters_to_local_tokens = scatters
communicator._ffn_output = StageOutput(
Layout(frozenset()),
group=SumGroup.MOE_OUTPUT,
leaves_for_next_layer=True,
leaves_for_reduce_scatter=True,
leaves_for_reduce_scatterv=True,
communicator._steps = ordinary_steps(
StageOutput(
Layout(frozenset()),
group=SumGroup.MOE_OUTPUT,
leaves_for_next_layer=True,
leaves_for_reduce_scatter=True,
leaves_for_reduce_scatterv=True,
),
returns_over_dp=scatters,
)
communicator._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: fuse
communicator._ffn_leaves_sum_to_reduce_scatter = lambda forward_batch, dp_step: (
@@ -223,7 +223,7 @@ class TestMlpInputOrder(CustomTestCase):
def test_a_gather_runs_the_order_chosen_at_construction(self):
for mlp_mode in (FULL, MOE_FULL):
with self.subTest(mlp_mode=mlp_mode):
c, (steps, may_fuse) = communicator(
c, (steps, fused) = communicator(
(TP_ATTN_FULL, TP_ATTN_FULL, mlp_mode, TP_ATTN_FULL),
make_context(tp=2),
Fusable(),
@@ -239,7 +239,7 @@ class TestMlpInputOrder(CustomTestCase):
gathers_residual=False,
fusions=(entry,),
)
self.assertTrue(may_fuse)
self.assertEqual([f.run for f in fused], [entry])
def test_no_fused_entry_without_a_fusable_norm_or_under_attention_dp(self):
for context, norm in (
@@ -247,10 +247,10 @@ class TestMlpInputOrder(CustomTestCase):
(make_context(dp=2, tp=2), Fusable()),
):
with self.subTest(dp=context.attn_dp_size):
_, (steps, may_fuse) = communicator(GATHER_LAYOUT, context, norm)
_, (steps, fused) = communicator(GATHER_LAYOUT, context, norm)
order = steps.keywords["order"]
self.assertEqual(order.keywords.get("fusions", ()), ())
self.assertFalse(may_fuse)
self.assertEqual(fused, ())
def test_other_kinds_take_their_steps(self):
tp = make_context(tp=2)
@@ -267,9 +267,9 @@ class TestMlpInputOrder(CustomTestCase):
),
):
with self.subTest(expected=expected.__name__):
_, (steps, may_fuse) = communicator(layout, context, Fusable())
_, (steps, fused) = communicator(layout, context, Fusable())
self.assertIs(steps, expected)
self.assertFalse(may_fuse)
self.assertEqual(fused, ())
_, (steps, _) = communicator(
(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable()
)
@@ -297,11 +297,11 @@ class TestMlpInputOrder(CustomTestCase):
c.layer_scatter_modes = modes(*layout)
c._context = make_context(tp=2)
c.post_attention_layernorm = Fusable()
steps, may_fuse = c._select_mlp_input()
steps, fused = c._select_mlp_input()
self.assert_order(
steps, func, residual_input_mode=TP_ATTN_FULL, mhc=mhc
)
self.assertFalse(may_fuse)
self.assertEqual(fused, ())
if __name__ == "__main__":
@@ -274,7 +274,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
declarations, or none do."""
def declared(self, communicator):
return getattr(communicator._mlp_input, "func", None) in (
return getattr(communicator._steps.ffn_input, "func", None) in (
comm._mlp_input_dp_partial,
comm._mlp_input_dp_replicate,
comm._mlp_input_without_dp,
@@ -288,10 +288,10 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
communicator = build(layer_facts(layer_id, 3), parallel)
self.assertTrue(self.declared(communicator))
self.assertIs(
communicator._communicate_simple_fn,
communicator._steps.attention_input,
comm.CommunicateSimpleFn._trivial,
)
self.assertTrue(communicator._postprocess_scatters_to_local_tokens)
self.assertTrue(communicator._steps.returns_over_dp)
def test_plain_tp(self):
for norm, fuses in ((Norm, False), (FusableNorm, True)):
@@ -302,18 +302,23 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
input_layernorm=norm(),
post_attention_layernorm=norm(),
)
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
self.assertIs(
communicator._steps.ffn_input.func, comm._mlp_input_without_dp
)
entry = (
communicator._mlp_input_reduce_output_and_update_and_read_residual
)
self.assertEqual(
communicator._mlp_input.keywords["fusions"],
communicator._steps.ffn_input.keywords["fusions"],
(entry,) if fuses else (),
)
self.assertIs(communicator._mlp_input_may_return_new_residual, fuses)
self.assertFalse(communicator._postprocess_scatters_to_local_tokens)
self.assertIs(
communicator._communicate_summable_tensor_pair_fn,
any(f.may_return_new_residual for f in communicator._steps.fused),
fuses,
)
self.assertFalse(communicator._steps.returns_over_dp)
self.assertIs(
communicator._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._trivial,
)
# One rank: the attention output is complete, only the norm is left.
@@ -321,7 +326,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
single = build(layer_facts(1, 3), one_rank)
with planning(one_rank):
self.assertIsNotNone(single._declared_sides())
self.assertIs(single._mlp_input, comm._mlp_input_norm)
self.assertIs(single._steps.ffn_input, comm._mlp_input_norm)
def test_the_order_follows_the_attention_output(self):
for attn_tp, force, order in (
@@ -335,7 +340,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
parallel_of(attn_dp=2, attn_tp=attn_tp),
force_layernorm_before_dp_gather=force,
)
self.assertIs(communicator._mlp_input.func, order)
self.assertIs(communicator._steps.ffn_input.func, order)
def test_a_dense_first_layer_before_sparse_ones(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
@@ -361,30 +366,32 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
]
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
if not a2a:
moe = layers[1]._ffn_output
moe = layers[1]._steps.ffn_output
self.assertIs(moe.group, SumGroup.MOE_OUTPUT)
self.assertTrue(moe.leaves_for_reduce_scatterv)
continue
# The first a2a layer slices the residual it takes from a
# dense layer; the next one takes it already sliced.
self.assertEqual(
[layers[i]._mlp_input.func for i in (1, 2)],
[layers[i]._steps.ffn_input.func for i in (1, 2)],
[comm._mlp_input_scatter] * 2,
)
self.assertEqual(
[
layers[i]._mlp_input.keywords["scatters_residual"]
layers[i]._steps.ffn_input.keywords["scatters_residual"]
for i in (1, 2)
],
[True, False],
)
self.assertIsNone(layers[1]._ffn_output.group)
self.assertIsNone(layers[1]._steps.ffn_output.group)
self.assertIs(
layers[2]._communicate_simple_fn,
layers[2]._steps.attention_input,
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
)
self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial)
self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"])
self.assertIs(
layers[3]._steps.ffn_input.func, comm._mlp_input_dp_partial
)
self.assertTrue(layers[3]._steps.ffn_input.keywords["gathers_residual"])
def test_the_last_a2a_layer_folds_the_residual_back(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
@@ -396,7 +403,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
a2a=True,
)
self.assertIs(
last._communicate_summable_tensor_pair_fn,
last._steps.ffn_output_move,
comm.CommunicateSummableTensorPairFn._gather,
)
@@ -407,7 +414,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
)
with patch.object(comm, "_use_ag_after_qlora", True):
layer = build(modes, parallel, a2a=True)
self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial)
self.assertIs(layer._steps.attention_input, comm.CommunicateSimpleFn._trivial)
def test_layers_that_keep_the_scatter_modes(self):
dp = parallel_of(attn_dp=2, attn_tp=2)
@@ -611,7 +618,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
input_layernorm=Norm(),
post_attention_layernorm=Norm(),
)
self.assertIs(communicator._mlp_input_may_return_new_residual, expected)
self.assertIs(
any(f.may_return_new_residual for f in communicator._steps.fused),
expected,
)
class TestTheSequenceParallelRegion(CustomTestCase):
@@ -624,8 +634,9 @@ class TestTheSequenceParallelRegion(CustomTestCase):
SP_STEPS = comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output,
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
ffn_sum_is_movable=False,
)
def test_the_region_declarations_choose_local_steps(self):
@@ -666,7 +677,31 @@ class TestTheSequenceParallelRegion(CustomTestCase):
post_attention_layernorm=Norm(),
)
self.assertEqual(communicator._sp_steps, self.SP_STEPS)
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
self.assertIs(
communicator._steps.ffn_input.func, comm._mlp_input_without_dp
)
def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self):
# Inside the region the FFN output is complete: no sum to move.
parallel = parallel_of(attn_dp=1, attn_tp=2)
communicator = build(layer_facts(1, 3), parallel, sp=True)
for active in (False, True):
with (
self.subTest(sp_active=active),
planning(parallel, sp=True),
patch.object(
comm, "get_forward", lambda: SimpleNamespace(sp_active=active)
),
patch.object(
comm,
"get_attn_tp_context",
lambda: SimpleNamespace(input_scattered=False),
),
patch.object(comm, "is_dp_attention_enabled", lambda: False),
):
self.assertIs(
communicator._ffn_sum_can_move_to_next_layer(), not active
)
def test_without_sp_there_is_no_region(self):
with planning(parallel_of(attn_dp=1, attn_tp=2)):
@@ -677,15 +712,19 @@ class TestTheSequenceParallelRegion(CustomTestCase):
)
self.assertIsNone(communicator._sp_steps)
def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self):
# A sum the FFN exit may hand to the next layer, or a move back over
# attention DP, waits on a per-batch decision.
for sizes, leaves_next in (
(self.SIZES, True),
({**self.SIZES, TokenAxis.ATTN_DP: 2}, False),
):
with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError):
comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next))
def test_the_same_selector_chooses_a_layer_s_ordinary_steps(self):
# Under attention DP the FFN exit chooses the move back per batch.
kept = comm._select_boundary_steps(sides_of(self.SIZES, leaves_next=True))
self.assertTrue(kept.ffn_output.leaves_for_next_layer)
self.assertTrue(kept.ffn_sum_is_movable)
self.assertIs(
kept.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial
)
returned = comm._select_boundary_steps(
sides_of({**self.SIZES, TokenAxis.ATTN_DP: 2})
)
self.assertTrue(returned.returns_over_dp)
self.assertIsNone(returned.ffn_output_move)
class TestInputScatteredAttention(CustomTestCase):
@@ -780,9 +819,40 @@ class TestInputScatteredAttention(CustomTestCase):
):
self.assertIs(
communicator._batch_steps(),
communicator._input_scattered_steps if scattered else None,
communicator._input_scattered_steps
if scattered
else communicator._steps,
)
def test_prepare_attn_completes_the_scattered_input(self):
scattered = []
group = SimpleNamespace(
name="tp",
reduce_scatter_tensor=lambda out, x: (
scattered.append(x.shape[0]) or out.copy_(x[: out.shape[0]] * 2)
),
)
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True, tp_group=group
)
communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
with (
patch.object(comm, "get_parallel", lambda: parallel),
patch.object(
comm,
"get_attn_tp_context",
lambda: SimpleNamespace(input_scattered=True),
),
patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)),
):
hidden, residual = communicator.prepare_attn(
torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None
)
self.assertEqual(scattered, [4])
# Norm: (2 * (h + r), h + r) on the slice, with h the completed sum.
torch.testing.assert_close(residual, torch.full((2, HIDDEN), 5.0))
torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0))
def test_the_last_layer_completes_its_own_sum(self):
parallel = parallel_of(
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
@@ -800,6 +870,64 @@ class TestInputScatteredAttention(CustomTestCase):
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
class TestOneRepresentation(CustomTestCase):
"""Every layer runs from one BoundarySteps per batch, whichever entry chose
its steps: none keeps the steps as separate attributes."""
SEPARATE = (
"_communicate_simple_fn",
"_mlp_input",
"_communicate_summable_tensor_pair_fn",
"_ffn_output",
"_postprocess_scatters_to_local_tokens",
"_ffn_sum_is_movable",
"_mlp_input_may_return_new_residual",
)
def test_both_entries_hold_steps_only(self):
parallel = parallel_of(attn_dp=2, attn_tp=2)
direct = LayerScatterModes(
layer_input_mode=ScatterMode.TP_ATTN_FULL,
attn_mode=ScatterMode.TP_ATTN_FULL,
mlp_mode=ScatterMode.FULL,
middle_residual_mode=ScatterMode.TP_ATTN_FULL,
layer_output_mode=ScatterMode.TP_ATTN_FULL,
)
for name, communicator in (
("declarations", build(layer_facts(1, 3), parallel)),
("scatter modes", build(direct, parallel)),
):
with self.subTest(name):
self.assertIsInstance(communicator._steps, comm.BoundarySteps)
for attribute in self.SEPARATE:
self.assertFalse(hasattr(communicator, attribute), attribute)
self.assertIs(communicator._batch_steps(), communicator._steps)
# Under attention DP both bring the FFN output back to this
# rank's tokens with the step the FFN exit chooses.
self.assertTrue(communicator._steps.returns_over_dp)
def test_the_scatter_mode_steps_complete_a_scattered_input(self):
# The scatter-mode path's input completion reads the batch fact itself.
steps = build(
LayerScatterModes(
layer_input_mode=ScatterMode.TP_ATTN_FULL,
attn_mode=ScatterMode.TP_ATTN_FULL,
mlp_mode=ScatterMode.FULL,
middle_residual_mode=ScatterMode.TP_ATTN_FULL,
layer_output_mode=ScatterMode.TP_ATTN_FULL,
),
parallel_of(attn_dp=1, attn_tp=2),
)._steps
self.assertIs(steps.layer_input, comm._complete_scattered_input)
hidden, residual = torch.ones(2, HIDDEN), torch.ones(2, HIDDEN)
with patch.object(
comm, "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False)
):
self.assertEqual(
steps.layer_input(hidden, residual, None), (hidden, residual)
)
# ---------------------------------------------------------------------------
# Two consecutive layers, rank by rank.
@@ -40,15 +40,32 @@ def sp_region_steps():
)
def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True):
return comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output=ffn_output or StageOutput(Layout(frozenset()), group=SumGroup.TP),
ffn_output_move=(
None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial
),
ffn_sum_is_movable=ffn_sum_is_movable,
)
def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator._speculative_algo = None
communicator.layer_scatter_modes = types.SimpleNamespace(mlp_mode=mlp_mode)
# Fixed at construction by the scatter-mode path from the MLP's mode.
communicator._ffn_sum_is_movable = mlp_mode not in (
ScatterMode.MOE_FULL,
ScatterMode.SCATTERED,
communicator._steps = _steps(
ffn_sum_is_movable=mlp_mode
not in (
ScatterMode.MOE_FULL,
ScatterMode.SCATTERED,
)
)
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator.is_last_layer = False
communicator._context = types.SimpleNamespace(tp_size=4)
return communicator
@@ -347,14 +364,16 @@ class TestDeferFfnReduction(CustomTestCase):
):
communicator = _fake_communicator()
communicator.is_last_layer = is_last_layer
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
communicator._sp_steps = sp_region_steps() if sp_active else None
communicator._input_scattered_steps = None
communicator._ffn_output = StageOutput(
Layout(frozenset()),
group=SumGroup.MOE_OUTPUT,
leaves_for_reduce_scatter=True,
leaves_for_reduce_scatterv=True,
communicator._steps = _steps(
ffn_output=StageOutput(
Layout(frozenset()),
group=SumGroup.MOE_OUTPUT,
leaves_for_reduce_scatter=True,
leaves_for_reduce_scatterv=True,
),
returns_over_dp=scatters_to_local_tokens,
)
forward_batch = types.SimpleNamespace(
input_ids=types.SimpleNamespace(shape=(batch_size,)),
@@ -15,6 +15,9 @@ from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp
from sglang.srt.layers import communicator as comm
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.boundary_layout import (
Layout,
StageOutput,
SumGroup,
TokenAxis,
sequence_parallel_layer_sides,
)
@@ -167,18 +170,28 @@ class TestSpRegionSteps(CustomTestCase):
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
c._sp_steps = sp_region_steps()
c._input_scattered_steps = None
c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer)
c.layer_scatter_modes = SimpleNamespace(
is_first_layer=first_layer, is_layer_sparse=False
)
c.allow_reduce_scatter = False
c._attn_input_fusions = ()
c.input_layernorm = _Norm()
c.post_attention_layernorm = _Norm()
c.qkv_latent_func = None
c._context = SimpleNamespace(cache=None)
c._communicate_simple_fn = MagicMock(side_effect=AssertionError("moved"))
c._communicate_summable_tensor_pair_fn = MagicMock(
side_effect=AssertionError("postprocess ran")
)
c._steps = self.ordinary_steps(MagicMock(side_effect=AssertionError("moved")))
return c
def ordinary_steps(self, attention_input):
"""The layer's steps outside the region, which must not run inside it."""
return comm.BoundarySteps(
attention_input=attention_input,
ffn_input=MagicMock(side_effect=AssertionError("ordinary FFN input ran")),
ffn_output=StageOutput(Layout(frozenset()), group=SumGroup.TP),
ffn_output_move=MagicMock(side_effect=AssertionError("postprocess ran")),
ffn_sum_is_movable=True,
)
def run_prepare_attn(self, communicator, mode, hidden, residual):
batch = SimpleNamespace(forward_mode=mode)
with (
@@ -208,20 +221,19 @@ class TestSpRegionSteps(CustomTestCase):
def test_decode_leaves_the_region_closed(self):
communicator = self.communicator(first_layer=True)
communicator._communicate_simple_fn = MagicMock(
side_effect=lambda **k: k["hidden_states"]
)
move = MagicMock(side_effect=lambda **k: k["hidden_states"])
communicator._steps = self.ordinary_steps(move)
(h, _), active, scatter = self.run_prepare_attn(
communicator, ForwardMode.DECODE, torch.ones(2, 4), None
)
self.assertFalse(active)
scatter.assert_not_called()
communicator._communicate_simple_fn.assert_called_once()
move.assert_called_once()
def test_a_later_layer_does_not_reopen_the_region(self):
communicator = self.communicator(first_layer=False)
communicator._communicate_simple_fn = MagicMock(
side_effect=lambda **k: k["hidden_states"]
communicator._steps = self.ordinary_steps(
MagicMock(side_effect=lambda **k: k["hidden_states"])
)
_, active, scatter = self.run_prepare_attn(
communicator, ForwardMode.EXTEND, torch.ones(2, 4), torch.ones(2, 4)
@@ -29,6 +29,18 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def steps(*, ffn_output_move):
"""A layer's steps with the given FFN output move; None sends the output
back over attention DP."""
return comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output=comm.StageOutput(comm.Layout(frozenset())),
ffn_output_move=ffn_output_move,
ffn_sum_is_movable=False,
)
@contextlib.contextmanager
def reduce_scatterv_applies():
with (
@@ -143,11 +155,10 @@ class TestPostprocessReduceScatterv(CustomTestCase):
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._context = None
communicator._postprocess_scatters_to_local_tokens = False
communicator.allow_reduce_scatter = False
communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True)
communicator._communicate_summable_tensor_pair_fn = lambda **kwargs: (
seen.update(kwargs) or (None, None)
communicator._steps = steps(
ffn_output_move=lambda **kwargs: seen.update(kwargs) or (None, None)
)
communicator.postprocess_layer(None, None, None)
self.assertIs(seen["is_layer_sparse"], True)
@@ -160,7 +171,7 @@ class TestPostprocessReduceScatterv(CustomTestCase):
communicator = LayerCommunicator.__new__(LayerCommunicator)
communicator._sp_steps = None
communicator._input_scattered_steps = None
communicator._postprocess_scatters_to_local_tokens = True
communicator._steps = steps(ffn_output_move=None)
communicator._postprocess_dp_step = lambda forward_batch: step
with patch.object(
comm, "_to_local_tokens", side_effect=lambda s, fb, h: (s, h)
@@ -48,7 +48,14 @@ def communicator(norm):
c._context = None
c.enable_fused_ar_quant = False
c.fused_ar_quant_keep_bf16 = False
c._communicate_simple_fn = lambda hidden_states, **_: hidden_states
# A layer whose attention takes its input as it is and owes nothing on it.
c._steps = comm.BoundarySteps(
attention_input=lambda hidden_states, **_: hidden_states,
ffn_input=comm._mlp_input_norm,
ffn_output=comm.StageOutput(comm.Layout(frozenset())),
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
ffn_sum_is_movable=False,
)
# Construction picks the fused entries; call it under the platform patches.
c._attn_input_fusions = c._select_attn_input_fusions()
return c
@@ -227,10 +227,10 @@ def test_communicator_publishes_layout_at_each_transition(
communicator.post_attention_layernorm = None
communicator.input_layernorm = lambda x: x
communicator.qkv_latent_func = None
communicator._communicate_simple_fn = lambda **kwargs: kwargs["hidden_states"]
communicator._mlp_input = lambda hidden_states, residual, *args: (
hidden_states,
residual,
communicator._steps = SimpleNamespace(
attention_input=lambda hidden_states, **kwargs: hidden_states,
layer_input=None,
ffn_input=lambda hidden_states, residual, *args: (hidden_states, residual),
)
communicator._attn_input_fusions = ()
monkeypatch.setattr(
@@ -46,17 +46,22 @@ class DeferringLayer(nn.Module):
super().__init__()
self.return_topk = return_topk
self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
self.layer_communicator._ffn_output = comm.StageOutput(
comm.Layout(frozenset()),
group=comm.SumGroup.TP,
leaves_for_next_layer=True,
self.layer_communicator._steps = comm.BoundarySteps(
attention_input=comm.CommunicateSimpleFn._trivial,
ffn_input=comm._mlp_input_norm,
ffn_output=comm.StageOutput(
comm.Layout(frozenset()),
group=comm.SumGroup.TP,
leaves_for_next_layer=True,
),
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
ffn_sum_is_movable=True,
)
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = (
lambda batch: defer
)
self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer
self.layer_communicator.is_last_layer = False
self.layer_communicator._postprocess_scatters_to_local_tokens = False
self.layer_communicator._sp_steps = None
self.layer_communicator._input_scattered_steps = None
self.layer_communicator.ffn_reduction_group = lambda: GROUP