[Refactor] Run every batch of a layer from one BoundarySteps (#41420)
This commit is contained in:
@@ -756,11 +756,17 @@ class LayerCommunicator:
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
force_layernorm_before_dp_gather
|
||||
)
|
||||
# The steps the layer's ordinary batches run.
|
||||
sides = self._declared_sides()
|
||||
if sides is not None:
|
||||
self._select_declared_boundaries(sides)
|
||||
else:
|
||||
self._select_boundaries_from_scatter_modes()
|
||||
self._steps = (
|
||||
_select_boundary_steps(
|
||||
sides,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
)
|
||||
if sides is not None
|
||||
else self._select_boundaries_from_scatter_modes()
|
||||
)
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
self._input_scattered_steps = (
|
||||
@@ -846,46 +852,13 @@ class LayerCommunicator:
|
||||
),
|
||||
)
|
||||
|
||||
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
|
||||
"""Choose the three boundary sides this layer owns from the
|
||||
declarations: the attention input, the attention -> FFN steps, and the
|
||||
FFN output's way on to the rows the layer hands on."""
|
||||
self._communicate_simple_fn = _select_attention_input_move(
|
||||
sides.input_rows, sides.attention
|
||||
)
|
||||
self._mlp_input, fused = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=sides.input_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
)
|
||||
self._mlp_input_may_return_new_residual = any(
|
||||
f.may_return_new_residual for f in fused
|
||||
)
|
||||
self._ffn_output = sides.ffn_output
|
||||
(
|
||||
self._postprocess_scatters_to_local_tokens,
|
||||
postprocess,
|
||||
) = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if postprocess is not None:
|
||||
self._communicate_summable_tensor_pair_fn = postprocess
|
||||
self._ffn_sum_is_movable = sides.ffn_output.group is not None
|
||||
|
||||
def _select_boundaries_from_scatter_modes(self) -> None:
|
||||
self._post_init_communicate()
|
||||
# Under attention DP, the base postprocess scatters the FFN output back
|
||||
# to this rank's tokens, which the next layer's input can run instead;
|
||||
# the MHC and DSA-CP postprocess do more, so theirs stays here.
|
||||
self._postprocess_scatters_to_local_tokens = (
|
||||
self._communicate_summable_tensor_pair_fn
|
||||
is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
)
|
||||
def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps":
|
||||
"""The layer's steps chosen from its scatter modes, for the layers the
|
||||
declarations do not cover yet."""
|
||||
attention_input, postprocess = self._post_init_communicate()
|
||||
ffn_input, fused = self._select_mlp_input()
|
||||
modes = self.layer_scatter_modes
|
||||
self._ffn_output = StageOutput(
|
||||
ffn_output = StageOutput(
|
||||
self._context.layouts[modes.mlp_mode],
|
||||
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
||||
@@ -897,30 +870,41 @@ class LayerCommunicator:
|
||||
self.allow_reduce_scatter or modes.is_layer_sparse
|
||||
),
|
||||
)
|
||||
# Whether the FFN sum is one the next layer's input can take: not when
|
||||
# the way back is the MoE-CP scatter, nor when a SCATTERED FFN computes
|
||||
# whole tokens with nothing left to sum.
|
||||
self._ffn_sum_is_movable = modes.mlp_mode not in (
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.SCATTERED,
|
||||
)
|
||||
self._mlp_input, self._mlp_input_may_return_new_residual = (
|
||||
self._select_mlp_input()
|
||||
return BoundarySteps(
|
||||
attention_input=attention_input,
|
||||
ffn_input=ffn_input,
|
||||
ffn_output=ffn_output,
|
||||
# Under attention DP the base postprocess scatters the FFN output
|
||||
# back to this rank's tokens, which the next layer's input can run
|
||||
# instead; the MHC and DSA-CP postprocess do more.
|
||||
ffn_output_move=(
|
||||
None
|
||||
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
else postprocess
|
||||
),
|
||||
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
|
||||
# FFN computes whole tokens with nothing left to sum.
|
||||
ffn_sum_is_movable=modes.mlp_mode
|
||||
not in (ScatterMode.MOE_FULL, ScatterMode.SCATTERED),
|
||||
fused=fused,
|
||||
layer_input=_complete_scattered_input,
|
||||
)
|
||||
|
||||
def _post_init_communicate(self):
|
||||
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
)
|
||||
self._communicate_summable_tensor_pair_fn = (
|
||||
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
|
||||
"""The attention input move and the postprocess the scatter modes
|
||||
choose."""
|
||||
return (
|
||||
CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
),
|
||||
CommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
||||
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
||||
output_mode=self.layer_scatter_modes.layer_output_mode,
|
||||
context=self._context,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
def prepare_attn_and_capture_last_layer_outputs(
|
||||
@@ -940,7 +924,7 @@ class LayerCommunicator:
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
gathered_last_layer_output = self._communicate_simple_fn(
|
||||
gathered_last_layer_output = self._batch_steps().attention_input(
|
||||
hidden_states=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
@@ -966,7 +950,7 @@ class LayerCommunicator:
|
||||
the batch is not input-scattered.
|
||||
"""
|
||||
return (
|
||||
self._mlp_input_may_return_new_residual
|
||||
any(f.may_return_new_residual for f in self._batch_steps().fused)
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
||||
)
|
||||
@@ -1028,16 +1012,11 @@ class LayerCommunicator:
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
steps = self._batch_steps()
|
||||
if steps is not None and steps.layer_input is not None:
|
||||
hidden_states, residual = steps.layer_input(
|
||||
layer_input = self._batch_steps().layer_input
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(
|
||||
hidden_states, residual, self._context
|
||||
)
|
||||
elif get_attn_tp_context().input_scattered:
|
||||
hidden_states, residual = self._tp_reduce_scatter(
|
||||
hidden_states,
|
||||
residual,
|
||||
)
|
||||
if hidden_states.shape[0] == 0:
|
||||
residual = hidden_states
|
||||
elif owed is not None:
|
||||
@@ -1059,9 +1038,9 @@ class LayerCommunicator:
|
||||
def _in_sp_region(self) -> bool:
|
||||
return self._sp_steps is not None and get_forward().sp_active
|
||||
|
||||
def _batch_steps(self) -> Optional["BoundarySteps"]:
|
||||
"""The steps this batch runs in place of the layer's ordinary ones: an
|
||||
active LayerNorm SP region's, or input-scattered attention's."""
|
||||
def _batch_steps(self) -> "BoundarySteps":
|
||||
"""The steps this batch runs: an active LayerNorm SP region's,
|
||||
input-scattered attention's, or the layer's ordinary ones."""
|
||||
if self._in_sp_region():
|
||||
return self._sp_steps
|
||||
if (
|
||||
@@ -1069,13 +1048,11 @@ class LayerCommunicator:
|
||||
and get_attn_tp_context().input_scattered
|
||||
):
|
||||
return self._input_scattered_steps
|
||||
return None
|
||||
return self._steps
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
steps = self._batch_steps()
|
||||
move = self._communicate_simple_fn if steps is None else steps.attention_input
|
||||
hidden_states = move(
|
||||
hidden_states = self._batch_steps().attention_input(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
@@ -1145,30 +1122,23 @@ class LayerCommunicator:
|
||||
hidden_states, residual, use_attn_tp_group=False
|
||||
)
|
||||
|
||||
def _tp_reduce_scatter(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return tp_reduce_scatter(hidden_states, residual, self._context)
|
||||
|
||||
def _select_mlp_input(self) -> Tuple[Callable, bool]:
|
||||
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
|
||||
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
|
||||
whether they include the aiter / flashinfer fused entry. The MHC and
|
||||
DSA-CP communicators pick their own implementations."""
|
||||
the fused kernels they try first. The MHC and DSA-CP communicators pick
|
||||
their own implementations."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
if kind is MlpInputKind.NORM:
|
||||
return _mlp_input_norm, False
|
||||
return _mlp_input_norm, ()
|
||||
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
|
||||
return _mlp_input_attn_tp_all_reduce, False
|
||||
return _mlp_input_attn_tp_all_reduce, ()
|
||||
if kind is MlpInputKind.SCATTER:
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
|
||||
),
|
||||
False,
|
||||
(),
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
fusions = (
|
||||
@@ -1182,7 +1152,7 @@ class LayerCommunicator:
|
||||
)
|
||||
if kind is MlpInputKind.GATHER_MOE_CP:
|
||||
steps = partial(_mlp_input_gather_moe_cp, gather=steps)
|
||||
return steps, any(f.may_return_new_residual for f in fusions)
|
||||
return steps, fusions
|
||||
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
@@ -1222,20 +1192,10 @@ class LayerCommunicator:
|
||||
cache=None,
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if self._in_sp_region():
|
||||
return self._sp_steps.ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.post_attention_layernorm,
|
||||
self._context,
|
||||
)
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
steps = self._batch_steps()
|
||||
mlp_input = self._mlp_input if steps is None else steps.ffn_input
|
||||
return mlp_input(
|
||||
return self._batch_steps().ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -1257,17 +1217,10 @@ class LayerCommunicator:
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self._postprocess_scatters_to_local_tokens:
|
||||
if steps.returns_over_dp:
|
||||
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
return self._communicate_summable_tensor_pair_fn(
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
@@ -1278,9 +1231,8 @@ class LayerCommunicator:
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(self) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, outside
|
||||
an active LayerNorm SP region."""
|
||||
return self._postprocess_scatters_to_local_tokens and not self._in_sp_region()
|
||||
output back to this rank's tokens: the base postprocess scatter."""
|
||||
return self._batch_steps().returns_over_dp
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1288,12 +1240,13 @@ class LayerCommunicator:
|
||||
"""The reduce-scatter that brings this layer's FFN output back to this
|
||||
rank's tokens under attention DP; None when the base postprocess would
|
||||
only scatter, or does not move tokens."""
|
||||
if not self._postprocess_scatters_to_local_tokens:
|
||||
steps = self._batch_steps()
|
||||
if not steps.returns_over_dp:
|
||||
return None
|
||||
return _reduce_and_redistribute_output_step(
|
||||
forward_batch,
|
||||
leaves_for_reduce_scatter=self._ffn_output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=self._ffn_output.leaves_for_reduce_scatterv,
|
||||
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
|
||||
)
|
||||
|
||||
def _ffn_leaves_sum_to_reduce_scatter(
|
||||
@@ -1302,10 +1255,7 @@ class LayerCommunicator:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output.leaves_for_reduce_scatter
|
||||
if not self._ffn_output.leaves_for_reduce_scatter:
|
||||
if not self._batch_steps().ffn_output.leaves_for_reduce_scatter:
|
||||
return False
|
||||
if dp_step is not None:
|
||||
return True
|
||||
@@ -1319,7 +1269,7 @@ class LayerCommunicator:
|
||||
def ffn_reduction_group(self) -> GroupCoordinator:
|
||||
"""The group this layer's FFN output owes its sum over: the MoE output's
|
||||
group on a sparse layer, the TP group a dense MLP reduces over."""
|
||||
return _sum_group(self._ffn_output.group)
|
||||
return _sum_group(self._batch_steps().ffn_output.group)
|
||||
|
||||
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion":
|
||||
"""Decide once, before the FFN runs, what it skips and what completes its
|
||||
@@ -1334,8 +1284,7 @@ class LayerCommunicator:
|
||||
dp_step=dp_step,
|
||||
)
|
||||
steps = self._batch_steps()
|
||||
ffn_output = self._ffn_output if steps is None else steps.ffn_output
|
||||
if not ffn_output.leaves_for_next_layer:
|
||||
if not steps.ffn_output.leaves_for_next_layer:
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
@@ -1349,7 +1298,7 @@ class LayerCommunicator:
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
group = self.ffn_reduction_group()
|
||||
if self._postprocess_scatters_to_local_tokens:
|
||||
if steps.returns_over_dp:
|
||||
# Under attention DP the next layer also brings the sum back to
|
||||
# this rank's tokens.
|
||||
wrap = partial(
|
||||
@@ -1435,7 +1384,7 @@ class LayerCommunicator:
|
||||
def _ffn_sum_can_move_to_next_layer(self) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if is_enable_moe_cp_allgather() or not self._ffn_sum_is_movable:
|
||||
if is_enable_moe_cp_allgather() or not self._batch_steps().ffn_sum_is_movable:
|
||||
return False
|
||||
|
||||
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
||||
@@ -1914,29 +1863,40 @@ def _token_axis_sizes() -> Dict[TokenAxis, int]:
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps one set of declarations chooses for a layer's boundaries:
|
||||
into the attention, from the attention output to the FFN input, and the
|
||||
FFN output on to the rows the layer hands on."""
|
||||
"""The steps a batch runs at a layer's boundaries: into the attention,
|
||||
from the attention output to the FFN input, and the FFN output on to the
|
||||
rows the layer hands on."""
|
||||
|
||||
attention_input: Callable
|
||||
ffn_input: Callable
|
||||
ffn_output_move: Callable
|
||||
# What the FFN exit reads in place of the layer's own FFN output declaration.
|
||||
# What the FFN exit reads: the FFN output's group and what it may leave.
|
||||
ffn_output: StageOutput
|
||||
# The postprocess that moves the FFN output on; None when it goes back over
|
||||
# attention DP, whose step the FFN exit and postprocess choose per batch.
|
||||
ffn_output_move: Optional[Callable]
|
||||
# Whether the next layer's input can take the FFN's sum.
|
||||
ffn_sum_is_movable: bool
|
||||
# The fused kernels ffn_input tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Completes what the layer's input owes before the input norm; None when it
|
||||
# owes nothing.
|
||||
layer_input: Optional[Callable] = None
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
return self.ffn_output_move is None
|
||||
|
||||
def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
|
||||
"""The steps for a batch's own declarations: nothing moves back over
|
||||
attention DP and no fused kernel runs, so the steps do not wait on a
|
||||
per-batch decision."""
|
||||
|
||||
def _select_boundary_steps(
|
||||
sides: DecoderLayerSides,
|
||||
*,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
) -> BoundarySteps:
|
||||
"""The steps a set of declarations chooses."""
|
||||
returns_over_dp, ffn_output_move = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if returns_over_dp or sides.ffn_output.leaves_for_next_layer:
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
rows = sides.input_rows
|
||||
layer_input = None
|
||||
if sides.input_owes is not None:
|
||||
@@ -1950,24 +1910,39 @@ def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
layer_input = tp_reduce_scatter
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
ffn_input, _ = _select_ffn_input(
|
||||
ffn_input, fused = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=False,
|
||||
fusions=(),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=_select_attention_input_move(rows, sides.attention),
|
||||
ffn_input=ffn_input,
|
||||
ffn_output_move=ffn_output_move,
|
||||
ffn_output=sides.ffn_output,
|
||||
ffn_output_move=None if returns_over_dp else ffn_output_move,
|
||||
ffn_sum_is_movable=sides.ffn_output.group is not None,
|
||||
fused=fused,
|
||||
layer_input=layer_input,
|
||||
)
|
||||
|
||||
|
||||
def _complete_scattered_input(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
context: "CommunicateContext",
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""The scatter-mode path's input completion: with input-scattered
|
||||
attention the layer's input is a TP partial that a reduce-scatter completes
|
||||
onto this rank's slice."""
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return tp_reduce_scatter(hidden_states, residual, context)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
|
||||
@@ -93,16 +93,18 @@ class DSACPLayerCommunicator(LayerCommunicator):
|
||||
assert self._context.attn_dp_size == 1, (
|
||||
f"dp_size should be 1 when moe_runner_backend is none"
|
||||
)
|
||||
self._communicate_simple_fn = DSACPCommunicateSimpleFn.get_fn(
|
||||
input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
)
|
||||
self._communicate_summable_tensor_pair_fn = DSACPCommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
|
||||
residual_input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
return (
|
||||
DSACPCommunicateSimpleFn.get_fn(
|
||||
input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
),
|
||||
DSACPCommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
|
||||
residual_input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
),
|
||||
)
|
||||
|
||||
def _select_mlp_input(self):
|
||||
@@ -113,7 +115,7 @@ class DSACPLayerCommunicator(LayerCommunicator):
|
||||
residual_output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
)
|
||||
return fn, False
|
||||
return fn, ()
|
||||
|
||||
|
||||
class DSACPCommunicateSimpleFn(CommunicateSimpleFn):
|
||||
|
||||
@@ -441,6 +441,8 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
# MHC's own prepare_attn and postprocess run these.
|
||||
return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
@@ -494,14 +496,14 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
fns = MHCCommunicateWithAllReduceAndLayerNormFn
|
||||
if kind is MlpInputKind.NORM:
|
||||
return partial(fns._simple, mhc=self.mhc), False
|
||||
return partial(fns._simple, mhc=self.mhc), ()
|
||||
if kind is MlpInputKind.GATHER:
|
||||
fn = fns._gather_hidden_states_and_residual
|
||||
elif kind is MlpInputKind.SCATTER:
|
||||
fn = fns._scatter_hidden_states_and_residual
|
||||
else:
|
||||
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
|
||||
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), False
|
||||
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), ()
|
||||
|
||||
def postprocess_layer(self, hidden_states, residual, forward_batch):
|
||||
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
|
||||
|
||||
@@ -45,6 +45,20 @@ def make_group(scale=3):
|
||||
return types.SimpleNamespace(all_reduce=MagicMock(side_effect=lambda h: h * scale))
|
||||
|
||||
|
||||
def ordinary_steps(ffn_output, *, returns_over_dp=False):
|
||||
"""A layer's ordinary steps, carrying the FFN output declaration the exit
|
||||
reads and whether the output goes back over attention DP."""
|
||||
return comm.BoundarySteps(
|
||||
attention_input=comm.CommunicateSimpleFn._trivial,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output=ffn_output,
|
||||
ffn_output_move=(
|
||||
None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial
|
||||
),
|
||||
ffn_sum_is_movable=True,
|
||||
)
|
||||
|
||||
|
||||
def make_communicator(
|
||||
*,
|
||||
fuse,
|
||||
@@ -58,8 +72,13 @@ def make_communicator(
|
||||
"""A communicator whose decisions and postprocess are stubbed, built
|
||||
without the process-wide parallel state."""
|
||||
communicator = cls.__new__(cls)
|
||||
communicator._ffn_output = StageOutput(
|
||||
Layout(frozenset()), group=SumGroup.TP, leaves_for_next_layer=allow_deferred
|
||||
communicator._steps = ordinary_steps(
|
||||
StageOutput(
|
||||
Layout(frozenset()),
|
||||
group=SumGroup.TP,
|
||||
leaves_for_next_layer=allow_deferred,
|
||||
),
|
||||
returns_over_dp=scatters_to_local_tokens,
|
||||
)
|
||||
if cls is LayerCommunicator:
|
||||
communicator._ffn_sum_moves_to_next_layer = MagicMock(return_value=fuse)
|
||||
@@ -72,7 +91,6 @@ def make_communicator(
|
||||
communicator.is_last_layer = False
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
|
||||
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
|
||||
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
|
||||
communicator.postprocess_layer = MagicMock(
|
||||
@@ -306,13 +324,15 @@ class TestSelectFfnCompletion(CustomTestCase):
|
||||
communicator.is_last_layer = is_last_layer
|
||||
communicator._sp_steps = sp_region_steps() if sp_region else None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters
|
||||
communicator._ffn_output = StageOutput(
|
||||
Layout(frozenset()),
|
||||
group=SumGroup.MOE_OUTPUT,
|
||||
leaves_for_next_layer=True,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
communicator._steps = ordinary_steps(
|
||||
StageOutput(
|
||||
Layout(frozenset()),
|
||||
group=SumGroup.MOE_OUTPUT,
|
||||
leaves_for_next_layer=True,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
),
|
||||
returns_over_dp=scatters,
|
||||
)
|
||||
communicator._ffn_sum_moves_to_next_layer = lambda forward_batch, **_: fuse
|
||||
communicator._ffn_leaves_sum_to_reduce_scatter = lambda forward_batch, dp_step: (
|
||||
|
||||
@@ -223,7 +223,7 @@ class TestMlpInputOrder(CustomTestCase):
|
||||
def test_a_gather_runs_the_order_chosen_at_construction(self):
|
||||
for mlp_mode in (FULL, MOE_FULL):
|
||||
with self.subTest(mlp_mode=mlp_mode):
|
||||
c, (steps, may_fuse) = communicator(
|
||||
c, (steps, fused) = communicator(
|
||||
(TP_ATTN_FULL, TP_ATTN_FULL, mlp_mode, TP_ATTN_FULL),
|
||||
make_context(tp=2),
|
||||
Fusable(),
|
||||
@@ -239,7 +239,7 @@ class TestMlpInputOrder(CustomTestCase):
|
||||
gathers_residual=False,
|
||||
fusions=(entry,),
|
||||
)
|
||||
self.assertTrue(may_fuse)
|
||||
self.assertEqual([f.run for f in fused], [entry])
|
||||
|
||||
def test_no_fused_entry_without_a_fusable_norm_or_under_attention_dp(self):
|
||||
for context, norm in (
|
||||
@@ -247,10 +247,10 @@ class TestMlpInputOrder(CustomTestCase):
|
||||
(make_context(dp=2, tp=2), Fusable()),
|
||||
):
|
||||
with self.subTest(dp=context.attn_dp_size):
|
||||
_, (steps, may_fuse) = communicator(GATHER_LAYOUT, context, norm)
|
||||
_, (steps, fused) = communicator(GATHER_LAYOUT, context, norm)
|
||||
order = steps.keywords["order"]
|
||||
self.assertEqual(order.keywords.get("fusions", ()), ())
|
||||
self.assertFalse(may_fuse)
|
||||
self.assertEqual(fused, ())
|
||||
|
||||
def test_other_kinds_take_their_steps(self):
|
||||
tp = make_context(tp=2)
|
||||
@@ -267,9 +267,9 @@ class TestMlpInputOrder(CustomTestCase):
|
||||
),
|
||||
):
|
||||
with self.subTest(expected=expected.__name__):
|
||||
_, (steps, may_fuse) = communicator(layout, context, Fusable())
|
||||
_, (steps, fused) = communicator(layout, context, Fusable())
|
||||
self.assertIs(steps, expected)
|
||||
self.assertFalse(may_fuse)
|
||||
self.assertEqual(fused, ())
|
||||
_, (steps, _) = communicator(
|
||||
(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable()
|
||||
)
|
||||
@@ -297,11 +297,11 @@ class TestMlpInputOrder(CustomTestCase):
|
||||
c.layer_scatter_modes = modes(*layout)
|
||||
c._context = make_context(tp=2)
|
||||
c.post_attention_layernorm = Fusable()
|
||||
steps, may_fuse = c._select_mlp_input()
|
||||
steps, fused = c._select_mlp_input()
|
||||
self.assert_order(
|
||||
steps, func, residual_input_mode=TP_ATTN_FULL, mhc=mhc
|
||||
)
|
||||
self.assertFalse(may_fuse)
|
||||
self.assertEqual(fused, ())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -274,7 +274,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
declarations, or none do."""
|
||||
|
||||
def declared(self, communicator):
|
||||
return getattr(communicator._mlp_input, "func", None) in (
|
||||
return getattr(communicator._steps.ffn_input, "func", None) in (
|
||||
comm._mlp_input_dp_partial,
|
||||
comm._mlp_input_dp_replicate,
|
||||
comm._mlp_input_without_dp,
|
||||
@@ -288,10 +288,10 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
communicator = build(layer_facts(layer_id, 3), parallel)
|
||||
self.assertTrue(self.declared(communicator))
|
||||
self.assertIs(
|
||||
communicator._communicate_simple_fn,
|
||||
communicator._steps.attention_input,
|
||||
comm.CommunicateSimpleFn._trivial,
|
||||
)
|
||||
self.assertTrue(communicator._postprocess_scatters_to_local_tokens)
|
||||
self.assertTrue(communicator._steps.returns_over_dp)
|
||||
|
||||
def test_plain_tp(self):
|
||||
for norm, fuses in ((Norm, False), (FusableNorm, True)):
|
||||
@@ -302,18 +302,23 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
input_layernorm=norm(),
|
||||
post_attention_layernorm=norm(),
|
||||
)
|
||||
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
|
||||
self.assertIs(
|
||||
communicator._steps.ffn_input.func, comm._mlp_input_without_dp
|
||||
)
|
||||
entry = (
|
||||
communicator._mlp_input_reduce_output_and_update_and_read_residual
|
||||
)
|
||||
self.assertEqual(
|
||||
communicator._mlp_input.keywords["fusions"],
|
||||
communicator._steps.ffn_input.keywords["fusions"],
|
||||
(entry,) if fuses else (),
|
||||
)
|
||||
self.assertIs(communicator._mlp_input_may_return_new_residual, fuses)
|
||||
self.assertFalse(communicator._postprocess_scatters_to_local_tokens)
|
||||
self.assertIs(
|
||||
communicator._communicate_summable_tensor_pair_fn,
|
||||
any(f.may_return_new_residual for f in communicator._steps.fused),
|
||||
fuses,
|
||||
)
|
||||
self.assertFalse(communicator._steps.returns_over_dp)
|
||||
self.assertIs(
|
||||
communicator._steps.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._trivial,
|
||||
)
|
||||
# One rank: the attention output is complete, only the norm is left.
|
||||
@@ -321,7 +326,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
single = build(layer_facts(1, 3), one_rank)
|
||||
with planning(one_rank):
|
||||
self.assertIsNotNone(single._declared_sides())
|
||||
self.assertIs(single._mlp_input, comm._mlp_input_norm)
|
||||
self.assertIs(single._steps.ffn_input, comm._mlp_input_norm)
|
||||
|
||||
def test_the_order_follows_the_attention_output(self):
|
||||
for attn_tp, force, order in (
|
||||
@@ -335,7 +340,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
parallel_of(attn_dp=2, attn_tp=attn_tp),
|
||||
force_layernorm_before_dp_gather=force,
|
||||
)
|
||||
self.assertIs(communicator._mlp_input.func, order)
|
||||
self.assertIs(communicator._steps.ffn_input.func, order)
|
||||
|
||||
def test_a_dense_first_layer_before_sparse_ones(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
@@ -361,30 +366,32 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
]
|
||||
self.assertEqual([self.declared(layer) for layer in layers], [True] * 4)
|
||||
if not a2a:
|
||||
moe = layers[1]._ffn_output
|
||||
moe = layers[1]._steps.ffn_output
|
||||
self.assertIs(moe.group, SumGroup.MOE_OUTPUT)
|
||||
self.assertTrue(moe.leaves_for_reduce_scatterv)
|
||||
continue
|
||||
# The first a2a layer slices the residual it takes from a
|
||||
# dense layer; the next one takes it already sliced.
|
||||
self.assertEqual(
|
||||
[layers[i]._mlp_input.func for i in (1, 2)],
|
||||
[layers[i]._steps.ffn_input.func for i in (1, 2)],
|
||||
[comm._mlp_input_scatter] * 2,
|
||||
)
|
||||
self.assertEqual(
|
||||
[
|
||||
layers[i]._mlp_input.keywords["scatters_residual"]
|
||||
layers[i]._steps.ffn_input.keywords["scatters_residual"]
|
||||
for i in (1, 2)
|
||||
],
|
||||
[True, False],
|
||||
)
|
||||
self.assertIsNone(layers[1]._ffn_output.group)
|
||||
self.assertIsNone(layers[1]._steps.ffn_output.group)
|
||||
self.assertIs(
|
||||
layers[2]._communicate_simple_fn,
|
||||
layers[2]._steps.attention_input,
|
||||
comm.CommunicateSimpleFn._scattered_to_tp_attn_full,
|
||||
)
|
||||
self.assertIs(layers[3]._mlp_input.func, comm._mlp_input_dp_partial)
|
||||
self.assertTrue(layers[3]._mlp_input.keywords["gathers_residual"])
|
||||
self.assertIs(
|
||||
layers[3]._steps.ffn_input.func, comm._mlp_input_dp_partial
|
||||
)
|
||||
self.assertTrue(layers[3]._steps.ffn_input.keywords["gathers_residual"])
|
||||
|
||||
def test_the_last_a2a_layer_folds_the_residual_back(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
@@ -396,7 +403,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
a2a=True,
|
||||
)
|
||||
self.assertIs(
|
||||
last._communicate_summable_tensor_pair_fn,
|
||||
last._steps.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._gather,
|
||||
)
|
||||
|
||||
@@ -407,7 +414,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
)
|
||||
with patch.object(comm, "_use_ag_after_qlora", True):
|
||||
layer = build(modes, parallel, a2a=True)
|
||||
self.assertIs(layer._communicate_simple_fn, comm.CommunicateSimpleFn._trivial)
|
||||
self.assertIs(layer._steps.attention_input, comm.CommunicateSimpleFn._trivial)
|
||||
|
||||
def test_layers_that_keep_the_scatter_modes(self):
|
||||
dp = parallel_of(attn_dp=2, attn_tp=2)
|
||||
@@ -611,7 +618,10 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase):
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
)
|
||||
self.assertIs(communicator._mlp_input_may_return_new_residual, expected)
|
||||
self.assertIs(
|
||||
any(f.may_return_new_residual for f in communicator._steps.fused),
|
||||
expected,
|
||||
)
|
||||
|
||||
|
||||
class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
@@ -624,8 +634,9 @@ class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
SP_STEPS = comm.BoundarySteps(
|
||||
attention_input=comm.CommunicateSimpleFn._trivial,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
|
||||
ffn_output=sequence_parallel_layer_sides(axis_sizes=SIZES).ffn_output,
|
||||
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
|
||||
ffn_sum_is_movable=False,
|
||||
)
|
||||
|
||||
def test_the_region_declarations_choose_local_steps(self):
|
||||
@@ -666,7 +677,31 @@ class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
post_attention_layernorm=Norm(),
|
||||
)
|
||||
self.assertEqual(communicator._sp_steps, self.SP_STEPS)
|
||||
self.assertIs(communicator._mlp_input.func, comm._mlp_input_without_dp)
|
||||
self.assertIs(
|
||||
communicator._steps.ffn_input.func, comm._mlp_input_without_dp
|
||||
)
|
||||
|
||||
def test_what_the_ffn_exit_reads_comes_from_the_batch_s_steps(self):
|
||||
# Inside the region the FFN output is complete: no sum to move.
|
||||
parallel = parallel_of(attn_dp=1, attn_tp=2)
|
||||
communicator = build(layer_facts(1, 3), parallel, sp=True)
|
||||
for active in (False, True):
|
||||
with (
|
||||
self.subTest(sp_active=active),
|
||||
planning(parallel, sp=True),
|
||||
patch.object(
|
||||
comm, "get_forward", lambda: SimpleNamespace(sp_active=active)
|
||||
),
|
||||
patch.object(
|
||||
comm,
|
||||
"get_attn_tp_context",
|
||||
lambda: SimpleNamespace(input_scattered=False),
|
||||
),
|
||||
patch.object(comm, "is_dp_attention_enabled", lambda: False),
|
||||
):
|
||||
self.assertIs(
|
||||
communicator._ffn_sum_can_move_to_next_layer(), not active
|
||||
)
|
||||
|
||||
def test_without_sp_there_is_no_region(self):
|
||||
with planning(parallel_of(attn_dp=1, attn_tp=2)):
|
||||
@@ -677,15 +712,19 @@ class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
)
|
||||
self.assertIsNone(communicator._sp_steps)
|
||||
|
||||
def test_declarations_left_to_the_ffn_exit_are_not_batch_steps(self):
|
||||
# A sum the FFN exit may hand to the next layer, or a move back over
|
||||
# attention DP, waits on a per-batch decision.
|
||||
for sizes, leaves_next in (
|
||||
(self.SIZES, True),
|
||||
({**self.SIZES, TokenAxis.ATTN_DP: 2}, False),
|
||||
):
|
||||
with self.subTest(sizes=sizes), self.assertRaises(NotImplementedError):
|
||||
comm._select_boundary_steps(sides_of(sizes, leaves_next=leaves_next))
|
||||
def test_the_same_selector_chooses_a_layer_s_ordinary_steps(self):
|
||||
# Under attention DP the FFN exit chooses the move back per batch.
|
||||
kept = comm._select_boundary_steps(sides_of(self.SIZES, leaves_next=True))
|
||||
self.assertTrue(kept.ffn_output.leaves_for_next_layer)
|
||||
self.assertTrue(kept.ffn_sum_is_movable)
|
||||
self.assertIs(
|
||||
kept.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial
|
||||
)
|
||||
returned = comm._select_boundary_steps(
|
||||
sides_of({**self.SIZES, TokenAxis.ATTN_DP: 2})
|
||||
)
|
||||
self.assertTrue(returned.returns_over_dp)
|
||||
self.assertIsNone(returned.ffn_output_move)
|
||||
|
||||
|
||||
class TestInputScatteredAttention(CustomTestCase):
|
||||
@@ -780,9 +819,40 @@ class TestInputScatteredAttention(CustomTestCase):
|
||||
):
|
||||
self.assertIs(
|
||||
communicator._batch_steps(),
|
||||
communicator._input_scattered_steps if scattered else None,
|
||||
communicator._input_scattered_steps
|
||||
if scattered
|
||||
else communicator._steps,
|
||||
)
|
||||
|
||||
def test_prepare_attn_completes_the_scattered_input(self):
|
||||
scattered = []
|
||||
group = SimpleNamespace(
|
||||
name="tp",
|
||||
reduce_scatter_tensor=lambda out, x: (
|
||||
scattered.append(x.shape[0]) or out.copy_(x[: out.shape[0]] * 2)
|
||||
),
|
||||
)
|
||||
parallel = parallel_of(
|
||||
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True, tp_group=group
|
||||
)
|
||||
communicator = build(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
|
||||
with (
|
||||
patch.object(comm, "get_parallel", lambda: parallel),
|
||||
patch.object(
|
||||
comm,
|
||||
"get_attn_tp_context",
|
||||
lambda: SimpleNamespace(input_scattered=True),
|
||||
),
|
||||
patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)),
|
||||
):
|
||||
hidden, residual = communicator.prepare_attn(
|
||||
torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None
|
||||
)
|
||||
self.assertEqual(scattered, [4])
|
||||
# Norm: (2 * (h + r), h + r) on the slice, with h the completed sum.
|
||||
torch.testing.assert_close(residual, torch.full((2, HIDDEN), 5.0))
|
||||
torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0))
|
||||
|
||||
def test_the_last_layer_completes_its_own_sum(self):
|
||||
parallel = parallel_of(
|
||||
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
|
||||
@@ -800,6 +870,64 @@ class TestInputScatteredAttention(CustomTestCase):
|
||||
self.assertIs(steps.ffn_output.leaves_for_reduce_scatter, hands_on)
|
||||
|
||||
|
||||
class TestOneRepresentation(CustomTestCase):
|
||||
"""Every layer runs from one BoundarySteps per batch, whichever entry chose
|
||||
its steps: none keeps the steps as separate attributes."""
|
||||
|
||||
SEPARATE = (
|
||||
"_communicate_simple_fn",
|
||||
"_mlp_input",
|
||||
"_communicate_summable_tensor_pair_fn",
|
||||
"_ffn_output",
|
||||
"_postprocess_scatters_to_local_tokens",
|
||||
"_ffn_sum_is_movable",
|
||||
"_mlp_input_may_return_new_residual",
|
||||
)
|
||||
|
||||
def test_both_entries_hold_steps_only(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
direct = LayerScatterModes(
|
||||
layer_input_mode=ScatterMode.TP_ATTN_FULL,
|
||||
attn_mode=ScatterMode.TP_ATTN_FULL,
|
||||
mlp_mode=ScatterMode.FULL,
|
||||
middle_residual_mode=ScatterMode.TP_ATTN_FULL,
|
||||
layer_output_mode=ScatterMode.TP_ATTN_FULL,
|
||||
)
|
||||
for name, communicator in (
|
||||
("declarations", build(layer_facts(1, 3), parallel)),
|
||||
("scatter modes", build(direct, parallel)),
|
||||
):
|
||||
with self.subTest(name):
|
||||
self.assertIsInstance(communicator._steps, comm.BoundarySteps)
|
||||
for attribute in self.SEPARATE:
|
||||
self.assertFalse(hasattr(communicator, attribute), attribute)
|
||||
self.assertIs(communicator._batch_steps(), communicator._steps)
|
||||
# Under attention DP both bring the FFN output back to this
|
||||
# rank's tokens with the step the FFN exit chooses.
|
||||
self.assertTrue(communicator._steps.returns_over_dp)
|
||||
|
||||
def test_the_scatter_mode_steps_complete_a_scattered_input(self):
|
||||
# The scatter-mode path's input completion reads the batch fact itself.
|
||||
steps = build(
|
||||
LayerScatterModes(
|
||||
layer_input_mode=ScatterMode.TP_ATTN_FULL,
|
||||
attn_mode=ScatterMode.TP_ATTN_FULL,
|
||||
mlp_mode=ScatterMode.FULL,
|
||||
middle_residual_mode=ScatterMode.TP_ATTN_FULL,
|
||||
layer_output_mode=ScatterMode.TP_ATTN_FULL,
|
||||
),
|
||||
parallel_of(attn_dp=1, attn_tp=2),
|
||||
)._steps
|
||||
self.assertIs(steps.layer_input, comm._complete_scattered_input)
|
||||
hidden, residual = torch.ones(2, HIDDEN), torch.ones(2, HIDDEN)
|
||||
with patch.object(
|
||||
comm, "get_attn_tp_context", lambda: SimpleNamespace(input_scattered=False)
|
||||
):
|
||||
self.assertEqual(
|
||||
steps.layer_input(hidden, residual, None), (hidden, residual)
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Two consecutive layers, rank by rank.
|
||||
|
||||
|
||||
@@ -40,15 +40,32 @@ def sp_region_steps():
|
||||
)
|
||||
|
||||
|
||||
def _steps(*, ffn_output=None, returns_over_dp=False, ffn_sum_is_movable=True):
|
||||
return comm.BoundarySteps(
|
||||
attention_input=comm.CommunicateSimpleFn._trivial,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output=ffn_output or StageOutput(Layout(frozenset()), group=SumGroup.TP),
|
||||
ffn_output_move=(
|
||||
None if returns_over_dp else comm.CommunicateSummableTensorPairFn._trivial
|
||||
),
|
||||
ffn_sum_is_movable=ffn_sum_is_movable,
|
||||
)
|
||||
|
||||
|
||||
def _fake_communicator(mlp_mode=ScatterMode.TP_ATTN_FULL):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._speculative_algo = None
|
||||
communicator.layer_scatter_modes = types.SimpleNamespace(mlp_mode=mlp_mode)
|
||||
# Fixed at construction by the scatter-mode path from the MLP's mode.
|
||||
communicator._ffn_sum_is_movable = mlp_mode not in (
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.SCATTERED,
|
||||
communicator._steps = _steps(
|
||||
ffn_sum_is_movable=mlp_mode
|
||||
not in (
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.SCATTERED,
|
||||
)
|
||||
)
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator.is_last_layer = False
|
||||
communicator._context = types.SimpleNamespace(tp_size=4)
|
||||
return communicator
|
||||
@@ -347,14 +364,16 @@ class TestDeferFfnReduction(CustomTestCase):
|
||||
):
|
||||
communicator = _fake_communicator()
|
||||
communicator.is_last_layer = is_last_layer
|
||||
communicator._postprocess_scatters_to_local_tokens = scatters_to_local_tokens
|
||||
communicator._sp_steps = sp_region_steps() if sp_active else None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._ffn_output = StageOutput(
|
||||
Layout(frozenset()),
|
||||
group=SumGroup.MOE_OUTPUT,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
communicator._steps = _steps(
|
||||
ffn_output=StageOutput(
|
||||
Layout(frozenset()),
|
||||
group=SumGroup.MOE_OUTPUT,
|
||||
leaves_for_reduce_scatter=True,
|
||||
leaves_for_reduce_scatterv=True,
|
||||
),
|
||||
returns_over_dp=scatters_to_local_tokens,
|
||||
)
|
||||
forward_batch = types.SimpleNamespace(
|
||||
input_ids=types.SimpleNamespace(shape=(batch_size,)),
|
||||
|
||||
@@ -15,6 +15,9 @@ from sglang.srt.arg_groups.layernorm_sp_hook import validate_layernorm_sp
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
Layout,
|
||||
StageOutput,
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
@@ -167,18 +170,28 @@ class TestSpRegionSteps(CustomTestCase):
|
||||
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
c._sp_steps = sp_region_steps()
|
||||
c._input_scattered_steps = None
|
||||
c.layer_scatter_modes = SimpleNamespace(is_first_layer=first_layer)
|
||||
c.layer_scatter_modes = SimpleNamespace(
|
||||
is_first_layer=first_layer, is_layer_sparse=False
|
||||
)
|
||||
c.allow_reduce_scatter = False
|
||||
c._attn_input_fusions = ()
|
||||
c.input_layernorm = _Norm()
|
||||
c.post_attention_layernorm = _Norm()
|
||||
c.qkv_latent_func = None
|
||||
c._context = SimpleNamespace(cache=None)
|
||||
c._communicate_simple_fn = MagicMock(side_effect=AssertionError("moved"))
|
||||
c._communicate_summable_tensor_pair_fn = MagicMock(
|
||||
side_effect=AssertionError("postprocess ran")
|
||||
)
|
||||
c._steps = self.ordinary_steps(MagicMock(side_effect=AssertionError("moved")))
|
||||
return c
|
||||
|
||||
def ordinary_steps(self, attention_input):
|
||||
"""The layer's steps outside the region, which must not run inside it."""
|
||||
return comm.BoundarySteps(
|
||||
attention_input=attention_input,
|
||||
ffn_input=MagicMock(side_effect=AssertionError("ordinary FFN input ran")),
|
||||
ffn_output=StageOutput(Layout(frozenset()), group=SumGroup.TP),
|
||||
ffn_output_move=MagicMock(side_effect=AssertionError("postprocess ran")),
|
||||
ffn_sum_is_movable=True,
|
||||
)
|
||||
|
||||
def run_prepare_attn(self, communicator, mode, hidden, residual):
|
||||
batch = SimpleNamespace(forward_mode=mode)
|
||||
with (
|
||||
@@ -208,20 +221,19 @@ class TestSpRegionSteps(CustomTestCase):
|
||||
|
||||
def test_decode_leaves_the_region_closed(self):
|
||||
communicator = self.communicator(first_layer=True)
|
||||
communicator._communicate_simple_fn = MagicMock(
|
||||
side_effect=lambda **k: k["hidden_states"]
|
||||
)
|
||||
move = MagicMock(side_effect=lambda **k: k["hidden_states"])
|
||||
communicator._steps = self.ordinary_steps(move)
|
||||
(h, _), active, scatter = self.run_prepare_attn(
|
||||
communicator, ForwardMode.DECODE, torch.ones(2, 4), None
|
||||
)
|
||||
self.assertFalse(active)
|
||||
scatter.assert_not_called()
|
||||
communicator._communicate_simple_fn.assert_called_once()
|
||||
move.assert_called_once()
|
||||
|
||||
def test_a_later_layer_does_not_reopen_the_region(self):
|
||||
communicator = self.communicator(first_layer=False)
|
||||
communicator._communicate_simple_fn = MagicMock(
|
||||
side_effect=lambda **k: k["hidden_states"]
|
||||
communicator._steps = self.ordinary_steps(
|
||||
MagicMock(side_effect=lambda **k: k["hidden_states"])
|
||||
)
|
||||
_, active, scatter = self.run_prepare_attn(
|
||||
communicator, ForwardMode.EXTEND, torch.ones(2, 4), torch.ones(2, 4)
|
||||
|
||||
@@ -29,6 +29,18 @@ from sglang.test.test_utils import CustomTestCase
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def steps(*, ffn_output_move):
|
||||
"""A layer's steps with the given FFN output move; None sends the output
|
||||
back over attention DP."""
|
||||
return comm.BoundarySteps(
|
||||
attention_input=comm.CommunicateSimpleFn._trivial,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output=comm.StageOutput(comm.Layout(frozenset())),
|
||||
ffn_output_move=ffn_output_move,
|
||||
ffn_sum_is_movable=False,
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def reduce_scatterv_applies():
|
||||
with (
|
||||
@@ -143,11 +155,10 @@ class TestPostprocessReduceScatterv(CustomTestCase):
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._context = None
|
||||
communicator._postprocess_scatters_to_local_tokens = False
|
||||
communicator.allow_reduce_scatter = False
|
||||
communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True)
|
||||
communicator._communicate_summable_tensor_pair_fn = lambda **kwargs: (
|
||||
seen.update(kwargs) or (None, None)
|
||||
communicator._steps = steps(
|
||||
ffn_output_move=lambda **kwargs: seen.update(kwargs) or (None, None)
|
||||
)
|
||||
communicator.postprocess_layer(None, None, None)
|
||||
self.assertIs(seen["is_layer_sparse"], True)
|
||||
@@ -160,7 +171,7 @@ class TestPostprocessReduceScatterv(CustomTestCase):
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
communicator._postprocess_scatters_to_local_tokens = True
|
||||
communicator._steps = steps(ffn_output_move=None)
|
||||
communicator._postprocess_dp_step = lambda forward_batch: step
|
||||
with patch.object(
|
||||
comm, "_to_local_tokens", side_effect=lambda s, fb, h: (s, h)
|
||||
|
||||
@@ -48,7 +48,14 @@ def communicator(norm):
|
||||
c._context = None
|
||||
c.enable_fused_ar_quant = False
|
||||
c.fused_ar_quant_keep_bf16 = False
|
||||
c._communicate_simple_fn = lambda hidden_states, **_: hidden_states
|
||||
# A layer whose attention takes its input as it is and owes nothing on it.
|
||||
c._steps = comm.BoundarySteps(
|
||||
attention_input=lambda hidden_states, **_: hidden_states,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output=comm.StageOutput(comm.Layout(frozenset())),
|
||||
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
|
||||
ffn_sum_is_movable=False,
|
||||
)
|
||||
# Construction picks the fused entries; call it under the platform patches.
|
||||
c._attn_input_fusions = c._select_attn_input_fusions()
|
||||
return c
|
||||
|
||||
@@ -227,10 +227,10 @@ def test_communicator_publishes_layout_at_each_transition(
|
||||
communicator.post_attention_layernorm = None
|
||||
communicator.input_layernorm = lambda x: x
|
||||
communicator.qkv_latent_func = None
|
||||
communicator._communicate_simple_fn = lambda **kwargs: kwargs["hidden_states"]
|
||||
communicator._mlp_input = lambda hidden_states, residual, *args: (
|
||||
hidden_states,
|
||||
residual,
|
||||
communicator._steps = SimpleNamespace(
|
||||
attention_input=lambda hidden_states, **kwargs: hidden_states,
|
||||
layer_input=None,
|
||||
ffn_input=lambda hidden_states, residual, *args: (hidden_states, residual),
|
||||
)
|
||||
communicator._attn_input_fusions = ()
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -46,17 +46,22 @@ class DeferringLayer(nn.Module):
|
||||
super().__init__()
|
||||
self.return_topk = return_topk
|
||||
self.layer_communicator = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
self.layer_communicator._ffn_output = comm.StageOutput(
|
||||
comm.Layout(frozenset()),
|
||||
group=comm.SumGroup.TP,
|
||||
leaves_for_next_layer=True,
|
||||
self.layer_communicator._steps = comm.BoundarySteps(
|
||||
attention_input=comm.CommunicateSimpleFn._trivial,
|
||||
ffn_input=comm._mlp_input_norm,
|
||||
ffn_output=comm.StageOutput(
|
||||
comm.Layout(frozenset()),
|
||||
group=comm.SumGroup.TP,
|
||||
leaves_for_next_layer=True,
|
||||
),
|
||||
ffn_output_move=comm.CommunicateSummableTensorPairFn._trivial,
|
||||
ffn_sum_is_movable=True,
|
||||
)
|
||||
self.layer_communicator.should_fuse_mlp_allreduce_with_next_layer = (
|
||||
lambda batch: defer
|
||||
)
|
||||
self.layer_communicator._ffn_sum_moves_to_next_layer = lambda batch, **_: defer
|
||||
self.layer_communicator.is_last_layer = False
|
||||
self.layer_communicator._postprocess_scatters_to_local_tokens = False
|
||||
self.layer_communicator._sp_steps = None
|
||||
self.layer_communicator._input_scattered_steps = None
|
||||
self.layer_communicator.ffn_reduction_group = lambda: GROUP
|
||||
|
||||
Reference in New Issue
Block a user