[Refactor] Run every batch of a layer from one BoundarySteps (#41420)
This commit is contained in:
@@ -756,11 +756,17 @@ class LayerCommunicator:
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
force_layernorm_before_dp_gather
|
||||
)
|
||||
# The steps the layer's ordinary batches run.
|
||||
sides = self._declared_sides()
|
||||
if sides is not None:
|
||||
self._select_declared_boundaries(sides)
|
||||
else:
|
||||
self._select_boundaries_from_scatter_modes()
|
||||
self._steps = (
|
||||
_select_boundary_steps(
|
||||
sides,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
)
|
||||
if sides is not None
|
||||
else self._select_boundaries_from_scatter_modes()
|
||||
)
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
self._input_scattered_steps = (
|
||||
@@ -846,46 +852,13 @@ class LayerCommunicator:
|
||||
),
|
||||
)
|
||||
|
||||
def _select_declared_boundaries(self, sides: DecoderLayerSides) -> None:
|
||||
"""Choose the three boundary sides this layer owns from the
|
||||
declarations: the attention input, the attention -> FFN steps, and the
|
||||
FFN output's way on to the rows the layer hands on."""
|
||||
self._communicate_simple_fn = _select_attention_input_move(
|
||||
sides.input_rows, sides.attention
|
||||
)
|
||||
self._mlp_input, fused = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=sides.input_rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=self.force_layernorm_before_dp_gather,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
)
|
||||
self._mlp_input_may_return_new_residual = any(
|
||||
f.may_return_new_residual for f in fused
|
||||
)
|
||||
self._ffn_output = sides.ffn_output
|
||||
(
|
||||
self._postprocess_scatters_to_local_tokens,
|
||||
postprocess,
|
||||
) = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if postprocess is not None:
|
||||
self._communicate_summable_tensor_pair_fn = postprocess
|
||||
self._ffn_sum_is_movable = sides.ffn_output.group is not None
|
||||
|
||||
def _select_boundaries_from_scatter_modes(self) -> None:
|
||||
self._post_init_communicate()
|
||||
# Under attention DP, the base postprocess scatters the FFN output back
|
||||
# to this rank's tokens, which the next layer's input can run instead;
|
||||
# the MHC and DSA-CP postprocess do more, so theirs stays here.
|
||||
self._postprocess_scatters_to_local_tokens = (
|
||||
self._communicate_summable_tensor_pair_fn
|
||||
is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
)
|
||||
def _select_boundaries_from_scatter_modes(self) -> "BoundarySteps":
|
||||
"""The layer's steps chosen from its scatter modes, for the layers the
|
||||
declarations do not cover yet."""
|
||||
attention_input, postprocess = self._post_init_communicate()
|
||||
ffn_input, fused = self._select_mlp_input()
|
||||
modes = self.layer_scatter_modes
|
||||
self._ffn_output = StageOutput(
|
||||
ffn_output = StageOutput(
|
||||
self._context.layouts[modes.mlp_mode],
|
||||
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
leaves_for_next_layer=self.allow_deferred_ffn_reduction,
|
||||
@@ -897,30 +870,41 @@ class LayerCommunicator:
|
||||
self.allow_reduce_scatter or modes.is_layer_sparse
|
||||
),
|
||||
)
|
||||
# Whether the FFN sum is one the next layer's input can take: not when
|
||||
# the way back is the MoE-CP scatter, nor when a SCATTERED FFN computes
|
||||
# whole tokens with nothing left to sum.
|
||||
self._ffn_sum_is_movable = modes.mlp_mode not in (
|
||||
ScatterMode.MOE_FULL,
|
||||
ScatterMode.SCATTERED,
|
||||
)
|
||||
self._mlp_input, self._mlp_input_may_return_new_residual = (
|
||||
self._select_mlp_input()
|
||||
return BoundarySteps(
|
||||
attention_input=attention_input,
|
||||
ffn_input=ffn_input,
|
||||
ffn_output=ffn_output,
|
||||
# Under attention DP the base postprocess scatters the FFN output
|
||||
# back to this rank's tokens, which the next layer's input can run
|
||||
# instead; the MHC and DSA-CP postprocess do more.
|
||||
ffn_output_move=(
|
||||
None
|
||||
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
else postprocess
|
||||
),
|
||||
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
|
||||
# FFN computes whole tokens with nothing left to sum.
|
||||
ffn_sum_is_movable=modes.mlp_mode
|
||||
not in (ScatterMode.MOE_FULL, ScatterMode.SCATTERED),
|
||||
fused=fused,
|
||||
layer_input=_complete_scattered_input,
|
||||
)
|
||||
|
||||
def _post_init_communicate(self):
|
||||
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
)
|
||||
self._communicate_summable_tensor_pair_fn = (
|
||||
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
|
||||
"""The attention input move and the postprocess the scatter modes
|
||||
choose."""
|
||||
return (
|
||||
CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
),
|
||||
CommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
||||
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
||||
output_mode=self.layer_scatter_modes.layer_output_mode,
|
||||
context=self._context,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
def prepare_attn_and_capture_last_layer_outputs(
|
||||
@@ -940,7 +924,7 @@ class LayerCommunicator:
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
if captured_last_layer_outputs is not None:
|
||||
gathered_last_layer_output = self._communicate_simple_fn(
|
||||
gathered_last_layer_output = self._batch_steps().attention_input(
|
||||
hidden_states=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
@@ -966,7 +950,7 @@ class LayerCommunicator:
|
||||
the batch is not input-scattered.
|
||||
"""
|
||||
return (
|
||||
self._mlp_input_may_return_new_residual
|
||||
any(f.may_return_new_residual for f in self._batch_steps().fused)
|
||||
and not get_attn_tp_context().input_scattered
|
||||
and apply_flashinfer_allreduce_fusion(residual.shape[0])
|
||||
)
|
||||
@@ -1028,16 +1012,11 @@ class LayerCommunicator:
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
steps = self._batch_steps()
|
||||
if steps is not None and steps.layer_input is not None:
|
||||
hidden_states, residual = steps.layer_input(
|
||||
layer_input = self._batch_steps().layer_input
|
||||
if layer_input is not None:
|
||||
hidden_states, residual = layer_input(
|
||||
hidden_states, residual, self._context
|
||||
)
|
||||
elif get_attn_tp_context().input_scattered:
|
||||
hidden_states, residual = self._tp_reduce_scatter(
|
||||
hidden_states,
|
||||
residual,
|
||||
)
|
||||
if hidden_states.shape[0] == 0:
|
||||
residual = hidden_states
|
||||
elif owed is not None:
|
||||
@@ -1059,9 +1038,9 @@ class LayerCommunicator:
|
||||
def _in_sp_region(self) -> bool:
|
||||
return self._sp_steps is not None and get_forward().sp_active
|
||||
|
||||
def _batch_steps(self) -> Optional["BoundarySteps"]:
|
||||
"""The steps this batch runs in place of the layer's ordinary ones: an
|
||||
active LayerNorm SP region's, or input-scattered attention's."""
|
||||
def _batch_steps(self) -> "BoundarySteps":
|
||||
"""The steps this batch runs: an active LayerNorm SP region's,
|
||||
input-scattered attention's, or the layer's ordinary ones."""
|
||||
if self._in_sp_region():
|
||||
return self._sp_steps
|
||||
if (
|
||||
@@ -1069,13 +1048,11 @@ class LayerCommunicator:
|
||||
and get_attn_tp_context().input_scattered
|
||||
):
|
||||
return self._input_scattered_steps
|
||||
return None
|
||||
return self._steps
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
steps = self._batch_steps()
|
||||
move = self._communicate_simple_fn if steps is None else steps.attention_input
|
||||
hidden_states = move(
|
||||
hidden_states = self._batch_steps().attention_input(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
@@ -1145,30 +1122,23 @@ class LayerCommunicator:
|
||||
hidden_states, residual, use_attn_tp_group=False
|
||||
)
|
||||
|
||||
def _tp_reduce_scatter(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
return tp_reduce_scatter(hidden_states, residual, self._context)
|
||||
|
||||
def _select_mlp_input(self) -> Tuple[Callable, bool]:
|
||||
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
|
||||
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
|
||||
whether they include the aiter / flashinfer fused entry. The MHC and
|
||||
DSA-CP communicators pick their own implementations."""
|
||||
the fused kernels they try first. The MHC and DSA-CP communicators pick
|
||||
their own implementations."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
if kind is MlpInputKind.NORM:
|
||||
return _mlp_input_norm, False
|
||||
return _mlp_input_norm, ()
|
||||
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
|
||||
return _mlp_input_attn_tp_all_reduce, False
|
||||
return _mlp_input_attn_tp_all_reduce, ()
|
||||
if kind is MlpInputKind.SCATTER:
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
|
||||
),
|
||||
False,
|
||||
(),
|
||||
)
|
||||
# Neither fused kernel runs under attention DP.
|
||||
fusions = (
|
||||
@@ -1182,7 +1152,7 @@ class LayerCommunicator:
|
||||
)
|
||||
if kind is MlpInputKind.GATHER_MOE_CP:
|
||||
steps = partial(_mlp_input_gather_moe_cp, gather=steps)
|
||||
return steps, any(f.may_return_new_residual for f in fusions)
|
||||
return steps, fusions
|
||||
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
@@ -1222,20 +1192,10 @@ class LayerCommunicator:
|
||||
cache=None,
|
||||
):
|
||||
self.publish_mlp_lora_layout()
|
||||
if self._in_sp_region():
|
||||
return self._sp_steps.ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
self.post_attention_layernorm,
|
||||
self._context,
|
||||
)
|
||||
if cache is not None:
|
||||
self._context.cache = cache
|
||||
|
||||
steps = self._batch_steps()
|
||||
mlp_input = self._mlp_input if steps is None else steps.ffn_input
|
||||
return mlp_input(
|
||||
return self._batch_steps().ffn_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch,
|
||||
@@ -1257,17 +1217,10 @@ class LayerCommunicator:
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self._postprocess_scatters_to_local_tokens:
|
||||
if steps.returns_over_dp:
|
||||
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
return self._communicate_summable_tensor_pair_fn(
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
@@ -1278,9 +1231,8 @@ class LayerCommunicator:
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(self) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, outside
|
||||
an active LayerNorm SP region."""
|
||||
return self._postprocess_scatters_to_local_tokens and not self._in_sp_region()
|
||||
output back to this rank's tokens: the base postprocess scatter."""
|
||||
return self._batch_steps().returns_over_dp
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1288,12 +1240,13 @@ class LayerCommunicator:
|
||||
"""The reduce-scatter that brings this layer's FFN output back to this
|
||||
rank's tokens under attention DP; None when the base postprocess would
|
||||
only scatter, or does not move tokens."""
|
||||
if not self._postprocess_scatters_to_local_tokens:
|
||||
steps = self._batch_steps()
|
||||
if not steps.returns_over_dp:
|
||||
return None
|
||||
return _reduce_and_redistribute_output_step(
|
||||
forward_batch,
|
||||
leaves_for_reduce_scatter=self._ffn_output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=self._ffn_output.leaves_for_reduce_scatterv,
|
||||
leaves_for_reduce_scatter=steps.ffn_output.leaves_for_reduce_scatter,
|
||||
leaves_for_reduce_scatterv=steps.ffn_output.leaves_for_reduce_scatterv,
|
||||
)
|
||||
|
||||
def _ffn_leaves_sum_to_reduce_scatter(
|
||||
@@ -1302,10 +1255,7 @@ class LayerCommunicator:
|
||||
"""Whether the FFN leaves its sum out because a reduce-scatter completes
|
||||
it: the attention-DP one ``dp_step`` names, or the CP / input-scattered
|
||||
one."""
|
||||
steps = self._batch_steps()
|
||||
if steps is not None:
|
||||
return steps.ffn_output.leaves_for_reduce_scatter
|
||||
if not self._ffn_output.leaves_for_reduce_scatter:
|
||||
if not self._batch_steps().ffn_output.leaves_for_reduce_scatter:
|
||||
return False
|
||||
if dp_step is not None:
|
||||
return True
|
||||
@@ -1319,7 +1269,7 @@ class LayerCommunicator:
|
||||
def ffn_reduction_group(self) -> GroupCoordinator:
|
||||
"""The group this layer's FFN output owes its sum over: the MoE output's
|
||||
group on a sparse layer, the TP group a dense MLP reduces over."""
|
||||
return _sum_group(self._ffn_output.group)
|
||||
return _sum_group(self._batch_steps().ffn_output.group)
|
||||
|
||||
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> "FfnCompletion":
|
||||
"""Decide once, before the FFN runs, what it skips and what completes its
|
||||
@@ -1334,8 +1284,7 @@ class LayerCommunicator:
|
||||
dp_step=dp_step,
|
||||
)
|
||||
steps = self._batch_steps()
|
||||
ffn_output = self._ffn_output if steps is None else steps.ffn_output
|
||||
if not ffn_output.leaves_for_next_layer:
|
||||
if not steps.ffn_output.leaves_for_next_layer:
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
@@ -1349,7 +1298,7 @@ class LayerCommunicator:
|
||||
)
|
||||
if fuse_mlp_allreduce:
|
||||
group = self.ffn_reduction_group()
|
||||
if self._postprocess_scatters_to_local_tokens:
|
||||
if steps.returns_over_dp:
|
||||
# Under attention DP the next layer also brings the sum back to
|
||||
# this rank's tokens.
|
||||
wrap = partial(
|
||||
@@ -1435,7 +1384,7 @@ class LayerCommunicator:
|
||||
def _ffn_sum_can_move_to_next_layer(self) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip postprocess_layer
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if is_enable_moe_cp_allgather() or not self._ffn_sum_is_movable:
|
||||
if is_enable_moe_cp_allgather() or not self._batch_steps().ffn_sum_is_movable:
|
||||
return False
|
||||
|
||||
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
||||
@@ -1914,29 +1863,40 @@ def _token_axis_sizes() -> Dict[TokenAxis, int]:
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps one set of declarations chooses for a layer's boundaries:
|
||||
into the attention, from the attention output to the FFN input, and the
|
||||
FFN output on to the rows the layer hands on."""
|
||||
"""The steps a batch runs at a layer's boundaries: into the attention,
|
||||
from the attention output to the FFN input, and the FFN output on to the
|
||||
rows the layer hands on."""
|
||||
|
||||
attention_input: Callable
|
||||
ffn_input: Callable
|
||||
ffn_output_move: Callable
|
||||
# What the FFN exit reads in place of the layer's own FFN output declaration.
|
||||
# What the FFN exit reads: the FFN output's group and what it may leave.
|
||||
ffn_output: StageOutput
|
||||
# The postprocess that moves the FFN output on; None when it goes back over
|
||||
# attention DP, whose step the FFN exit and postprocess choose per batch.
|
||||
ffn_output_move: Optional[Callable]
|
||||
# Whether the next layer's input can take the FFN's sum.
|
||||
ffn_sum_is_movable: bool
|
||||
# The fused kernels ffn_input tries first.
|
||||
fused: Tuple["FusedMlpInput", ...] = ()
|
||||
# Completes what the layer's input owes before the input norm; None when it
|
||||
# owes nothing.
|
||||
layer_input: Optional[Callable] = None
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
return self.ffn_output_move is None
|
||||
|
||||
def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
|
||||
"""The steps for a batch's own declarations: nothing moves back over
|
||||
attention DP and no fused kernel runs, so the steps do not wait on a
|
||||
per-batch decision."""
|
||||
|
||||
def _select_boundary_steps(
|
||||
sides: DecoderLayerSides,
|
||||
*,
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
) -> BoundarySteps:
|
||||
"""The steps a set of declarations chooses."""
|
||||
returns_over_dp, ffn_output_move = _select_ffn_output_move(
|
||||
sides.ffn_output, residual=sides.ffn_residual_rows, to=sides.output_rows
|
||||
)
|
||||
if returns_over_dp or sides.ffn_output.leaves_for_next_layer:
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
rows = sides.input_rows
|
||||
layer_input = None
|
||||
if sides.input_owes is not None:
|
||||
@@ -1950,24 +1910,39 @@ def _select_boundary_steps(sides: DecoderLayerSides) -> BoundarySteps:
|
||||
raise NotImplementedError(f"{sides=}")
|
||||
layer_input = tp_reduce_scatter
|
||||
rows = Layout(frozenset({TokenAxis.ATTN_TP_SCATTER}))
|
||||
ffn_input, _ = _select_ffn_input(
|
||||
ffn_input, fused = _select_ffn_input(
|
||||
sides.attention_output,
|
||||
residual=rows,
|
||||
residual_to=sides.ffn_residual_rows,
|
||||
need=sides.ffn,
|
||||
force_layernorm_before_gather=False,
|
||||
fusions=(),
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=_select_attention_input_move(rows, sides.attention),
|
||||
ffn_input=ffn_input,
|
||||
ffn_output_move=ffn_output_move,
|
||||
ffn_output=sides.ffn_output,
|
||||
ffn_output_move=None if returns_over_dp else ffn_output_move,
|
||||
ffn_sum_is_movable=sides.ffn_output.group is not None,
|
||||
fused=fused,
|
||||
layer_input=layer_input,
|
||||
)
|
||||
|
||||
|
||||
def _complete_scattered_input(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
context: "CommunicateContext",
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""The scatter-mode path's input completion: with input-scattered
|
||||
attention the layer's input is a TP partial that a reduce-scatter completes
|
||||
onto this rank's slice."""
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return tp_reduce_scatter(hidden_states, residual, context)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _select_attention_input_move(rows: Layout, need: StageInput) -> Callable:
|
||||
"""How the rows a layer takes become its attention's input: as they are,
|
||||
or gathered over attention TP from each rank's slice, unless the attention
|
||||
|
||||
@@ -93,16 +93,18 @@ class DSACPLayerCommunicator(LayerCommunicator):
|
||||
assert self._context.attn_dp_size == 1, (
|
||||
f"dp_size should be 1 when moe_runner_backend is none"
|
||||
)
|
||||
self._communicate_simple_fn = DSACPCommunicateSimpleFn.get_fn(
|
||||
input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
)
|
||||
self._communicate_summable_tensor_pair_fn = DSACPCommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
|
||||
residual_input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
return (
|
||||
DSACPCommunicateSimpleFn.get_fn(
|
||||
input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
),
|
||||
DSACPCommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, # SCATTERED, FULL
|
||||
residual_input_mode=ScatterMode.SCATTERED,
|
||||
output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
),
|
||||
)
|
||||
|
||||
def _select_mlp_input(self):
|
||||
@@ -113,7 +115,7 @@ class DSACPLayerCommunicator(LayerCommunicator):
|
||||
residual_output_mode=ScatterMode.SCATTERED,
|
||||
context=self._context,
|
||||
)
|
||||
return fn, False
|
||||
return fn, ()
|
||||
|
||||
|
||||
class DSACPCommunicateSimpleFn(CommunicateSimpleFn):
|
||||
|
||||
@@ -441,6 +441,8 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
# MHC's own prepare_attn and postprocess run these.
|
||||
return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
@@ -494,14 +496,14 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
fns = MHCCommunicateWithAllReduceAndLayerNormFn
|
||||
if kind is MlpInputKind.NORM:
|
||||
return partial(fns._simple, mhc=self.mhc), False
|
||||
return partial(fns._simple, mhc=self.mhc), ()
|
||||
if kind is MlpInputKind.GATHER:
|
||||
fn = fns._gather_hidden_states_and_residual
|
||||
elif kind is MlpInputKind.SCATTER:
|
||||
fn = fns._scatter_hidden_states_and_residual
|
||||
else:
|
||||
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
|
||||
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), False
|
||||
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), ()
|
||||
|
||||
def postprocess_layer(self, hidden_states, residual, forward_batch):
|
||||
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
|
||||
|
||||
Reference in New Issue
Block a user