[Refactor] Run MHC layers on the shared boundary steps with MHC's residual operations (#41425)
This commit is contained in:
@@ -138,6 +138,52 @@ def sequence_parallel_layer_sides(
|
||||
)
|
||||
|
||||
|
||||
def scattered_residual_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
is_first_layer: bool,
|
||||
is_last_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on an input-scattered batch whose residual stays on
|
||||
each attention-TP rank's slice (MHC). The attention and the FFN compute on
|
||||
the full rows. The attention takes the slice and gathers it itself (its
|
||||
QKV hook, or the handoff before an attention that needs the rows); the
|
||||
boundary brings the FFN its rows and moves both outputs back onto the
|
||||
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
|
||||
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
|
||||
is the embedding's partial sum on the full rows; the last layer hands on
|
||||
the full rows again."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
ffn_output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=attention if is_last_layer else local,
|
||||
input_owes=SumGroup.TP if is_first_layer else None,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
|
||||
@@ -16,7 +16,7 @@ from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union
|
||||
from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.layers.boundary_layout import (
|
||||
TokenAxis,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
scattered_residual_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
@@ -651,6 +652,92 @@ def _attn_input_update_and_read_residual(quant_format: str):
|
||||
return _update_and_read_residual_plain
|
||||
|
||||
|
||||
class ResidualOps(Protocol):
|
||||
"""How a layer writes each stage's output into its residual and reads the
|
||||
next stage's input from it. The boundary steps complete sums and move
|
||||
tokens around these operations. AddAndNorm adds and normalizes; MHC's
|
||||
hyper-connection streams implement the same operations (communicator_mhc)."""
|
||||
|
||||
# The write-back is a plain add, which one rank may run before the sum it
|
||||
# adds into completes: the DP partial order, and a residual that joins the
|
||||
# attention output's sum.
|
||||
adds_plainly: bool
|
||||
# The layer writes its FFN output into the residual itself instead of
|
||||
# leaving that to the next layer's input.
|
||||
updates_residual_after_ffn: bool
|
||||
|
||||
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
|
||||
"""The attention input and the residual, from a residual that already
|
||||
holds the previous layer's output."""
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
|
||||
) -> Tuple:
|
||||
"""Write the previous layer's output into the residual, then read the
|
||||
attention input from it."""
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
|
||||
"""Write the attention output into the residual, then read the FFN
|
||||
input from it."""
|
||||
|
||||
def update_residual(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the FFN output written into it."""
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
|
||||
"""This attention-TP rank's slice of the residual."""
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
|
||||
"""The residual gathered from every attention-TP rank's slice."""
|
||||
|
||||
|
||||
class AddAndNorm:
|
||||
"""A plain residual: a stage's output is added into it, and the next
|
||||
stage's input is its norm. An empty batch skips the norms."""
|
||||
|
||||
adds_plainly = True
|
||||
updates_residual_after_ffn = False
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if residual.shape[0] == 0:
|
||||
return residual, residual
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, residual, None, None
|
||||
)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
)
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return _redistribute_to_attn_tp_shards(residual, context)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return _redistribute_from_attn_tp_shards(residual)
|
||||
|
||||
|
||||
ADD_AND_NORM = AddAndNorm()
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes its sum, left for the next layer's input.
|
||||
Hand it to the next layer, or pass it through reduce_output() before reading
|
||||
@@ -730,6 +817,8 @@ class LayerCommunicator:
|
||||
# Whether this class's boundary steps may be chosen from both sides'
|
||||
# declarations; a subclass that picks its own steps says no.
|
||||
_takes_declared_boundaries = True
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual_ops: ResidualOps = ADD_AND_NORM
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -744,6 +833,8 @@ class LayerCommunicator:
|
||||
fused_ar_quant_keep_bf16: bool = False,
|
||||
# False for a layer whose FFN always completes its own reduction.
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
# How the layer writes its residual and reads its stages' inputs.
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.input_layernorm = input_layernorm
|
||||
@@ -755,6 +846,7 @@ class LayerCommunicator:
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
||||
self._residual_ops = residual_ops
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
@@ -762,17 +854,8 @@ class LayerCommunicator:
|
||||
)
|
||||
# The steps the layer's ordinary batches run.
|
||||
sides = self._declared_sides()
|
||||
if (
|
||||
sides is None
|
||||
and self._takes_declared_boundaries
|
||||
and (_generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp())
|
||||
):
|
||||
# The scatter-mode steps have no attention-CP gather.
|
||||
raise NotImplementedError(
|
||||
"a DSA or MLA prefill CP layer outside the declarations"
|
||||
)
|
||||
self._steps = (
|
||||
_select_boundary_steps(
|
||||
self._steps_from_declarations(
|
||||
sides,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
@@ -783,7 +866,7 @@ class LayerCommunicator:
|
||||
# The steps a batch that shards its tokens over attention CP runs; None
|
||||
# without CP or where the steps come from the scatter modes.
|
||||
self._cp_steps = (
|
||||
_select_boundary_steps(
|
||||
self._steps_from_declarations(
|
||||
self._declared_sides(cp_active=True),
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
@@ -795,14 +878,7 @@ class LayerCommunicator:
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
self._input_scattered_steps = (
|
||||
_select_boundary_steps(
|
||||
input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=allow_reduce_scatter
|
||||
and not layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
)
|
||||
self._steps_for_input_scattered(sides)
|
||||
if sides is not None and self._input_can_be_scattered()
|
||||
else None
|
||||
)
|
||||
@@ -820,7 +896,8 @@ class LayerCommunicator:
|
||||
# q_lora, which input-scattered attention needs.
|
||||
self._sp_steps = (
|
||||
_select_boundary_steps(
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes())
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()),
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
if layernorm_sp.layernorm_sp_enabled()
|
||||
else None
|
||||
@@ -918,6 +995,7 @@ class LayerCommunicator:
|
||||
attention_input, postprocess = self._post_init_communicate()
|
||||
ffn_input, fused = self._select_mlp_input()
|
||||
modes = self.layer_scatter_modes
|
||||
ops = self._residual_ops
|
||||
ffn_output = StageOutput(
|
||||
self._context.layouts[modes.mlp_mode],
|
||||
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
@@ -940,7 +1018,7 @@ class LayerCommunicator:
|
||||
ffn_output_move=(
|
||||
None
|
||||
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
else postprocess
|
||||
else partial(postprocess, residual_ops=ops)
|
||||
),
|
||||
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
|
||||
# FFN computes whole tokens with nothing left to sum.
|
||||
@@ -950,9 +1028,46 @@ class LayerCommunicator:
|
||||
layer_input=_complete_scattered_input,
|
||||
)
|
||||
|
||||
def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps":
|
||||
"""The steps a batch with input-scattered attention runs at this layer,
|
||||
for the layer's ordinary declarations ``sides``. A plain residual comes
|
||||
back to the full rows inside the attention output's sum; one whose
|
||||
write-back is not a plain add stays on each rank's slice."""
|
||||
if self._residual_ops.adds_plainly:
|
||||
scattered = input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
|
||||
)
|
||||
handoff = _hand_qkv_hook_its_input
|
||||
else:
|
||||
scattered = scattered_residual_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
is_first_layer=self.layer_scatter_modes.is_first_layer,
|
||||
is_last_layer=self.is_last_layer,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
)
|
||||
# DSA and hook-less attention take the slice gathered.
|
||||
handoff = _hand_scattered_input_to_attention
|
||||
return _select_boundary_steps(
|
||||
scattered, residual_ops=self._residual_ops, attention_handoff=handoff
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
self, sides: DecoderLayerSides, **kwargs
|
||||
) -> "BoundarySteps":
|
||||
"""The steps this layer runs for a set of declarations."""
|
||||
return _select_boundary_steps(sides, residual_ops=self._residual_ops, **kwargs)
|
||||
|
||||
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
|
||||
"""The attention input move and the postprocess the scatter modes
|
||||
choose."""
|
||||
if _generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp():
|
||||
# These tables have no attention-CP gather.
|
||||
raise NotImplementedError(
|
||||
"a DSA or MLA prefill CP layer outside the declarations"
|
||||
)
|
||||
return (
|
||||
CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
@@ -1048,7 +1163,8 @@ class LayerCommunicator:
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
self.publish_attn_lora_layout()
|
||||
|
||||
# The layer stack's first layer starts its residual from its input.
|
||||
enters = residual is None and self.layer_scatter_modes.is_first_layer
|
||||
# What the previous layer left to complete: an UnreducedOutput, or a
|
||||
# producer's handoff that one of the fused entries consumes.
|
||||
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
|
||||
@@ -1084,17 +1200,24 @@ class LayerCommunicator:
|
||||
hidden_states, residual = layer_input(
|
||||
hidden_states, residual, self._context
|
||||
)
|
||||
if hidden_states.shape[0] == 0:
|
||||
residual = hidden_states
|
||||
elif owed is not None:
|
||||
ops = self._residual_ops
|
||||
if enters:
|
||||
hidden_states, residual = ops.enter(hidden_states), None
|
||||
if owed is not None and hidden_states.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
hidden_states, residual = _attn_input_update_and_read_residual(
|
||||
quant_format
|
||||
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
hidden_states, residual = ops.read_attention_input(
|
||||
hidden_states, self.input_layernorm, quant_format
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = _attn_input_update_and_read_residual(
|
||||
quant_format
|
||||
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
||||
hidden_states, residual = ops.update_and_read_attention_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.input_layernorm,
|
||||
quant_format,
|
||||
post_residual_addition,
|
||||
)
|
||||
|
||||
return self._finish_prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
@@ -1122,24 +1245,27 @@ class LayerCommunicator:
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
hidden_states = self._batch_steps(forward_batch).attention_input(
|
||||
steps = self._batch_steps(forward_batch)
|
||||
hidden_states = steps.attention_input(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
)
|
||||
get_attn_tp_context().set_attn_inputs(attn_inputs)
|
||||
hidden_states = steps.attention_handoff(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete what the previous layer left together
|
||||
with the residual update and the input norm, in the order they are tried.
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch."""
|
||||
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly."""
|
||||
if not (
|
||||
self._residual_ops.adds_plainly
|
||||
and hasattr(self.input_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
return ()
|
||||
self._attn_input_fuses_quant = (
|
||||
self.enable_fused_ar_quant
|
||||
@@ -1194,19 +1320,20 @@ class LayerCommunicator:
|
||||
|
||||
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
|
||||
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
|
||||
the fused kernels they try first. The MHC and DSA-CP communicators pick
|
||||
their own implementations."""
|
||||
the fused kernels they try first."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
ops = self._residual_ops
|
||||
if kind is MlpInputKind.NORM:
|
||||
return _mlp_input_norm, ()
|
||||
return partial(_mlp_input_norm, residual_ops=ops), ()
|
||||
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
|
||||
return _mlp_input_attn_tp_all_reduce, ()
|
||||
return partial(_mlp_input_attn_tp_all_reduce, residual_ops=ops), ()
|
||||
if kind is MlpInputKind.SCATTER:
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
|
||||
residual_ops=ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -1217,7 +1344,10 @@ class LayerCommunicator:
|
||||
steps = partial(
|
||||
_mlp_input_gather,
|
||||
order=_mlp_input_order(
|
||||
self._context, residual_input_mode, tuple(f.run for f in fusions)
|
||||
self._context,
|
||||
residual_input_mode,
|
||||
tuple(f.run for f in fusions),
|
||||
residual_ops=ops,
|
||||
),
|
||||
)
|
||||
if kind is MlpInputKind.GATHER_MOE_CP:
|
||||
@@ -1226,8 +1356,11 @@ class LayerCommunicator:
|
||||
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried."""
|
||||
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
|
||||
order they are tried. They add the residual plainly."""
|
||||
if not (
|
||||
self._residual_ops.adds_plainly
|
||||
and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
return ()
|
||||
return (
|
||||
FusedMlpInput(
|
||||
@@ -1286,25 +1419,23 @@ class LayerCommunicator:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
steps = self._batch_steps(forward_batch)
|
||||
if steps.returns_over_dp:
|
||||
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
return self._complete_ffn_output_now(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
dp_step=self._postprocess_dp_step(forward_batch),
|
||||
)
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter."""
|
||||
return self._batch_steps(forward_batch).returns_over_dp
|
||||
output back to this rank's tokens: the base postprocess scatter, when
|
||||
the next layer's input also writes the output into the residual."""
|
||||
return (
|
||||
self._batch_steps(forward_batch).returns_over_dp
|
||||
and not self._residual_ops.updates_residual_after_ffn
|
||||
)
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1418,16 +1549,27 @@ class LayerCommunicator:
|
||||
forward_batch: ForwardBatch,
|
||||
dp_step: Optional[Callable],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This layer's postprocess, run with the attention-DP step the FFN exit
|
||||
chose."""
|
||||
if self._local_token_move_can_go_to_next_layer(forward_batch):
|
||||
return (
|
||||
_to_local_tokens(
|
||||
dp_step or _redistribute_output, forward_batch, hidden_states
|
||||
),
|
||||
residual,
|
||||
"""This layer's postprocess, run with the attention-DP step already
|
||||
chosen: the move back to where the next layer reads the FFN output, then
|
||||
the write-back into the residual for a layer that does it itself."""
|
||||
steps = self._batch_steps(forward_batch)
|
||||
if steps.returns_over_dp:
|
||||
hidden_states = _to_local_tokens(
|
||||
dp_step or _redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
return self.postprocess_layer(hidden_states, residual, forward_batch)
|
||||
else:
|
||||
hidden_states, residual = steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
)
|
||||
if residual is not None and self._residual_ops.updates_residual_after_ffn:
|
||||
hidden_states = self._residual_ops.update_residual(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
@@ -1751,7 +1893,9 @@ class CommunicateSimpleFn:
|
||||
|
||||
|
||||
def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
|
||||
gathered = get_local_dp_buffer(get_parallel().attn_tp_group)
|
||||
gathered = get_local_dp_buffer(
|
||||
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(gathered, tensor)
|
||||
return gathered
|
||||
|
||||
@@ -1808,10 +1952,11 @@ def _redistribute_input_to_moe_cp(
|
||||
|
||||
|
||||
def _mlp_input_reduce_output(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
not forward_batch.forward_mode.is_decode_or_idle()
|
||||
may_quantize
|
||||
and not forward_batch.forward_mode.is_decode_or_idle()
|
||||
and get_exec().comm.enable_quant_communications
|
||||
):
|
||||
return attention_tensor_model_parallel_quant_all_reduce(hidden_states)
|
||||
@@ -1873,17 +2018,21 @@ def _mlp_input_without_dp(
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch)
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
|
||||
)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
return layernorm(hidden_states, residual)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_dp_replicate(
|
||||
@@ -1896,12 +2045,14 @@ def _mlp_input_dp_replicate(
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, add and
|
||||
normalize locally, then gather. With ``places_cp_shards`` each CP rank puts
|
||||
its shard of the DP group's tokens beside the others' in the group's slot."""
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, write the
|
||||
output into the residual and read the FFN input locally, then gather. With
|
||||
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
|
||||
beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
@@ -1909,7 +2060,13 @@ def _mlp_input_dp_replicate(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
)
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
return (
|
||||
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
|
||||
@@ -1948,6 +2105,7 @@ def _mlp_input_order(
|
||||
context: CommunicateContext,
|
||||
residual_input_mode: ScatterMode,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Callable:
|
||||
"""The steps from the attention output to the FFN input, chosen from facts
|
||||
fixed at construction."""
|
||||
@@ -1956,13 +2114,21 @@ def _mlp_input_order(
|
||||
)
|
||||
if context.attn_dp_size == 1:
|
||||
return partial(
|
||||
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=fusions,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1:
|
||||
if (
|
||||
context.force_layernorm_before_dp_gather
|
||||
or context.attn_tp_size == 1
|
||||
or not residual_ops.adds_plainly
|
||||
):
|
||||
return partial(
|
||||
_mlp_input_dp_replicate,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=context.attn_tp_size > 1,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
|
||||
|
||||
@@ -2049,6 +2215,53 @@ def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool:
|
||||
return sorted(a.ranks) == sorted(b.ranks)
|
||||
|
||||
|
||||
def _hand_qkv_hook_its_input(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Give the attention's QKV hook the attention input."""
|
||||
if qkv_latent_func is not None:
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _hand_scattered_input_to_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Input-scattered attention takes each rank's slice, and its QKV hook
|
||||
gathers the rows after the projection. DSA and attention without a hook
|
||||
consume full hidden states, so those are gathered here, and the hook is
|
||||
told they are."""
|
||||
ctx = get_attn_tp_context()
|
||||
if ctx.is_dsa or qkv_latent_func is None:
|
||||
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
|
||||
if qkv_latent_func is not None:
|
||||
ctx.set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
qkv_latent_func,
|
||||
is_pre_gathered=ctx.is_dsa,
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _tp_all_gather_scattered_rows(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
# Input-scattered attention keeps the same number of tokens on every TP rank.
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps a batch runs at a layer's boundaries: into the attention,
|
||||
from the attention output to the FFN input, and the FFN output on to the
|
||||
@@ -2070,6 +2283,9 @@ class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
# Completes what the layer's input owes before the input norm; None when it
|
||||
# owes nothing.
|
||||
layer_input: Optional[Callable] = None
|
||||
# Hands the attention its input once attention_input has moved it:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
@@ -2082,14 +2298,17 @@ def _select_boundary_steps(
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input,
|
||||
) -> BoundarySteps:
|
||||
"""The steps a set of declarations chooses; ``cp_moves`` for the ones
|
||||
that gather over attention CP."""
|
||||
"""The steps a set of declarations chooses, around the layer's residual
|
||||
operations; ``cp_moves`` for the ones that gather over attention CP."""
|
||||
returns_over_dp, ffn_output_move, completes_sum = _select_ffn_output_move(
|
||||
sides.ffn_output,
|
||||
residual=sides.ffn_residual_rows,
|
||||
to=sides.output_rows,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
rows = sides.input_rows
|
||||
layer_input = None
|
||||
@@ -2113,6 +2332,7 @@ def _select_boundary_steps(
|
||||
fusions=fusions,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=_select_attention_input_move(rows, sides.attention),
|
||||
@@ -2123,6 +2343,7 @@ def _select_boundary_steps(
|
||||
ffn_sum_is_movable=sides.ffn_output.group is not None,
|
||||
fused=fused,
|
||||
layer_input=layer_input,
|
||||
attention_handoff=attention_handoff,
|
||||
)
|
||||
|
||||
|
||||
@@ -2164,13 +2385,16 @@ def _select_ffn_input(
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
the rows it has while the FFN runs, add it and normalize, and bring the rows
|
||||
to what the FFN's group needs: a gather over attention DP, or each rank's
|
||||
own slice. A kernel in ``fusions`` is tried only when nothing is gathered or
|
||||
sliced, and only if it completes the sum the attention output owes."""
|
||||
the rows it has while the FFN runs, write the output into it and read the
|
||||
FFN input, and bring the rows to what the FFN's group needs: a gather over
|
||||
attention DP, or each rank's own slice. A kernel in ``fusions`` is tried
|
||||
only when nothing is gathered or sliced, and only if it completes the sum
|
||||
the attention output owes. A write-back that is not a plain add runs only
|
||||
after the sum completes."""
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
# always left by the output projection, or nothing.
|
||||
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
||||
@@ -2193,9 +2417,19 @@ def _select_ffn_input(
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(_mlp_input_scatter, scatters_residual=residual != residual_to),
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual != residual_to,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# The residual stays on each rank's slice while the FFN takes the
|
||||
# attention's rows (MHC on an input-scattered batch).
|
||||
if gathered or not owes_attention_tp or residual != residual_to:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
@@ -2208,6 +2442,7 @@ def _select_ffn_input(
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused
|
||||
if (
|
||||
@@ -2226,10 +2461,12 @@ def _select_ffn_input(
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return _mlp_input_norm, ()
|
||||
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if not residual_ops.adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return _mlp_input_residual_into_sum, ()
|
||||
fused = tuple(f for f in fusions if f.completes is produced.group)
|
||||
return (
|
||||
@@ -2237,6 +2474,7 @@ def _select_ffn_input(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
fused,
|
||||
)
|
||||
@@ -2246,7 +2484,11 @@ def _select_ffn_input(
|
||||
# Over attention DP and CP, the DP gather puts each CP rank's shard in its
|
||||
# DP group's slot, so the one DP sum gathers both axes.
|
||||
places_cp_shards = TokenAxis.ATTN_CP in gathered
|
||||
if owes_attention_tp and not force_layernorm_before_gather:
|
||||
if (
|
||||
owes_attention_tp
|
||||
and not force_layernorm_before_gather
|
||||
and residual_ops.adds_plainly
|
||||
):
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_dp_partial,
|
||||
@@ -2261,6 +2503,7 @@ def _select_ffn_input(
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
places_cp_shards=places_cp_shards,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -2272,21 +2515,38 @@ def _select_ffn_output_move(
|
||||
residual: Layout,
|
||||
to: Layout,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Tuple[bool, Optional[Callable], bool]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
that step), or else the postprocess that moves it, None when there is none
|
||||
to choose here; and whether that move also completes the sum the FFN
|
||||
leaves."""
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
if produced.layout == residual:
|
||||
if to == residual:
|
||||
return False, CommunicateSummableTensorPairFn._trivial, False
|
||||
return False, pair._trivial, False
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: fold the residual
|
||||
# into the output, then gather over attention TP.
|
||||
return False, CommunicateSummableTensorPairFn._gather, False
|
||||
# Each rank's slice back to the attention's rows: write the output
|
||||
# into the residual, then gather over attention TP.
|
||||
return False, partial(pair._gather, residual_ops=residual_ops), False
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
|
||||
# The residual stays on each rank's slice (MHC on an input-scattered
|
||||
# batch): a reduce-scatter onto the slice completes the sum the FFN
|
||||
# leaves; a complete output is only sliced.
|
||||
sums = produced.leaves_for_reduce_scatter
|
||||
return (
|
||||
False,
|
||||
partial(
|
||||
pair._onto_residual_shard,
|
||||
sums=sums,
|
||||
gathers_back=to != residual,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
sums,
|
||||
)
|
||||
if to != residual or not produced.layout.sharded <= residual.sharded:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
if returned == {TokenAxis.ATTN_CP}:
|
||||
@@ -2371,11 +2631,10 @@ def _mlp_input_norm(
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
# TODO move these `if shape != 0` into LayerNorm itself
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_attn_tp_all_reduce(
|
||||
@@ -2384,6 +2643,8 @@ def _mlp_input_attn_tp_all_reduce(
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
"""All-reduce hidden states inside the attention TP group, then layernorm.
|
||||
|
||||
@@ -2392,9 +2653,7 @@ def _mlp_input_attn_tp_all_reduce(
|
||||
``TP_ATTN_FULL`` across the boundary.
|
||||
"""
|
||||
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
@@ -2405,14 +2664,35 @@ def _mlp_input_scatter(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states, context
|
||||
)
|
||||
if scatters_residual:
|
||||
residual = _redistribute_to_attn_tp_shards(residual, context)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps,
|
||||
):
|
||||
"""The residual stays on each rank's slice while the FFN takes the full
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
its sum, write it into the residual and read the FFN input there, then
|
||||
gather the input back into the attention output's rows."""
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -2681,11 +2961,41 @@ class CommunicateSummableTensorPairFn:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states += residual
|
||||
hidden_states = residual_ops.update_residual(hidden_states, residual)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
def _onto_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
residual_ops: ResidualOps,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
rank's slice of the complete output. With ``gathers_back`` it is
|
||||
written into the residual there and the full rows are gathered back."""
|
||||
if sums:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
|
||||
else:
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = residual_ops.update_residual(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -12,62 +12,28 @@
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.distributed.communication_op import (
|
||||
attention_tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis
|
||||
from sglang.srt.layers.communicator import (
|
||||
AttentionInputs,
|
||||
CommunicateContext,
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
FfnCompletion,
|
||||
BoundarySteps,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
MlpInputKind,
|
||||
ScatterMode,
|
||||
get_attn_tp_context,
|
||||
mlp_input_kind,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
dp_gather_replicate,
|
||||
dp_reduce_scatter_tensor,
|
||||
dp_scatter,
|
||||
get_dp_global_num_tokens,
|
||||
get_global_dp_buffer,
|
||||
get_local_dp_buffer_mhc,
|
||||
is_allocation_symmetric,
|
||||
)
|
||||
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def tp_all_gather_hidden_states(hidden_states, forward_batch):
|
||||
assert get_attn_tp_context().input_scattered, (
|
||||
"Input scattered guarantees same num tokens in TP group."
|
||||
)
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@dataclass
|
||||
class MHCState:
|
||||
"""Parameters belong to the owning layer; this state only holds scratch
|
||||
"""A layer's residual as hyper-connection streams, with the residual
|
||||
operations the boundary steps run (communicator.ResidualOps): the residual
|
||||
is hc_mult streams, a stage's output is written in with hc_post and the
|
||||
next stage's input read with hc_pre and the norm. A read produces the
|
||||
h_res / h_post the next write-back consumes; they move with the tokens.
|
||||
Parameters belong to the owning layer; this state only holds scratch
|
||||
shared across communication stages."""
|
||||
|
||||
hc_mult: int
|
||||
@@ -75,9 +41,19 @@ class MHCState:
|
||||
hc_ffn_pre: Callable
|
||||
hc_post: Callable
|
||||
hc_ffn_post_pre: Optional[Callable] = None
|
||||
# The last layer's write-back also contracts the streams into the hidden
|
||||
# states the layer stack hands on.
|
||||
is_last_layer: bool = False
|
||||
h_res: Optional[torch.Tensor] = None
|
||||
h_post: Optional[torch.Tensor] = None
|
||||
|
||||
# hc_post is not a plain add, so it runs only once the sum it writes in
|
||||
# is complete.
|
||||
adds_plainly = False
|
||||
# The FFN output's write-back uses the coefficients this layer's FFN input
|
||||
# read produced, so the layer runs it itself.
|
||||
updates_residual_after_ffn = True
|
||||
|
||||
@staticmethod
|
||||
def _resolve_out_norm(out_norm):
|
||||
if out_norm is None:
|
||||
@@ -131,261 +107,45 @@ class MHCState:
|
||||
self.h_res = None
|
||||
self.h_post = None
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hc_expand(hidden_states, self.hc_mult)
|
||||
|
||||
class MHCCommunicateWithAllReduceAndLayerNormFn:
|
||||
@staticmethod
|
||||
def _scatter_hidden_states_and_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_input_mode,
|
||||
mhc: MHCState,
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if quant_format:
|
||||
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
|
||||
return self.attn_split(residual, out_norm=norm)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
input_hidden_states = hidden_states
|
||||
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
attn_tp_reduce_scatter_tensor(hidden_states, input_hidden_states)
|
||||
if residual_input_mode == ScatterMode.TP_ATTN_FULL:
|
||||
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
||||
mhc.h_res = mhc.h_res.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
mhc.h_post = mhc.h_post.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
# An MHC layer's input arrives with the previous FFN output written in.
|
||||
raise NotImplementedError("an MHC layer takes its input written back")
|
||||
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
return self.attn_to_mlp(hidden_states, residual, out_norm=norm)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
hidden_states = self.mlp_combine(hidden_states, residual)
|
||||
self.reset_aux()
|
||||
if self.is_last_layer:
|
||||
hidden_states = hc_contract(hidden_states, self.hc_mult)
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
rank, size = context.attn_tp_rank, context.attn_tp_size
|
||||
self.h_res = self.h_res.tensor_split(size)[rank]
|
||||
self.h_post = self.h_post.tensor_split(size)[rank]
|
||||
return residual.tensor_split(size)[rank]
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
raise NotImplementedError(
|
||||
"Unsupported: h_res/h_post allgather not implemented."
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _simple(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
):
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
|
||||
scatter_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(scatter_states, hidden_states)
|
||||
|
||||
scatter_states, residual = mhc.attn_to_mlp(
|
||||
scatter_states, residual, out_norm=layernorm
|
||||
)
|
||||
|
||||
attn_tp_all_gather_into_tensor(hidden_states, scatter_states)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _gather_hidden_states_and_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_input_mode,
|
||||
mhc: MHCState,
|
||||
):
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return MHCCommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
layernorm,
|
||||
context,
|
||||
mhc=mhc,
|
||||
)
|
||||
|
||||
if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Unsupported: h_res/h_post allgather not implemented."
|
||||
)
|
||||
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
if context.attn_dp_size != 1:
|
||||
if hidden_states.shape[0] != 0:
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = mhc.attn_to_mlp(hidden_states, residual)
|
||||
|
||||
hidden_states, local_hidden_states = (
|
||||
get_global_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
)
|
||||
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class MHCCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn):
|
||||
@staticmethod
|
||||
def get_fn(
|
||||
hidden_states_input_mode: ScatterMode,
|
||||
residual_input_mode: ScatterMode,
|
||||
output_mode: ScatterMode,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
fn = CommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode,
|
||||
residual_input_mode,
|
||||
output_mode,
|
||||
context,
|
||||
)
|
||||
replacements = {
|
||||
CommunicateSummableTensorPairFn._trivial: MHCCommunicateSummableTensorPairFn._trivial,
|
||||
CommunicateSummableTensorPairFn._scatter_hidden_states: MHCCommunicateSummableTensorPairFn._scatter_hidden_states,
|
||||
CommunicateSummableTensorPairFn._gather: MHCCommunicateSummableTensorPairFn._gather,
|
||||
CommunicateSummableTensorPairFn._scatter: MHCCommunicateSummableTensorPairFn._scatter,
|
||||
}
|
||||
return replacements.get(fn, fn)
|
||||
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
if get_attn_tp_context().input_scattered:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
|
||||
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
if not is_last_layer:
|
||||
return hidden_states, None
|
||||
|
||||
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
|
||||
if get_attn_tp_context().input_scattered:
|
||||
local_states = hidden_states
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter_hidden_states(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
allow_reduce_scatter: bool = False,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states, global_hidden_states = (
|
||||
get_local_dp_buffer_mhc(get_parallel().tp_group, 1),
|
||||
hidden_states,
|
||||
)
|
||||
# MoE skips its post-expert all-reduce with reduce_scatterv, so this
|
||||
# scatter must reduce while combining local-expert partial sums.
|
||||
if should_use_dp_reduce_scatterv():
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
global_hidden_states,
|
||||
output=hidden_states,
|
||||
sizes=get_dp_global_num_tokens(),
|
||||
)
|
||||
elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len():
|
||||
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
|
||||
else:
|
||||
dp_scatter(hidden_states, global_hidden_states, forward_batch)
|
||||
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
if not is_last_layer:
|
||||
return hidden_states, None
|
||||
|
||||
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
if is_last_layer:
|
||||
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
|
||||
|
||||
hidden_states, local_hidden_states = (
|
||||
get_local_dp_buffer_mhc(
|
||||
get_parallel().tp_group, 1 if is_last_layer else mhc.hc_mult
|
||||
),
|
||||
hidden_states,
|
||||
)
|
||||
|
||||
attn_tp_all_gather_into_tensor(hidden_states, local_hidden_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
||||
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
|
||||
return hidden_states, None
|
||||
|
||||
|
||||
class MHCLayerCommunicator(LayerCommunicator):
|
||||
# Chooses its own boundary steps, not from the declarations.
|
||||
_takes_declared_boundaries = False
|
||||
"""A layer whose residual is hyper-connection streams: the shared boundary
|
||||
steps, run with MHCState's residual operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -395,158 +155,50 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
allow_reduce_scatter: bool = False,
|
||||
qkv_latent_func: Optional[Callable] = None,
|
||||
*,
|
||||
is_first_layer: bool,
|
||||
hc_mult: int,
|
||||
hc_attn_pre: Callable,
|
||||
hc_ffn_pre: Callable,
|
||||
hc_post: Callable,
|
||||
hc_ffn_post_pre: Optional[Callable] = None,
|
||||
):
|
||||
self.is_first_layer = is_first_layer
|
||||
self.mhc = MHCState(
|
||||
hc_mult=hc_mult,
|
||||
hc_attn_pre=hc_attn_pre,
|
||||
hc_ffn_pre=hc_ffn_pre,
|
||||
hc_post=hc_post,
|
||||
hc_ffn_post_pre=hc_ffn_post_pre,
|
||||
is_last_layer=layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
|
||||
# The postprocess writes the FFN output into the streams, so the FFN's
|
||||
# sum never waits for the next layer.
|
||||
super().__init__(
|
||||
layer_scatter_modes,
|
||||
input_layernorm,
|
||||
post_attention_layernorm,
|
||||
allow_reduce_scatter,
|
||||
qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
residual_ops=self.mhc,
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
self, sides: DecoderLayerSides, **kwargs
|
||||
) -> BoundarySteps:
|
||||
# MHC has not been run with an FFN input gathered over attention CP.
|
||||
if (
|
||||
TokenAxis.ATTN_CP
|
||||
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
|
||||
)
|
||||
return super()._steps_from_declarations(sides, **kwargs)
|
||||
|
||||
def _post_init_communicate(self):
|
||||
# Base MOE_FULL callables do not accept ``mhc``, so reject this
|
||||
# combination at construction.
|
||||
if self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL:
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator does not support MOE_FULL "
|
||||
"(moe_dp_size < attention_context_parallel_size). Increase "
|
||||
"moe_dp_size to match attention_context_parallel_size."
|
||||
)
|
||||
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
)
|
||||
self._communicate_summable_tensor_pair_fn = (
|
||||
MHCCommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
||||
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
||||
output_mode=self.layer_scatter_modes.layer_output_mode,
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
# MHC's own prepare_attn and postprocess run these.
|
||||
return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
self.publish_attn_lora_layout()
|
||||
if self.is_first_layer:
|
||||
if get_attn_tp_context().input_scattered:
|
||||
hidden_states, _ = tp_reduce_scatter(
|
||||
hidden_states,
|
||||
None,
|
||||
self._context,
|
||||
)
|
||||
hidden_states = hc_expand(hidden_states, self.mhc.hc_mult)
|
||||
|
||||
hidden_states, residual = self.mhc.attn_split(
|
||||
hidden_states, out_norm=self.input_layernorm
|
||||
)
|
||||
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
|
||||
# DSA and attention without a QKV hook consume full hidden states, so
|
||||
# gather them before attention.
|
||||
ctx = get_attn_tp_context()
|
||||
dsa_pre_gather = ctx.input_scattered and ctx.is_dsa
|
||||
no_qkv_latent_pre_gather = ctx.input_scattered and self.qkv_latent_func is None
|
||||
if dsa_pre_gather or no_qkv_latent_pre_gather:
|
||||
hidden_states = tp_all_gather_hidden_states(hidden_states, forward_batch)
|
||||
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
self.qkv_latent_func,
|
||||
is_pre_gathered=dsa_pre_gather,
|
||||
)
|
||||
ctx.set_attn_inputs(attn_inputs)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_mlp_input(self):
|
||||
"""MHC's own implementation of each boundary kind, applied to this
|
||||
layer's MHC state."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
fns = MHCCommunicateWithAllReduceAndLayerNormFn
|
||||
if kind is MlpInputKind.NORM:
|
||||
return partial(fns._simple, mhc=self.mhc), ()
|
||||
if kind is MlpInputKind.GATHER:
|
||||
fn = fns._gather_hidden_states_and_residual
|
||||
elif kind is MlpInputKind.SCATTER:
|
||||
fn = fns._scatter_hidden_states_and_residual
|
||||
else:
|
||||
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
|
||||
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), ()
|
||||
|
||||
def postprocess_layer(self, hidden_states, residual, forward_batch):
|
||||
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
mhc=self.mhc,
|
||||
is_last_layer=self.is_last_layer,
|
||||
)
|
||||
self.mhc.reset_aux()
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
|
||||
"""An MHC layer's own postprocess completes its FFN output, combining the
|
||||
hyper-connection streams; nothing is left to the next layer."""
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=self.should_use_reduce_scatter(forward_batch),
|
||||
complete=partial(
|
||||
self._complete_ffn_output_now,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=None,
|
||||
),
|
||||
)
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
if not self.allow_reduce_scatter:
|
||||
return False
|
||||
if (
|
||||
self._communicate_summable_tensor_pair_fn
|
||||
is MHCCommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
):
|
||||
# reduce_scatterv already combines expert outputs; returning False
|
||||
# would make RowParallelLinear perform an extra all-reduce.
|
||||
if should_use_dp_reduce_scatterv():
|
||||
return True
|
||||
if forward_batch.dp_padding_mode.is_max_len():
|
||||
return True
|
||||
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return True
|
||||
return False
|
||||
return super()._post_init_communicate()
|
||||
|
||||
@@ -258,27 +258,14 @@ class _DpGatheredBufferWrapper:
|
||||
return buffer
|
||||
|
||||
@classmethod
|
||||
def get_local_dp_buffer(cls, group: GroupCoordinator) -> torch.Tensor:
|
||||
|
||||
dp = get_flags().dp
|
||||
with use_symmetric_memory(group, disabled=not cls._dp_max_padding):
|
||||
buffer = torch.empty(
|
||||
(cls._local_dp_buffer_len, dp.buffer_hidden_size),
|
||||
dtype=dp.buffer_dtype,
|
||||
device=dp.buffer_device,
|
||||
)
|
||||
return buffer
|
||||
|
||||
@classmethod
|
||||
def get_local_dp_buffer_mhc(
|
||||
cls, group: GroupCoordinator, n: int = 1
|
||||
def get_local_dp_buffer(
|
||||
cls, group: GroupCoordinator, hidden_size: Optional[int] = None
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.runtime_context import get_flags
|
||||
|
||||
dp = get_flags().dp
|
||||
with use_symmetric_memory(group, disabled=not cls._dp_max_padding):
|
||||
buffer = torch.empty(
|
||||
(cls._local_dp_buffer_len, dp.buffer_hidden_size * n),
|
||||
(cls._local_dp_buffer_len, hidden_size or dp.buffer_hidden_size),
|
||||
dtype=dp.buffer_dtype,
|
||||
device=dp.buffer_device,
|
||||
)
|
||||
@@ -344,12 +331,14 @@ def get_global_dp_buffer(group: GroupCoordinator) -> torch.Tensor:
|
||||
return _DpGatheredBufferWrapper.get_global_dp_buffer(group=group)
|
||||
|
||||
|
||||
def get_local_dp_buffer(group: GroupCoordinator) -> torch.Tensor:
|
||||
return _DpGatheredBufferWrapper.get_local_dp_buffer(group=group)
|
||||
|
||||
|
||||
def get_local_dp_buffer_mhc(group: GroupCoordinator, n: int = 1) -> torch.Tensor:
|
||||
return _DpGatheredBufferWrapper.get_local_dp_buffer_mhc(group=group, n=n)
|
||||
def get_local_dp_buffer(
|
||||
group: GroupCoordinator, hidden_size: Optional[int] = None
|
||||
) -> torch.Tensor:
|
||||
"""A buffer for this rank's local DP rows, ``hidden_size`` wide (the model's
|
||||
hidden size by default)."""
|
||||
return _DpGatheredBufferWrapper.get_local_dp_buffer(
|
||||
group=group, hidden_size=hidden_size
|
||||
)
|
||||
|
||||
|
||||
def get_global_dp_buffer_len() -> int:
|
||||
|
||||
@@ -795,7 +795,6 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
|
||||
if self.config.mhc:
|
||||
mhc_kwargs: Dict[str, Any] = dict(
|
||||
is_first_layer=(self.layer_id == 0),
|
||||
hc_mult=config.hc_mult,
|
||||
hc_attn_pre=self.hc_attn_pre,
|
||||
hc_ffn_pre=self.hc_ffn_pre,
|
||||
|
||||
@@ -94,11 +94,8 @@ def make_communicator(
|
||||
communicator._cp_steps = None
|
||||
communicator._postprocess_dp_step = MagicMock(return_value=reduce_scatter_step)
|
||||
communicator.ffn_reduction_group = MagicMock(return_value=group or make_group())
|
||||
communicator.postprocess_layer = MagicMock(
|
||||
side_effect=lambda hidden_states, residual, forward_batch: (
|
||||
hidden_states + 1,
|
||||
residual,
|
||||
)
|
||||
communicator._complete_ffn_output_now = MagicMock(
|
||||
side_effect=lambda hidden_states, residual, **_: (hidden_states + 1, residual)
|
||||
)
|
||||
return communicator
|
||||
|
||||
@@ -126,7 +123,7 @@ class TestFfnExit(CustomTestCase):
|
||||
self.assertEqual(seen, (True, False))
|
||||
self.assertIsInstance(hidden_states, UnreducedOutput)
|
||||
self.assertIs(residual, self.residual)
|
||||
communicator.postprocess_layer.assert_not_called()
|
||||
communicator._complete_ffn_output_now.assert_not_called()
|
||||
|
||||
def test_reduction_left_to_next_layer_declares_its_group(self):
|
||||
group = make_group()
|
||||
@@ -152,7 +149,7 @@ class TestFfnExit(CustomTestCase):
|
||||
self.assertIs(bound.func, comm._to_local_tokens)
|
||||
self.assertEqual(bound.args, (step, self.forward_batch))
|
||||
self.assertIs(residual, self.residual)
|
||||
communicator.postprocess_layer.assert_not_called()
|
||||
communicator._complete_ffn_output_now.assert_not_called()
|
||||
step.assert_not_called()
|
||||
|
||||
def test_a_deferred_sum_carries_the_scatter_back_under_attention_dp(self):
|
||||
@@ -165,7 +162,7 @@ class TestFfnExit(CustomTestCase):
|
||||
bound = hidden_states.reduce_and_redistribute
|
||||
self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens)
|
||||
self.assertEqual(bound.args, (group, self.forward_batch))
|
||||
communicator.postprocess_layer.assert_not_called()
|
||||
communicator._complete_ffn_output_now.assert_not_called()
|
||||
group.all_reduce.assert_not_called()
|
||||
|
||||
def test_postprocess_completes_other_exits(self):
|
||||
@@ -177,7 +174,7 @@ class TestFfnExit(CustomTestCase):
|
||||
seen, (hidden_states, residual) = self.run_exit(communicator)
|
||||
self.assertEqual(seen, (False, reduce_scatter))
|
||||
self.assertNotIsInstance(hidden_states, UnreducedOutput)
|
||||
communicator.postprocess_layer.assert_called_once()
|
||||
communicator._complete_ffn_output_now.assert_called_once()
|
||||
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
|
||||
self.assertIs(residual, self.residual)
|
||||
|
||||
@@ -189,7 +186,7 @@ class TestFfnExit(CustomTestCase):
|
||||
|
||||
self.assertEqual(seen, (False, True))
|
||||
communicator._ffn_sum_moves_to_next_layer.assert_not_called()
|
||||
communicator.postprocess_layer.assert_called_once()
|
||||
communicator._complete_ffn_output_now.assert_called_once()
|
||||
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
|
||||
|
||||
def test_finish_applies_the_decision_the_ffn_saw(self):
|
||||
@@ -206,7 +203,7 @@ class TestFfnExit(CustomTestCase):
|
||||
hidden_states, _ = ffn_exit.finish(hidden_states, self.residual)
|
||||
self.assertEqual(seen, (fuse, False))
|
||||
self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse)
|
||||
self.assertEqual(communicator.postprocess_layer.called, not fuse)
|
||||
self.assertEqual(communicator._complete_ffn_output_now.called, not fuse)
|
||||
|
||||
def test_flags_are_restored_after_the_ffn(self):
|
||||
before = published_flags()
|
||||
@@ -223,7 +220,7 @@ class TestFfnExit(CustomTestCase):
|
||||
)
|
||||
seen, _ = self.run_exit(communicator)
|
||||
self.assertEqual(seen, (False, False))
|
||||
communicator.postprocess_layer.assert_called_once()
|
||||
communicator._complete_ffn_output_now.assert_called_once()
|
||||
|
||||
def test_deferral_implies_fusion_and_passes_the_handoff_through(self):
|
||||
"""A deferring communicator publishes both flags, and a non-tensor
|
||||
@@ -244,7 +241,7 @@ class TestFfnExit(CustomTestCase):
|
||||
self.assertEqual(
|
||||
ffn_exit.finish(handoff, self.residual), (handoff, self.residual)
|
||||
)
|
||||
communicator.postprocess_layer.assert_not_called()
|
||||
communicator._complete_ffn_output_now.assert_not_called()
|
||||
self.assertFalse(get_forward().defer_moe_finalize)
|
||||
|
||||
# The MoE may decline per forward; a tensor then takes the fused exit.
|
||||
|
||||
@@ -184,10 +184,12 @@ GATHER_LAYOUT = (TP_ATTN_FULL, TP_ATTN_FULL, FULL, TP_ATTN_FULL)
|
||||
|
||||
|
||||
class TestMlpInputOrder(CustomTestCase):
|
||||
def assert_order(self, order, func, **keywords):
|
||||
def assert_order(self, order, func, residual_ops=comm.ADD_AND_NORM, **keywords):
|
||||
self.assertIsInstance(order, partial)
|
||||
self.assertIs(order.func, func)
|
||||
self.assertEqual(order.keywords, keywords)
|
||||
bound = dict(order.keywords)
|
||||
self.assertIs(bound.pop("residual_ops", comm.ADD_AND_NORM), residual_ops)
|
||||
self.assertEqual(bound, keywords)
|
||||
|
||||
def test_facts_fixed_at_construction_pick_the_order(self):
|
||||
fusions = (object(),)
|
||||
@@ -268,41 +270,63 @@ class TestMlpInputOrder(CustomTestCase):
|
||||
):
|
||||
with self.subTest(expected=expected.__name__):
|
||||
_, (steps, fused) = communicator(layout, context, Fusable())
|
||||
self.assertIs(steps, expected)
|
||||
self.assert_order(steps, expected)
|
||||
self.assertEqual(fused, ())
|
||||
_, (steps, _) = communicator(
|
||||
(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED), tp, Fusable()
|
||||
)
|
||||
self.assert_order(steps, comm._mlp_input_scatter, scatters_residual=True)
|
||||
|
||||
def test_mhc_picks_its_own_implementation_of_the_kind(self):
|
||||
from sglang.srt.layers.communicator_mhc import (
|
||||
MHCCommunicateWithAllReduceAndLayerNormFn as MHC,
|
||||
)
|
||||
def test_mhc_runs_the_shared_steps_with_its_residual(self):
|
||||
from sglang.srt.layers.communicator_mhc import (
|
||||
MHCLayerCommunicator,
|
||||
MHCState,
|
||||
)
|
||||
|
||||
mhc = object()
|
||||
for layout, func in (
|
||||
(GATHER_LAYOUT, MHC._gather_hidden_states_and_residual),
|
||||
mhc = MHCState(hc_mult=2, hc_attn_pre=None, hc_ffn_pre=None, hc_post=None)
|
||||
for layout, func, order, keywords in (
|
||||
(
|
||||
GATHER_LAYOUT,
|
||||
comm._mlp_input_gather,
|
||||
comm._mlp_input_without_dp,
|
||||
# The fused add + RMSNorm kernels do not write hc_post.
|
||||
dict(gathers_residual=False, fusions=()),
|
||||
),
|
||||
(
|
||||
(TP_ATTN_FULL, TP_ATTN_FULL, SCATTERED, SCATTERED),
|
||||
MHC._scatter_hidden_states_and_residual,
|
||||
comm._mlp_input_scatter,
|
||||
None,
|
||||
dict(scatters_residual=True),
|
||||
),
|
||||
):
|
||||
with self.subTest(func=func.__name__):
|
||||
c = MHCLayerCommunicator.__new__(MHCLayerCommunicator)
|
||||
c.mhc = mhc
|
||||
c._residual_ops = mhc
|
||||
c.layer_scatter_modes = modes(*layout)
|
||||
c._context = make_context(tp=2)
|
||||
c.post_attention_layernorm = Fusable()
|
||||
steps, fused = c._select_mlp_input()
|
||||
self.assert_order(
|
||||
steps, func, residual_input_mode=TP_ATTN_FULL, mhc=mhc
|
||||
)
|
||||
if order is None:
|
||||
self.assert_order(steps, func, residual_ops=mhc, **keywords)
|
||||
else:
|
||||
self.assertIs(steps.func, func)
|
||||
self.assert_order(
|
||||
steps.keywords["order"], order, residual_ops=mhc, **keywords
|
||||
)
|
||||
self.assertEqual(fused, ())
|
||||
|
||||
def test_mhc_gathers_over_dp_after_its_write_back(self):
|
||||
from sglang.srt.layers.communicator_mhc import MHCState
|
||||
|
||||
mhc = MHCState(hc_mult=2, hc_attn_pre=None, hc_ffn_pre=None, hc_post=None)
|
||||
self.assert_order(
|
||||
comm._mlp_input_order(make_context(dp=2, tp=2), SCATTERED, (), mhc),
|
||||
comm._mlp_input_dp_replicate,
|
||||
residual_ops=mhc,
|
||||
gathers_residual=True,
|
||||
reduces_attention_tp=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -15,11 +15,13 @@ import unittest
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers import communicator as comm
|
||||
from sglang.srt.layers import communicator_mhc as mhc_module
|
||||
from sglang.srt.layers.boundary_layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
@@ -365,7 +367,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
single = build(layer_facts(1, 3), one_rank)
|
||||
with planning(one_rank):
|
||||
self.assertIsNotNone(single._declared_sides())
|
||||
self.assertIs(single._steps.ffn_input, comm._mlp_input_norm)
|
||||
self.assertIs(single._steps.ffn_input.func, comm._mlp_input_norm)
|
||||
|
||||
def test_the_order_follows_the_attention_output(self):
|
||||
for attn_tp, force, order in (
|
||||
@@ -482,7 +484,7 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
a2a=True,
|
||||
)
|
||||
self.assertIs(
|
||||
last._steps.ffn_output_move,
|
||||
last._steps.ffn_output_move.func,
|
||||
comm.CommunicateSummableTensorPairFn._gather,
|
||||
)
|
||||
|
||||
@@ -534,10 +536,288 @@ class TestWhichLayersUseDeclarations(CustomTestCase):
|
||||
self.assertFalse(self.declared(build(direct, dp)))
|
||||
|
||||
def test_subclasses_that_pick_their_own_steps(self):
|
||||
for cls in (MHCLayerCommunicator, CuteDSLFusionLayerCommunicator):
|
||||
self.assertFalse(CuteDSLFusionLayerCommunicator._takes_declared_boundaries)
|
||||
for cls in (LayerCommunicator, MHCLayerCommunicator):
|
||||
with self.subTest(cls.__name__):
|
||||
self.assertFalse(cls._takes_declared_boundaries)
|
||||
self.assertTrue(LayerCommunicator._takes_declared_boundaries)
|
||||
self.assertTrue(cls._takes_declared_boundaries)
|
||||
|
||||
|
||||
def build_mhc(
|
||||
modes, parallel, *, a2a=False, dsa_cp=False, two_batch_overlap=False, **kwargs
|
||||
):
|
||||
overlap = SimpleNamespace(
|
||||
overlap=SimpleNamespace(enable_two_batch_overlap=two_batch_overlap)
|
||||
)
|
||||
with (
|
||||
planning(parallel, a2a=a2a, dsa_cp=dsa_cp),
|
||||
patch.object(comm, "get_exec", lambda: overlap),
|
||||
):
|
||||
return MHCLayerCommunicator(
|
||||
layer_scatter_modes=modes,
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
hc_mult=2,
|
||||
hc_attn_pre=lambda *a: None,
|
||||
hc_ffn_pre=lambda *a: None,
|
||||
**{"hc_post": lambda *a: None, **kwargs},
|
||||
)
|
||||
|
||||
|
||||
class TestMhcOnTheDeclarations(CustomTestCase):
|
||||
"""An MHC layer takes the same declarations and runs the same steps as any
|
||||
other layer; only the residual operations the steps run are MHC's."""
|
||||
|
||||
def assert_step(self, step, func, communicator, **keywords):
|
||||
self.assertIs(step.func, func)
|
||||
self.assertIs(step.keywords["residual_ops"], communicator.mhc)
|
||||
for key, value in keywords.items():
|
||||
self.assertEqual(step.keywords[key], value, key)
|
||||
|
||||
def test_each_boundary_runs_the_shared_step(self):
|
||||
for name, parallel, ffn_input, output_move in (
|
||||
(
|
||||
"attention TP 1",
|
||||
parallel_of(attn_dp=1, attn_tp=1),
|
||||
comm._mlp_input_norm,
|
||||
comm.CommunicateSummableTensorPairFn._trivial,
|
||||
),
|
||||
(
|
||||
"the attention-TP sum",
|
||||
parallel_of(attn_dp=1, attn_tp=2),
|
||||
comm._mlp_input_without_dp,
|
||||
comm.CommunicateSummableTensorPairFn._trivial,
|
||||
),
|
||||
# The DP gather runs after hc_post, which is not a plain add.
|
||||
(
|
||||
"a DP gather",
|
||||
parallel_of(attn_dp=2, attn_tp=2),
|
||||
comm._mlp_input_dp_replicate,
|
||||
None,
|
||||
),
|
||||
):
|
||||
with self.subTest(name):
|
||||
communicator = build_mhc(layer_facts(1, 3), parallel)
|
||||
self.assert_step(communicator._steps.ffn_input, ffn_input, communicator)
|
||||
self.assertIs(communicator._steps.ffn_output_move, output_move)
|
||||
# The fused add + RMSNorm kernels do not write hc_post.
|
||||
self.assertEqual(communicator._steps.fused, ())
|
||||
self.assertEqual(communicator._attn_input_fusions, ())
|
||||
|
||||
def test_the_move_back_over_dp_stays_with_the_layer(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
communicator = build_mhc(layer_facts(1, 3), parallel, allow_reduce_scatter=True)
|
||||
self.assertTrue(communicator._steps.returns_over_dp)
|
||||
self.assertFalse(communicator._steps.ffn_output.leaves_for_next_layer)
|
||||
# The move back also writes the FFN output into the streams.
|
||||
self.assertFalse(communicator._local_token_move_can_go_to_next_layer(None))
|
||||
max_len = SimpleNamespace(
|
||||
dp_padding_mode=SimpleNamespace(is_max_len=lambda: True),
|
||||
forward_mode=SimpleNamespace(is_context_parallel_extend=lambda: False),
|
||||
)
|
||||
with (
|
||||
planning(parallel),
|
||||
patch.object(comm, "get_forward", lambda: SimpleNamespace(sp_active=False)),
|
||||
patch.object(comm, "should_use_dp_reduce_scatterv", lambda: False),
|
||||
patch.object(comm, "can_use_dp_reduce_scatter", lambda: True),
|
||||
):
|
||||
self.assertTrue(communicator.should_use_reduce_scatter(max_len))
|
||||
|
||||
def test_the_exit_runs_the_move_back_over_dp_it_chose(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
for name, reduce_scatterv, is_max_len, move in (
|
||||
("SUM_LEN", True, False, comm._reduce_and_redistribute_output_varlen),
|
||||
("MAX_LEN", False, True, comm._reduce_and_redistribute_output_max_len),
|
||||
):
|
||||
with self.subTest(name):
|
||||
hc_post = MagicMock(side_effect=lambda h, r, h_res, h_post: h + r)
|
||||
communicator = build_mhc(
|
||||
layer_facts(1, 3),
|
||||
parallel,
|
||||
allow_reduce_scatter=True,
|
||||
hc_post=hc_post,
|
||||
)
|
||||
communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2)
|
||||
forward_batch = SimpleNamespace(
|
||||
dp_padding_mode=SimpleNamespace(is_max_len=lambda: is_max_len),
|
||||
forward_mode=SimpleNamespace(
|
||||
is_context_parallel_extend=lambda: False
|
||||
),
|
||||
)
|
||||
choose = MagicMock(wraps=comm._reduce_and_redistribute_output_step)
|
||||
to_local_tokens = MagicMock(side_effect=lambda step, fb, h: h[:2])
|
||||
with (
|
||||
planning(parallel),
|
||||
patch.object(
|
||||
comm, "should_use_dp_reduce_scatterv", lambda: reduce_scatterv
|
||||
),
|
||||
patch.object(comm, "can_use_dp_reduce_scatter", lambda: True),
|
||||
patch.object(comm, "_reduce_and_redistribute_output_step", choose),
|
||||
patch.object(comm, "_to_local_tokens", to_local_tokens),
|
||||
):
|
||||
with communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
self.assertTrue(ffn_exit.mlp_reduce_scatter)
|
||||
hidden, residual = ffn_exit.finish(
|
||||
torch.ones(4, 4), torch.full((2, 4), 2.0)
|
||||
)
|
||||
choose.assert_called_once()
|
||||
to_local_tokens.assert_called_once()
|
||||
self.assertIs(to_local_tokens.call_args.args[0], move)
|
||||
hc_post.assert_called_once()
|
||||
self.assertIsNone(residual)
|
||||
torch.testing.assert_close(hidden, torch.full((2, 4), 3.0))
|
||||
|
||||
def test_a2a_layers_take_their_slice_with_the_coefficients(self):
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2)
|
||||
layers = [
|
||||
build_mhc(
|
||||
planned_modes(
|
||||
i,
|
||||
3,
|
||||
sparse=sparse,
|
||||
previous_sparse=previous,
|
||||
parallel=parallel,
|
||||
a2a=True,
|
||||
),
|
||||
parallel,
|
||||
a2a=True,
|
||||
)
|
||||
for i, (sparse, previous) in enumerate(
|
||||
((False, False), (True, False), (True, True))
|
||||
)
|
||||
]
|
||||
first_a2a, last = layers[1], layers[2]
|
||||
self.assert_step(
|
||||
first_a2a._steps.ffn_input,
|
||||
comm._mlp_input_scatter,
|
||||
first_a2a,
|
||||
scatters_residual=True,
|
||||
)
|
||||
self.assert_step(
|
||||
last._steps.ffn_input,
|
||||
comm._mlp_input_scatter,
|
||||
last,
|
||||
scatters_residual=False,
|
||||
)
|
||||
self.assert_step(
|
||||
last._steps.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._gather,
|
||||
last,
|
||||
)
|
||||
state = first_a2a.mhc
|
||||
state.h_res, state.h_post = torch.arange(4.0), torch.arange(4.0) + 10
|
||||
context = SimpleNamespace(attn_tp_rank=1, attn_tp_size=2)
|
||||
residual = state.residual_to_attn_tp_shard(torch.arange(4.0) + 20, context)
|
||||
self.assertEqual(residual.tolist(), [22.0, 23.0])
|
||||
self.assertEqual(state.h_res.tolist(), [2.0, 3.0])
|
||||
self.assertEqual(state.h_post.tolist(), [12.0, 13.0])
|
||||
|
||||
def test_an_input_scattered_batch_keeps_the_residual_on_the_slice(self):
|
||||
parallel = parallel_of(
|
||||
attn_dp=1, attn_tp=2, enable_attn_tp_input_scattered=True
|
||||
)
|
||||
for layer_id in range(3):
|
||||
with self.subTest(layer_id=layer_id):
|
||||
communicator = build_mhc(
|
||||
layer_facts(layer_id, 3), parallel, allow_reduce_scatter=True
|
||||
)
|
||||
steps = communicator._input_scattered_steps
|
||||
self.assert_step(
|
||||
steps.ffn_input, comm._mlp_input_on_residual_shard, communicator
|
||||
)
|
||||
self.assert_step(
|
||||
steps.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._onto_residual_shard,
|
||||
communicator,
|
||||
sums=True,
|
||||
gathers_back=layer_id == 2,
|
||||
)
|
||||
self.assertTrue(steps.ffn_output_move_completes_sum)
|
||||
completes = build_mhc(
|
||||
layer_facts(layer_id, 3), parallel, allow_reduce_scatter=False
|
||||
)._input_scattered_steps
|
||||
self.assertFalse(completes.ffn_output_move.keywords["sums"])
|
||||
self.assertFalse(completes.ffn_output_move_completes_sum)
|
||||
# Only the first layer's input, the embedding's partial sum,
|
||||
# is completed onto the slice.
|
||||
self.assertIs(
|
||||
steps.layer_input,
|
||||
comm.tp_reduce_scatter if layer_id == 0 else None,
|
||||
)
|
||||
self.assertIs(steps.attention_input, comm.CommunicateSimpleFn._trivial)
|
||||
self.assertIs(
|
||||
steps.attention_handoff, comm._hand_scattered_input_to_attention
|
||||
)
|
||||
self.assertIs(
|
||||
communicator._steps.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._trivial,
|
||||
)
|
||||
|
||||
def test_two_batch_overlap_keeps_the_scatter_modes(self):
|
||||
# A dense layer before a sparse one gathers its output for the split,
|
||||
# so dense layers on every rank keep their scatter-mode steps.
|
||||
parallel = parallel_of(attn_dp=2, attn_tp=2, moe_dense_tp_size=1)
|
||||
modes = planned_modes(
|
||||
1, 3, sparse=False, previous_sparse=False, parallel=parallel
|
||||
)
|
||||
declared = build_mhc(modes, parallel)
|
||||
kept = build_mhc(modes, parallel, two_batch_overlap=True)
|
||||
# Both run the shared slice onto each rank's rows with MHC's residual.
|
||||
for communicator in (declared, kept):
|
||||
self.assert_step(
|
||||
communicator._steps.ffn_input, comm._mlp_input_scatter, communicator
|
||||
)
|
||||
# The declared layer never reads a scatter mode; the kept one does.
|
||||
build_mhc(layer_facts(1, 3), parallel)
|
||||
with self.assertRaises(AssertionError):
|
||||
build_mhc(layer_facts(1, 3), parallel, two_batch_overlap=True)
|
||||
|
||||
def test_the_postprocess_writes_the_output_into_the_streams(self):
|
||||
# The last layer also contracts the streams into the hidden states.
|
||||
parallel = parallel_of(attn_dp=1, attn_tp=1)
|
||||
for layer_id, contracted in ((1, False), (2, True)):
|
||||
with self.subTest(layer_id=layer_id):
|
||||
communicator = build_mhc(
|
||||
layer_facts(layer_id, 3),
|
||||
parallel,
|
||||
hc_post=lambda h, r, h_res, h_post: h + r,
|
||||
)
|
||||
communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2)
|
||||
with (
|
||||
planning(parallel),
|
||||
patch.object(
|
||||
comm, "get_forward", lambda: SimpleNamespace(sp_active=False)
|
||||
),
|
||||
patch.object(
|
||||
mhc_module, "hc_contract", lambda h, hc_mult: h.sum(-1)
|
||||
),
|
||||
):
|
||||
hidden, residual = communicator.postprocess_layer(
|
||||
torch.ones(2, 4), torch.full((2, 4), 2.0), None
|
||||
)
|
||||
self.assertIsNone(residual)
|
||||
expected = torch.full((2, 4), 3.0)
|
||||
torch.testing.assert_close(
|
||||
hidden, expected.sum(-1) if contracted else expected
|
||||
)
|
||||
# The write-back consumes this layer's coefficients.
|
||||
self.assertIsNone(communicator.mhc.h_res)
|
||||
|
||||
def test_a_gather_over_attention_cp_is_rejected(self):
|
||||
# A MoE on the TP group under DSA prefill CP gathers its input over
|
||||
# attention CP, which MHC has not been run with.
|
||||
parallel = parallel_of(
|
||||
attn_dp=1, attn_tp=1, attn_cp=2, enable_prefill_cp=True, moe_dense_tp_size=1
|
||||
)
|
||||
modes = planned_modes(
|
||||
1, 3, sparse=True, previous_sparse=False, parallel=parallel, dsa_cp=True
|
||||
)
|
||||
with self.assertRaises(NotImplementedError):
|
||||
build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True)
|
||||
# A dense layer on every rank computes on its own shard: nothing moves.
|
||||
modes = planned_modes(
|
||||
1, 3, sparse=False, previous_sparse=False, parallel=parallel, dsa_cp=True
|
||||
)
|
||||
build_mhc(modes, parallel, dsa_cp=True, allow_reduce_scatter=True)
|
||||
|
||||
|
||||
class TestTheAttentionOutputDecidesItsSum(CustomTestCase):
|
||||
@@ -723,6 +1003,11 @@ class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
ffn_sum_is_movable=False,
|
||||
)
|
||||
|
||||
def unbound(self, steps):
|
||||
"""``steps`` with the plain residual's binding taken off the FFN input."""
|
||||
self.assertEqual(steps.ffn_input.keywords, {"residual_ops": comm.ADD_AND_NORM})
|
||||
return msgspec.structs.replace(steps, ffn_input=steps.ffn_input.func)
|
||||
|
||||
def test_the_region_declarations_choose_local_steps(self):
|
||||
for attn_tp in (2, 4):
|
||||
with self.subTest(attn_tp=attn_tp):
|
||||
@@ -739,7 +1024,7 @@ class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
self.assertEqual(sides.ffn.gathers_itself, local)
|
||||
self.assertIsNone(sides.attention_output.group)
|
||||
self.assertIsNone(sides.ffn_output.group)
|
||||
steps = comm._select_boundary_steps(sides)
|
||||
steps = self.unbound(comm._select_boundary_steps(sides))
|
||||
self.assertEqual(
|
||||
(steps.attention_input, steps.ffn_input, steps.ffn_output_move),
|
||||
(
|
||||
@@ -760,7 +1045,7 @@ class TestTheSequenceParallelRegion(CustomTestCase):
|
||||
input_layernorm=Norm(),
|
||||
post_attention_layernorm=Norm(),
|
||||
)
|
||||
self.assertEqual(communicator._sp_steps, self.SP_STEPS)
|
||||
self.assertEqual(self.unbound(communicator._sp_steps), self.SP_STEPS)
|
||||
self.assertIs(
|
||||
communicator._steps.ffn_input.func, comm._mlp_input_without_dp
|
||||
)
|
||||
@@ -1042,7 +1327,7 @@ class TestPrefillCP(CustomTestCase):
|
||||
communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True)
|
||||
cp, ordinary = communicator._cp_steps, communicator._steps
|
||||
self.assertIs(cp.ffn_input.func, comm._mlp_input_gather_attention_cp)
|
||||
self.assertIs(cp.ffn_input.keywords["gather"], comm._mlp_input_norm)
|
||||
self.assertIs(cp.ffn_input.keywords["gather"].func, comm._mlp_input_norm)
|
||||
self.assertIs(
|
||||
cp.ffn_output_move,
|
||||
comm.CommunicateSummableTensorPairFn._reduce_scatter_over_cp,
|
||||
@@ -1052,7 +1337,7 @@ class TestPrefillCP(CustomTestCase):
|
||||
# The dense layer before it ran on the same shard: nothing to move.
|
||||
self.assertIs(cp.attention_input, comm.CommunicateSimpleFn._trivial)
|
||||
# Other batches hold every token on each CP rank: the MoE sums itself.
|
||||
self.assertIs(ordinary.ffn_input, comm._mlp_input_norm)
|
||||
self.assertIs(ordinary.ffn_input.func, comm._mlp_input_norm)
|
||||
self.assertIs(
|
||||
ordinary.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial
|
||||
)
|
||||
@@ -1109,7 +1394,7 @@ class TestPrefillCP(CustomTestCase):
|
||||
)
|
||||
communicator = build(modes, parallel, dsa_cp=True, allow_reduce_scatter=True)
|
||||
for steps in (communicator._steps, communicator._cp_steps):
|
||||
self.assertIs(steps.ffn_input, comm._mlp_input_norm)
|
||||
self.assertIs(steps.ffn_input.func, comm._mlp_input_norm)
|
||||
self.assertIsNone(steps.ffn_output.group)
|
||||
self.assertIs(
|
||||
steps.ffn_output_move, comm.CommunicateSummableTensorPairFn._trivial
|
||||
@@ -1393,8 +1678,8 @@ def running(*, reduce_scatterv, a2a=False):
|
||||
"get_global_dp_buffer": lambda g: torch.zeros(
|
||||
state().global_rows, HIDDEN, dtype=torch.double
|
||||
),
|
||||
"get_local_dp_buffer": lambda g: torch.zeros(
|
||||
state().local_rows, HIDDEN, dtype=torch.double
|
||||
"get_local_dp_buffer": lambda g, hidden_size=None: torch.zeros(
|
||||
state().local_rows, hidden_size or HIDDEN, dtype=torch.double
|
||||
),
|
||||
"dp_gather_replicate": partial(fake_dp_gather, is_partial=False),
|
||||
"dp_gather_partial": partial(fake_dp_gather, is_partial=True),
|
||||
@@ -1672,5 +1957,47 @@ class TestTwoLayers(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestTheFfnInputReduction(CustomTestCase):
|
||||
"""With quantized communications a prefill reduces the FFN input quantized
|
||||
for a plain residual; MHC sums its streams in full precision."""
|
||||
|
||||
def test_only_a_plain_residual_reduces_quantized(self):
|
||||
exec_ = SimpleNamespace(comm=SimpleNamespace(enable_quant_communications=True))
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=SimpleNamespace(is_decode_or_idle=lambda: False)
|
||||
)
|
||||
for adds_plainly, reduction in ((True, "quant"), (False, "full")):
|
||||
with self.subTest(adds_plainly=adds_plainly):
|
||||
ops = SimpleNamespace(
|
||||
adds_plainly=adds_plainly,
|
||||
update_and_read_ffn_input=lambda h, r, norm: (h, r),
|
||||
)
|
||||
calls = []
|
||||
with (
|
||||
patch.object(comm, "get_exec", lambda: exec_),
|
||||
patch.object(
|
||||
comm,
|
||||
"attention_tensor_model_parallel_quant_all_reduce",
|
||||
lambda h: calls.append("quant") or h,
|
||||
),
|
||||
patch.object(
|
||||
comm,
|
||||
"attention_tensor_model_parallel_all_reduce",
|
||||
lambda h: calls.append("full") or h,
|
||||
),
|
||||
):
|
||||
comm._mlp_input_without_dp(
|
||||
torch.ones(2, HIDDEN),
|
||||
torch.ones(2, HIDDEN),
|
||||
batch,
|
||||
None,
|
||||
SimpleNamespace(cache=None),
|
||||
gathers_residual=False,
|
||||
fusions=(),
|
||||
residual_ops=ops,
|
||||
)
|
||||
self.assertEqual(calls, [reduction])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -231,7 +231,9 @@ class TestDpCpGather(CustomTestCase):
|
||||
),
|
||||
(
|
||||
(comm, "get_local_dp_buffer"),
|
||||
lambda group: torch.empty(local_buffer_len, HIDDEN).double(),
|
||||
lambda group, hidden_size=None: torch.empty(
|
||||
local_buffer_len, hidden_size or HIDDEN
|
||||
).double(),
|
||||
),
|
||||
]:
|
||||
stack.enter_context(patch.object(*target, value))
|
||||
|
||||
@@ -42,6 +42,8 @@ class Norm:
|
||||
def communicator(norm):
|
||||
c = comm.LayerCommunicator.__new__(comm.LayerCommunicator)
|
||||
c.input_layernorm = norm
|
||||
# A layer inside the stack: an absent residual was written back before.
|
||||
c.layer_scatter_modes = SimpleNamespace(is_first_layer=False)
|
||||
c._sp_steps = None
|
||||
c._input_scattered_steps = None
|
||||
c._cp_steps = None
|
||||
|
||||
@@ -220,7 +220,9 @@ def test_communicator_publishes_layout_at_each_transition(
|
||||
expected_mlp = expected_attn = initial
|
||||
communicator = LayerCommunicator.__new__(LayerCommunicator)
|
||||
communicator._publish_lora_layout = publish_lora_layout
|
||||
communicator.layer_scatter_modes = SimpleNamespace(mlp_mode=mlp_mode)
|
||||
communicator.layer_scatter_modes = SimpleNamespace(
|
||||
mlp_mode=mlp_mode, is_first_layer=False
|
||||
)
|
||||
communicator._context = SimpleNamespace()
|
||||
communicator._sp_steps = None
|
||||
communicator._input_scattered_steps = None
|
||||
@@ -231,6 +233,7 @@ def test_communicator_publishes_layout_at_each_transition(
|
||||
communicator._steps = SimpleNamespace(
|
||||
attention_input=lambda hidden_states, **kwargs: hidden_states,
|
||||
layer_input=None,
|
||||
attention_handoff=lambda hidden_states, *args: hidden_states,
|
||||
ffn_input=lambda hidden_states, residual, *args: (hidden_states, residual),
|
||||
)
|
||||
communicator._attn_input_fusions = ()
|
||||
|
||||
@@ -69,9 +69,8 @@ class DeferringLayer(nn.Module):
|
||||
self.layer_communicator._ffn_leaves_sum_to_reduce_scatter = (
|
||||
lambda batch, dp_step: False
|
||||
)
|
||||
self.layer_communicator.postprocess_layer = lambda hidden, residual, batch: (
|
||||
all_reduce(hidden),
|
||||
residual,
|
||||
self.layer_communicator._complete_ffn_output_now = (
|
||||
lambda hidden, residual, **_: (all_reduce(hidden), residual)
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -59,7 +59,7 @@ class TestLongcatShortcut(CustomTestCase):
|
||||
patch.object(
|
||||
comm,
|
||||
"get_local_dp_buffer",
|
||||
side_effect=lambda group: torch.empty(rows, 3),
|
||||
side_effect=lambda group, hidden_size=None: torch.empty(rows, 3),
|
||||
),
|
||||
patch.object(
|
||||
comm,
|
||||
|
||||
Reference in New Issue
Block a user