[Refactor] Run MHC layers on the shared boundary steps with MHC's residual operations (#41425)
This commit is contained in:
@@ -138,6 +138,52 @@ def sequence_parallel_layer_sides(
|
||||
)
|
||||
|
||||
|
||||
def scattered_residual_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
ffn_group: SumGroup,
|
||||
is_first_layer: bool,
|
||||
is_last_layer: bool,
|
||||
leaves_for_reduce_scatter: bool,
|
||||
) -> DecoderLayerSides:
|
||||
"""A decoder layer on an input-scattered batch whose residual stays on
|
||||
each attention-TP rank's slice (MHC). The attention and the FFN compute on
|
||||
the full rows. The attention takes the slice and gathers it itself (its
|
||||
QKV hook, or the handoff before an attention that needs the rows); the
|
||||
boundary brings the FFN its rows and moves both outputs back onto the
|
||||
slice: the attention output's TP sum by a reduce-scatter, the FFN's too
|
||||
when it leaves it (``leaves_for_reduce_scatter``). The first layer's input
|
||||
is the embedding's partial sum on the full rows; the last layer hands on
|
||||
the full rows again."""
|
||||
attention = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP, TokenAxis.ATTN_CP, axis_sizes=axis_sizes
|
||||
)
|
||||
local = Layout.sharded_over(
|
||||
TokenAxis.ATTN_DP,
|
||||
TokenAxis.ATTN_CP,
|
||||
TokenAxis.ATTN_TP_SCATTER,
|
||||
axis_sizes=axis_sizes,
|
||||
)
|
||||
return DecoderLayerSides(
|
||||
input_rows=attention if is_first_layer else local,
|
||||
attention=StageInput(
|
||||
attention, gathers_itself=frozenset({TokenAxis.ATTN_TP_SCATTER})
|
||||
),
|
||||
attention_output=StageOutput(
|
||||
attention, group=SumGroup.ATTN_TP, always_leaves=True
|
||||
),
|
||||
ffn=StageInput(Layout.sharded_over(axis_sizes=axis_sizes)),
|
||||
ffn_output=StageOutput(
|
||||
attention,
|
||||
group=ffn_group,
|
||||
leaves_for_reduce_scatter=leaves_for_reduce_scatter,
|
||||
),
|
||||
ffn_residual_rows=local,
|
||||
output_rows=attention if is_last_layer else local,
|
||||
input_owes=SumGroup.TP if is_first_layer else None,
|
||||
)
|
||||
|
||||
|
||||
def input_scattered_layer_sides(
|
||||
*,
|
||||
axis_sizes: Mapping[TokenAxis, int],
|
||||
|
||||
@@ -16,7 +16,7 @@ from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property, partial
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union
|
||||
from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.layers.boundary_layout import (
|
||||
TokenAxis,
|
||||
decoder_layer_sides,
|
||||
input_scattered_layer_sides,
|
||||
scattered_residual_layer_sides,
|
||||
sequence_parallel_layer_sides,
|
||||
)
|
||||
from sglang.srt.layers.communicator_dsa_cp import (
|
||||
@@ -651,6 +652,92 @@ def _attn_input_update_and_read_residual(quant_format: str):
|
||||
return _update_and_read_residual_plain
|
||||
|
||||
|
||||
class ResidualOps(Protocol):
|
||||
"""How a layer writes each stage's output into its residual and reads the
|
||||
next stage's input from it. The boundary steps complete sums and move
|
||||
tokens around these operations. AddAndNorm adds and normalizes; MHC's
|
||||
hyper-connection streams implement the same operations (communicator_mhc)."""
|
||||
|
||||
# The write-back is a plain add, which one rank may run before the sum it
|
||||
# adds into completes: the DP partial order, and a residual that joins the
|
||||
# attention output's sum.
|
||||
adds_plainly: bool
|
||||
# The layer writes its FFN output into the residual itself instead of
|
||||
# leaving that to the next layer's input.
|
||||
updates_residual_after_ffn: bool
|
||||
|
||||
def enter(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""The residual the layer stack starts from, given its input."""
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format: str) -> Tuple:
|
||||
"""The attention input and the residual, from a residual that already
|
||||
holds the previous layer's output."""
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format: str, post_residual_addition
|
||||
) -> Tuple:
|
||||
"""Write the previous layer's output into the residual, then read the
|
||||
attention input from it."""
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm) -> Tuple:
|
||||
"""Write the attention output into the residual, then read the FFN
|
||||
input from it."""
|
||||
|
||||
def update_residual(self, hidden_states, residual) -> torch.Tensor:
|
||||
"""The residual with the FFN output written into it."""
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context) -> torch.Tensor:
|
||||
"""This attention-TP rank's slice of the residual."""
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual) -> torch.Tensor:
|
||||
"""The residual gathered from every attention-TP rank's slice."""
|
||||
|
||||
|
||||
class AddAndNorm:
|
||||
"""A plain residual: a stage's output is added into it, and the next
|
||||
stage's input is its norm. An empty batch skips the norms."""
|
||||
|
||||
adds_plainly = True
|
||||
updates_residual_after_ffn = False
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if residual.shape[0] == 0:
|
||||
return residual, residual
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, residual, None, None
|
||||
)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
return _attn_input_update_and_read_residual(quant_format)(
|
||||
norm, hidden_states, residual, post_residual_addition
|
||||
)
|
||||
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, residual
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
hidden_states += residual
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
return _redistribute_to_attn_tp_shards(residual, context)
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
return _redistribute_from_attn_tp_shards(residual)
|
||||
|
||||
|
||||
ADD_AND_NORM = AddAndNorm()
|
||||
|
||||
|
||||
class UnreducedOutput(msgspec.Struct, frozen=True):
|
||||
"""A layer output that still owes its sum, left for the next layer's input.
|
||||
Hand it to the next layer, or pass it through reduce_output() before reading
|
||||
@@ -730,6 +817,8 @@ class LayerCommunicator:
|
||||
# Whether this class's boundary steps may be chosen from both sides'
|
||||
# declarations; a subclass that picks its own steps says no.
|
||||
_takes_declared_boundaries = True
|
||||
# A plain residual unless the layer is built with its own.
|
||||
_residual_ops: ResidualOps = ADD_AND_NORM
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -744,6 +833,8 @@ class LayerCommunicator:
|
||||
fused_ar_quant_keep_bf16: bool = False,
|
||||
# False for a layer whose FFN always completes its own reduction.
|
||||
allow_deferred_ffn_reduction: bool = True,
|
||||
# How the layer writes its residual and reads its stages' inputs.
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
self.layer_scatter_modes = layer_scatter_modes
|
||||
self.input_layernorm = input_layernorm
|
||||
@@ -755,6 +846,7 @@ class LayerCommunicator:
|
||||
self.enable_fused_ar_quant = enable_fused_ar_quant
|
||||
self.fused_ar_quant_keep_bf16 = fused_ar_quant_keep_bf16
|
||||
self.allow_deferred_ffn_reduction = allow_deferred_ffn_reduction
|
||||
self._residual_ops = residual_ops
|
||||
|
||||
self._context = CommunicateContext.init_new()
|
||||
self._context.force_layernorm_before_dp_gather = (
|
||||
@@ -762,17 +854,8 @@ class LayerCommunicator:
|
||||
)
|
||||
# The steps the layer's ordinary batches run.
|
||||
sides = self._declared_sides()
|
||||
if (
|
||||
sides is None
|
||||
and self._takes_declared_boundaries
|
||||
and (_generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp())
|
||||
):
|
||||
# The scatter-mode steps have no attention-CP gather.
|
||||
raise NotImplementedError(
|
||||
"a DSA or MLA prefill CP layer outside the declarations"
|
||||
)
|
||||
self._steps = (
|
||||
_select_boundary_steps(
|
||||
self._steps_from_declarations(
|
||||
sides,
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
@@ -783,7 +866,7 @@ class LayerCommunicator:
|
||||
# The steps a batch that shards its tokens over attention CP runs; None
|
||||
# without CP or where the steps come from the scatter modes.
|
||||
self._cp_steps = (
|
||||
_select_boundary_steps(
|
||||
self._steps_from_declarations(
|
||||
self._declared_sides(cp_active=True),
|
||||
fusions=self._select_mlp_input_fusions(),
|
||||
force_layernorm_before_gather=force_layernorm_before_dp_gather,
|
||||
@@ -795,14 +878,7 @@ class LayerCommunicator:
|
||||
# The steps a batch with input-scattered attention runs; None where it
|
||||
# cannot run or the steps come from the scatter modes.
|
||||
self._input_scattered_steps = (
|
||||
_select_boundary_steps(
|
||||
input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=allow_reduce_scatter
|
||||
and not layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
)
|
||||
self._steps_for_input_scattered(sides)
|
||||
if sides is not None and self._input_can_be_scattered()
|
||||
else None
|
||||
)
|
||||
@@ -820,7 +896,8 @@ class LayerCommunicator:
|
||||
# q_lora, which input-scattered attention needs.
|
||||
self._sp_steps = (
|
||||
_select_boundary_steps(
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes())
|
||||
sequence_parallel_layer_sides(axis_sizes=_token_axis_sizes()),
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
if layernorm_sp.layernorm_sp_enabled()
|
||||
else None
|
||||
@@ -918,6 +995,7 @@ class LayerCommunicator:
|
||||
attention_input, postprocess = self._post_init_communicate()
|
||||
ffn_input, fused = self._select_mlp_input()
|
||||
modes = self.layer_scatter_modes
|
||||
ops = self._residual_ops
|
||||
ffn_output = StageOutput(
|
||||
self._context.layouts[modes.mlp_mode],
|
||||
group=SumGroup.MOE_OUTPUT if modes.is_layer_sparse else SumGroup.TP,
|
||||
@@ -940,7 +1018,7 @@ class LayerCommunicator:
|
||||
ffn_output_move=(
|
||||
None
|
||||
if postprocess is CommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
else postprocess
|
||||
else partial(postprocess, residual_ops=ops)
|
||||
),
|
||||
# Not when the way back is the MoE-CP scatter, nor when a SCATTERED
|
||||
# FFN computes whole tokens with nothing left to sum.
|
||||
@@ -950,9 +1028,46 @@ class LayerCommunicator:
|
||||
layer_input=_complete_scattered_input,
|
||||
)
|
||||
|
||||
def _steps_for_input_scattered(self, sides: DecoderLayerSides) -> "BoundarySteps":
|
||||
"""The steps a batch with input-scattered attention runs at this layer,
|
||||
for the layer's ordinary declarations ``sides``. A plain residual comes
|
||||
back to the full rows inside the attention output's sum; one whose
|
||||
write-back is not a plain add stays on each rank's slice."""
|
||||
if self._residual_ops.adds_plainly:
|
||||
scattered = input_scattered_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
hands_on_partial=self.allow_reduce_scatter and not self.is_last_layer,
|
||||
)
|
||||
handoff = _hand_qkv_hook_its_input
|
||||
else:
|
||||
scattered = scattered_residual_layer_sides(
|
||||
axis_sizes=_token_axis_sizes(),
|
||||
ffn_group=sides.ffn_output.group,
|
||||
is_first_layer=self.layer_scatter_modes.is_first_layer,
|
||||
is_last_layer=self.is_last_layer,
|
||||
leaves_for_reduce_scatter=self.allow_reduce_scatter,
|
||||
)
|
||||
# DSA and hook-less attention take the slice gathered.
|
||||
handoff = _hand_scattered_input_to_attention
|
||||
return _select_boundary_steps(
|
||||
scattered, residual_ops=self._residual_ops, attention_handoff=handoff
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
self, sides: DecoderLayerSides, **kwargs
|
||||
) -> "BoundarySteps":
|
||||
"""The steps this layer runs for a set of declarations."""
|
||||
return _select_boundary_steps(sides, residual_ops=self._residual_ops, **kwargs)
|
||||
|
||||
def _post_init_communicate(self) -> Tuple[Callable, Callable]:
|
||||
"""The attention input move and the postprocess the scatter modes
|
||||
choose."""
|
||||
if _generic_prefill_cp_shards_tokens() and _gathers_over_attention_cp():
|
||||
# These tables have no attention-CP gather.
|
||||
raise NotImplementedError(
|
||||
"a DSA or MLA prefill CP layer outside the declarations"
|
||||
)
|
||||
return (
|
||||
CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
@@ -1048,7 +1163,8 @@ class LayerCommunicator:
|
||||
post_residual_addition: Optional[torch.Tensor] = None,
|
||||
):
|
||||
self.publish_attn_lora_layout()
|
||||
|
||||
# The layer stack's first layer starts its residual from its input.
|
||||
enters = residual is None and self.layer_scatter_modes.is_first_layer
|
||||
# What the previous layer left to complete: an UnreducedOutput, or a
|
||||
# producer's handoff that one of the fused entries consumes.
|
||||
owed = None if isinstance(hidden_states, torch.Tensor) else hidden_states
|
||||
@@ -1084,17 +1200,24 @@ class LayerCommunicator:
|
||||
hidden_states, residual = layer_input(
|
||||
hidden_states, residual, self._context
|
||||
)
|
||||
if hidden_states.shape[0] == 0:
|
||||
residual = hidden_states
|
||||
elif owed is not None:
|
||||
ops = self._residual_ops
|
||||
if enters:
|
||||
hidden_states, residual = ops.enter(hidden_states), None
|
||||
if owed is not None and hidden_states.shape[0] != 0:
|
||||
hidden_states = reduce_output(owed)
|
||||
hidden_states, residual = _attn_input_update_and_read_residual(
|
||||
quant_format
|
||||
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
||||
if residual is None:
|
||||
# The previous layer already wrote its output into the residual.
|
||||
hidden_states, residual = ops.read_attention_input(
|
||||
hidden_states, self.input_layernorm, quant_format
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = _attn_input_update_and_read_residual(
|
||||
quant_format
|
||||
)(self.input_layernorm, hidden_states, residual, post_residual_addition)
|
||||
hidden_states, residual = ops.update_and_read_attention_input(
|
||||
hidden_states,
|
||||
residual,
|
||||
self.input_layernorm,
|
||||
quant_format,
|
||||
post_residual_addition,
|
||||
)
|
||||
|
||||
return self._finish_prepare_attn(
|
||||
hidden_states=hidden_states,
|
||||
@@ -1122,24 +1245,27 @@ class LayerCommunicator:
|
||||
|
||||
def _finish_prepare_attn(self, hidden_states, residual, forward_batch):
|
||||
"""Tail every prepare_attn path must run, or ``attn_inputs`` is unset."""
|
||||
hidden_states = self._batch_steps(forward_batch).attention_input(
|
||||
steps = self._batch_steps(forward_batch)
|
||||
hidden_states = steps.attention_input(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
)
|
||||
get_attn_tp_context().set_attn_inputs(attn_inputs)
|
||||
hidden_states = steps.attention_handoff(
|
||||
hidden_states, forward_batch, self.qkv_latent_func
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_attn_input_fusions(self) -> Tuple[Callable, ...]:
|
||||
"""The fused kernels that complete what the previous layer left together
|
||||
with the residual update and the input norm, in the order they are tried.
|
||||
Each takes (owed, residual, forward_batch, post_residual_addition) and
|
||||
returns None when it does not take the batch."""
|
||||
if not hasattr(self.input_layernorm, "forward_with_allreduce_fusion"):
|
||||
returns None when it does not take the batch. They add the residual
|
||||
plainly."""
|
||||
if not (
|
||||
self._residual_ops.adds_plainly
|
||||
and hasattr(self.input_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
return ()
|
||||
self._attn_input_fuses_quant = (
|
||||
self.enable_fused_ar_quant
|
||||
@@ -1194,19 +1320,20 @@ class LayerCommunicator:
|
||||
|
||||
def _select_mlp_input(self) -> Tuple[Callable, Tuple["FusedMlpInput", ...]]:
|
||||
"""The attention-TP -> FFN boundary's steps, chosen from the layouts, and
|
||||
the fused kernels they try first. The MHC and DSA-CP communicators pick
|
||||
their own implementations."""
|
||||
the fused kernels they try first."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
ops = self._residual_ops
|
||||
if kind is MlpInputKind.NORM:
|
||||
return _mlp_input_norm, ()
|
||||
return partial(_mlp_input_norm, residual_ops=ops), ()
|
||||
if kind is MlpInputKind.ATTN_TP_ALL_REDUCE:
|
||||
return _mlp_input_attn_tp_all_reduce, ()
|
||||
return partial(_mlp_input_attn_tp_all_reduce, residual_ops=ops), ()
|
||||
if kind is MlpInputKind.SCATTER:
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual_input_mode == ScatterMode.TP_ATTN_FULL,
|
||||
residual_ops=ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -1217,7 +1344,10 @@ class LayerCommunicator:
|
||||
steps = partial(
|
||||
_mlp_input_gather,
|
||||
order=_mlp_input_order(
|
||||
self._context, residual_input_mode, tuple(f.run for f in fusions)
|
||||
self._context,
|
||||
residual_input_mode,
|
||||
tuple(f.run for f in fusions),
|
||||
residual_ops=ops,
|
||||
),
|
||||
)
|
||||
if kind is MlpInputKind.GATHER_MOE_CP:
|
||||
@@ -1226,8 +1356,11 @@ class LayerCommunicator:
|
||||
|
||||
def _select_mlp_input_fusions(self) -> Tuple["FusedMlpInput", ...]:
|
||||
"""The fused kernels that can take the attention -> FFN steps, in the
|
||||
order they are tried."""
|
||||
if not hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion"):
|
||||
order they are tried. They add the residual plainly."""
|
||||
if not (
|
||||
self._residual_ops.adds_plainly
|
||||
and hasattr(self.post_attention_layernorm, "forward_with_allreduce_fusion")
|
||||
):
|
||||
return ()
|
||||
return (
|
||||
FusedMlpInput(
|
||||
@@ -1286,25 +1419,23 @@ class LayerCommunicator:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
steps = self._batch_steps(forward_batch)
|
||||
if steps.returns_over_dp:
|
||||
step = self._postprocess_dp_step(forward_batch) or _redistribute_output
|
||||
return _to_local_tokens(step, forward_batch, hidden_states), residual
|
||||
return steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
return self._complete_ffn_output_now(
|
||||
hidden_states,
|
||||
residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
dp_step=self._postprocess_dp_step(forward_batch),
|
||||
)
|
||||
|
||||
def _local_token_move_can_go_to_next_layer(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter."""
|
||||
return self._batch_steps(forward_batch).returns_over_dp
|
||||
output back to this rank's tokens: the base postprocess scatter, when
|
||||
the next layer's input also writes the output into the residual."""
|
||||
return (
|
||||
self._batch_steps(forward_batch).returns_over_dp
|
||||
and not self._residual_ops.updates_residual_after_ffn
|
||||
)
|
||||
|
||||
def _postprocess_dp_step(
|
||||
self, forward_batch: ForwardBatch
|
||||
@@ -1418,16 +1549,27 @@ class LayerCommunicator:
|
||||
forward_batch: ForwardBatch,
|
||||
dp_step: Optional[Callable],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This layer's postprocess, run with the attention-DP step the FFN exit
|
||||
chose."""
|
||||
if self._local_token_move_can_go_to_next_layer(forward_batch):
|
||||
return (
|
||||
_to_local_tokens(
|
||||
dp_step or _redistribute_output, forward_batch, hidden_states
|
||||
),
|
||||
residual,
|
||||
"""This layer's postprocess, run with the attention-DP step already
|
||||
chosen: the move back to where the next layer reads the FFN output, then
|
||||
the write-back into the residual for a layer that does it itself."""
|
||||
steps = self._batch_steps(forward_batch)
|
||||
if steps.returns_over_dp:
|
||||
hidden_states = _to_local_tokens(
|
||||
dp_step or _redistribute_output, forward_batch, hidden_states
|
||||
)
|
||||
return self.postprocess_layer(hidden_states, residual, forward_batch)
|
||||
else:
|
||||
hidden_states, residual = steps.ffn_output_move(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
is_layer_sparse=self.layer_scatter_modes.is_layer_sparse,
|
||||
)
|
||||
if residual is not None and self._residual_ops.updates_residual_after_ffn:
|
||||
hidden_states = self._residual_ops.update_residual(hidden_states, residual)
|
||||
residual = None
|
||||
return hidden_states, residual
|
||||
|
||||
def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit":
|
||||
"""Decide once how this layer's FFN output reduction completes. Use the
|
||||
@@ -1751,7 +1893,9 @@ class CommunicateSimpleFn:
|
||||
|
||||
|
||||
def _redistribute_from_attn_tp_shards(tensor: torch.Tensor) -> torch.Tensor:
|
||||
gathered = get_local_dp_buffer(get_parallel().attn_tp_group)
|
||||
gathered = get_local_dp_buffer(
|
||||
get_parallel().attn_tp_group, hidden_size=tensor.shape[-1]
|
||||
)
|
||||
attn_tp_all_gather_into_tensor(gathered, tensor)
|
||||
return gathered
|
||||
|
||||
@@ -1808,10 +1952,11 @@ def _redistribute_input_to_moe_cp(
|
||||
|
||||
|
||||
def _mlp_input_reduce_output(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
not forward_batch.forward_mode.is_decode_or_idle()
|
||||
may_quantize
|
||||
and not forward_batch.forward_mode.is_decode_or_idle()
|
||||
and get_exec().comm.enable_quant_communications
|
||||
):
|
||||
return attention_tensor_model_parallel_quant_all_reduce(hidden_states)
|
||||
@@ -1873,17 +2018,21 @@ def _mlp_input_without_dp(
|
||||
*,
|
||||
gathers_residual: bool,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
for fused in fusions:
|
||||
result = fused(hidden_states, residual, forward_batch)
|
||||
if result is not None:
|
||||
return result
|
||||
hidden_states = _mlp_input_reduce_output(hidden_states, forward_batch)
|
||||
# MHC sums its streams in full precision.
|
||||
hidden_states = _mlp_input_reduce_output(
|
||||
hidden_states, forward_batch, may_quantize=residual_ops.adds_plainly
|
||||
)
|
||||
if _is_npu and context.cache is not None:
|
||||
_ = prepare_weight_cache(hidden_states, context.cache)
|
||||
return layernorm(hidden_states, residual)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_dp_replicate(
|
||||
@@ -1896,12 +2045,14 @@ def _mlp_input_dp_replicate(
|
||||
gathers_residual: bool,
|
||||
reduces_attention_tp: bool,
|
||||
places_cp_shards: bool = False,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, add and
|
||||
normalize locally, then gather. With ``places_cp_shards`` each CP rank puts
|
||||
its shard of the DP group's tokens beside the others' in the group's slot."""
|
||||
"""Attention DP: complete the attention-TP sum if it is owed, write the
|
||||
output into the residual and read the FFN input locally, then gather. With
|
||||
``places_cp_shards`` each CP rank puts its shard of the DP group's tokens
|
||||
beside the others' in the group's slot."""
|
||||
if gathers_residual:
|
||||
residual = _redistribute_from_attn_tp_shards(residual)
|
||||
residual = residual_ops.residual_from_attn_tp_shards(residual)
|
||||
if hidden_states.shape[0] != 0:
|
||||
if reduces_attention_tp:
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
@@ -1909,7 +2060,13 @@ def _mlp_input_dp_replicate(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = residual_ops.update_and_read_ffn_input(
|
||||
hidden_states, residual, layernorm
|
||||
)
|
||||
cp_shard_counts = _cp_shard_token_rows(forward_batch) if places_cp_shards else None
|
||||
return (
|
||||
_redistribute_input_to_dp(hidden_states, forward_batch, cp_shard_counts),
|
||||
@@ -1948,6 +2105,7 @@ def _mlp_input_order(
|
||||
context: CommunicateContext,
|
||||
residual_input_mode: ScatterMode,
|
||||
fusions: Tuple[Callable, ...],
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Callable:
|
||||
"""The steps from the attention output to the FFN input, chosen from facts
|
||||
fixed at construction."""
|
||||
@@ -1956,13 +2114,21 @@ def _mlp_input_order(
|
||||
)
|
||||
if context.attn_dp_size == 1:
|
||||
return partial(
|
||||
_mlp_input_without_dp, gathers_residual=gathers_residual, fusions=fusions
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=fusions,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
if context.force_layernorm_before_dp_gather or context.attn_tp_size == 1:
|
||||
if (
|
||||
context.force_layernorm_before_dp_gather
|
||||
or context.attn_tp_size == 1
|
||||
or not residual_ops.adds_plainly
|
||||
):
|
||||
return partial(
|
||||
_mlp_input_dp_replicate,
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=context.attn_tp_size > 1,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(_mlp_input_dp_partial, gathers_residual=gathers_residual)
|
||||
|
||||
@@ -2049,6 +2215,53 @@ def _same_ranks(a: GroupCoordinator, b: GroupCoordinator) -> bool:
|
||||
return sorted(a.ranks) == sorted(b.ranks)
|
||||
|
||||
|
||||
def _hand_qkv_hook_its_input(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Give the attention's QKV hook the attention input."""
|
||||
if qkv_latent_func is not None:
|
||||
get_attn_tp_context().set_attn_inputs(
|
||||
AttentionInputs(hidden_states, forward_batch, qkv_latent_func)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _hand_scattered_input_to_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
qkv_latent_func: Optional[Callable],
|
||||
) -> torch.Tensor:
|
||||
"""Input-scattered attention takes each rank's slice, and its QKV hook
|
||||
gathers the rows after the projection. DSA and attention without a hook
|
||||
consume full hidden states, so those are gathered here, and the hook is
|
||||
told they are."""
|
||||
ctx = get_attn_tp_context()
|
||||
if ctx.is_dsa or qkv_latent_func is None:
|
||||
hidden_states = _tp_all_gather_scattered_rows(hidden_states, forward_batch)
|
||||
if qkv_latent_func is not None:
|
||||
ctx.set_attn_inputs(
|
||||
AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
qkv_latent_func,
|
||||
is_pre_gathered=ctx.is_dsa,
|
||||
)
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _tp_all_gather_scattered_rows(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
# Input-scattered attention keeps the same number of tokens on every TP rank.
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
"""The steps a batch runs at a layer's boundaries: into the attention,
|
||||
from the attention output to the FFN input, and the FFN output on to the
|
||||
@@ -2070,6 +2283,9 @@ class BoundarySteps(msgspec.Struct, frozen=True):
|
||||
# Completes what the layer's input owes before the input norm; None when it
|
||||
# owes nothing.
|
||||
layer_input: Optional[Callable] = None
|
||||
# Hands the attention its input once attention_input has moved it:
|
||||
# (hidden_states, forward_batch, qkv_latent_func) -> hidden_states.
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input
|
||||
|
||||
@property
|
||||
def returns_over_dp(self) -> bool:
|
||||
@@ -2082,14 +2298,17 @@ def _select_boundary_steps(
|
||||
fusions: Tuple["FusedMlpInput", ...] = (),
|
||||
force_layernorm_before_gather: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
attention_handoff: Callable = _hand_qkv_hook_its_input,
|
||||
) -> BoundarySteps:
|
||||
"""The steps a set of declarations chooses; ``cp_moves`` for the ones
|
||||
that gather over attention CP."""
|
||||
"""The steps a set of declarations chooses, around the layer's residual
|
||||
operations; ``cp_moves`` for the ones that gather over attention CP."""
|
||||
returns_over_dp, ffn_output_move, completes_sum = _select_ffn_output_move(
|
||||
sides.ffn_output,
|
||||
residual=sides.ffn_residual_rows,
|
||||
to=sides.output_rows,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
rows = sides.input_rows
|
||||
layer_input = None
|
||||
@@ -2113,6 +2332,7 @@ def _select_boundary_steps(
|
||||
fusions=fusions,
|
||||
residual_joins_sum=sides.residual_joins_attention_sum,
|
||||
cp_moves=cp_moves,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return BoundarySteps(
|
||||
attention_input=_select_attention_input_move(rows, sides.attention),
|
||||
@@ -2123,6 +2343,7 @@ def _select_boundary_steps(
|
||||
ffn_sum_is_movable=sides.ffn_output.group is not None,
|
||||
fused=fused,
|
||||
layer_input=layer_input,
|
||||
attention_handoff=attention_handoff,
|
||||
)
|
||||
|
||||
|
||||
@@ -2164,13 +2385,16 @@ def _select_ffn_input(
|
||||
fusions: Tuple[FusedMlpInput, ...],
|
||||
residual_joins_sum: bool = False,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Tuple[Callable, Tuple[FusedMlpInput, ...]]:
|
||||
"""The steps from the attention output to the FFN input, and the fused
|
||||
kernels they try first: complete the attention-TP sum, move the residual to
|
||||
the rows it has while the FFN runs, add it and normalize, and bring the rows
|
||||
to what the FFN's group needs: a gather over attention DP, or each rank's
|
||||
own slice. A kernel in ``fusions`` is tried only when nothing is gathered or
|
||||
sliced, and only if it completes the sum the attention output owes."""
|
||||
the rows it has while the FFN runs, write the output into it and read the
|
||||
FFN input, and bring the rows to what the FFN's group needs: a gather over
|
||||
attention DP, or each rank's own slice. A kernel in ``fusions`` is tried
|
||||
only when nothing is gathered or sliced, and only if it completes the sum
|
||||
the attention output owes. A write-back that is not a plain add runs only
|
||||
after the sum completes."""
|
||||
# What the attention output owes decides the steps: the attention-TP sum,
|
||||
# always left by the output projection, or nothing.
|
||||
owes_attention_tp = produced.group is SumGroup.ATTN_TP
|
||||
@@ -2193,9 +2417,19 @@ def _select_ffn_input(
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return (
|
||||
partial(_mlp_input_scatter, scatters_residual=residual != residual_to),
|
||||
partial(
|
||||
_mlp_input_scatter,
|
||||
scatters_residual=residual != residual_to,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# The residual stays on each rank's slice while the FFN takes the
|
||||
# attention's rows (MHC on an input-scattered batch).
|
||||
if gathered or not owes_attention_tp or residual != residual_to:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return partial(_mlp_input_on_residual_shard, residual_ops=residual_ops), ()
|
||||
if gathered == {TokenAxis.ATTN_CP}:
|
||||
# Each CP rank completes its own chunk, then the CP moves gather them.
|
||||
if cp_moves is None:
|
||||
@@ -2208,6 +2442,7 @@ def _select_ffn_input(
|
||||
force_layernorm_before_gather=force_layernorm_before_gather,
|
||||
fusions=fusions,
|
||||
residual_joins_sum=residual_joins_sum,
|
||||
residual_ops=residual_ops,
|
||||
)
|
||||
return partial(cp_moves.gather, gather=on_chunk), fused
|
||||
if (
|
||||
@@ -2226,10 +2461,12 @@ def _select_ffn_input(
|
||||
gathers_residual = residual != residual_to
|
||||
if not gathered:
|
||||
if not owes_attention_tp:
|
||||
return _mlp_input_norm, ()
|
||||
return partial(_mlp_input_norm, residual_ops=residual_ops), ()
|
||||
if gathers_residual and residual_joins_sum:
|
||||
# Each rank adds its slice of the residual into its share of the
|
||||
# sum, so the all-reduce also brings the residual back to every row.
|
||||
if not residual_ops.adds_plainly:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
return _mlp_input_residual_into_sum, ()
|
||||
fused = tuple(f for f in fusions if f.completes is produced.group)
|
||||
return (
|
||||
@@ -2237,6 +2474,7 @@ def _select_ffn_input(
|
||||
_mlp_input_without_dp,
|
||||
gathers_residual=gathers_residual,
|
||||
fusions=tuple(f.run for f in fused),
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
fused,
|
||||
)
|
||||
@@ -2246,7 +2484,11 @@ def _select_ffn_input(
|
||||
# Over attention DP and CP, the DP gather puts each CP rank's shard in its
|
||||
# DP group's slot, so the one DP sum gathers both axes.
|
||||
places_cp_shards = TokenAxis.ATTN_CP in gathered
|
||||
if owes_attention_tp and not force_layernorm_before_gather:
|
||||
if (
|
||||
owes_attention_tp
|
||||
and not force_layernorm_before_gather
|
||||
and residual_ops.adds_plainly
|
||||
):
|
||||
return (
|
||||
partial(
|
||||
_mlp_input_dp_partial,
|
||||
@@ -2261,6 +2503,7 @@ def _select_ffn_input(
|
||||
gathers_residual=gathers_residual,
|
||||
reduces_attention_tp=owes_attention_tp,
|
||||
places_cp_shards=places_cp_shards,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
(),
|
||||
)
|
||||
@@ -2272,21 +2515,38 @@ def _select_ffn_output_move(
|
||||
residual: Layout,
|
||||
to: Layout,
|
||||
cp_moves: Optional[CpMoves] = None,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
) -> Tuple[bool, Optional[Callable], bool]:
|
||||
"""How the FFN output reaches the rows the layer hands on: whether it goes
|
||||
back by undoing the attention-DP gather (the FFN exit and postprocess run
|
||||
that step), or else the postprocess that moves it, None when there is none
|
||||
to choose here; and whether that move also completes the sum the FFN
|
||||
leaves."""
|
||||
pair = CommunicateSummableTensorPairFn
|
||||
if produced.layout == residual:
|
||||
if to == residual:
|
||||
return False, CommunicateSummableTensorPairFn._trivial, False
|
||||
return False, pair._trivial, False
|
||||
if to.sharded == residual.sharded - {TokenAxis.ATTN_TP_SCATTER}:
|
||||
# Each rank's slice back to the attention's rows: fold the residual
|
||||
# into the output, then gather over attention TP.
|
||||
return False, CommunicateSummableTensorPairFn._gather, False
|
||||
# Each rank's slice back to the attention's rows: write the output
|
||||
# into the residual, then gather over attention TP.
|
||||
return False, partial(pair._gather, residual_ops=residual_ops), False
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
returned = residual.sharded - produced.layout.sharded
|
||||
if returned == {TokenAxis.ATTN_TP_SCATTER} and to in (residual, produced.layout):
|
||||
# The residual stays on each rank's slice (MHC on an input-scattered
|
||||
# batch): a reduce-scatter onto the slice completes the sum the FFN
|
||||
# leaves; a complete output is only sliced.
|
||||
sums = produced.leaves_for_reduce_scatter
|
||||
return (
|
||||
False,
|
||||
partial(
|
||||
pair._onto_residual_shard,
|
||||
sums=sums,
|
||||
gathers_back=to != residual,
|
||||
residual_ops=residual_ops,
|
||||
),
|
||||
sums,
|
||||
)
|
||||
if to != residual or not produced.layout.sharded <= residual.sharded:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
if returned == {TokenAxis.ATTN_CP}:
|
||||
@@ -2371,11 +2631,10 @@ def _mlp_input_norm(
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
# TODO move these `if shape != 0` into LayerNorm itself
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_attn_tp_all_reduce(
|
||||
@@ -2384,6 +2643,8 @@ def _mlp_input_attn_tp_all_reduce(
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
"""All-reduce hidden states inside the attention TP group, then layernorm.
|
||||
|
||||
@@ -2392,9 +2653,7 @@ def _mlp_input_attn_tp_all_reduce(
|
||||
``TP_ATTN_FULL`` across the boundary.
|
||||
"""
|
||||
hidden_states = get_parallel().attn_tp_group.all_reduce(hidden_states)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
return hidden_states, residual
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_scatter(
|
||||
@@ -2405,14 +2664,35 @@ def _mlp_input_scatter(
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
scatters_residual: bool,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
):
|
||||
hidden_states = _reduce_and_redistribute_output_to_attn_tp_shards(
|
||||
hidden_states, context
|
||||
)
|
||||
if scatters_residual:
|
||||
residual = _redistribute_to_attn_tp_shards(residual, context)
|
||||
if hidden_states.shape[0] != 0:
|
||||
hidden_states, residual = layernorm(hidden_states, residual)
|
||||
residual = residual_ops.residual_to_attn_tp_shard(residual, context)
|
||||
return residual_ops.update_and_read_ffn_input(hidden_states, residual, layernorm)
|
||||
|
||||
|
||||
def _mlp_input_on_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_ops: ResidualOps,
|
||||
):
|
||||
"""The residual stays on each rank's slice while the FFN takes the full
|
||||
rows: reduce-scatter the attention output onto the slice, which completes
|
||||
its sum, write it into the residual and read the FFN input there, then
|
||||
gather the input back into the attention output's rows."""
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
shard = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(shard, hidden_states)
|
||||
shard, residual = residual_ops.update_and_read_ffn_input(shard, residual, layernorm)
|
||||
attn_tp_all_gather_into_tensor(hidden_states, shard)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -2681,11 +2961,41 @@ class CommunicateSummableTensorPairFn:
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
residual_ops: ResidualOps = ADD_AND_NORM,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states += residual
|
||||
hidden_states = residual_ops.update_residual(hidden_states, residual)
|
||||
return _redistribute_from_attn_tp_shards(hidden_states), None
|
||||
|
||||
@staticmethod
|
||||
def _onto_residual_shard(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
sums: bool,
|
||||
gathers_back: bool,
|
||||
residual_ops: ResidualOps,
|
||||
**kwargs,
|
||||
):
|
||||
"""Bring the FFN output onto the slice of the rows the residual is on:
|
||||
a reduce-scatter that also completes its sum when ``sums``, else this
|
||||
rank's slice of the complete output. With ``gathers_back`` it is
|
||||
written into the residual there and the full rows are gathered back."""
|
||||
if sums:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
|
||||
else:
|
||||
hidden_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
if not gathers_back:
|
||||
return hidden_states, residual
|
||||
local_states = residual_ops.update_residual(hidden_states, residual)
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -12,62 +12,28 @@
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_contract, hc_expand
|
||||
from sglang.srt.distributed.communication_op import (
|
||||
attention_tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.boundary_layout import DecoderLayerSides, TokenAxis
|
||||
from sglang.srt.layers.communicator import (
|
||||
AttentionInputs,
|
||||
CommunicateContext,
|
||||
CommunicateSimpleFn,
|
||||
CommunicateSummableTensorPairFn,
|
||||
FfnCompletion,
|
||||
BoundarySteps,
|
||||
LayerCommunicator,
|
||||
LayerScatterModes,
|
||||
MlpInputKind,
|
||||
ScatterMode,
|
||||
get_attn_tp_context,
|
||||
mlp_input_kind,
|
||||
tp_reduce_scatter,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_reduce_scatter_tensor,
|
||||
dp_gather_replicate,
|
||||
dp_reduce_scatter_tensor,
|
||||
dp_scatter,
|
||||
get_dp_global_num_tokens,
|
||||
get_global_dp_buffer,
|
||||
get_local_dp_buffer_mhc,
|
||||
is_allocation_symmetric,
|
||||
)
|
||||
from sglang.srt.layers.moe import should_use_dp_reduce_scatterv
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def tp_all_gather_hidden_states(hidden_states, forward_batch):
|
||||
assert get_attn_tp_context().input_scattered, (
|
||||
"Input scattered guarantees same num tokens in TP group."
|
||||
)
|
||||
total_tokens = forward_batch.input_ids.shape[0]
|
||||
output = hidden_states.new_empty((total_tokens, hidden_states.shape[-1]))
|
||||
get_parallel().tp_group.all_gather_into_tensor(output, hidden_states)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
@dataclass
|
||||
class MHCState:
|
||||
"""Parameters belong to the owning layer; this state only holds scratch
|
||||
"""A layer's residual as hyper-connection streams, with the residual
|
||||
operations the boundary steps run (communicator.ResidualOps): the residual
|
||||
is hc_mult streams, a stage's output is written in with hc_post and the
|
||||
next stage's input read with hc_pre and the norm. A read produces the
|
||||
h_res / h_post the next write-back consumes; they move with the tokens.
|
||||
Parameters belong to the owning layer; this state only holds scratch
|
||||
shared across communication stages."""
|
||||
|
||||
hc_mult: int
|
||||
@@ -75,9 +41,19 @@ class MHCState:
|
||||
hc_ffn_pre: Callable
|
||||
hc_post: Callable
|
||||
hc_ffn_post_pre: Optional[Callable] = None
|
||||
# The last layer's write-back also contracts the streams into the hidden
|
||||
# states the layer stack hands on.
|
||||
is_last_layer: bool = False
|
||||
h_res: Optional[torch.Tensor] = None
|
||||
h_post: Optional[torch.Tensor] = None
|
||||
|
||||
# hc_post is not a plain add, so it runs only once the sum it writes in
|
||||
# is complete.
|
||||
adds_plainly = False
|
||||
# The FFN output's write-back uses the coefficients this layer's FFN input
|
||||
# read produced, so the layer runs it itself.
|
||||
updates_residual_after_ffn = True
|
||||
|
||||
@staticmethod
|
||||
def _resolve_out_norm(out_norm):
|
||||
if out_norm is None:
|
||||
@@ -131,261 +107,45 @@ class MHCState:
|
||||
self.h_res = None
|
||||
self.h_post = None
|
||||
|
||||
def enter(self, hidden_states):
|
||||
return hc_expand(hidden_states, self.hc_mult)
|
||||
|
||||
class MHCCommunicateWithAllReduceAndLayerNormFn:
|
||||
@staticmethod
|
||||
def _scatter_hidden_states_and_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_input_mode,
|
||||
mhc: MHCState,
|
||||
def read_attention_input(self, residual, norm, quant_format):
|
||||
if quant_format:
|
||||
raise NotImplementedError(f"an MHC attention input in {quant_format=}")
|
||||
return self.attn_split(residual, out_norm=norm)
|
||||
|
||||
def update_and_read_attention_input(
|
||||
self, hidden_states, residual, norm, quant_format, post_residual_addition
|
||||
):
|
||||
input_hidden_states = hidden_states
|
||||
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
attn_tp_reduce_scatter_tensor(hidden_states, input_hidden_states)
|
||||
if residual_input_mode == ScatterMode.TP_ATTN_FULL:
|
||||
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
||||
mhc.h_res = mhc.h_res.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
mhc.h_post = mhc.h_post.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
# An MHC layer's input arrives with the previous FFN output written in.
|
||||
raise NotImplementedError("an MHC layer takes its input written back")
|
||||
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
def update_and_read_ffn_input(self, hidden_states, residual, norm):
|
||||
return self.attn_to_mlp(hidden_states, residual, out_norm=norm)
|
||||
|
||||
def update_residual(self, hidden_states, residual):
|
||||
hidden_states = self.mlp_combine(hidden_states, residual)
|
||||
self.reset_aux()
|
||||
if self.is_last_layer:
|
||||
hidden_states = hc_contract(hidden_states, self.hc_mult)
|
||||
return hidden_states
|
||||
|
||||
def residual_to_attn_tp_shard(self, residual, context):
|
||||
rank, size = context.attn_tp_rank, context.attn_tp_size
|
||||
self.h_res = self.h_res.tensor_split(size)[rank]
|
||||
self.h_post = self.h_post.tensor_split(size)[rank]
|
||||
return residual.tensor_split(size)[rank]
|
||||
|
||||
def residual_from_attn_tp_shards(self, residual):
|
||||
raise NotImplementedError(
|
||||
"Unsupported: h_res/h_post allgather not implemented."
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _simple(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
):
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _tp_all_reduce_with_scattered_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
):
|
||||
if hidden_states.shape[0] == 0:
|
||||
return hidden_states, hidden_states
|
||||
|
||||
scatter_states = hidden_states.tensor_split(context.tp_size)[context.tp_rank]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(scatter_states, hidden_states)
|
||||
|
||||
scatter_states, residual = mhc.attn_to_mlp(
|
||||
scatter_states, residual, out_norm=layernorm
|
||||
)
|
||||
|
||||
attn_tp_all_gather_into_tensor(hidden_states, scatter_states)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
@staticmethod
|
||||
def _gather_hidden_states_and_residual(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
layernorm: torch.nn.Module,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
residual_input_mode,
|
||||
mhc: MHCState,
|
||||
):
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return MHCCommunicateWithAllReduceAndLayerNormFn._tp_all_reduce_with_scattered_residual(
|
||||
hidden_states,
|
||||
residual,
|
||||
layernorm,
|
||||
context,
|
||||
mhc=mhc,
|
||||
)
|
||||
|
||||
if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Unsupported: h_res/h_post allgather not implemented."
|
||||
)
|
||||
|
||||
hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states)
|
||||
if context.attn_dp_size != 1:
|
||||
if hidden_states.shape[0] != 0:
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group,
|
||||
disabled=not is_allocation_symmetric(),
|
||||
):
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
)
|
||||
else:
|
||||
hidden_states, residual = mhc.attn_to_mlp(hidden_states, residual)
|
||||
|
||||
hidden_states, local_hidden_states = (
|
||||
get_global_dp_buffer(get_parallel().tp_group),
|
||||
hidden_states,
|
||||
)
|
||||
dp_gather_replicate(hidden_states, local_hidden_states, forward_batch)
|
||||
else:
|
||||
hidden_states, residual = mhc.attn_to_mlp(
|
||||
hidden_states, residual, out_norm=layernorm
|
||||
)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class MHCCommunicateSummableTensorPairFn(CommunicateSummableTensorPairFn):
|
||||
@staticmethod
|
||||
def get_fn(
|
||||
hidden_states_input_mode: ScatterMode,
|
||||
residual_input_mode: ScatterMode,
|
||||
output_mode: ScatterMode,
|
||||
context: CommunicateContext,
|
||||
):
|
||||
fn = CommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode,
|
||||
residual_input_mode,
|
||||
output_mode,
|
||||
context,
|
||||
)
|
||||
replacements = {
|
||||
CommunicateSummableTensorPairFn._trivial: MHCCommunicateSummableTensorPairFn._trivial,
|
||||
CommunicateSummableTensorPairFn._scatter_hidden_states: MHCCommunicateSummableTensorPairFn._scatter_hidden_states,
|
||||
CommunicateSummableTensorPairFn._gather: MHCCommunicateSummableTensorPairFn._gather,
|
||||
CommunicateSummableTensorPairFn._scatter: MHCCommunicateSummableTensorPairFn._scatter,
|
||||
}
|
||||
return replacements.get(fn, fn)
|
||||
|
||||
@staticmethod
|
||||
def _trivial(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
if get_attn_tp_context().input_scattered:
|
||||
hidden_states, _ = tp_reduce_scatter(hidden_states, None, context)
|
||||
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
if not is_last_layer:
|
||||
return hidden_states, None
|
||||
|
||||
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
|
||||
if get_attn_tp_context().input_scattered:
|
||||
local_states = hidden_states
|
||||
hidden_states = local_states.new_empty(
|
||||
local_states.shape[0] * context.tp_size, *local_states.shape[1:]
|
||||
)
|
||||
get_parallel().tp_group.all_gather_into_tensor(hidden_states, local_states)
|
||||
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter_hidden_states(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
allow_reduce_scatter: bool = False,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states, global_hidden_states = (
|
||||
get_local_dp_buffer_mhc(get_parallel().tp_group, 1),
|
||||
hidden_states,
|
||||
)
|
||||
# MoE skips its post-expert all-reduce with reduce_scatterv, so this
|
||||
# scatter must reduce while combining local-expert partial sums.
|
||||
if should_use_dp_reduce_scatterv():
|
||||
get_parallel().tp_group.reduce_scatterv(
|
||||
global_hidden_states,
|
||||
output=hidden_states,
|
||||
sizes=get_dp_global_num_tokens(),
|
||||
)
|
||||
elif allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len():
|
||||
dp_reduce_scatter_tensor(hidden_states, global_hidden_states)
|
||||
else:
|
||||
dp_scatter(hidden_states, global_hidden_states, forward_batch)
|
||||
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
if not is_last_layer:
|
||||
return hidden_states, None
|
||||
|
||||
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _gather(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
if is_last_layer:
|
||||
hidden_states = hc_contract(hidden_states, mhc.hc_mult)
|
||||
|
||||
hidden_states, local_hidden_states = (
|
||||
get_local_dp_buffer_mhc(
|
||||
get_parallel().tp_group, 1 if is_last_layer else mhc.hc_mult
|
||||
),
|
||||
hidden_states,
|
||||
)
|
||||
|
||||
attn_tp_all_gather_into_tensor(hidden_states, local_hidden_states)
|
||||
return hidden_states, None
|
||||
|
||||
@staticmethod
|
||||
def _scatter(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
context: CommunicateContext,
|
||||
*,
|
||||
mhc: MHCState,
|
||||
is_last_layer: bool,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = hidden_states.tensor_split(context.attn_tp_size)[
|
||||
context.attn_tp_rank
|
||||
]
|
||||
residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank]
|
||||
|
||||
hidden_states = mhc.mlp_combine(hidden_states, residual)
|
||||
|
||||
return hidden_states, None
|
||||
|
||||
|
||||
class MHCLayerCommunicator(LayerCommunicator):
|
||||
# Chooses its own boundary steps, not from the declarations.
|
||||
_takes_declared_boundaries = False
|
||||
"""A layer whose residual is hyper-connection streams: the shared boundary
|
||||
steps, run with MHCState's residual operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -395,158 +155,50 @@ class MHCLayerCommunicator(LayerCommunicator):
|
||||
allow_reduce_scatter: bool = False,
|
||||
qkv_latent_func: Optional[Callable] = None,
|
||||
*,
|
||||
is_first_layer: bool,
|
||||
hc_mult: int,
|
||||
hc_attn_pre: Callable,
|
||||
hc_ffn_pre: Callable,
|
||||
hc_post: Callable,
|
||||
hc_ffn_post_pre: Optional[Callable] = None,
|
||||
):
|
||||
self.is_first_layer = is_first_layer
|
||||
self.mhc = MHCState(
|
||||
hc_mult=hc_mult,
|
||||
hc_attn_pre=hc_attn_pre,
|
||||
hc_ffn_pre=hc_ffn_pre,
|
||||
hc_post=hc_post,
|
||||
hc_ffn_post_pre=hc_ffn_post_pre,
|
||||
is_last_layer=layer_scatter_modes.is_last_layer,
|
||||
)
|
||||
|
||||
# The postprocess writes the FFN output into the streams, so the FFN's
|
||||
# sum never waits for the next layer.
|
||||
super().__init__(
|
||||
layer_scatter_modes,
|
||||
input_layernorm,
|
||||
post_attention_layernorm,
|
||||
allow_reduce_scatter,
|
||||
qkv_latent_func,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
residual_ops=self.mhc,
|
||||
)
|
||||
|
||||
def _steps_from_declarations(
|
||||
self, sides: DecoderLayerSides, **kwargs
|
||||
) -> BoundarySteps:
|
||||
# MHC has not been run with an FFN input gathered over attention CP.
|
||||
if (
|
||||
TokenAxis.ATTN_CP
|
||||
in sides.attention_output.layout.sharded - sides.ffn.layout.sharded
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"MHCLayerCommunicator with a gather over attention CP: {sides=}"
|
||||
)
|
||||
return super()._steps_from_declarations(sides, **kwargs)
|
||||
|
||||
def _post_init_communicate(self):
|
||||
# Base MOE_FULL callables do not accept ``mhc``, so reject this
|
||||
# combination at construction.
|
||||
if self.layer_scatter_modes.mlp_mode == ScatterMode.MOE_FULL:
|
||||
raise NotImplementedError(
|
||||
"MHCLayerCommunicator does not support MOE_FULL "
|
||||
"(moe_dp_size < attention_context_parallel_size). Increase "
|
||||
"moe_dp_size to match attention_context_parallel_size."
|
||||
)
|
||||
self._communicate_simple_fn = CommunicateSimpleFn.get_fn(
|
||||
input_mode=self.layer_scatter_modes.layer_input_mode,
|
||||
output_mode=self.layer_scatter_modes.attn_mode,
|
||||
context=self._context,
|
||||
)
|
||||
self._communicate_summable_tensor_pair_fn = (
|
||||
MHCCommunicateSummableTensorPairFn.get_fn(
|
||||
hidden_states_input_mode=self.layer_scatter_modes.mlp_mode,
|
||||
residual_input_mode=self.layer_scatter_modes.middle_residual_mode,
|
||||
output_mode=self.layer_scatter_modes.layer_output_mode,
|
||||
context=self._context,
|
||||
)
|
||||
)
|
||||
# MHC's own prepare_attn and postprocess run these.
|
||||
return self._communicate_simple_fn, self._communicate_summable_tensor_pair_fn
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
):
|
||||
self.publish_attn_lora_layout()
|
||||
if self.is_first_layer:
|
||||
if get_attn_tp_context().input_scattered:
|
||||
hidden_states, _ = tp_reduce_scatter(
|
||||
hidden_states,
|
||||
None,
|
||||
self._context,
|
||||
)
|
||||
hidden_states = hc_expand(hidden_states, self.mhc.hc_mult)
|
||||
|
||||
hidden_states, residual = self.mhc.attn_split(
|
||||
hidden_states, out_norm=self.input_layernorm
|
||||
)
|
||||
|
||||
hidden_states = self._communicate_simple_fn(
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
)
|
||||
|
||||
# DSA and attention without a QKV hook consume full hidden states, so
|
||||
# gather them before attention.
|
||||
ctx = get_attn_tp_context()
|
||||
dsa_pre_gather = ctx.input_scattered and ctx.is_dsa
|
||||
no_qkv_latent_pre_gather = ctx.input_scattered and self.qkv_latent_func is None
|
||||
if dsa_pre_gather or no_qkv_latent_pre_gather:
|
||||
hidden_states = tp_all_gather_hidden_states(hidden_states, forward_batch)
|
||||
|
||||
if self.qkv_latent_func is not None:
|
||||
attn_inputs = AttentionInputs(
|
||||
hidden_states,
|
||||
forward_batch,
|
||||
self.qkv_latent_func,
|
||||
is_pre_gathered=dsa_pre_gather,
|
||||
)
|
||||
ctx.set_attn_inputs(attn_inputs)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_mlp_input(self):
|
||||
"""MHC's own implementation of each boundary kind, applied to this
|
||||
layer's MHC state."""
|
||||
kind = mlp_input_kind(self.layer_scatter_modes, self._context)
|
||||
residual_input_mode = self.layer_scatter_modes.layer_input_mode
|
||||
fns = MHCCommunicateWithAllReduceAndLayerNormFn
|
||||
if kind is MlpInputKind.NORM:
|
||||
return partial(fns._simple, mhc=self.mhc), ()
|
||||
if kind is MlpInputKind.GATHER:
|
||||
fn = fns._gather_hidden_states_and_residual
|
||||
elif kind is MlpInputKind.SCATTER:
|
||||
fn = fns._scatter_hidden_states_and_residual
|
||||
else:
|
||||
raise NotImplementedError(f"MHCLayerCommunicator does not support {kind}")
|
||||
return partial(fn, residual_input_mode=residual_input_mode, mhc=self.mhc), ()
|
||||
|
||||
def postprocess_layer(self, hidden_states, residual, forward_batch):
|
||||
hidden_states, residual = self._communicate_summable_tensor_pair_fn(
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
forward_batch=forward_batch,
|
||||
context=self._context,
|
||||
allow_reduce_scatter=self.allow_reduce_scatter,
|
||||
mhc=self.mhc,
|
||||
is_last_layer=self.is_last_layer,
|
||||
)
|
||||
self.mhc.reset_aux()
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
def _select_ffn_completion(self, forward_batch: ForwardBatch) -> FfnCompletion:
|
||||
"""An MHC layer's own postprocess completes its FFN output, combining the
|
||||
hyper-connection streams; nothing is left to the next layer."""
|
||||
return FfnCompletion(
|
||||
defer_moe_finalize=False,
|
||||
fuse_mlp_allreduce=False,
|
||||
mlp_reduce_scatter=self.should_use_reduce_scatter(forward_batch),
|
||||
complete=partial(
|
||||
self._complete_ffn_output_now,
|
||||
forward_batch=forward_batch,
|
||||
dp_step=None,
|
||||
),
|
||||
)
|
||||
|
||||
def should_use_reduce_scatter(self, forward_batch: ForwardBatch):
|
||||
if not self.allow_reduce_scatter:
|
||||
return False
|
||||
if (
|
||||
self._communicate_summable_tensor_pair_fn
|
||||
is MHCCommunicateSummableTensorPairFn._scatter_hidden_states
|
||||
):
|
||||
# reduce_scatterv already combines expert outputs; returning False
|
||||
# would make RowParallelLinear perform an extra all-reduce.
|
||||
if should_use_dp_reduce_scatterv():
|
||||
return True
|
||||
if forward_batch.dp_padding_mode.is_max_len():
|
||||
return True
|
||||
|
||||
if get_attn_tp_context().input_scattered:
|
||||
return True
|
||||
return False
|
||||
return super()._post_init_communicate()
|
||||
|
||||
@@ -258,27 +258,14 @@ class _DpGatheredBufferWrapper:
|
||||
return buffer
|
||||
|
||||
@classmethod
|
||||
def get_local_dp_buffer(cls, group: GroupCoordinator) -> torch.Tensor:
|
||||
|
||||
dp = get_flags().dp
|
||||
with use_symmetric_memory(group, disabled=not cls._dp_max_padding):
|
||||
buffer = torch.empty(
|
||||
(cls._local_dp_buffer_len, dp.buffer_hidden_size),
|
||||
dtype=dp.buffer_dtype,
|
||||
device=dp.buffer_device,
|
||||
)
|
||||
return buffer
|
||||
|
||||
@classmethod
|
||||
def get_local_dp_buffer_mhc(
|
||||
cls, group: GroupCoordinator, n: int = 1
|
||||
def get_local_dp_buffer(
|
||||
cls, group: GroupCoordinator, hidden_size: Optional[int] = None
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.runtime_context import get_flags
|
||||
|
||||
dp = get_flags().dp
|
||||
with use_symmetric_memory(group, disabled=not cls._dp_max_padding):
|
||||
buffer = torch.empty(
|
||||
(cls._local_dp_buffer_len, dp.buffer_hidden_size * n),
|
||||
(cls._local_dp_buffer_len, hidden_size or dp.buffer_hidden_size),
|
||||
dtype=dp.buffer_dtype,
|
||||
device=dp.buffer_device,
|
||||
)
|
||||
@@ -344,12 +331,14 @@ def get_global_dp_buffer(group: GroupCoordinator) -> torch.Tensor:
|
||||
return _DpGatheredBufferWrapper.get_global_dp_buffer(group=group)
|
||||
|
||||
|
||||
def get_local_dp_buffer(group: GroupCoordinator) -> torch.Tensor:
|
||||
return _DpGatheredBufferWrapper.get_local_dp_buffer(group=group)
|
||||
|
||||
|
||||
def get_local_dp_buffer_mhc(group: GroupCoordinator, n: int = 1) -> torch.Tensor:
|
||||
return _DpGatheredBufferWrapper.get_local_dp_buffer_mhc(group=group, n=n)
|
||||
def get_local_dp_buffer(
|
||||
group: GroupCoordinator, hidden_size: Optional[int] = None
|
||||
) -> torch.Tensor:
|
||||
"""A buffer for this rank's local DP rows, ``hidden_size`` wide (the model's
|
||||
hidden size by default)."""
|
||||
return _DpGatheredBufferWrapper.get_local_dp_buffer(
|
||||
group=group, hidden_size=hidden_size
|
||||
)
|
||||
|
||||
|
||||
def get_global_dp_buffer_len() -> int:
|
||||
|
||||
@@ -795,7 +795,6 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
|
||||
if self.config.mhc:
|
||||
mhc_kwargs: Dict[str, Any] = dict(
|
||||
is_first_layer=(self.layer_id == 0),
|
||||
hc_mult=config.hc_mult,
|
||||
hc_attn_pre=self.hc_attn_pre,
|
||||
hc_ffn_pre=self.hc_ffn_pre,
|
||||
|
||||
Reference in New Issue
Block a user